diff --git a/modules/sync/kazimir/DocumentToken.py b/modules/sync/kazimir/DocumentToken.py index e24b5072..c9c97c25 100644 --- a/modules/sync/kazimir/DocumentToken.py +++ b/modules/sync/kazimir/DocumentToken.py @@ -1,5 +1,4 @@ from kazimir.Token import parse_file, is_document, Token -import math class DocumentToken(Token): @@ -16,44 +15,3 @@ class DocumentToken(Token): data = parse_file(self.data) document = await self.build(data) return document - - async def compile(self): - document = await self.get_data() - - return create_document(document) - - -def create_document(document): - file_size = get_size(document['file_size'], 1) - file_format = 'ПДФ' - - return f''' -
-
- - -
- {document['title']} -
- -
- - {file_format} ({file_size}) - -
-
-
- '''.strip() - - -def get_size(number_of_bytes, precision=1): - units = ['байт', 'КБ', 'МБ', 'ГБ', 'ТБ', 'ПБ'] - number = math.floor(math.log(number_of_bytes) / math.log(1024)) - size = (number_of_bytes / math.pow(1024, math.floor(number))) - size = round(size, precision) - - return f'{size} {units[number]}' diff --git a/modules/sync/kazimir/InstagramToken.py b/modules/sync/kazimir/InstagramToken.py index ad7757ea..e3d3fc78 100644 --- a/modules/sync/kazimir/InstagramToken.py +++ b/modules/sync/kazimir/InstagramToken.py @@ -18,11 +18,21 @@ class InstagramToken(UrlToken): url = urlparse(data) return url.netloc in ['www.instagram.com', 'instagram.com'] - async def compile(self): - match = pattr.match(self.data) - if match: - m = match.group('media') - return await embed_instagram(m) + def __init__(self, data: str) -> None: + super().__init__(data=data) + self.name = 'instagram' + + async def get_data(self): + m = pattr.match(self.data) + embed = None + if m: + m = m.group('media') + embed = await embed_instagram(m) + + return { + 'url': self.data, + 'embed': embed, + } async def embed_instagram(media): @@ -51,20 +61,23 @@ async def embed_instagram(media): """ - url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media) + url = f'https://api.instagram.com/oembed?url=http://instagr.am/p/{media}&hidecaption=true&omitscript=true' req = requests.get(url) if req.status_code == 200: html = req.json()['html'] html = fix_html(html) - i = etree.fromstring(html) - obj = etree.Element('div') - obj.set('class', "kazimir__embed") - obj.append(i) - return etree.tostring(obj).decode() + return html + # i = etree.fromstring(html) + + # obj = etree.Element('div') + # obj.set('class', "kazimir__embed") + # obj.append(i) + # return etree.tostring(obj).decode() else: raise Exception( - 'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code) + 'Failed to compile Instagram {media}: Status Code: {status}'.format( + media=media, status=req.status_code) ) diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py index 3b81c9bb..ee013743 100644 --- a/modules/sync/kazimir/Marker.py +++ b/modules/sync/kazimir/Marker.py @@ -8,14 +8,10 @@ class Marker: def __init__(self) -> None: super().__init__() self.tokens = [] - self.tree_middlewares = [] def add_token_factory(self, token): self.tokens.append(token) - def add_tree_middleware(self, callback): - self.tree_middlewares.append(callback) - def parse(self, text: str): tokens = text.split('\n') tokens = self.markup_tokens(tokens) @@ -24,16 +20,6 @@ class Marker: return tokens - async def create_tree(self, tokens: list): - compiled = await compile_tokens(tokens) - tree = lxml.html.fromstring(compiled) - return await self.process_tree(tree) - - async def process_tree(self, tree): - for fn in self.tree_middlewares: - tree = await fn(tree) - return tree - def markup_tokens(self, tokens: list): marked_tokens = [] @@ -95,18 +81,3 @@ def merge_tokens(tokens: list): token = token.merge(x) tokens_out.append(token) return tokens_out - - -async def compile_tokens(tokens: list): - result = [] - for token in tokens: - data = await token.compile() - result.append(data) - return '\n'.join(result) - - - - - - - diff --git a/modules/sync/kazimir/Token.py b/modules/sync/kazimir/Token.py index 4de5ae35..0ac9a9c6 100644 --- a/modules/sync/kazimir/Token.py +++ b/modules/sync/kazimir/Token.py @@ -41,10 +41,6 @@ class Token: async def get_data(self): return self.data - async def compile(self) -> str: - data = await self.get_data() - return str(data) - def merge(self, token): return self @@ -76,9 +72,19 @@ class TextToken(Token): data += token.data return TextToken(data.strip()) - async def compile(self): - data = await self.get_data() - return markdown_to_html(data) + +class HtmlToken(Token): + @staticmethod + def test(data: str): + """ + Result of compiling markdown to html is the same as source data meaning that the source is html + """ + html = markdown(data) + return html == data + + def __init__(self, data: str) -> None: + super().__init__(name='html', data=data) + self.joinable = False class FileToken(Token): @@ -93,12 +99,6 @@ class FileToken(Token): async def get_data(self): return parse_file(self.data) - async def compile(self): - data = await self.get_data() - url = data['file'] - text = data['caption'] if data['caption'] else url - return f'{text}' - class ImageToken(Token): @staticmethod @@ -110,74 +110,11 @@ class ImageToken(Token): self.joinable = False self.build = None - def merge(self, token: Token): - merged_token = ImageCollectionToken([self.data, token.data]) - merged_token.build = self.build - return merged_token - async def get_data(self): data = parse_file(self.data) data = await self.build(data) return data - async def compile(self): - s = 1500 # ('big', 1500, 667) - img = await self.get_data() - alt = img['alt'] - caption = markdown_to_html(img['caption']) - src = img['src'] - src = f'https://images.weserv.nl/?url={src}&w={s}&h={s}' - - tpl = f''' -
-
- {alt} -
{caption}
-
-
- ''' - return tpl.format(img).strip() - - -class ImageCollectionToken(ImageToken): - def __init__(self, data) -> None: - super().__init__(data) - self.name = 'image_collection' - self.build = None - - def merge(self, token: Token): - if isinstance(token.data, list): - data = token.data - else: - data = [token.data] - merged_token = ImageCollectionToken([*self.data, *data]) - merged_token.build = self.build - return merged_token - - async def get_data(self): - items = [parse_file(x) for x in self.data] - images = [] - for x in items: - img = await self.build(x) - images.append(img) - return images - - async def compile(self): - images = await self.get_data() - images = [x['src'] for x in images] - images = [f'' for x in images] - images = '\n'.join(images) - - return f''' -
-
- {images} -
-
- ''' class UrlToken(Token): @staticmethod diff --git a/modules/sync/kazimir/YoutubeToken.py b/modules/sync/kazimir/YoutubeToken.py index c6133d52..2157d957 100644 --- a/modules/sync/kazimir/YoutubeToken.py +++ b/modules/sync/kazimir/YoutubeToken.py @@ -11,22 +11,11 @@ class YoutubeToken(UrlToken): url = urlparse(data) return url.netloc in ['www.youtube.com', 'youtube.com'] - async def compile(self): - """ -
- -
- """ + def __init__(self, data: str) -> None: + super().__init__(data=data) + self.name = 'youtube' - url = urlparse(self.data) - query = parse_qs(url.query) - - if 'v' not in query: - return '' - - video_id = query['v'][0] - - url = f'//www.youtube.com/embed/{video_id}' - return f''' -
- '''.strip() + async def get_data(self): + return { + 'url': self.data + } diff --git a/modules/sync/kazimir/__init__.py b/modules/sync/kazimir/__init__.py deleted file mode 100644 index fa0281ed..00000000 --- a/modules/sync/kazimir/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -import lxml.html -import re -from markdown import markdown - -from kazimir.figure import FigureExtension -from kazimir.instagram import InstagramExtension -from sync.data import request - -def extract_files(text: str) -> [str]: - html = markdown(text) - post_html = lxml.html.fromstring(html) - - images = post_html.cssselect('img') - images = [img.get('src') for img in images] - - return images, [] - - -def html_from_tree(tree): - html = lxml.html.tostring(tree, encoding='unicode') - - return html - - -async def typo(text): - from kazimir import artlebedev - result = await artlebedev.typograf(text) - if result == 'Размер текста ограничен 32 КБ': - return text - - return result - # url = 'http://www.typograf.ru/webservice/' - # params = dict(text=text) - # return await request.post(url, params, {'chr': 'utf-8'}) diff --git a/modules/sync/kazimir/artlebedev.py b/modules/sync/kazimir/artlebedev.py deleted file mode 100644 index c1c42289..00000000 --- a/modules/sync/kazimir/artlebedev.py +++ /dev/null @@ -1,45 +0,0 @@ -from sync.data import request -from lxml import etree - - -async def typograf(text, options={}): - text = text.replace('&', '&') - text = text.replace('<', '<') - text = text.replace('>', '>') - - body = ''' - - - - - {text} - 1 - 0 - 1 - 3 - - - - '''.strip().format(text=text) - - url = 'http://typograf.artlebedev.ru/webservices/typograf.asmx' - resp = await request.post(url, body, {}, { - 'Content-Type': 'text/xml', - 'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText', - }) - resp_text = resp.encode() - - try: - # create xml tree from response - response_tree = etree.fromstring(resp_text) - - # look for ProcessTextResult - typograf_response = response_tree.getchildren()[0].getchildren()[0].getchildren()[0].text - except Exception as response_tree: - print(response_tree) - return text - - typograf_response = typograf_response.replace('&', '&') - typograf_response = typograf_response.replace('<', '<') - typograf_response = typograf_response.replace('>', '>') - return typograf_response diff --git a/modules/sync/kazimir/figure.py b/modules/sync/kazimir/figure.py deleted file mode 100644 index 1d0a9668..00000000 --- a/modules/sync/kazimir/figure.py +++ /dev/null @@ -1,44 +0,0 @@ -from markdown import Extension -from markdown.inlinepatterns import IMAGE_LINK_RE, IMAGE_REFERENCE_RE -from markdown.blockprocessors import BlockProcessor -from markdown.util import etree -import re - -FIGURES = [ - u'^\s*' + IMAGE_LINK_RE + u'\s*$', - u'^\s*' + IMAGE_REFERENCE_RE + u'\s*$', -] - -TAG_FIGURE = 'figure' -TAG_FIGCAPTION = 'figcaption' -NAME = 'FigureAltCaption' - - -class FigureProcessor(BlockProcessor): - FIGURES_RE = re.compile('|'.join(f for f in FIGURES)) - - def test(self, parent, block): - is_image = bool(self.FIGURES_RE.search(block)) - is_single_list = (len(block.splitlines()) == 1) - is_in_figure = parent.tag == TAG_FIGURE - - return is_image and is_single_list and not is_in_figure - - def run(self, parent, blocks): - raw_block = blocks.pop(0) - caption_text = self.FIGURES_RE.search(raw_block).group(1) - - figure = etree.SubElement(parent, TAG_FIGURE) - figure.text = raw_block - - figcaption_elem = etree.SubElement(figure, TAG_FIGCAPTION) - figcaption_elem.text = caption_text - - -class FigureExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.parser.blockprocessors.add( - NAME, - FigureProcessor(md.parser), - ' [s]ample tex[t] -> (s, t) -> match pair from quote_pair - ? (s, t) - : None - - :param text: - :return: - """ - if len(text) < 2: - return None - - first = text[0] - last = text[-1] - - for pair_first, pair_last in quote_pairs: - if pair_first == first and pair_last == last: - return first, last - - return None diff --git a/modules/sync/kazimir/instagram.py b/modules/sync/kazimir/instagram.py deleted file mode 100644 index ee5f8c0c..00000000 --- a/modules/sync/kazimir/instagram.py +++ /dev/null @@ -1,68 +0,0 @@ -import requests -from markdown import Extension -from markdown.inlinepatterns import Pattern -from markdown.util import etree - - -class InstagramExtension(Extension): - def add_inline(self, md, name, klass, re): - pattern = klass(re) - pattern.md = md - pattern.ext = self - md.inlinePatterns.add(name, pattern, "[\w\d]+)/?(.*)$' - self.add_inline(md, 'instagram', Instagram, match_url) - - -class Instagram(Pattern): - def handleMatch(self, m): - return embed_instagram(m.group('media')) - - -def embed_instagram(media): - """ -
-
-
-
-
-
- -

- - Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg - -

- -

- A post shared by Шлиссельбургская ДХШ (@hudozka) on - - -

-
-
- """ - - url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media) - req = requests.get(url) - if req.status_code == 200: - html = req.json()['html'] - html = fix_html(html) - i = etree.fromstring(html) - - obj = etree.Element('div') - obj.set('class', "kazimir__embed") - obj.append(i) - return obj - else: - raise Exception( - 'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code) - ) - - -def fix_html(html: str): - return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""') diff --git a/modules/sync/settings.py b/modules/sync/settings.py index 808b1c08..13d79699 100644 --- a/modules/sync/settings.py +++ b/modules/sync/settings.py @@ -65,7 +65,7 @@ collection_images = 'images' collection_documents = 'documents' collection_pages = 'pages' -hash_salt_pages = '4' +hash_salt_pages = '6' hash_salt_documents = '4' hash_salt_images = '5' diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py index 5ccfef90..4ad4e024 100644 --- a/modules/sync/sync/__init__.py +++ b/modules/sync/sync/__init__.py @@ -10,7 +10,7 @@ from kazimir.DocumentToken import DocumentToken from kazimir.YoutubeToken import YoutubeToken from kazimir.InstagramToken import InstagramToken from kazimir.Marker import Marker -from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken +from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, HtmlToken from sync.data import Provider from sync.models import Model from sync.models.Document import Document @@ -66,8 +66,6 @@ async def untouched(items: [Model]) -> [Model]: async def create_post(provider: Provider, folder: str, md: str, sizes): - from kazimir.fix_links_quotes import fix_links_quotes - images = [] documents = [] @@ -128,14 +126,10 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document)) m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file)) # m.add_token_factory(TokenFactory(FileToken)) + m.add_token_factory(TokenFactory(HtmlToken)) m.add_token_factory(TokenFactory(TextToken)) - m.add_tree_middleware(fix_links_quotes) tokens = m.parse(md) - tree = await m.create_tree(tokens) - - post = kazimir.html_from_tree(tree) - post = await kazimir.typo(post) encoded_tokens = [] for t in tokens: @@ -145,11 +139,4 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): 'data': data, }) - return post, images, documents, encoded_tokens - - -def title_from_html(md): - html = lxml.html.fromstring(md) - titles = html.cssselect('h1') - if len(titles): - return titles[0].text + return images, documents, encoded_tokens diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py index aa2b22e6..aec6a836 100644 --- a/modules/sync/sync/models/Page.py +++ b/modules/sync/sync/models/Page.py @@ -2,21 +2,36 @@ import os import logging from lxml import etree import lxml.html -import kazimir import settings from db import collection from sync.data import request from sync.models import Model -from sync import create_date, title_from_html, create_post +from sync import create_date, create_post from sync.models.Image import Image from utils.hash import hash_str from utils.front_matter import parse_yaml_front_matter from utils.text.transform import url_encode_text +from markdown import markdown + logger = logging.getLogger(settings.name + '.Page') store = collection(settings.collection_pages) +def extract_files(html) -> [str]: + images = html.cssselect('img') + images = [img.get('src') for img in images] + + return images, [] + + +def title_from_html(html): + titles = html.cssselect('h1') + if len(titles): + return titles[0].text + return '' + + def find(store, item_id: str): q = {'id': item_id} try: @@ -103,7 +118,6 @@ class Page(Model): self.images = [] self.documents = [] self.title = None - self.data = None self.preview = None self.tokens = None @@ -114,7 +128,12 @@ class Page(Model): super().__init__(provider, store, file, params=params) def init(self): - images, documents = kazimir.extract_files(self.get_param('content')) + content = self.get_param('content') + html = markdown(content) + html_tree = lxml.html.fromstring(html) + images, documents = extract_files(html_tree) + self.title = title_from_html(html_tree) + self.params['images'] = images self.params['documents'] = documents @@ -136,9 +155,7 @@ class Page(Model): sizes = kwargs['sizes'] markdown_post = self.get_param('content') - post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) - self.title = title_from_html(post) - self.data = post + images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) self.images = images self.documents = documents self.tokens = tokens @@ -158,7 +175,6 @@ class Page(Model): 'hash': self.hash, 'url': self.url, 'file': self.file, - 'data': self.data, 'images': images, 'documents': documents, 'title': self.title,