From 46968ef0cf0bb0b7a005e92260c34eee690a8058 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sun, 22 Oct 2017 14:21:40 +0300 Subject: [PATCH] [Sync] Added Marker (refactored Kazimir) --- utils/Sync/kazimir/InstagramToken.py | 72 +++++++ utils/Sync/kazimir/Marker.py | 199 ++++++++++++++++++++ utils/Sync/kazimir/Token.py | 251 +++++++++++++++++++++++++ utils/Sync/kazimir/YoutubeToken.py | 32 ++++ utils/Sync/kazimir/fix_links_quotes.py | 50 +++++ 5 files changed, 604 insertions(+) create mode 100644 utils/Sync/kazimir/InstagramToken.py create mode 100644 utils/Sync/kazimir/Marker.py create mode 100644 utils/Sync/kazimir/Token.py create mode 100644 utils/Sync/kazimir/YoutubeToken.py create mode 100644 utils/Sync/kazimir/fix_links_quotes.py diff --git a/utils/Sync/kazimir/InstagramToken.py b/utils/Sync/kazimir/InstagramToken.py new file mode 100644 index 00000000..f11f3ae8 --- /dev/null +++ b/utils/Sync/kazimir/InstagramToken.py @@ -0,0 +1,72 @@ +from urllib.parse import urlparse + +import re +import requests +from markdown.util import etree + +from kazimir.Token import UrlToken, is_url + +pattr = re.compile(r'https?://www\.instagram\.com/p/(?P[\w\d]+)/?(.*)$') + + +class InstagramToken(UrlToken): + @staticmethod + def test(data: str): + if not is_url(data): + return False + + url = urlparse(data) + return url.netloc in ['www.instagram.com', 'instagram.com'] + + async def compile(self): + match = pattr.match(self.data) + if match: + m = match.group('media') + return await embed_instagram(m) + + +async def embed_instagram(media): + """ +
+
+
+
+
+
+ +

+ + Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg + +

+ +

+ A post shared by Шлиссельбургская ДХШ (@hudozka) on + + +

+
+
+ """ + + url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media) + req = requests.get(url) + if req.status_code == 200: + html = req.json()['html'] + html = fix_html(html) + i = etree.fromstring(html) + + obj = etree.Element('div') + obj.set('class', "kazimir__embed") + obj.append(i) + return etree.tostring(obj).decode() + else: + raise Exception( + 'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code) + ) + + +def fix_html(html: str): + return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""') diff --git a/utils/Sync/kazimir/Marker.py b/utils/Sync/kazimir/Marker.py new file mode 100644 index 00000000..bd0d40f1 --- /dev/null +++ b/utils/Sync/kazimir/Marker.py @@ -0,0 +1,199 @@ +import asyncio +import lxml.html +from markdown import markdown +from kazimir.InstagramToken import InstagramToken +from kazimir.YoutubeToken import YoutubeToken +from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, DocumentToken, TokenFactory, \ + DocumentTokenFactory + + +class Marker: + def __init__(self) -> None: + super().__init__() + self.tokens = [] + self.tree_middlewares = [] + + def add_token_factory(self, token): + self.tokens.append(token) + + def add_tree_middleware(self, callback): + self.tree_middlewares.append(callback) + + async def create_tree(self, text: str): + tokens = text.split('\n') + tokens = self.markup_tokens(tokens) + tokens = join_tokens(tokens) + tokens = merge_tokens(tokens) + + compiled = await compile_tokens(tokens) + tree = lxml.html.fromstring(compiled) + return await self.process_tree(tree) + + async def process_tree(self, tree): + for fn in self.tree_middlewares: + tree = await fn(tree) + return tree + + def markup_tokens(self, tokens: list): + marked_tokens = [] + + for data in tokens: + t = self.create_token(data) + if t: + marked_tokens.append(t) + else: + raise Exception(f'Token not recognized for data {data}') + return marked_tokens + + def create_token(self, data): + for f in self.tokens: + if f.token.test(data): + return f.create(data=data) + return None + + +def join_tokens(tokens): + is_split = lambda x: isinstance(x, SplitToken) + + buffer_tag = None + buffer = [] + ts = [] + for token in tokens: + tag = token.name + changed = buffer_tag is not None and tag != buffer_tag + + flush = is_split(token) + if changed: + flush = True + + if not token.joinable: + flush = True + + if flush: + if buffer: + ts.append(buffer) + if is_split(token): + buffer = [] + buffer_tag = None + else: + buffer = [token] + buffer_tag = tag + else: + buffer_tag = tag + buffer.append(token) + + if len(buffer): + ts.append(buffer) + return ts + + +def merge_tokens(tokens: list): + tokens_out = [] + for ts in tokens: + token = ts[0] + for x in ts[1:]: + token = token.merge(x) + tokens_out.append(token) + return tokens_out + + +async def compile_tokens(tokens: list): + result = [] + for token in tokens: + data = await token.compile() + result.append(data) + return '\n'.join(result) + + +def extract_files(text: str) -> [str]: + html = markdown(text) + post_html = lxml.html.fromstring(html) + + images = post_html.cssselect('img') + images = [img.get('src') for img in images] + + return images, [] + + +def html_from_tree(tree): + html = lxml.html.tostring(tree, encoding='unicode') + + return html + + +if __name__ == '__main__': + sample_text = ''' +Hello! + +https://www.instagram1.com/p/BUelE04l1kU + +https://www.youtube.com/watch?v=qnguOqWpraI +https://www.youtube.com/watch?v=qnguOqWpraI +![](1.jpg) +![](2.jpg) + +LOL NOOB +![](12.jpg) +![](22.jpg) +WTF OMG + +![File](file.pdf) +hello.pdf +file.doc +sheet.xlsx + +print.docx +image.jpg 1 +image.png 2 + +image.jpeg Hi + +Bye! +Fye! + +![](1.jpg) + +["Ya"](http://ya.ru) + +More text + ''' + + + async def build_document(data): + text = data['caption'] if data['caption'] else data['file'] + # document = await self.build() + document = { + 'url': data['file'], + 'image_url': data['file'], + 'file_url': data['file'], + 'title': text, + 'file_size': '0 KB', + 'file_format': 'PDF', + } + return document + + + async def run(): + from kazimir.fix_links_quotes import fix_links_quotes + + m = Marker() + m.add_token_factory(TokenFactory(SplitToken)) + m.add_token_factory(TokenFactory(YoutubeToken)) + m.add_token_factory(TokenFactory(InstagramToken)) + m.add_token_factory(TokenFactory(UrlToken)) + m.add_token_factory(TokenFactory(ImageToken)) + m.add_token_factory(DocumentTokenFactory(build=build_document)) + m.add_token_factory(TokenFactory(FileToken)) + m.add_token_factory(TokenFactory(TextToken)) + m.add_tree_middleware(fix_links_quotes) + sample_tree = await m.create_tree(sample_text) + + # for x in sample_tree: + # print(x) + + print(html_from_tree(sample_tree)) + + + loop = asyncio.get_event_loop() + loop.run_until_complete(run()) + loop.close() diff --git a/utils/Sync/kazimir/Token.py b/utils/Sync/kazimir/Token.py new file mode 100644 index 00000000..1ecd33e1 --- /dev/null +++ b/utils/Sync/kazimir/Token.py @@ -0,0 +1,251 @@ +import os +import re +from markdown import markdown +from urllib.parse import urlparse + +image_extensions = ['.jpg', '.jpeg', '.png'] +document_extensions = ['.doc', '.docx', '.pdf', '.xls', '.xlsx'] + + +class TokenFactory: + def __init__(self, token) -> None: + super().__init__() + self.token = token + + def create(self, data): + return self.token(data=data) + + +class DocumentTokenFactory(TokenFactory): + def __init__(self, build) -> None: + super().__init__(DocumentToken) + self.build = build + + def create(self, data): + x = self.token(data=data) + x.build = self.build + return x + + +class Token: + @staticmethod + def test(data: str): + return True + + def __init__(self, name, data) -> None: + super().__init__() + self.name = name + self.data = data + self.joinable = True + + async def compile(self): + return str(self.data) + + def merge(self, token): + return self + + def __repr__(self) -> str: + return f'[Token {self.name} {self.data}]' + + +class SplitToken(Token): + @staticmethod + def test(data: str): + data = data.strip() + return data == '' + + def __init__(self, data) -> None: + super().__init__(name='split', data=None) + + +class TextToken(Token): + @staticmethod + def test(data: str): + return True + + def __init__(self, data) -> None: + super().__init__(name='text', data=data) + + def merge(self, token: Token): + data = self.data + data += '\n' + data += token.data + return TextToken(data.strip()) + + async def compile(self): + return markdown_to_html(self.data) + + +class FileToken(Token): + @staticmethod + def test(data: str): + return is_file(data, document_extensions + image_extensions) + + def __init__(self, data) -> None: + super().__init__(name='file', data=data) + self.joinable = False + + async def compile(self): + data = parse_file(self.data) + url = data['file'] + text = data['caption'] if data['caption'] else url + return f'{text}' + + +class DocumentToken(Token): + @staticmethod + def test(data: str): + return is_document(data) + + def __init__(self, data) -> None: + super().__init__(name='file', data=data) + self.joinable = False + self.build = None + + async def compile(self): + data = parse_file(self.data) + document = await self.build(data) + + return f''' +
+
+ + +
+ {document['title']} +
+ +
+ + {document['file_format']} ({document['file_size']}) + +
+
+
+ '''.strip() + + +class ImageToken(Token): + @staticmethod + def test(data: str): + return is_image(data) + + def __init__(self, data) -> None: + super().__init__(name='image', data=data) + self.joinable = True + + def merge(self, token: Token): + return ImageCollectionToken([self.data, token.data]) + + async def compile(self): + img = self.parse_data() + caption = img['caption'] + file = img['file'] + + tpl = f''' +
+
+ {caption} +
{caption}
+
+
+ ''' + return tpl.format(img).strip() + + def parse_data(self): + return parse_file(self.data) + + +class ImageCollectionToken(ImageToken): + def __init__(self, data) -> None: + super().__init__(data) + self.name = 'image_collection' + + def merge(self, token: Token): + if isinstance(token.data, list): + data = token.data + else: + data = [token.data] + return ImageCollectionToken([*self.data, *data]) + + async def compile(self): + data = self.parse_data() + images = [f'' for x in data] + images = '\n'.join(images) + + return f''' +
+
+ {images} +
+
+ ''' + + def parse_data(self): + images = [parse_file(x) for x in self.data] + return [x['file'] for x in images] + + +class UrlToken(Token): + @staticmethod + def test(data: str): + return is_url(data) + + def __init__(self, data) -> None: + super().__init__(name='url', data=data) + self.joinable = False + + +def markdown_to_html(text): + return markdown(text) + + +def is_url(sample: str): + url = urlparse(sample) + + return url.scheme and url.netloc + + +def is_image(sample: str): + file = parse_file(sample)['file'] + return is_file(file, image_extensions) + + +def is_document(sample: str): + file = parse_file(sample)['file'] + return is_file(file, document_extensions) + + +def is_file(sample: str, extensions): + name, ext = os.path.splitext(sample) + ext = ext.lower() + return ext in extensions + + +def parse_file(sample: str): + """ + ![Caption](file.jpg) -> {file, caption} + file.jpg Caption -> {file, caption} + + :param sample: + :return: + """ + p = re.match(r'!\[(.*)\]\((.*)\)', sample) + if p: + caption = p.group(1) + file = p.group(2) + else: + sample = sample.split(' ') + file = sample[0] + caption = ' '.join(sample[1:]) + return { + 'file': file, + 'caption': caption, + } diff --git a/utils/Sync/kazimir/YoutubeToken.py b/utils/Sync/kazimir/YoutubeToken.py new file mode 100644 index 00000000..bb381a75 --- /dev/null +++ b/utils/Sync/kazimir/YoutubeToken.py @@ -0,0 +1,32 @@ +import re +from urllib.parse import urlparse +from kazimir.Token import UrlToken, is_url + +pattr = re.compile(r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P[A-Za-z0-9_&=-]+)\S*') + + +class YoutubeToken(UrlToken): + @staticmethod + def test(data: str): + if not is_url(data): + return False + + url = urlparse(data) + return url.netloc in ['www.youtube.com', 'youtube.com'] + + async def compile(self): + """ +
+ +
+ """ + + match = pattr.match(self.data) + if match: + video_id = match.group(2) + url = f'//www.youtube.com/embed/{video_id}' + return f''' +
+ '''.strip() + else: + return '' diff --git a/utils/Sync/kazimir/fix_links_quotes.py b/utils/Sync/kazimir/fix_links_quotes.py new file mode 100644 index 00000000..c51e30a9 --- /dev/null +++ b/utils/Sync/kazimir/fix_links_quotes.py @@ -0,0 +1,50 @@ +quote_pairs = [ + ('"', '"'), + ('«', '»'), +] + + +async def fix_links_quotes(tree): + safe = lambda x: x if x else '' + + for link in tree.iterlinks(): + a = link[0] + if a.tag != 'a': + continue + + quote = get_quote_pair(a.text) + if not quote: + continue + + # New text of A without quotes + a.text = a.text[1:-1] + + # Move first quote out of A to the prefix + a.getparent().text = safe(a.getparent().text) + quote[0] + + # Move last quote out of A to the suffix + a.tail = quote[1] + safe(a.tail) + + return tree + + +def get_quote_pair(text: str): + """ + sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair + ? (s, t) + : None + + :param text: + :return: + """ + if len(text) < 2: + return None + + first = text[0] + last = text[-1] + + for pair_first, pair_last in quote_pairs: + if pair_first == first and pair_last == last: + return first, last + + return None