From 4ae002b1ffaef6298d94a4483825e99e145b3e4a Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Fri, 27 Oct 2017 15:01:32 +0300 Subject: [PATCH] [Sync] 1. marker refactoring; 2. added document url token --- utils/Sync/kazimir/DocumentToken.py | 42 +++++++++ utils/Sync/kazimir/DocumentUrlToken.py | 37 ++++++++ utils/Sync/kazimir/Marker.py | 96 +-------------------- utils/Sync/kazimir/MarkerRun.py | 113 +++++++++++++++++++++++++ utils/Sync/kazimir/Token.py | 60 +++++-------- 5 files changed, 215 insertions(+), 133 deletions(-) create mode 100644 utils/Sync/kazimir/DocumentToken.py create mode 100644 utils/Sync/kazimir/DocumentUrlToken.py create mode 100644 utils/Sync/kazimir/MarkerRun.py diff --git a/utils/Sync/kazimir/DocumentToken.py b/utils/Sync/kazimir/DocumentToken.py new file mode 100644 index 00000000..60e6cb72 --- /dev/null +++ b/utils/Sync/kazimir/DocumentToken.py @@ -0,0 +1,42 @@ +from kazimir.Token import parse_file, is_document, Token + + +class DocumentToken(Token): + @staticmethod + def test(data: str): + return is_document(data) + + def __init__(self, data) -> None: + super().__init__(name='file', data=data) + self.joinable = False + self.build = None + + async def compile(self): + data = parse_file(self.data) + document = await self.build(data) + + return create_document(document) + + +def create_document(document): + return f''' +
+
+ + +
+ {document['title']} +
+ +
+ + {document['file_format']} ({document['file_size']}) + +
+
+
+ '''.strip() diff --git a/utils/Sync/kazimir/DocumentUrlToken.py b/utils/Sync/kazimir/DocumentUrlToken.py new file mode 100644 index 00000000..e290c61d --- /dev/null +++ b/utils/Sync/kazimir/DocumentUrlToken.py @@ -0,0 +1,37 @@ +from kazimir.Token import UrlToken, parse_link + + +class DocumentUrlToken(UrlToken): + @staticmethod + def test(data: str): + link = parse_link(data) + if not link: + return False + url = link['url'] + host = url.netloc + path = url.path + return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org') + + def __init__(self, data) -> None: + super().__init__(data) + self.build = None + + async def compile(self): + from kazimir.DocumentToken import create_document + + link = parse_link(self.data) + url = link['url'] + doc_id = url.path.replace('/document/', '') + doc = await self.build(doc_id) + + if not doc: + raise Exception(f'Document {doc_id} not found') + + # return f'''DOCUMENT: {# doc['title']}''' + return create_document(doc) + + def merge(self, token): + return self + + def __repr__(self) -> str: + return f'[Token {self.name} {self.data}]' diff --git a/utils/Sync/kazimir/Marker.py b/utils/Sync/kazimir/Marker.py index c351797f..ddf21502 100644 --- a/utils/Sync/kazimir/Marker.py +++ b/utils/Sync/kazimir/Marker.py @@ -1,12 +1,7 @@ -import asyncio import lxml.html from markdown import markdown -from kazimir.CSVToken import CSVToken -from kazimir.InstagramToken import InstagramToken -from kazimir.YoutubeToken import YoutubeToken -from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, DocumentToken, TokenFactory, \ - BuildTokenFactory +from kazimir.Token import SplitToken class Marker: @@ -121,92 +116,3 @@ def html_from_tree(tree): html = lxml.html.tostring(tree, encoding='unicode') return html - - -if __name__ == '__main__': - sample_text = ''' -Hello! - -https://www.instagram1.com/p/BUelE04l1kU - -https://www.youtube.com/watch?v=qnguOqWpraI -https://www.youtube.com/watch?v=qnguOqWpraI -![](1.jpg) -![](2.jpg) - -LOL NOOB -![](12.jpg) -![](22.jpg) -WTF OMG - -![File](file.pdf) -hello.pdf -file.doc -sheet.xlsx - -print.docx -image.jpg 1 -image.png 2 - -image.jpeg Hi - -Bye! -Fye! - -![](1.jpg) - -["Ya"](http://ya.ru) - -More text - -file.csv - ''' - - - async def build_document(data): - text = data['caption'] if data['caption'] else data['file'] - # document = await self.build() - document = { - 'url': data['file'], - 'image_url': data['file'], - 'file_url': data['file'], - 'title': text, - 'file_size': '0 KB', - 'file_format': 'PDF', - } - return document - - async def read_csv(data): - return '\n'.join([ - 'N,Name', - f'0,{data}', - '1,Pop', - '2,Top', - ]) - - - async def run(): - from kazimir.fix_links_quotes import fix_links_quotes - - m = Marker() - m.add_token_factory(TokenFactory(SplitToken)) - m.add_token_factory(TokenFactory(YoutubeToken)) - m.add_token_factory(TokenFactory(InstagramToken)) - m.add_token_factory(TokenFactory(UrlToken)) - m.add_token_factory(TokenFactory(ImageToken)) - m.add_token_factory(BuildTokenFactory(build=build_document)) - m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv)) - m.add_token_factory(TokenFactory(FileToken)) - m.add_token_factory(TokenFactory(TextToken)) - m.add_tree_middleware(fix_links_quotes) - sample_tree = await m.create_tree(sample_text) - - # for x in sample_tree: - # print(x) - - print(html_from_tree(sample_tree)) - - - loop = asyncio.get_event_loop() - loop.run_until_complete(run()) - loop.close() diff --git a/utils/Sync/kazimir/MarkerRun.py b/utils/Sync/kazimir/MarkerRun.py new file mode 100644 index 00000000..846736b4 --- /dev/null +++ b/utils/Sync/kazimir/MarkerRun.py @@ -0,0 +1,113 @@ +import asyncio + +from kazimir.DocumentUrlToken import DocumentUrlToken +from kazimir.DocumentToken import DocumentToken +from kazimir.CSVToken import CSVToken +from kazimir.InstagramToken import InstagramToken +from kazimir.Marker import Marker, html_from_tree +from kazimir.YoutubeToken import YoutubeToken +from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \ + BuildTokenFactory +from sync import Document + +if __name__ == '__main__': + sample_text = ''' +Hello! + +https://www.instagram1.com/p/BUelE04l1kU + +https://www.youtube.com/watch?v=qnguOqWpraI +https://www.youtube.com/watch?v=qnguOqWpraI +![](1.jpg) +![](2.jpg) + +LOL NOOB +![](12.jpg) +![](22.jpg) +WTF OMG + +![](/document/documents-arkhiv-ustav-2013-pdf) + +![File](file.pdf) +hello.pdf +file.doc +sheet.xlsx + +print.docx +image.jpg 1 +image.png 2 + +image.jpeg Hi + +Bye! +Fye! + +![](1.jpg) + +["Ya"](http://ya.ru) + +More text + +file.csv + ''' + + + async def build_document(data): + text = data['caption'] if data['caption'] else data['file'] + # document = await self.build() + document = { + 'url': data['file'], + 'image_url': data['file'], + 'file_url': data['file'], + 'title': text, + 'file_size': '0 KB', + 'file_format': 'PDF', + } + return document + + async def read_csv(data): + return '\n'.join([ + 'N,Name', + f'0,{data}', + '1,Pop', + '2,Top', + ]) + + async def read_document(data): + doc = await Document.find_one({'id': data}) + return { + 'url': doc['url'], + 'image_url': doc['url'], + 'file_url': doc['url'], + 'title': doc['title'], + 'file_size': '0 KB', + 'file_format': 'PDF', + } + + + async def run(): + from kazimir.fix_links_quotes import fix_links_quotes + + m = Marker() + m.add_token_factory(TokenFactory(SplitToken)) + m.add_token_factory(TokenFactory(YoutubeToken)) + m.add_token_factory(TokenFactory(InstagramToken)) + m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document)) + m.add_token_factory(TokenFactory(UrlToken)) + # m.add_token_factory(TokenFactory(ImageToken)) + m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document)) + m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv)) + m.add_token_factory(TokenFactory(FileToken)) + m.add_token_factory(TokenFactory(TextToken)) + m.add_tree_middleware(fix_links_quotes) + sample_tree = await m.create_tree(sample_text) + + # for x in sample_tree: + # print(x) + + print(html_from_tree(sample_tree)) + + + loop = asyncio.get_event_loop() + loop.run_until_complete(run()) + loop.close() diff --git a/utils/Sync/kazimir/Token.py b/utils/Sync/kazimir/Token.py index 3d09f9df..ba7aab04 100644 --- a/utils/Sync/kazimir/Token.py +++ b/utils/Sync/kazimir/Token.py @@ -92,43 +92,6 @@ class FileToken(Token): return f'{text}' -class DocumentToken(Token): - @staticmethod - def test(data: str): - return is_document(data) - - def __init__(self, data) -> None: - super().__init__(name='file', data=data) - self.joinable = False - self.build = None - - async def compile(self): - data = parse_file(self.data) - document = await self.build(data) - - return f''' -
-
- - -
- {document['title']} -
- -
- - {document['file_format']} ({document['file_size']}) - -
-
-
- '''.strip() - - class ImageToken(Token): @staticmethod def test(data: str): @@ -221,8 +184,11 @@ def markdown_to_html(text): def is_url(sample: str): - url = urlparse(sample) + link = parse_link(sample) + if not link: + return False + url = link['url'] return url.scheme and url.netloc @@ -262,3 +228,21 @@ def parse_file(sample: str): 'file': file, 'caption': caption, } + + +def parse_link(sample: str): + """ + ![Alt](href) -> {href, alt} + + :param sample: + :return: + """ + p = re.match(r'!\[(.*)\]\((.*)\)', sample) + if not p: + return None + + url = p.group(2) + return { + 'url': urlparse(url), + 'alt': p.group(1), + }