[Sync] 1. marker refactoring; 2. added document url token
This commit is contained in:
Roman Timashev committed
1 parent
6e93875377
commit
4ae002b1ff
5 files changed
+215
-133
No files matched your search
@@ -0,0 +1,42 @@
|
||||
from kazimir.Token import parse_file, is_document, Token
|
||||
|
||||
|
||||
class DocumentToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return is_document(data)
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='file', data=data)
|
||||
self.joinable = False
|
||||
self.build = None
|
||||
|
||||
async def compile(self):
|
||||
data = parse_file(self.data)
|
||||
document = await self.build(data)
|
||||
|
||||
return create_document(document)
|
||||
|
||||
|
||||
def create_document(document):
|
||||
return f'''
|
||||
<div>
|
||||
<div class="document-row">
|
||||
<a href="{document['url']}" class="invisible">
|
||||
<div class="document-row__image">
|
||||
<img src="{document['image_url']}" alt="{document['title']}">
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<div class="document-row__file">
|
||||
<a href="{document['url']}">{document['title']}</a>
|
||||
</div>
|
||||
|
||||
<div class="document-row__file-info">
|
||||
<a href="{document['file_url']}" target="_blank">
|
||||
{document['file_format']} ({document['file_size']})
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
'''.strip()
|
||||
@@ -0,0 +1,37 @@
|
||||
from kazimir.Token import UrlToken, parse_link
|
||||
|
||||
|
||||
class DocumentUrlToken(UrlToken):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
link = parse_link(data)
|
||||
if not link:
|
||||
return False
|
||||
url = link['url']
|
||||
host = url.netloc
|
||||
path = url.path
|
||||
return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org')
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(data)
|
||||
self.build = None
|
||||
|
||||
async def compile(self):
|
||||
from kazimir.DocumentToken import create_document
|
||||
|
||||
link = parse_link(self.data)
|
||||
url = link['url']
|
||||
doc_id = url.path.replace('/document/', '')
|
||||
doc = await self.build(doc_id)
|
||||
|
||||
if not doc:
|
||||
raise Exception(f'Document {doc_id} not found')
|
||||
|
||||
# return f'''DOCUMENT: {# doc['title']}'''
|
||||
return create_document(doc)
|
||||
|
||||
def merge(self, token):
|
||||
return self
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'[Token {self.name} {self.data}]'
|
||||
@@ -1,12 +1,7 @@
|
||||
import asyncio
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
|
||||
from kazimir.CSVToken import CSVToken
|
||||
from kazimir.InstagramToken import InstagramToken
|
||||
from kazimir.YoutubeToken import YoutubeToken
|
||||
from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, DocumentToken, TokenFactory, \
|
||||
BuildTokenFactory
|
||||
from kazimir.Token import SplitToken
|
||||
|
||||
|
||||
class Marker:
|
||||
@@ -121,92 +116,3 @@ def html_from_tree(tree):
|
||||
html = lxml.html.tostring(tree, encoding='unicode')
|
||||
|
||||
return html
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sample_text = '''
|
||||
Hello!
|
||||
|
||||
https://www.instagram1.com/p/BUelE04l1kU
|
||||
|
||||
https://www.youtube.com/watch?v=qnguOqWpraI
|
||||
https://www.youtube.com/watch?v=qnguOqWpraI
|
||||
|
||||
|
||||
|
||||
LOL NOOB
|
||||
|
||||
|
||||
WTF OMG
|
||||
|
||||
|
||||
hello.pdf
|
||||
file.doc
|
||||
sheet.xlsx
|
||||
|
||||
print.docx
|
||||
image.jpg 1
|
||||
image.png 2
|
||||
|
||||
image.jpeg Hi
|
||||
|
||||
Bye!
|
||||
Fye!
|
||||
|
||||
|
||||
|
||||
["Ya"](http://ya.ru)
|
||||
|
||||
More text
|
||||
|
||||
file.csv
|
||||
'''
|
||||
|
||||
|
||||
async def build_document(data):
|
||||
text = data['caption'] if data['caption'] else data['file']
|
||||
# document = await self.build()
|
||||
document = {
|
||||
'url': data['file'],
|
||||
'image_url': data['file'],
|
||||
'file_url': data['file'],
|
||||
'title': text,
|
||||
'file_size': '0 KB',
|
||||
'file_format': 'PDF',
|
||||
}
|
||||
return document
|
||||
|
||||
async def read_csv(data):
|
||||
return '\n'.join([
|
||||
'N,Name',
|
||||
f'0,{data}',
|
||||
'1,Pop',
|
||||
'2,Top',
|
||||
])
|
||||
|
||||
|
||||
async def run():
|
||||
from kazimir.fix_links_quotes import fix_links_quotes
|
||||
|
||||
m = Marker()
|
||||
m.add_token_factory(TokenFactory(SplitToken))
|
||||
m.add_token_factory(TokenFactory(YoutubeToken))
|
||||
m.add_token_factory(TokenFactory(InstagramToken))
|
||||
m.add_token_factory(TokenFactory(UrlToken))
|
||||
m.add_token_factory(TokenFactory(ImageToken))
|
||||
m.add_token_factory(BuildTokenFactory(build=build_document))
|
||||
m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv))
|
||||
m.add_token_factory(TokenFactory(FileToken))
|
||||
m.add_token_factory(TokenFactory(TextToken))
|
||||
m.add_tree_middleware(fix_links_quotes)
|
||||
sample_tree = await m.create_tree(sample_text)
|
||||
|
||||
# for x in sample_tree:
|
||||
# print(x)
|
||||
|
||||
print(html_from_tree(sample_tree))
|
||||
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(run())
|
||||
loop.close()
|
||||
@@ -0,0 +1,113 @@
|
||||
import asyncio
|
||||
|
||||
from kazimir.DocumentUrlToken import DocumentUrlToken
|
||||
from kazimir.DocumentToken import DocumentToken
|
||||
from kazimir.CSVToken import CSVToken
|
||||
from kazimir.InstagramToken import InstagramToken
|
||||
from kazimir.Marker import Marker, html_from_tree
|
||||
from kazimir.YoutubeToken import YoutubeToken
|
||||
from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \
|
||||
BuildTokenFactory
|
||||
from sync import Document
|
||||
|
||||
if __name__ == '__main__':
|
||||
sample_text = '''
|
||||
Hello!
|
||||
|
||||
https://www.instagram1.com/p/BUelE04l1kU
|
||||
|
||||
https://www.youtube.com/watch?v=qnguOqWpraI
|
||||
https://www.youtube.com/watch?v=qnguOqWpraI
|
||||
|
||||
|
||||
|
||||
LOL NOOB
|
||||
|
||||
|
||||
WTF OMG
|
||||
|
||||
|
||||
|
||||
|
||||
hello.pdf
|
||||
file.doc
|
||||
sheet.xlsx
|
||||
|
||||
print.docx
|
||||
image.jpg 1
|
||||
image.png 2
|
||||
|
||||
image.jpeg Hi
|
||||
|
||||
Bye!
|
||||
Fye!
|
||||
|
||||
|
||||
|
||||
["Ya"](http://ya.ru)
|
||||
|
||||
More text
|
||||
|
||||
file.csv
|
||||
'''
|
||||
|
||||
|
||||
async def build_document(data):
|
||||
text = data['caption'] if data['caption'] else data['file']
|
||||
# document = await self.build()
|
||||
document = {
|
||||
'url': data['file'],
|
||||
'image_url': data['file'],
|
||||
'file_url': data['file'],
|
||||
'title': text,
|
||||
'file_size': '0 KB',
|
||||
'file_format': 'PDF',
|
||||
}
|
||||
return document
|
||||
|
||||
async def read_csv(data):
|
||||
return '\n'.join([
|
||||
'N,Name',
|
||||
f'0,{data}',
|
||||
'1,Pop',
|
||||
'2,Top',
|
||||
])
|
||||
|
||||
async def read_document(data):
|
||||
doc = await Document.find_one({'id': data})
|
||||
return {
|
||||
'url': doc['url'],
|
||||
'image_url': doc['url'],
|
||||
'file_url': doc['url'],
|
||||
'title': doc['title'],
|
||||
'file_size': '0 KB',
|
||||
'file_format': 'PDF',
|
||||
}
|
||||
|
||||
|
||||
async def run():
|
||||
from kazimir.fix_links_quotes import fix_links_quotes
|
||||
|
||||
m = Marker()
|
||||
m.add_token_factory(TokenFactory(SplitToken))
|
||||
m.add_token_factory(TokenFactory(YoutubeToken))
|
||||
m.add_token_factory(TokenFactory(InstagramToken))
|
||||
m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document))
|
||||
m.add_token_factory(TokenFactory(UrlToken))
|
||||
# m.add_token_factory(TokenFactory(ImageToken))
|
||||
m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
|
||||
m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv))
|
||||
m.add_token_factory(TokenFactory(FileToken))
|
||||
m.add_token_factory(TokenFactory(TextToken))
|
||||
m.add_tree_middleware(fix_links_quotes)
|
||||
sample_tree = await m.create_tree(sample_text)
|
||||
|
||||
# for x in sample_tree:
|
||||
# print(x)
|
||||
|
||||
print(html_from_tree(sample_tree))
|
||||
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(run())
|
||||
loop.close()
|
||||
+22
-38
@@ -92,43 +92,6 @@ class FileToken(Token):
|
||||
return f'<a href="{url}">{text}</a>'
|
||||
|
||||
|
||||
class DocumentToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return is_document(data)
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='file', data=data)
|
||||
self.joinable = False
|
||||
self.build = None
|
||||
|
||||
async def compile(self):
|
||||
data = parse_file(self.data)
|
||||
document = await self.build(data)
|
||||
|
||||
return f'''
|
||||
<div>
|
||||
<div class="document-row">
|
||||
<a href="{document['url']}" class="invisible">
|
||||
<div class="document-row__image">
|
||||
<img src="{document['image_url']}" alt="{document['title']}">
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<div class="document-row__file">
|
||||
<a href="{document['url']}">{document['title']}</a>
|
||||
</div>
|
||||
|
||||
<div class="document-row__file-info">
|
||||
<a href="{document['file_url']}" target="_blank">
|
||||
{document['file_format']} ({document['file_size']})
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
'''.strip()
|
||||
|
||||
|
||||
class ImageToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
@@ -221,8 +184,11 @@ def markdown_to_html(text):
|
||||
|
||||
|
||||
def is_url(sample: str):
|
||||
url = urlparse(sample)
|
||||
link = parse_link(sample)
|
||||
if not link:
|
||||
return False
|
||||
|
||||
url = link['url']
|
||||
return url.scheme and url.netloc
|
||||
|
||||
|
||||
@@ -262,3 +228,21 @@ def parse_file(sample: str):
|
||||
'file': file,
|
||||
'caption': caption,
|
||||
}
|
||||
|
||||
|
||||
def parse_link(sample: str):
|
||||
"""
|
||||
 -> {href, alt}
|
||||
|
||||
:param sample:
|
||||
:return:
|
||||
"""
|
||||
p = re.match(r'!\[(.*)\]\((.*)\)', sample)
|
||||
if not p:
|
||||
return None
|
||||
|
||||
url = p.group(2)
|
||||
return {
|
||||
'url': urlparse(url),
|
||||
'alt': p.group(1),
|
||||
}
|
||||
Reference in new issue
Block a user