diff --git a/modules/sync/kazimir/DocumentToken.py b/modules/sync/kazimir/DocumentToken.py
index 60e6cb72..259eda98 100644
--- a/modules/sync/kazimir/DocumentToken.py
+++ b/modules/sync/kazimir/DocumentToken.py
@@ -10,10 +10,14 @@ class DocumentToken(Token):
super().__init__(name='file', data=data)
self.joinable = False
self.build = None
-
- async def compile(self):
+
+ async def get_data(self):
data = parse_file(self.data)
document = await self.build(data)
+ return document
+
+ async def compile(self):
+ document = await self.get_data()
return create_document(document)
diff --git a/modules/sync/kazimir/DocumentUrlToken.py b/modules/sync/kazimir/DocumentUrlToken.py
deleted file mode 100644
index e290c61d..00000000
--- a/modules/sync/kazimir/DocumentUrlToken.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from kazimir.Token import UrlToken, parse_link
-
-
-class DocumentUrlToken(UrlToken):
- @staticmethod
- def test(data: str):
- link = parse_link(data)
- if not link:
- return False
- url = link['url']
- host = url.netloc
- path = url.path
- return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org')
-
- def __init__(self, data) -> None:
- super().__init__(data)
- self.build = None
-
- async def compile(self):
- from kazimir.DocumentToken import create_document
-
- link = parse_link(self.data)
- url = link['url']
- doc_id = url.path.replace('/document/', '')
- doc = await self.build(doc_id)
-
- if not doc:
- raise Exception(f'Document {doc_id} not found')
-
- # return f'''DOCUMENT: {# doc['title']}'''
- return create_document(doc)
-
- def merge(self, token):
- return self
-
- def __repr__(self) -> str:
- return f'[Token {self.name} {self.data}]'
diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py
index ddf21502..3b81c9bb 100644
--- a/modules/sync/kazimir/Marker.py
+++ b/modules/sync/kazimir/Marker.py
@@ -16,12 +16,15 @@ class Marker:
def add_tree_middleware(self, callback):
self.tree_middlewares.append(callback)
- async def create_tree(self, text: str):
+ def parse(self, text: str):
tokens = text.split('\n')
tokens = self.markup_tokens(tokens)
tokens = join_tokens(tokens)
tokens = merge_tokens(tokens)
+ return tokens
+
+ async def create_tree(self, tokens: list):
compiled = await compile_tokens(tokens)
tree = lxml.html.fromstring(compiled)
return await self.process_tree(tree)
@@ -102,17 +105,8 @@ async def compile_tokens(tokens: list):
return '\n'.join(result)
-def extract_files(text: str) -> [str]:
- html = markdown(text)
- post_html = lxml.html.fromstring(html)
-
- images = post_html.cssselect('img')
- images = [img.get('src') for img in images]
-
- return images, []
-def html_from_tree(tree):
- html = lxml.html.tostring(tree, encoding='unicode')
- return html
+
+
diff --git a/modules/sync/kazimir/MarkerRun.py b/modules/sync/kazimir/MarkerRun.py
deleted file mode 100644
index 846736b4..00000000
--- a/modules/sync/kazimir/MarkerRun.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import asyncio
-
-from kazimir.DocumentUrlToken import DocumentUrlToken
-from kazimir.DocumentToken import DocumentToken
-from kazimir.CSVToken import CSVToken
-from kazimir.InstagramToken import InstagramToken
-from kazimir.Marker import Marker, html_from_tree
-from kazimir.YoutubeToken import YoutubeToken
-from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \
- BuildTokenFactory
-from sync import Document
-
-if __name__ == '__main__':
- sample_text = '''
-Hello!
-
-https://www.instagram1.com/p/BUelE04l1kU
-
-https://www.youtube.com/watch?v=qnguOqWpraI
-https://www.youtube.com/watch?v=qnguOqWpraI
-
-
-
-LOL NOOB
-
-
-WTF OMG
-
-
-
-
-hello.pdf
-file.doc
-sheet.xlsx
-
-print.docx
-image.jpg 1
-image.png 2
-
-image.jpeg Hi
-
-Bye!
-Fye!
-
-
-
-["Ya"](http://ya.ru)
-
-More text
-
-file.csv
- '''
-
-
- async def build_document(data):
- text = data['caption'] if data['caption'] else data['file']
- # document = await self.build()
- document = {
- 'url': data['file'],
- 'image_url': data['file'],
- 'file_url': data['file'],
- 'title': text,
- 'file_size': '0 KB',
- 'file_format': 'PDF',
- }
- return document
-
- async def read_csv(data):
- return '\n'.join([
- 'N,Name',
- f'0,{data}',
- '1,Pop',
- '2,Top',
- ])
-
- async def read_document(data):
- doc = await Document.find_one({'id': data})
- return {
- 'url': doc['url'],
- 'image_url': doc['url'],
- 'file_url': doc['url'],
- 'title': doc['title'],
- 'file_size': '0 KB',
- 'file_format': 'PDF',
- }
-
-
- async def run():
- from kazimir.fix_links_quotes import fix_links_quotes
-
- m = Marker()
- m.add_token_factory(TokenFactory(SplitToken))
- m.add_token_factory(TokenFactory(YoutubeToken))
- m.add_token_factory(TokenFactory(InstagramToken))
- m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document))
- m.add_token_factory(TokenFactory(UrlToken))
- # m.add_token_factory(TokenFactory(ImageToken))
- m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
- m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv))
- m.add_token_factory(TokenFactory(FileToken))
- m.add_token_factory(TokenFactory(TextToken))
- m.add_tree_middleware(fix_links_quotes)
- sample_tree = await m.create_tree(sample_text)
-
- # for x in sample_tree:
- # print(x)
-
- print(html_from_tree(sample_tree))
-
-
- loop = asyncio.get_event_loop()
- loop.run_until_complete(run())
- loop.close()
diff --git a/modules/sync/kazimir/Token.py b/modules/sync/kazimir/Token.py
index 8c8d2c37..dcdc2963 100644
--- a/modules/sync/kazimir/Token.py
+++ b/modules/sync/kazimir/Token.py
@@ -38,8 +38,12 @@ class Token:
self.data = data
self.joinable = True
- async def compile(self):
- return str(self.data)
+ async def get_data(self):
+ return self.data
+
+ async def compile(self) -> str:
+ data = await self.get_data()
+ return str(data)
def merge(self, token):
return self
@@ -73,7 +77,8 @@ class TextToken(Token):
return TextToken(data.strip())
async def compile(self):
- return markdown_to_html(self.data)
+ data = await self.get_data()
+ return markdown_to_html(data)
class FileToken(Token):
@@ -85,8 +90,11 @@ class FileToken(Token):
super().__init__(name='file', data=data)
self.joinable = False
+ async def get_data(self):
+ return parse_file(self.data)
+
async def compile(self):
- data = parse_file(self.data)
+ data = await self.get_data()
url = data['file']
text = data['caption'] if data['caption'] else url
return f'{text}'
@@ -107,9 +115,13 @@ class ImageToken(Token):
merged_token.build = self.build
return merged_token
+ async def get_data(self):
+ data = parse_file(self.data)
+ data = await self.build(data)
+ return data
+
async def compile(self):
- img = self.parse_data()
- img = await self.build(img)
+ img = await self.get_data()
alt = img['alt']
caption = markdown_to_html(img['caption'])
src = img['src']
@@ -124,9 +136,6 @@ class ImageToken(Token):
'''
return tpl.format(img).strip()
- def parse_data(self):
- return parse_file(self.data)
-
class ImageCollectionToken(ImageToken):
def __init__(self, data) -> None:
@@ -143,13 +152,16 @@ class ImageCollectionToken(ImageToken):
merged_token.build = self.build
return merged_token
- async def compile(self):
- data = self.parse_data()
+ async def get_data(self):
+ items = [parse_file(x) for x in self.data]
images = []
- for x in data:
+ for x in items:
img = await self.build(x)
images.append(img)
+ return images
+ async def compile(self):
+ images = await self.get_data()
images = [x['src'] for x in images]
images = [f'' for x in images]
images = '\n'.join(images)
@@ -165,10 +177,6 @@ class ImageCollectionToken(ImageToken):
'''
- def parse_data(self):
- return [parse_file(x) for x in self.data]
-
-
class UrlToken(Token):
@staticmethod
def test(data: str):
diff --git a/modules/sync/kazimir/__init__.py b/modules/sync/kazimir/__init__.py
index 881bfc75..fa0281ed 100644
--- a/modules/sync/kazimir/__init__.py
+++ b/modules/sync/kazimir/__init__.py
@@ -6,19 +6,6 @@ from kazimir.figure import FigureExtension
from kazimir.instagram import InstagramExtension
from sync.data import request
-quote_pairs = [
- ('"', '"'),
- ('«', '»'),
-]
-
-
-def markdown_to_html(text):
- return markdown(text, extensions=[
- FigureExtension(),
- InstagramExtension(),
- ])
-
-
def extract_files(text: str) -> [str]:
html = markdown(text)
post_html = lxml.html.fromstring(html)
@@ -29,67 +16,6 @@ def extract_files(text: str) -> [str]:
return images, []
-def create_tree(text: str) -> str:
- tokens = text.split('\n')
- tokens = markup_tokens(tokens)
- tokens = join_tokens(tokens)
- tokens = merge_tokens(tokens)
-
- kz = compile_tokens(tokens)
- tree = lxml.html.fromstring(kz)
- return clean_tree(tree)
-
-
-def clean_tree(tree):
- return fix_links_quotes(tree)
-
-
-def fix_links_quotes(tree):
- safe = lambda x: x if x else ''
-
- for link in tree.iterlinks():
- a = link[0]
- if a.tag != 'a':
- continue
-
- quote = get_quote_pair(a.text)
- if not quote:
- continue
-
- # New text of A without quotes
- a.text = a.text[1:-1]
-
- # Move first quote out of A to the prefix
- a.getparent().text = safe(a.getparent().text) + quote[0]
-
- # Move last quote out of A to the suffix
- a.tail = quote[1] + safe(a.tail)
-
- return tree
-
-
-def get_quote_pair(text: str):
- """
- sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
- ? (s, t)
- : None
-
- :param text:
- :return:
- """
- if len(text) < 2:
- return None
-
- first = text[0]
- last = text[-1]
-
- for pair_first, pair_last in quote_pairs:
- if pair_first == first and pair_last == last:
- return first, last
-
- return None
-
-
def html_from_tree(tree):
html = lxml.html.tostring(tree, encoding='unicode')
@@ -98,223 +24,11 @@ def html_from_tree(tree):
async def typo(text):
from kazimir import artlebedev
- return await artlebedev.typograf(text)
+ result = await artlebedev.typograf(text)
+ if result == 'Размер текста ограничен 32 КБ':
+ return text
+
+ return result
# url = 'http://www.typograf.ru/webservice/'
# params = dict(text=text)
# return await request.post(url, params, {'chr': 'utf-8'})
-
-
-def join_tokens(tokens):
- buffer_tag = None
- buffer = []
- ts = []
- for token in tokens:
- tag = token['tag']
- changed = buffer_tag is not None and tag != buffer_tag
-
- flush = is_tag_split(tag)
- if changed:
- flush = True
-
- if not is_tag_joinable(tag):
- flush = True
-
- if flush:
- if buffer:
- ts.append(buffer)
- if is_tag_split(tag):
- buffer = []
- buffer_tag = None
- else:
- buffer = [token]
- buffer_tag = tag
- else:
- buffer_tag = tag
- buffer.append(token)
-
- if len(buffer):
- ts.append(buffer)
- return ts
-
-
-def merge_tokens(tokens: list):
- tokens_out = []
- for ts in tokens:
- tag = ts[0]['tag']
- fn = None
- if tag == 'image':
- fn = merge_image
- elif tag == 'text':
- fn = merge_text
- else:
- fn = merge_any
- tokens_out.append(fn(ts))
- return tokens_out
-
-
-def compile_tokens(tokens: list):
- compiled = []
- for ts in tokens:
- tag = ts['tag']
- fn = None
- if tag == 'text':
- fn = compile_text
- elif tag == 'image_collection':
- fn = compile_image_collection
- elif tag == 'image':
- fn = compile_image
- elif tag == 'url':
- fn = compile_url
- else:
- fn = compile_any
- compiled.append(fn(ts))
- return '\n'.join(compiled)
-
-
-def markup_tokens(tokens: list):
- marked_tokens = []
- for token in tokens:
- tag = 'text'
- if is_url(token):
- tag = 'url'
- elif is_image(token):
- tag = 'image'
- elif not token:
- tag = 'split'
- token = None
-
- marked_tokens.append({
- 'tag': tag,
- 'data': token,
- })
- return marked_tokens
-
-
-def is_tag_split(tag):
- return tag == 'split'
-
-
-def is_tag_joinable(tag: str):
- return tag not in ['url']
-
-
-def is_url(sample: str):
- from urllib.parse import urlparse
- url = urlparse(sample)
-
- return url.scheme and url.netloc
-
-
-def is_image(sample: str):
- return sample[:2] == '![' and sample[-1:] == ')'
-
-
-def merge_any(tokens):
- return tokens[0]
-
-
-def merge_text(tokens):
- data = ''
-
- for t in tokens:
- data += t['data']
- data += '\n'
-
- return {
- 'tag': 'text',
- 'data': data.strip(),
- }
-
-
-def merge_image(tokens):
- if len(tokens) == 1:
- return tokens[0]
- else:
- data = [t['data'] for t in tokens]
- return {
- 'tag': 'image_collection',
- 'data': data,
- }
-
-
-def compile_text(token):
- return markdown_to_html(token['data'])
-
-
-def compile_image_collection(token):
- file = lambda i: i[4:][:-1]
- imgs = ['
'.format(file(i)) for i in token['data']]
- return '''
-