From 58c317cc6140def1aa9f6a24655327c4b1e066b5 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:53:43 +0300 Subject: [PATCH 01/13] artlebedev typograph fallback for big pages --- modules/sync/kazimir/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/sync/kazimir/__init__.py b/modules/sync/kazimir/__init__.py index 881bfc75..43cafd8c 100644 --- a/modules/sync/kazimir/__init__.py +++ b/modules/sync/kazimir/__init__.py @@ -98,7 +98,11 @@ def html_from_tree(tree): async def typo(text): from kazimir import artlebedev - return await artlebedev.typograf(text) + result = await artlebedev.typograf(text) + if result == 'Размер текста ограничен 32 КБ': + return text + + return result # url = 'http://www.typograf.ru/webservice/' # params = dict(text=text) # return await request.post(url, params, {'chr': 'utf-8'}) From 5d56d308b54ceb58d21d4488442a9fac0526ccec Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:56:09 +0300 Subject: [PATCH 02/13] remove kazimir code moved to marker in the past --- modules/sync/kazimir/__init__.py | 290 ------------------------------- 1 file changed, 290 deletions(-) diff --git a/modules/sync/kazimir/__init__.py b/modules/sync/kazimir/__init__.py index 43cafd8c..fa0281ed 100644 --- a/modules/sync/kazimir/__init__.py +++ b/modules/sync/kazimir/__init__.py @@ -6,19 +6,6 @@ from kazimir.figure import FigureExtension from kazimir.instagram import InstagramExtension from sync.data import request -quote_pairs = [ - ('"', '"'), - ('«', '»'), -] - - -def markdown_to_html(text): - return markdown(text, extensions=[ - FigureExtension(), - InstagramExtension(), - ]) - - def extract_files(text: str) -> [str]: html = markdown(text) post_html = lxml.html.fromstring(html) @@ -29,67 +16,6 @@ def extract_files(text: str) -> [str]: return images, [] -def create_tree(text: str) -> str: - tokens = text.split('\n') - tokens = markup_tokens(tokens) - tokens = join_tokens(tokens) - tokens = merge_tokens(tokens) - - kz = compile_tokens(tokens) - tree = lxml.html.fromstring(kz) - return clean_tree(tree) - - -def clean_tree(tree): - return fix_links_quotes(tree) - - -def fix_links_quotes(tree): - safe = lambda x: x if x else '' - - for link in tree.iterlinks(): - a = link[0] - if a.tag != 'a': - continue - - quote = get_quote_pair(a.text) - if not quote: - continue - - # New text of A without quotes - a.text = a.text[1:-1] - - # Move first quote out of A to the prefix - a.getparent().text = safe(a.getparent().text) + quote[0] - - # Move last quote out of A to the suffix - a.tail = quote[1] + safe(a.tail) - - return tree - - -def get_quote_pair(text: str): - """ - sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair - ? (s, t) - : None - - :param text: - :return: - """ - if len(text) < 2: - return None - - first = text[0] - last = text[-1] - - for pair_first, pair_last in quote_pairs: - if pair_first == first and pair_last == last: - return first, last - - return None - - def html_from_tree(tree): html = lxml.html.tostring(tree, encoding='unicode') @@ -106,219 +32,3 @@ async def typo(text): # url = 'http://www.typograf.ru/webservice/' # params = dict(text=text) # return await request.post(url, params, {'chr': 'utf-8'}) - - -def join_tokens(tokens): - buffer_tag = None - buffer = [] - ts = [] - for token in tokens: - tag = token['tag'] - changed = buffer_tag is not None and tag != buffer_tag - - flush = is_tag_split(tag) - if changed: - flush = True - - if not is_tag_joinable(tag): - flush = True - - if flush: - if buffer: - ts.append(buffer) - if is_tag_split(tag): - buffer = [] - buffer_tag = None - else: - buffer = [token] - buffer_tag = tag - else: - buffer_tag = tag - buffer.append(token) - - if len(buffer): - ts.append(buffer) - return ts - - -def merge_tokens(tokens: list): - tokens_out = [] - for ts in tokens: - tag = ts[0]['tag'] - fn = None - if tag == 'image': - fn = merge_image - elif tag == 'text': - fn = merge_text - else: - fn = merge_any - tokens_out.append(fn(ts)) - return tokens_out - - -def compile_tokens(tokens: list): - compiled = [] - for ts in tokens: - tag = ts['tag'] - fn = None - if tag == 'text': - fn = compile_text - elif tag == 'image_collection': - fn = compile_image_collection - elif tag == 'image': - fn = compile_image - elif tag == 'url': - fn = compile_url - else: - fn = compile_any - compiled.append(fn(ts)) - return '\n'.join(compiled) - - -def markup_tokens(tokens: list): - marked_tokens = [] - for token in tokens: - tag = 'text' - if is_url(token): - tag = 'url' - elif is_image(token): - tag = 'image' - elif not token: - tag = 'split' - token = None - - marked_tokens.append({ - 'tag': tag, - 'data': token, - }) - return marked_tokens - - -def is_tag_split(tag): - return tag == 'split' - - -def is_tag_joinable(tag: str): - return tag not in ['url'] - - -def is_url(sample: str): - from urllib.parse import urlparse - url = urlparse(sample) - - return url.scheme and url.netloc - - -def is_image(sample: str): - return sample[:2] == '![' and sample[-1:] == ')' - - -def merge_any(tokens): - return tokens[0] - - -def merge_text(tokens): - data = '' - - for t in tokens: - data += t['data'] - data += '\n' - - return { - 'tag': 'text', - 'data': data.strip(), - } - - -def merge_image(tokens): - if len(tokens) == 1: - return tokens[0] - else: - data = [t['data'] for t in tokens] - return { - 'tag': 'image_collection', - 'data': data, - } - - -def compile_text(token): - return markdown_to_html(token['data']) - - -def compile_image_collection(token): - file = lambda i: i[4:][:-1] - imgs = [''.format(file(i)) for i in token['data']] - return ''' -
-
- {} -
-
- '''.format('\n'.join(imgs)) - - -def compile_image(token): - img = markdown_to_html(token['data']) - tpl = '''
{}
''' - return tpl.format(img) - - -def compile_url(token): - return compile_youtube(token) - - -def compile_youtube(token): - """ -
- -
- """ - - youtube = r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P[A-Za-z0-9_&=-]+)\S*' - youtube_pattr = re.compile(youtube) - match = youtube_pattr.match(token['data']) - if match: - video_id = match.group(2) - url = f'//www.youtube.com/embed/{video_id}' - tpl = '''
''' - - return tpl.format(url=url) - else: - return '' - - -def compile_any(token): - return markdown_to_html(token['data']) - - -if __name__ == '__main__': - sample_text = ''' -Hello! - -https://www.instagram.com/p/BUelE04l1kU -ku! - -https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka -https://www.youtube.com/watch?v=qnguOqWpraI -![](1.jpg) -![](2.jpg) -![](3.jpg) - -LOL NOOB -![](12.jpg) -![](22.jpg) -WTF OMG - -Bye! -Fye! - -![](1.jpg) - -More text - ''' - - sample_tree = create_tree(sample_text) - print(html_from_tree(sample_tree)) From 95925cffe5f3e35fff5b077f4ccd487235410681 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:56:33 +0300 Subject: [PATCH 03/13] rm DocumentUrlToken and MarkerRun --- modules/sync/kazimir/DocumentUrlToken.py | 37 -------- modules/sync/kazimir/MarkerRun.py | 113 ----------------------- 2 files changed, 150 deletions(-) delete mode 100644 modules/sync/kazimir/DocumentUrlToken.py delete mode 100644 modules/sync/kazimir/MarkerRun.py diff --git a/modules/sync/kazimir/DocumentUrlToken.py b/modules/sync/kazimir/DocumentUrlToken.py deleted file mode 100644 index e290c61d..00000000 --- a/modules/sync/kazimir/DocumentUrlToken.py +++ /dev/null @@ -1,37 +0,0 @@ -from kazimir.Token import UrlToken, parse_link - - -class DocumentUrlToken(UrlToken): - @staticmethod - def test(data: str): - link = parse_link(data) - if not link: - return False - url = link['url'] - host = url.netloc - path = url.path - return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org') - - def __init__(self, data) -> None: - super().__init__(data) - self.build = None - - async def compile(self): - from kazimir.DocumentToken import create_document - - link = parse_link(self.data) - url = link['url'] - doc_id = url.path.replace('/document/', '') - doc = await self.build(doc_id) - - if not doc: - raise Exception(f'Document {doc_id} not found') - - # return f'''DOCUMENT: {# doc['title']}''' - return create_document(doc) - - def merge(self, token): - return self - - def __repr__(self) -> str: - return f'[Token {self.name} {self.data}]' diff --git a/modules/sync/kazimir/MarkerRun.py b/modules/sync/kazimir/MarkerRun.py deleted file mode 100644 index 846736b4..00000000 --- a/modules/sync/kazimir/MarkerRun.py +++ /dev/null @@ -1,113 +0,0 @@ -import asyncio - -from kazimir.DocumentUrlToken import DocumentUrlToken -from kazimir.DocumentToken import DocumentToken -from kazimir.CSVToken import CSVToken -from kazimir.InstagramToken import InstagramToken -from kazimir.Marker import Marker, html_from_tree -from kazimir.YoutubeToken import YoutubeToken -from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \ - BuildTokenFactory -from sync import Document - -if __name__ == '__main__': - sample_text = ''' -Hello! - -https://www.instagram1.com/p/BUelE04l1kU - -https://www.youtube.com/watch?v=qnguOqWpraI -https://www.youtube.com/watch?v=qnguOqWpraI -![](1.jpg) -![](2.jpg) - -LOL NOOB -![](12.jpg) -![](22.jpg) -WTF OMG - -![](/document/documents-arkhiv-ustav-2013-pdf) - -![File](file.pdf) -hello.pdf -file.doc -sheet.xlsx - -print.docx -image.jpg 1 -image.png 2 - -image.jpeg Hi - -Bye! -Fye! - -![](1.jpg) - -["Ya"](http://ya.ru) - -More text - -file.csv - ''' - - - async def build_document(data): - text = data['caption'] if data['caption'] else data['file'] - # document = await self.build() - document = { - 'url': data['file'], - 'image_url': data['file'], - 'file_url': data['file'], - 'title': text, - 'file_size': '0 KB', - 'file_format': 'PDF', - } - return document - - async def read_csv(data): - return '\n'.join([ - 'N,Name', - f'0,{data}', - '1,Pop', - '2,Top', - ]) - - async def read_document(data): - doc = await Document.find_one({'id': data}) - return { - 'url': doc['url'], - 'image_url': doc['url'], - 'file_url': doc['url'], - 'title': doc['title'], - 'file_size': '0 KB', - 'file_format': 'PDF', - } - - - async def run(): - from kazimir.fix_links_quotes import fix_links_quotes - - m = Marker() - m.add_token_factory(TokenFactory(SplitToken)) - m.add_token_factory(TokenFactory(YoutubeToken)) - m.add_token_factory(TokenFactory(InstagramToken)) - m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document)) - m.add_token_factory(TokenFactory(UrlToken)) - # m.add_token_factory(TokenFactory(ImageToken)) - m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document)) - m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv)) - m.add_token_factory(TokenFactory(FileToken)) - m.add_token_factory(TokenFactory(TextToken)) - m.add_tree_middleware(fix_links_quotes) - sample_tree = await m.create_tree(sample_text) - - # for x in sample_tree: - # print(x) - - print(html_from_tree(sample_tree)) - - - loop = asyncio.get_event_loop() - loop.run_until_complete(run()) - loop.close() From 71f4ca8a2c9c524373ba3e16e6178ddd26c46f3e Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:57:08 +0300 Subject: [PATCH 04/13] rm unused functions from Marker --- modules/sync/kazimir/Marker.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py index ddf21502..cad715f6 100644 --- a/modules/sync/kazimir/Marker.py +++ b/modules/sync/kazimir/Marker.py @@ -102,17 +102,8 @@ async def compile_tokens(tokens: list): return '\n'.join(result) -def extract_files(text: str) -> [str]: - html = markdown(text) - post_html = lxml.html.fromstring(html) - - images = post_html.cssselect('img') - images = [img.get('src') for img in images] - - return images, [] -def html_from_tree(tree): - html = lxml.html.tostring(tree, encoding='unicode') - return html + + From dcea20b046222f042be87c074ce6c545633dcf17 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:58:12 +0300 Subject: [PATCH 05/13] rm SyncDocuments (now it in the page only) --- modules/sync/sync.py | 8 -------- modules/sync/sync/core/SyncDocument.py | 14 -------------- 2 files changed, 22 deletions(-) delete mode 100644 modules/sync/sync/core/SyncDocument.py diff --git a/modules/sync/sync.py b/modules/sync/sync.py index b15dc929..5b59c929 100644 --- a/modules/sync/sync.py +++ b/modules/sync/sync.py @@ -1,6 +1,5 @@ import settings from sync.core import Sync -from sync.core.SyncDocument import SyncDocument from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider @@ -41,12 +40,6 @@ async def run_sync(sync: Sync, **kwargs): async def run(run_interval=0): io = lambda root: get_provider(settings.provider_name, root) - sync_documents = SyncDocument( - provider=io(settings.provider_root), - sizes=settings.image_sizes, - ) - sync_documents.strict_origin = True - sync_pages = Sync( provider=io(settings.provider_root), model=Page, @@ -62,7 +55,6 @@ async def run(run_interval=0): while True: await asyncio.wait([ - run_sync(sync_documents), run_sync(sync_pages), run_sync(sync_articles), ]) diff --git a/modules/sync/sync/core/SyncDocument.py b/modules/sync/sync/core/SyncDocument.py deleted file mode 100644 index 13b55301..00000000 --- a/modules/sync/sync/core/SyncDocument.py +++ /dev/null @@ -1,14 +0,0 @@ -from sync.core import Sync -from sync.data import Provider -from sync.models.Document import Document - - -class SyncDocument(Sync): - def __init__(self, provider: Provider, sizes): - super().__init__(provider, Document) - self.sizes = sizes - - def _get_build_args(self): - return { - 'sizes': self.sizes, - } From abbb977b8e9839fe9235a480158bd3b9c870974e Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:58:24 +0300 Subject: [PATCH 06/13] rm unused from sync io lib --- modules/sync/utils/io.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/sync/utils/io.py b/modules/sync/utils/io.py index 138bf8e7..0c94a6cd 100644 --- a/modules/sync/utils/io.py +++ b/modules/sync/utils/io.py @@ -1,14 +1,10 @@ import re import yaml -from kazimir import markdown_to_html FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE) -def read_yaml_md(data): - yaml_front_matter, content = split_yaml_front_matter(data) - return yaml_front_matter, markdown_to_html(content) def parse_yaml_front_matter(data: str): From a2a73034da37038da0109f31fc4f158bf81d80cb Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 00:58:58 +0300 Subject: [PATCH 07/13] Document model pdf title stability improved (drop 0 byte from title value) --- modules/sync/sync/models/Document.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/sync/sync/models/Document.py b/modules/sync/sync/models/Document.py index 0c989f5c..2eeb30ba 100644 --- a/modules/sync/sync/models/Document.py +++ b/modules/sync/sync/models/Document.py @@ -181,14 +181,21 @@ async def get_pdf_title(provider, file): from PyPDF2.generic import TextStringObject from PyPDF2.generic import IndirectObject + def t(value): + value = str(value) + value = value.replace('\u0000', '') + if value == 'None': + return None + return value + pdf = PdfFileReader(provider.read(file)) info = pdf.getDocumentInfo() if info: if type(info.title) == TextStringObject: - return str(info.title) + return t(info.title) if type(info.title_raw) == IndirectObject: o = pdf.getObject(info.title_raw) - return str(o) + return t(info.title) return None From 092fa4578bdbdc6f2fc8bdd3ebe9edd5be5f5e95 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 01:02:04 +0300 Subject: [PATCH 08/13] move create_post_from_image_list to Article (usage only there) --- modules/sync/sync/__init__.py | 10 ---------- modules/sync/sync/models/Article.py | 13 +++++++++++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py index baabdbe2..77aeaea7 100644 --- a/modules/sync/sync/__init__.py +++ b/modules/sync/sync/__init__.py @@ -3,8 +3,6 @@ import re from datetime import datetime import lxml.html -from markdown import markdown - import kazimir import settings from kazimir.CSVToken import CSVToken @@ -181,11 +179,3 @@ def title_from_html(md): titles = html.cssselect('h1') if len(titles): return titles[0].text - - -create_post_from_image_list = lambda images: markdown( - '\n'.join(map( - lambda i: '![]({img})'.format(img=i), - images - )) -) diff --git a/modules/sync/sync/models/Article.py b/modules/sync/sync/models/Article.py index 8d943b3f..68b364df 100644 --- a/modules/sync/sync/models/Article.py +++ b/modules/sync/sync/models/Article.py @@ -5,8 +5,8 @@ import os import kazimir import settings from db import collection -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, \ - create_post +from markdown import markdown +from sync import create_date_and_title_from_folder_name, create_date, create_post from sync.data import list_images from sync.models import Model from sync.models.Image import Image @@ -172,3 +172,12 @@ def read_manifest(provider, path): return { **manifest, } + + +def create_post_from_image_list(images): + return markdown( + '\n'.join(map( + lambda i: '![]({img})'.format(img=i), + images + )) + ) From ec3e09b509f533cf40de5919fe2e89a91567f2b3 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 01:02:20 +0300 Subject: [PATCH 09/13] fix Page imports --- modules/sync/sync/models/Page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py index 043bec1b..afd6b500 100644 --- a/modules/sync/sync/models/Page.py +++ b/modules/sync/sync/models/Page.py @@ -14,7 +14,7 @@ from sync.models import Model from sync import create_date, images_from_html, title_from_html, create_post from sync.models.Image import Image from utils.hash import hash_str -from utils.io import read_yaml_md, parse_yaml_front_matter +from utils.io import parse_yaml_front_matter from utils.text.transform import url_encode_text logger = logging.getLogger(settings.name + '.Page') From 1b5fc82e5964727bae89217d2d6f94a1ed9da924 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 01:03:09 +0300 Subject: [PATCH 10/13] Token compile refactored --- modules/sync/kazimir/DocumentToken.py | 8 ++++-- modules/sync/kazimir/Token.py | 40 ++++++++++++++++----------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/modules/sync/kazimir/DocumentToken.py b/modules/sync/kazimir/DocumentToken.py index 60e6cb72..259eda98 100644 --- a/modules/sync/kazimir/DocumentToken.py +++ b/modules/sync/kazimir/DocumentToken.py @@ -10,10 +10,14 @@ class DocumentToken(Token): super().__init__(name='file', data=data) self.joinable = False self.build = None - - async def compile(self): + + async def get_data(self): data = parse_file(self.data) document = await self.build(data) + return document + + async def compile(self): + document = await self.get_data() return create_document(document) diff --git a/modules/sync/kazimir/Token.py b/modules/sync/kazimir/Token.py index 8c8d2c37..dcdc2963 100644 --- a/modules/sync/kazimir/Token.py +++ b/modules/sync/kazimir/Token.py @@ -38,8 +38,12 @@ class Token: self.data = data self.joinable = True - async def compile(self): - return str(self.data) + async def get_data(self): + return self.data + + async def compile(self) -> str: + data = await self.get_data() + return str(data) def merge(self, token): return self @@ -73,7 +77,8 @@ class TextToken(Token): return TextToken(data.strip()) async def compile(self): - return markdown_to_html(self.data) + data = await self.get_data() + return markdown_to_html(data) class FileToken(Token): @@ -85,8 +90,11 @@ class FileToken(Token): super().__init__(name='file', data=data) self.joinable = False + async def get_data(self): + return parse_file(self.data) + async def compile(self): - data = parse_file(self.data) + data = await self.get_data() url = data['file'] text = data['caption'] if data['caption'] else url return f'{text}' @@ -107,9 +115,13 @@ class ImageToken(Token): merged_token.build = self.build return merged_token + async def get_data(self): + data = parse_file(self.data) + data = await self.build(data) + return data + async def compile(self): - img = self.parse_data() - img = await self.build(img) + img = await self.get_data() alt = img['alt'] caption = markdown_to_html(img['caption']) src = img['src'] @@ -124,9 +136,6 @@ class ImageToken(Token): ''' return tpl.format(img).strip() - def parse_data(self): - return parse_file(self.data) - class ImageCollectionToken(ImageToken): def __init__(self, data) -> None: @@ -143,13 +152,16 @@ class ImageCollectionToken(ImageToken): merged_token.build = self.build return merged_token - async def compile(self): - data = self.parse_data() + async def get_data(self): + items = [parse_file(x) for x in self.data] images = [] - for x in data: + for x in items: img = await self.build(x) images.append(img) + return images + async def compile(self): + images = await self.get_data() images = [x['src'] for x in images] images = [f'' for x in images] images = '\n'.join(images) @@ -165,10 +177,6 @@ class ImageCollectionToken(ImageToken): ''' - def parse_data(self): - return [parse_file(x) for x in self.data] - - class UrlToken(Token): @staticmethod def test(data: str): From 55f44fab8c0a4d6df74c17dd6dacc80d5b1a7205 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 01:03:42 +0300 Subject: [PATCH 11/13] include tokens in result data --- modules/sync/kazimir/Marker.py | 5 ++++- modules/sync/sync/__init__.py | 14 ++++++++++++-- modules/sync/sync/models/Article.py | 5 ++++- modules/sync/sync/models/Page.py | 5 ++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py index cad715f6..3b81c9bb 100644 --- a/modules/sync/kazimir/Marker.py +++ b/modules/sync/kazimir/Marker.py @@ -16,12 +16,15 @@ class Marker: def add_tree_middleware(self, callback): self.tree_middlewares.append(callback) - async def create_tree(self, text: str): + def parse(self, text: str): tokens = text.split('\n') tokens = self.markup_tokens(tokens) tokens = join_tokens(tokens) tokens = merge_tokens(tokens) + return tokens + + async def create_tree(self, tokens: list): compiled = await compile_tokens(tokens) tree = lxml.html.fromstring(compiled) return await self.process_tree(tree) diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py index 77aeaea7..209d6afe 100644 --- a/modules/sync/sync/__init__.py +++ b/modules/sync/sync/__init__.py @@ -153,12 +153,22 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): # m.add_token_factory(TokenFactory(FileToken)) m.add_token_factory(TokenFactory(TextToken)) m.add_tree_middleware(fix_links_quotes) - tree = await m.create_tree(md) + + tokens = m.parse(md) + tree = await m.create_tree(tokens) post = kazimir.html_from_tree(tree) post = await kazimir.typo(post) - return post, images, documents + encoded_tokens = [] + for t in tokens: + data = await t.get_data() + encoded_tokens.append({ + 'token': t.name, + 'data': data, + }) + + return post, images, documents, encoded_tokens def images_from_html(md): diff --git a/modules/sync/sync/models/Article.py b/modules/sync/sync/models/Article.py index 68b364df..40d003ce 100644 --- a/modules/sync/sync/models/Article.py +++ b/modules/sync/sync/models/Article.py @@ -71,6 +71,7 @@ class Article(Model): self.preview = None self.images = None self.documents = None + self.tokens = None super().__init__(provider, store, file, params=params) def init(self): @@ -103,10 +104,11 @@ class Article(Model): else: markdown_post = create_post_from_image_list(self.get_param('images')) - post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) + post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) self.post = post self.images = images self.documents = documents + self.tokens = tokens if self.has_param('preview'): preview = self.get_param('preview') @@ -124,6 +126,7 @@ class Article(Model): 'file': self.file, 'origin': self.origin, 'version': self.version, + 'tokens': self.tokens, 'preview': self.preview.ref if self.preview else None, } diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py index afd6b500..e72923d4 100644 --- a/modules/sync/sync/models/Page.py +++ b/modules/sync/sync/models/Page.py @@ -96,6 +96,7 @@ class Page(Model): self.title = None self.data = None self.preview = None + self.tokens = None self.__id_template: str = '{category}-{file}' self.__url_template: str = settings.document_url_template @@ -126,11 +127,12 @@ class Page(Model): sizes = kwargs['sizes'] markdown_post = self.get_param('content') - post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) + post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) self.title = title_from_html(post) self.data = post self.images = images self.documents = documents + self.tokens = tokens if self.has_param('preview'): preview = self.get_param('preview') @@ -151,6 +153,7 @@ class Page(Model): 'images': images, 'documents': documents, 'title': self.title, + 'tokens': self.tokens, 'preview': self.preview.ref if self.preview else None, } From 263f18a1df0cbce854647e63a8d3431ba0651b9e Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 01:04:28 +0300 Subject: [PATCH 12/13] drop documents page (now it is a regular page) --- pages/documents.tsx | 138 -------------------------- scripts/sitemap.js | 1 - src/components/DocumentList/index.tsx | 22 ---- src/types.ts | 2 +- src/utils/common.ts | 20 ---- 5 files changed, 1 insertion(+), 182 deletions(-) delete mode 100644 pages/documents.tsx delete mode 100644 src/components/DocumentList/index.tsx diff --git a/pages/documents.tsx b/pages/documents.tsx deleted file mode 100644 index 12c1f8c6..00000000 --- a/pages/documents.tsx +++ /dev/null @@ -1,138 +0,0 @@ -import Head from 'next/head' -import { App } from 'src/components/App' -import { Meta } from 'src/components/Meta' -import { meta } from 'src/lib/meta' -import { IDocument } from 'src/types' -import { DocumentList } from 'src/components/DocumentList' -import { unique } from 'src/utils/common' -import { createApiUrl, requestGet, IResponseItems } from 'src/next-lib' -import { NextPage } from 'next' - -interface IDocumentCategory { - name: string - items: IDocument[] -} - -function getSorted(documents: IDocument[]): IDocumentCategory[] { - const compose = (...fns) => value => fns - .map(fn => fn) - .reverse() - .reduce((acc, fn) => fn(acc), value) - - const pass = fn => value => { - fn(value) - return value - } - - const subtract = (a, b) => a - b - - const uniqueCategories = unique(i => i.category) - - const priority = { - 'Основные документы': [ - 'https://static.shlisselburg.org/art/uploads/ustav-2015.pdf', - 'https://static.shlisselburg.org/art/uploads/obrazovatelnaia-litsenziia-2015.pdf', - 'https://static.shlisselburg.org/art/uploads/svidetelstvo-na-pomeshchenie.pdf', - 'https://static.shlisselburg.org/art/uploads/egriul.pdf', - 'https://static.shlisselburg.org/art/uploads/inn.pdf', - 'https://static.shlisselburg.org/art/uploads/reshenie-o-sozdanii-shkoly.pdf', - 'https://static.shlisselburg.org/art/uploads/kopiia-resheniia-o-naznachenii-rukovoditelia.pdf', - 'https://static.shlisselburg.org/art/uploads/postanovlenie-o-sozdanii-munitsipalnogo-obrazovatelnogo-uchrezhdeniia-kultury-detskaia-khudozhestvennaia-shkola.pdf', - ], - 'Учебные программы': [ - 'https://static.shlisselburg.org/art/uploads/uchebnaia-programma-zhivopis.pdf', - ] - } - - const CATEGORY_PRIORITY = [ - 'Основные документы', - 'Планы ФХД', - 'Самообследование деятельности', - 'Инструкции', - 'Образование', - 'Учебные программы', - 'Платные образовательные услуги', - 'Дополнительные общеразвивающие программы', - 'Документы для поступления', - ] - - function sortBy(fn: (value: T) => number): (a: T, b: T) => number { - return (a, b) => [a, b].map(fn).reduce(subtract) - } - - function getPriorityFn(m: number, list: T[]): (value: T) => number { - return (value: T) => { - const has = list.includes(value) - return has ? list.indexOf(value) : m - } - } - - const safeSelect = (d, store, key) => key in store - ? store[key] - : d - - const categoryPriority = getPriorityFn(10000, CATEGORY_PRIORITY) - - const documentPriority = ({ category, fileUrl }) => compose( - getPriorityFn.bind(null, 10000), - safeSelect.bind(null, [], priority) - )(category)(fileUrl) - - const documentsOf = category => documents.filter(i => i.category === category) - const sorted = documents => documents.sort(sortBy(documentPriority)) - const collectDocumentsByCategory = compose(sorted, documentsOf) - - return uniqueCategories(documents) - .sort(sortBy(categoryPriority)) - .reduce((acc, name) => [...acc, { - name, - items: collectDocumentsByCategory(name) - }], []) -} - -interface IProps { - collections: IDocumentCategory[] - title: string - meta: any -} - -const Page: NextPage = props => ( - - - {props.title} - - - -

{props.title}

- - {props.collections.map((collection, index) => ( - - ))} - -) - -export const unstable_getStaticProps = async (ctx) => { - const res = await requestGet>(createApiUrl('/api/files'), null) - const files = res.items - const title = 'Документы' - const collections = getSorted(files) - - return { - props: { - collections, - title, - meta: meta({ - title, - description: 'Документы Шлиссельбургской ДХШ', - }) - } - } -} - -export default Page diff --git a/scripts/sitemap.js b/scripts/sitemap.js index 23b1d96a..ad704d64 100644 --- a/scripts/sitemap.js +++ b/scripts/sitemap.js @@ -12,7 +12,6 @@ async function getMenuUrls(frequency = 'daily') { const menu = [ '/', '/schedule', - '/documents', '/collective', ] return menu diff --git a/src/components/DocumentList/index.tsx b/src/components/DocumentList/index.tsx deleted file mode 100644 index 541e1d12..00000000 --- a/src/components/DocumentList/index.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import * as React from 'react' -import { IDocument } from '../../types' -import { DocumentListItem } from '../DocumentListItem' - -export interface IDocumentListProps { - documents: IDocument[] - name: string -} - -export const DocumentList: React.FC = props => ( -
-
-

{props.name}

-
- -
- {props.documents.map((d, i) => ( - - ))} -
-
-) diff --git a/src/types.ts b/src/types.ts index baf86823..7fe64ce1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -20,7 +20,7 @@ export interface IPage { } export interface IDocument { - category: string + // category: string fileName: string fileSize: number fileUrl: string diff --git a/src/utils/common.ts b/src/utils/common.ts index 00920c56..b06647d2 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -36,26 +36,6 @@ export function choise(scope, selector) { return scope.reduce(selector, null); } -export function unique(fn: (value: T) => U): (list: T[]) => U[] { - return list => Array.from( - new Set(list.map(fn)) - ) -} - -export function branch(fn) { - let keys = unique(fn); - - return list => keys(list) - .reduce((dict: any, key: any) => { - dict[key] = list.filter( - i => fn(i) === key - ); - - return dict; - }, {}) - -} - export function assign(dict, key, value) { dict[key] = value; return dict; From de20ed726fe1738b424b509fcc6e2bad18cdcbaa Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 29 Feb 2020 01:19:04 +0300 Subject: [PATCH 13/13] sync cleanup --- modules/sync/settings.py | 1 - modules/sync/sync/models/Document.py | 8 +------- modules/sync/sync/models/Image.py | 9 +-------- modules/sync/test/sync/test_document.py | 27 ------------------------- 4 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 modules/sync/test/sync/test_document.py diff --git a/modules/sync/settings.py b/modules/sync/settings.py index 65475075..547f70a1 100644 --- a/modules/sync/settings.py +++ b/modules/sync/settings.py @@ -91,7 +91,6 @@ provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') f = abs_fn(provider_root) -dir_documents = './Documents' dir_articles = './Articles' dir_pages = './Pages' diff --git a/modules/sync/sync/models/Document.py b/modules/sync/sync/models/Document.py index 2eeb30ba..5c04d327 100644 --- a/modules/sync/sync/models/Document.py +++ b/modules/sync/sync/models/Document.py @@ -3,7 +3,7 @@ from tempfile import mkstemp import settings from db import collection -from sync.data import scan_subdirs, request +from sync.data import request from sync.models import Model from sync.models.DocumentPreviewImage import DocumentPreviewImage from utils.hash import hash_str, md5 @@ -25,12 +25,6 @@ class Document(Model): async def delete(query): return store.find_one_and_delete(query) - @staticmethod - async def scan(provider): - scan_dir = settings.dir_documents - paths = scan_subdirs(provider, '.pdf', scan_dir) - return [Document.read_path(provider, path) for path in paths] - @staticmethod def read_path(provider, path): name = os.path.basename(path) diff --git a/modules/sync/sync/models/Image.py b/modules/sync/sync/models/Image.py index 0d0eb911..c4954474 100644 --- a/modules/sync/sync/models/Image.py +++ b/modules/sync/sync/models/Image.py @@ -7,7 +7,7 @@ from typing import Optional, List import settings from db import collection -from sync.data import request, list_images +from sync.data import request from sync.models import Model from utils.hash import md5, hash_str from utils.image import image_magick_resize @@ -54,13 +54,6 @@ class Image(Model): return img - @staticmethod - async def scan(provider): - scan_path = settings.dir_images - documents = list_images(provider, scan_path) - - return [Image.read_path(provider, i) for i in documents] - @staticmethod def read_path(provider, path): return Image( diff --git a/modules/sync/test/sync/test_document.py b/modules/sync/test/sync/test_document.py deleted file mode 100644 index 4db32b89..00000000 --- a/modules/sync/test/sync/test_document.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest - -import settings -from sync.data import scan_subdirs -from sync.data.fs import FSProvider - - -class MyTestCase(unittest.TestCase): - def test_document(self): - p = FSProvider(settings.dir_documents) - - documents = scan_subdirs(p, '.pdf') - # documents = lmap( - # lambda document: create_document(p, document), - # documents - # ) - - self.assertEqual(True, True) - - def test_scan(self): - p = FSProvider(settings.dir_documents) - documents = scan_subdirs(p, '.pdf') - self.assertIsInstance(documents, list) - - -if __name__ == '__main__': - unittest.main()