From a5fd1d7339549415ce1932fdeb705b8c50c69898 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sun, 22 Oct 2017 23:24:07 +0300 Subject: [PATCH] [Sync] Document refactoring + attaching to post --- utils/Sync/settings.py | 3 +- utils/Sync/sync.py | 2 +- utils/Sync/sync/__init__.py | 61 ++++++++++++++++--- utils/Sync/sync/models/Document.py | 97 ++++++++++++++++++++---------- 4 files changed, 122 insertions(+), 41 deletions(-) diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 9c444400..f98fe892 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -132,6 +132,7 @@ collection_pages = 'pages' hash_salt_articles = '3' hash_salt_pages = '1' hash_salt_albums = '0' +hash_salt_documents = '0' provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs') provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') @@ -139,7 +140,7 @@ provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') f = abs_fn(provider_root) dir_settings = f('Settings') -dir_documents = f('Documents') +dir_documents = './Documents' dir_awards = f('Awards') dir_schedules = f('Schedules') dir_articles = f('Articles') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index ba960d73..72b89cdb 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -47,7 +47,7 @@ async def run(run_interval=0): io = lambda root: get_provider(settings.provider_name, root) sync_documents = SyncDocument( - provider=io(settings.dir_documents), + provider=io(settings.provider_root), sizes=settings.image_sizes, ) diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index ed3b64a3..4c98a5d1 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -7,9 +7,16 @@ from markdown import markdown import kazimir import settings +from kazimir.YoutubeToken import YoutubeToken +from kazimir.InstagramToken import InstagramToken +from kazimir.Marker import Marker +from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, DocumentToken from sync.data import Provider from sync.models import Model +from sync.models.Document import Document from sync.models.Image import Image +from utils.text.file import get_size +from utils.text.file import get_ext def create_date(date_str, date_formats=None): @@ -82,10 +89,34 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None): async def create_post(provider: Provider, folder: str, md: str, sizes): - html = kazimir.create_tree(md) + from kazimir.fix_links_quotes import fix_links_quotes + images = [] - for img in html.cssselect('img'): - src = img.get('src') + documents = [] + + async def build_document(data): + filepath = os.path.join(folder, data['file']) + document = await Document.new(provider, filepath, settings.image_sizes) + + if not document: + raise Exception(f'Failed to build Document {filepath}') + + text = data['caption'] if data['caption'] else data['file'] + url = f'/document/{document.id}' + + image_url = document.preview.get_size('small')['url'] + + return { + 'url': url, + 'image_url': image_url, + 'file_url': document.url, + 'title': document.title, + 'file_size': get_size(document.file_size, 1), + 'file_format': get_ext(data['file']), + } + + async def build_image(data): + src = data['file'] relative_image_path = os.path.join(folder, src) image_path = provider.get_local(relative_image_path) @@ -94,14 +125,30 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): if image: url = image.get_size('big')['url'] images.append(image) - img.set('src', url) - else: - raise Exception('Fail to get Image', folder, src) + + return { + 'src': url, + 'alt': data['caption'], + 'caption': data['caption'], + } + raise Exception('Fail to get Image', relative_image_path) + + m = Marker() + m.add_token_factory(TokenFactory(SplitToken)) + m.add_token_factory(TokenFactory(YoutubeToken)) + m.add_token_factory(TokenFactory(InstagramToken)) + m.add_token_factory(TokenFactory(UrlToken)) + m.add_token_factory(BuildTokenFactory(ImageToken, build=build_image)) + m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document)) + # m.add_token_factory(TokenFactory(FileToken)) + m.add_token_factory(TokenFactory(TextToken)) + m.add_tree_middleware(fix_links_quotes) + html = await m.create_tree(md) post = kazimir.html_from_tree(html) post = await kazimir.typo(post) - return post, images, [] + return post, images, documents def images_from_html(md): diff --git a/utils/Sync/sync/models/Document.py b/utils/Sync/sync/models/Document.py index 029cc8b4..144b52d2 100644 --- a/utils/Sync/sync/models/Document.py +++ b/utils/Sync/sync/models/Document.py @@ -6,14 +6,17 @@ from db import collection from sync.data import scan_subdirs, request from sync.models import Model from sync.models.Image import Image -from utils.hash import hash_str +from utils.hash import hash_str, md5 from utils.image import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file store = collection(settings.collection_documents) class Document(Model): + @staticmethod + async def find_one(query): + return store.find_one(query) + @staticmethod async def find(query): return store.find(query) @@ -24,7 +27,8 @@ class Document(Model): @staticmethod async def scan(provider): - paths = scan_subdirs(provider, '.pdf') + scan_dir = settings.dir_documents + paths = scan_subdirs(provider, '.pdf', scan_dir) return [Document.read_path(provider, path) for path in paths] @staticmethod @@ -39,28 +43,53 @@ class Document(Model): return Document( provider, path, - **document + document ) - def __init__(self, provider, file, category=None, title=None): + @staticmethod + async def new(provider, file, sizes): + document = Document.read_path(provider, file) + changed = await document.is_changed() + if changed: + await document.build(sizes=sizes) + await document.upload() + await document.save() + else: + doc = await Document.find_one({'file': file}) + document = Document(provider, file, doc) + document.ref = doc['_id'] + await document.create_preview(sizes) + + return document + + def __init__(self, provider, file, params=None): + self.__hash_salt = settings.hash_salt_documents self.__hash_keys = ['id', 'url'] self.__id_template: str = '{category}-{file}' self.__url_template: str = settings.document_url_template self.__url_preview_template: str = settings.document_url_preview_template - self.category = category - self.title = title + self.title = None + self.file_name = None + self.file_size = None self.type = 'document' # 'award' self.preview = None - super().__init__(provider, store, file) + super().__init__(provider, store, file, params=params) def init(self): self.__set_id() self.__set_url() self.__set_hash() + filename = os.path.basename(self.file) + self.file_name = filename + self.file_size = self.provider.size(self.file) + + if self.has_param('title'): + self.title = self.get_param('title') + async def build(self, **kwargs): sizes = kwargs['sizes'] await self.create_preview(sizes) @@ -75,7 +104,10 @@ class Document(Model): query = {'id': document['id']} try: self.store.update_one(query, {'$set': document}, upsert=True) - # return self.read(document) + + doc = await Document.find_one(query) + self.ref = doc['_id'] + return self except ValueError: pass @@ -89,58 +121,59 @@ class Document(Model): await request.upload(url, filepath) def bake(self): - file = self.file - filename = os.path.basename(file) - return { + **self.params, 'id': self.id, + 'file': self.file, 'hash': self.hash, 'url': self.url, 'type': self.type, - 'file': { - 'name': filename, - 'size': self.provider.size(file) - }, - 'preview': self.preview, 'title': self.title, + 'preview': self.preview.ref, + 'fileInfo': { + 'name': self.file_name, + 'size': self.file_size, + }, } def __filename(self): return os.path.basename(self.file) def __set_id(self): - self.id = self.__id_template.format( - category=url_encode_text(self.category), - file=url_encode_text(self.__filename()) - ) + self.id = md5(self.file) def __set_url(self): + s = os.path.splitext(self.file) + file = f'{self.id}{s[1]}' + self.url = self.__url_template.format( - file=url_encode_file(self.__filename()) + file=file ) def __set_hash(self): doc = [getattr(self, key) for key in self.__hash_keys] self.hash = hash_str( - hash_str(doc) + self.provider.hash(self.file) + self.__hash_salt + hash_str(doc) + self.provider.hash(self.file) ) async def create_preview(self, sizes): - def url(_, size, ext): - file = url_encode_text(self.__filename()) - return self.__url_preview_template.format(id=file, size=size, ext=ext) + if False and self.has_param('preview'): + preview = self.get_param('preview') + preview = await Image.find_one({'_id': preview}) + else: + def url(_, size, ext): + return self.__url_preview_template.format(id=self.id, size=size, ext=ext) - temp_preview_path = await pdf_to_jpg(self.provider, self.file) - preview = await Image.new(self.provider, temp_preview_path, sizes, url) + temp_preview_path = await pdf_to_jpg(self.provider, self.file) + preview = await Image.new(self.provider, temp_preview_path, sizes, url) + os.remove(temp_preview_path) if preview: - self.preview = preview.id - - os.remove(temp_preview_path) + self.preview = preview def __str__(self): - return ''.format(self.hash, self.file, self.id) + return f'' async def pdf_to_jpg(provider, pdf):