diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 771ef63b..b79f1409 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,6 +1,7 @@ import settings from db import collection from sync.core import Sync +from sync.core.SyncArticle import SyncArticle from sync.core.SyncDocument import SyncDocument from sync.core.SyncPage import SyncPage from sync.core.SyncPerson import SyncPerson @@ -57,17 +58,22 @@ async def run(run_interval=0): collection=c(settings.collection_collective), ) + articles = SyncArticle( + provider=io(settings.dir_articles), + collection=c(settings.collection_articles), + ) + while True: # main(io(settings.dir_documents), sync_documents, settings.collection_documents) # main(io(settings.dir_awards), sync_documents, settings.collection_awards) # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) - # main(io(settings.dir_articles), sync_posts, settings.collection_articles) # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) await asyncio.wait([ main(documents), main(pages), main(persons), + main(articles), ]) if run_interval == 0: diff --git a/utils/Sync/sync/core/SyncArticle.py b/utils/Sync/sync/core/SyncArticle.py new file mode 100644 index 00000000..666079a5 --- /dev/null +++ b/utils/Sync/sync/core/SyncArticle.py @@ -0,0 +1,58 @@ +import os + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import Provider +from sync.models.Article import Article +from sync.models.Person import Person + +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Article') + + +class SyncArticle(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Article.scan(self.provider) + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build( + sizes=settings.image_sizes, + ) + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() \ No newline at end of file diff --git a/utils/Sync/sync/core/post.py b/utils/Sync/sync/core/post.py deleted file mode 100644 index 51f8f094..00000000 --- a/utils/Sync/sync/core/post.py +++ /dev/null @@ -1,92 +0,0 @@ -import os -from tempfile import mkstemp - -import lxml.html - -import kazimir -from sync.core import Sync -from utils.fn import last_good, lmap, combine -from utils.hash import hash_str, hash_file -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class SyncPost(Sync): - def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin): - super().__init__() - self.collection = collection - self.provider = provider - self.sizes = sizes - self.type = document_type - self.image_url_base = image_url_base - self.dir_local_images = dir_local_images - self.origin = origin - - def create_id(self, document): - new_id = url_encode_text(document['title']) - document['id'] = document['id'] if 'id' in document else new_id - return document - - def create_hash(self, document): - files = sorted(document['images']) - hashes = lmap( - self.provider.hash, - lmap( - lambda i: os.path.join(document['folder'], i), - files - ) - ) - - document['hash'] = hash_str( - combine([hash_str(document)] + hashes) - ) - - return document - - def create(self, document): - if 'date' not in document or not document['date']: - return None - - if 'title' not in document or not document['title']: - return None - - if self.type: - document['type'] = self.type - - post_html = lxml.html.fromstring(document['post']) - images = [] - for img in post_html.cssselect('img'): - src = img.get('src') - relative_image_path = os.path.join(document['folder'], src) - - img_path = self.provider.get_local(relative_image_path) - if os.path.exists(img_path): - img_id = url_encode_text(os.path.splitext(src)[0]) - url_fn = lambda size, ext: self.image_url_base.format( - id=document['id'], - img=img_id, - size=size, - ext=ext.lower() - ) - - image = create_image(img_path, self.sizes, url_fn, self.dir_local_images) - if image: - images.append(image) - img.set('src', image['data']['big']['url']) - else: - print('fail: ', document['folder'], src) - - document['post'] = kazimir.html_from_tree(post_html) - document['images'] = images - document['origin'] = self.origin - return document - - def create_remove_query(self, query): - if self.type: - query = { - **query, - 'type': self.type, - 'origin': self.origin - } - return query diff --git a/utils/Sync/sync/models/Article.py b/utils/Sync/sync/models/Article.py new file mode 100644 index 00000000..1197e196 --- /dev/null +++ b/utils/Sync/sync/models/Article.py @@ -0,0 +1,192 @@ +import logging + +import os + +import lxml.html + +import kazimir +import settings +from db import collection +from kazimir import kazimir_to_html +from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date +from sync.data import Provider, list_images +from sync.models import Model +from sync.models.Image import Image + +from utils.fn import swap_ext, first, map_cases, constant, combine +from utils.hash import hash_str +from utils.io import read_yaml_md, parse_yaml_front_matter +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Person') +store = collection(settings.collection_articles) + + +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Article(Model): + @staticmethod + async def scan(provider): + documents = [i for i in provider.scan('.') if provider.is_dir(i)] + documents = [Article.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + file_time_formats = ['%Y.%m.%d'] + + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats) + document = { + 'folder': path, + 'date': doc_date, + 'title': doc_title, + } + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + images = list_images(provider, path) + images = [os.path.relpath(i, path) for i in images] + document['images'] = images + document['post'] = create_post_from_image_list(images) + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + manifest_file = md_from_folder(provider, path) + if manifest_file: + manifest = get_manifest(provider, manifest_file) + else: + manifest = {} + + return Article( + provider, + path, + params={ + **document, + **manifest, + }, + ) + + def __init__(self, provider, file, params=None): + # self.__id_template: str = '{category}-{file}' + # self.__url_template: str = settings.document_url_template + # self.__url_preview_template: str = settings.document_url_preview_template + + self.params = params if params else {} + self.origin = settings.origin + self.version = '2' + self.post = None + self.images = None + super().__init__(provider, store, file) + + def init(self): + self.__set_id() + self.__set_hash() + + def validate(self): + document = self.params + if 'date' not in document or not document['date']: + return None + + if 'title' not in document or not document['title']: + return None + + async def build(self, **kwargs): + sizes = kwargs['sizes'] + post, images = await create_post(self.provider, self.id, self.params, sizes) + self.post = post + self.images = images + + post, images = await create_post(self.provider, self.id, self.params, sizes) + self.post = kazimir.html_from_tree(post) + self.images = images + + def bake(self): + return { + **self.params, + 'images': [i.id for i in self.images], + 'post': self.post, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + 'origin': self.origin, + 'version': self.version, + } + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + self.id = url_encode_text(self.params['title']) + + def __set_hash(self): + f = self.params['folder'] + images = sorted(self.params['images']) + images = [os.path.join(f, i) for i in images] + images = [self.provider.hash(i) for i in images] + + self.hash = hash_str( + combine([hash_str(self.params)] + images) + ) + + def __str__(self): + return '
'.format(self.hash, self.file, self.id) + + +def md_from_folder(provider, i): + return first(provider.type_filter(i, '.md')) + + +def get_manifest(provider, path): + data = provider.read(path).read().decode('utf-8') + manifest = parse_yaml_front_matter(data) + + if 'date' in manifest: + manifest['date'] = create_date(manifest['date'], settings.date_formats) + + if 'until' in manifest: + manifest['until'] = create_date(manifest['until'], settings.date_formats) + + if 'id' in manifest: + manifest['id'] = str(manifest['id']) + + html = kazimir_to_html(manifest['content']) + return { + **manifest, + 'post': html, + 'images': images_from_html(html) # Add images for hashing + } + + +async def create_post(provider, doc_id, document, sizes): + image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}' + + post_html = lxml.html.fromstring(document['post']) + images = [] + for img in post_html.cssselect('img'): + src = img.get('src') + relative_image_path = os.path.join(document['folder'], src) + + img_path = provider.get_local(relative_image_path) + if os.path.exists(img_path): + img_id = url_encode_text(os.path.splitext(src)[0]) + url_fn = lambda file, size, ext: image_url_base.format( + id=doc_id, + img=img_id, + size=size, + ext=ext.lower() + ) + + image = await Image.new(provider, img_path, sizes, url_fn) + if image: + url = image.get_size('big')['url'] + images.append(image) + img.set('src', url) + else: + logger.error('Fail to get Image', document['folder'], src) + return post_html, images diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py deleted file mode 100644 index e1579bd0..00000000 --- a/utils/Sync/sync/post.py +++ /dev/null @@ -1,192 +0,0 @@ -import os - -import settings -from kazimir import kazimir_to_html -from sync.data import list_images -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ - synced_images_ids, untouched -from sync.core.post import SyncPost -from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant -from utils.io import read_yaml_md, parse_yaml_front_matter - - -def create_document_from_folder(sync, path, file_time_formats): - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats) - document = { - 'folder': path, - 'date': doc_date, - 'title': doc_title, - } - - # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - images = lmap( - lambda img: os.path.relpath(img, path), - list_images(sync.provider, path) - ) - post = create_post_from_image_list(images) - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - params = map_cases( - document, - [( - lambda doc: md_from_folder(sync.provider, path), - lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)), - )], - constant({}) - ) - - document = { - 'images': images, - 'post': post, - **document, - **params, - } - - document = sync.create_id(document) - document = sync.create_hash(document) - return document - - -def get_manifest(provider, path): - data = provider.read(path).read().decode('utf-8') - manifest = parse_yaml_front_matter(data) - - if 'date' in manifest: - manifest['date'] = create_date(manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date(manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - html = kazimir_to_html(manifest['content']) - return { - **manifest, - 'post': html, - 'images': images_from_html(html) # Add images for hashing - } - - -def md_from_folder(provider, i): - return first(provider.type_filter(i, '.md')) - - -def get_folder_documents(sync, file_time_formats): - # GET EVENT FOLDERS - documents = lmap( - lambda i: i, - filter( - lambda i: sync.provider.is_dir(i), - sync.provider.scan('.') - ) - ) - - return lmap( - lambda doc: create_document_from_folder(sync, doc, file_time_formats), - documents - ) - - -# GET EVENT FILES -def get_file_documents(sync, file_time_formats): - documents = sync.provider.type_filter('.', '.md') - documents = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats), - documents - ) - - documents = lmap( - lambda i: { - 'file': i[0], - 'date': i[1], - 'title': i[2], - }, - documents - ) - - # OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST - documents = lmap( - lambda i: { - **i, - **get_manifest(sync.provider, i['file']) - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) - - # CREATE HASH OF DOCUMENT FILE - documents = lmap(sync.create_hash, documents) - - return documents - - -def main(sync, file_time_formats, update_documents=True, delete_documents=True): - documents = get_folder_documents(sync, file_time_formats) - documents += get_file_documents(sync, file_time_formats) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmapfn(documents)( - lambda i: i['id'] - ) - - # SKIP UNTOUCHED DOCUMENTS - # documents = untouched(documents, sync) - - # MAP EVENT MANIFEST -> EVENT_OBJECT - documents = lmap( - sync.create, - documents - ) - - # FILTER INCORRECT EVENTS - documents = list(filter(None, documents)) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) - - # SYNC EVENT_OBJECT WITH DB - if update_documents: - documents = lmap( - sync.update, - documents - ) - - documents_to_delete = [] - if delete_documents: - remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}}) - documents_to_delete = sync.query(remove_query) - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - - return documents, documents_to_delete - - -def sync_posts(provider, collection, update=True, delete=True): - image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}' - - return main( - SyncPost( - collection, - provider, - 'post', - image_url_base, - settings.dir_static_images, - settings.event_image_sizes, - origin=settings.origin - ), - file_time_formats=['%Y.%m.%d'], - update_documents=update, - delete_documents=delete - )