From 8bf1ea37c83af286b12c298b4e0e0de12d36c02a Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Mon, 11 Sep 2017 18:24:00 +0300 Subject: [PATCH] [Sync] + new Page sync --- utils/Sync/sync.py | 8 +- utils/Sync/sync/core/SyncPage.py | 77 +++++++++++ utils/Sync/sync/models/Page.py | 203 +++++++++++++++++++++++++++ utils/Sync/sync/page.py | 226 ------------------------------- 4 files changed, 287 insertions(+), 227 deletions(-) create mode 100644 utils/Sync/sync/core/SyncPage.py create mode 100644 utils/Sync/sync/models/Page.py delete mode 100644 utils/Sync/sync/page.py diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 1cf0d9ce..499d542d 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -4,6 +4,7 @@ from sync.album import sync_albums from sync.core import Sync from sync.core.SyncDocument import SyncDocument from sync.data import Provider +from sync.core.SyncPage import SyncPage from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider from sync.document import sync_documents @@ -55,6 +56,11 @@ async def run(run_interval=0): sizes=settings.image_sizes, ) + pages = SyncPage( + provider=io(settings.dir_pages), + collection=c(settings.collection_pages), + ) + while True: # main(io(settings.dir_documents), sync_documents, settings.collection_documents) # main(io(settings.dir_awards), sync_documents, settings.collection_awards) @@ -62,10 +68,10 @@ async def run(run_interval=0): # main(io(settings.dir_articles), sync_posts, settings.collection_articles) # main(io(settings.dir_collective), sync_persons, settings.collection_collective) # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) - # main(io(settings.dir_pages), sync_pages, settings.collection_pages) await asyncio.wait([ main(documents), + main(pages), ]) if run_interval == 0: diff --git a/utils/Sync/sync/core/SyncPage.py b/utils/Sync/sync/core/SyncPage.py new file mode 100644 index 00000000..f38f0b48 --- /dev/null +++ b/utils/Sync/sync/core/SyncPage.py @@ -0,0 +1,77 @@ +import os + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import Provider +from sync.models.Page import Page + +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Page') + + +class SyncPage(Sync): + def __init__(self, provider: Provider, collection): + super().__init__(provider, collection) + + async def clean(self): + pass + + async def run(self): + logger.info('Checking for update') + + documents = await Page.scan(self.provider) + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Page(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Pages(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Pages to update') + + for document in documents: + logger.info('Updating Page {}'.format(document.id)) + + await document.setup_images(settings.image_sizes, get_image_url_fn(document.id)) + await document.save() + + logger.info('Updated Page {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Pages to delete') + else: + for document in documents_delete: + logger.info('Deleting Page {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + await self.clean() + + +def get_image_url_fn(page_id: str): + image_id = lambda filename: url_encode_text( + os.path.splitext(filename)[0] + ) + + def img_url_fn(file, size, ext): + return settings.page_url_base.format( + page=page_id, + id=image_id(file), + size=size, + ext=ext + ) + + return img_url_fn diff --git a/utils/Sync/sync/models/Page.py b/utils/Sync/sync/models/Page.py new file mode 100644 index 00000000..1c4c1802 --- /dev/null +++ b/utils/Sync/sync/models/Page.py @@ -0,0 +1,203 @@ +import os + +import logging + +from lxml import etree +import lxml.html + +import settings +from db import collection +from sync.data import request +from sync.models import Model + +from sync import create_date, images_from_html, title_from_html +from sync.models.Image import Image +from utils.hash import hash_str +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Page') + + +def find(store, item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Page(Model): + @staticmethod + async def scan(provider): + documents = [i for i in provider.scan('.') if provider.is_dir(i)] + documents = [provider.type_filter(i, '.md') for i in documents] + documents = [i[0] for i in documents if len(i)] + documents = [Page.read_path(provider, i) for i in documents] + documents = [i for i in documents if i] + + return documents + + @staticmethod + def read_path(provider, path): + params = { + 'file': os.path.basename(path), + 'folder': os.path.dirname(path), + 'title': os.path.basename(path), + } + + manifest = get_manifest(provider, path) + params = { + **params, + **manifest, + } + + if 'url' not in params: + logger.warning('URL not specified for Page {}'.format(path)) + return None + + return Page( + provider, + path, + params=params, + ) + + def __init__(self, provider, file, params=None): + store = collection(settings.collection_pages) + + self.__hash_keys = ['id', 'url'] + + self.__id_template: str = '{category}-{file}' + self.__url_template: str = settings.document_url_template + self.__url_preview_template: str = settings.document_url_preview_template + + self.params = params if params else {} + + super().__init__(provider, store, file) + + def init(self): + self.__set_id() + self.__set_url() + self.__set_hash() + + async def is_changed(self): + i = find(self.store, self.id) + if not i: + return True + + if 'hash' not in i: + return True + + return not (self.hash == i['hash']) + + async def save(self): + document = self.bake() + + query = {'id': document['id']} + try: + self.store.update_one(query, {'$set': document}, upsert=True) + return self + except ValueError: + pass + return None + + async def setup_images(self, sizes, url_factory): + folder = self.params['folder'] + images = [] + for i in self.params['images']: + img_file = os.path.join(folder, i) + + img = await Image.new(self.provider, img_file, sizes, url_factory) + if img: + images.append(img) + self.images = images + + def bake(self): + images = [image.id for image in self.images] + + data = create_post(self.params['data'], lambda src: self.__image_url_by_src(src)) + + return { + **self.params, + 'id': self.id, + 'hash': self.hash, + 'url': self.url, + 'file': self.file, + 'data': data, + 'images': images, + } + + def __image_url_by_src(self, src): + for img in self.images: + if img.file.endswith(src): + s = img.get_size('big') + if s: + return s['url'] + else: + return None + + return None + + def __filename(self): + return os.path.basename(self.file) + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + self.id = url_encode_text(self.params['url']) + + def __set_url(self): + self.url = self.params['url'] + + def __set_hash(self): + images = [os.path.join(self.params['folder'], i) for i in self.params['images']] + files = sorted([self.file] + images) + hashes = [self.provider.hash(i) for i in files] + + self.hash = hash_str(''.join(hashes)) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def create_post(md, image_path_fn): + html = lxml.html.fromstring(md) + + for img in html.cssselect('img'): + src = img.get('src') + path = image_path_fn(src) + + if path: + img.set('src', path) + img.set('class', settings.album_html_img_class) + img.set('data-file', src) + return etree.tounicode(html) + + +def get_manifest(provider, path): + dirname = os.path.dirname(path) + data = provider.read(path).read().decode('utf-8') + manifest, body = read_yaml_md(data) + manifest = manifest if manifest else {} + + if 'date' in manifest: + manifest['date'] = create_date(manifest['date'], settings.date_formats) + + if 'until' in manifest: + manifest['until'] = create_date(manifest['until'], settings.date_formats) + + if 'id' in manifest: + manifest['id'] = str(manifest['id']) + + if 'content' in manifest: + body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8') + del manifest['content'] + + return { + **manifest, + 'data': body, + 'title': title_from_html(body), + 'images': images_from_html(body) + } diff --git a/utils/Sync/sync/page.py b/utils/Sync/sync/page.py deleted file mode 100644 index bedf8a55..00000000 --- a/utils/Sync/sync/page.py +++ /dev/null @@ -1,226 +0,0 @@ -import os - -import settings -from sync.core import Sync -from sync.data import list_images, Provider -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ - synced_images_ids, untouched, title_from_html, create_post -from sync.core.post import SyncPost -from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant -from utils.hash import hash_str -from utils.image import create_image -from utils.io import read_yaml_md -from utils.text.transform import url_encode_text - - -class ImageCreator: - def __init__(self, provider: Provider, sizes: list, output_dir: str, cache: dict): - super().__init__() - self.provider = provider - self.sizes = sizes - self.output_dir = output_dir - self.cache = cache if cache else set() - - def create(self, file, name_fn): - filehash = self.provider.hash(file) - cached = self.cached(filehash) - if cached: - return cached - return create_image(self.provider.get_abs(file), self.sizes, name_fn, self.output_dir, skip_processing=False) - - def cached(self, name): - return self.cache[name] if name in self.cache else None - - -def create_document_from_folder(provider, path): - document = { - 'file': os.path.basename(path), - 'folder': os.path.dirname(path), - 'title': os.path.basename(path), - } - - manifest = get_manifest(provider, path) - document = { - **document, - **manifest, - } - - if 'url' not in document: - return None - - document = create_id(document) - document = create_hash(provider, document) - return document - - -def get_manifest(provider, path): - dirname = os.path.dirname(path) - data = provider.read(path).read().decode('utf-8') - manifest, body = read_yaml_md(data) - manifest = manifest if manifest else {} - - if 'date' in manifest: - manifest['date'] = create_date(manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date(manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - if 'content' in manifest: - body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8') - del manifest['content'] - - return { - **manifest, - 'data': body, - 'title': title_from_html(body), - 'images': images_from_html(body) - } - - -def create_id(document): - new_id = url_encode_text(document['url']) - return { - **document, - 'id': document['id'] if 'id' in document else new_id - } - - -def create_hash(provider, document): - files = sorted([document['file']] + document['images']) - hashes = lmap( - provider.hash, - lmap( - lambda x: os.path.join(document['folder'], x), - files - ) - ) - - return { - **document, - 'hash': hash_str(''.join(hashes)) - } - - -def create_page(provider: Provider, img: ImageCreator, document): - image_path = lambda x: os.path.join(document['folder'], x) - image_id = lambda filename: url_encode_text( - os.path.splitext(filename)[0] - ) - - def process_image(src: str): - path = image_path(src) - - if provider.exists(path): - url = lambda s, e: settings.url_page_base_preview.format( - page=document['id'], - id=image_id(src), - size=s, - ext=e - ) - - image = img.create(path, url) - if image: - images.append(image) - return image['data']['big']['url'] - return None - - images = [] - post_html = create_post(document['data'], process_image) - - document['data'] = post_html - document['images'] = images - return document - - -def check_url(documents): - urls = list(map( - lambda i: i['url'], - documents - )) - - unique_urls = set(urls) - if len(urls) != len(unique_urls): - raise Exception('URL of pages should be unique') - - -def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True): - img_sizes = settings.album_image_sizes - output_dir = settings.dir_static_images - - sync = Sync() - sync.collection = collection - image_creator = ImageCreator(provider, img_sizes, output_dir, {}) - - documents = list(filter( - lambda x: provider.is_dir(x), - provider.scan('.') - )) - - documents = lmap( - lambda x: provider.type_filter(x, '.md'), - documents - ) - - documents = list(filter( - lambda x: len(x), - documents - )) - - documents = lmap( - lambda x: create_document_from_folder(provider, x[0]), - documents - ) - - documents = list(filter(None, documents)) - - # CREATE SCOPE OF CURRENT SESSION - documents_ids = lmapfn(documents)( - lambda i: i['id'] - ) - - check_url(documents) - - # SKIP UNTOUCHED DOCUMENTS - if skip_unchanged: - documents = untouched(documents, sync) - - # # MAP EVENT MANIFEST -> EVENT_OBJECT - # documents = lmap( - # sync.create, - # documents - # ) - - documents = lmap( - lambda document: create_page(provider, image_creator, document), - documents - ) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) - - # SYNC EVENT_OBJECT WITH DB - if update: - documents = lmap( - sync.update, - documents - ) - - documents_to_delete = [] - if delete: - remove_query = sync.create_remove_query({'id': {'$nin': documents_ids}}) - documents_to_delete = sync.query(remove_query) - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - - return documents, documents_to_delete