From a4b4e0768723573c9f3d4ede4d06cc5fcd8c229d Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Wed, 23 Nov 2016 02:10:26 +0300 Subject: [PATCH] [sync] updated; page images --- utils/Sync/sync.py | 12 +++---- utils/Sync/sync/__init__.py | 16 +++++++++ utils/Sync/sync/album.py | 13 ++++--- utils/Sync/sync/core/album.py | 22 +++++------- utils/Sync/sync/page.py | 66 +++++++++++++++++++++++++++++++++-- utils/Sync/utils/fn.py | 3 +- 6 files changed, 104 insertions(+), 28 deletions(-) diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 533a4359..8febad88 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -37,10 +37,10 @@ def main(provider, fn, collection_name): if __name__ == '__main__': io = lambda root: get_provider(settings.sync_provider_type, root) - main(io(settings.dir_documents), sync_documents, settings.collection_documents) - main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) - main(io(settings.dir_events), sync_posts, settings.collection_events) - main(io(settings.dir_news), sync_posts, settings.collection_news) - main(io(settings.dir_collective), sync_persons, settings.collection_collective) - main(io(settings.dir_gallery), sync_albums, settings.collection_albums) + # main(io(settings.dir_documents), sync_documents, settings.collection_documents) + # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) + # main(io(settings.dir_events), sync_posts, settings.collection_events) + # main(io(settings.dir_news), sync_posts, settings.collection_news) + # main(io(settings.dir_collective), sync_persons, settings.collection_collective) + # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) main(io(settings.dir_pages), sync_pages, settings.collection_pages) diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index 879a2b4b..2dbfcf55 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -2,10 +2,12 @@ import os import re from datetime import datetime from glob import glob +from lxml import etree import lxml.html from markdown import markdown +import settings from sync.image import sync_image from utils.fn import lmap, combine @@ -80,6 +82,20 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None): return date, title +def create_post(md, image_path_fn): + html = lxml.html.fromstring(md) + + for img in html.cssselect('img'): + src = img.get('src') + path = image_path_fn(src) + + if path: + img.set('src', path) + img.set('class', settings.album_html_img_class) + img.set('data-file', src) + return etree.tounicode(html) + + def images_from_html(md): if not md: return [] diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py index a924632c..71a97d41 100644 --- a/utils/Sync/sync/album.py +++ b/utils/Sync/sync/album.py @@ -2,7 +2,7 @@ import os import settings from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \ - synced_images_ids + synced_images_ids, synced_image_id from sync.core.album import SyncAlbum from sync.data import list_images from utils.fn import lmap, lmapfn, key_mapper @@ -19,10 +19,11 @@ class ImageCreator: def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False): filehash = self.provider.hash(file) cached = self.cached(filehash) - return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached + return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, + skip_processing=skip_processing) if not cached else cached - def cached(self, hash): - return self.cache[hash] if hash in self.cache else None + def cached(self, name): + return self.cache[name] if name in self.cache else None def get_manifest(provider, filepath): @@ -121,6 +122,10 @@ def main(sync, do_update, do_delete): key_mapper('images', synced_images_ids), documents ) + documents = lmap( + key_mapper('preview', synced_image_id), + documents + ) # SYNC OBJECT WITH DB if do_update: diff --git a/utils/Sync/sync/core/album.py b/utils/Sync/sync/core/album.py index d68a571b..7c80a109 100644 --- a/utils/Sync/sync/core/album.py +++ b/utils/Sync/sync/core/album.py @@ -3,6 +3,7 @@ from lxml import etree import lxml.html import settings +from sync import create_post from sync.core import Sync from sync.models.product import create_product from utils.fn import lmap, iterate_iter_over_fns, combine @@ -10,20 +11,6 @@ from utils.hash import hash_file, hash_str from utils.text.transform import url_encode_text -def create_post(md, image_path_fn): - html = lxml.html.fromstring(md) - - for img in html.cssselect('img'): - src = img.get('src') - path = image_path_fn(src) - - if path: - img.set('src', path) - img.set('class', settings.album_html_img_class) - img.set('data-file', src) - return etree.tounicode(html) - - def create_url(a, i, s, e): return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( album=a, @@ -120,6 +107,13 @@ class SyncAlbum(Sync): images = [] post_html = create_post(doc['post'], process_post_image) + if 'preview' in doc: + preview = os.path.join(doc['folder'], doc['preview']) + doc['preview'] = self.create_image(preview, url_creator( + doc['id'], + image_id(doc['preview']) + )) + doc['post'] = post_html doc['images'] = images return doc diff --git a/utils/Sync/sync/page.py b/utils/Sync/sync/page.py index 3c0217a8..e8deb5dd 100644 --- a/utils/Sync/sync/page.py +++ b/utils/Sync/sync/page.py @@ -2,16 +2,36 @@ import os import settings from sync.core import Sync -from sync.data import list_images +from sync.data import list_images, Provider from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ - synced_images_ids, untouched, title_from_html + synced_images_ids, untouched, title_from_html, create_post from sync.core.post import SyncPost from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant from utils.hash import hash_str +from utils.image import create_image from utils.io import read_yaml_md from utils.text.transform import url_encode_text +class ImageCreator: + def __init__(self, provider: Provider, sizes: list, output_dir: str, cache: dict): + super().__init__() + self.provider = provider + self.sizes = sizes + self.output_dir = output_dir + self.cache = cache if cache else set() + + def create(self, file, name_fn): + filehash = self.provider.hash(file) + cached = self.cached(filehash) + if cached: + return cached + return create_image(self.provider.get_abs(file), self.sizes, name_fn, self.output_dir, skip_processing=False) + + def cached(self, name): + return self.cache[name] if name in self.cache else None + + def create_document_from_folder(provider, path): document = { 'file': os.path.basename(path), @@ -75,13 +95,48 @@ def create_hash(provider, document): return { **document, - 'hash':hash_str(''.join(hashes)) + 'hash': hash_str(''.join(hashes)) } +def create_page(provider: Provider, img: ImageCreator, document): + image_path = lambda x: os.path.join(document['folder'], x) + image_id = lambda filename: url_encode_text( + os.path.splitext(filename)[0] + ) + + def process_image(src: str): + path = image_path(src) + + if provider.exists(path): + url = lambda s, e: 'https://static.shburg.org/art/images/page-{page}-{id}-{size}{ext}'.format( + page=document['id'], + id=image_id(src), + size=s, + ext=e + ) + + image = img.create(path, url) + if image: + images.append(image) + return image['data']['big']['url'] + return None + + images = [] + post_html = create_post(document['data'], process_image) + + document['data'] = post_html + document['images'] = images + return document + + def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True): + img_sizes = settings.album_image_sizes + output_dir = settings.dir_static_images + sync = Sync() sync.collection = collection + image_creator = ImageCreator(provider, img_sizes, output_dir, {}) documents = list(filter( lambda x: provider.is_dir(x), @@ -120,6 +175,11 @@ def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=Tr # documents # ) + documents = lmap( + lambda document: create_page(provider, image_creator, document), + documents + ) + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB documents = lmap( key_mapper('images', synced_images_ids), diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index 11ad079b..7cd0b53a 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -61,7 +61,8 @@ def map_item_key(item, key, map_fn): :param map_fn: :return: """ - item[key] = map_fn(item[key]) + if key in item: + item[key] = map_fn(item[key]) return item