[sync] updated; page images

utils/Sync/sync.py

@@ -37,10 +37,10 @@ def main(provider, fn, collection_name):
 if __name__ == '__main__':
     io = lambda root: get_provider(settings.sync_provider_type, root)
 
-    main(io(settings.dir_documents), sync_documents, settings.collection_documents)
-    main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
-    main(io(settings.dir_events), sync_posts, settings.collection_events)
-    main(io(settings.dir_news), sync_posts, settings.collection_news)
-    main(io(settings.dir_collective), sync_persons, settings.collection_collective)
-    main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
+    # main(io(settings.dir_documents), sync_documents, settings.collection_documents)
+    # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
+    # main(io(settings.dir_events), sync_posts, settings.collection_events)
+    # main(io(settings.dir_news), sync_posts, settings.collection_news)
+    # main(io(settings.dir_collective), sync_persons, settings.collection_collective)
+    # main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
     main(io(settings.dir_pages), sync_pages, settings.collection_pages)

utils/Sync/sync/__init__.py

@@ -2,10 +2,12 @@ import os
 import re
 from datetime import datetime
 from glob import glob
+from lxml import etree
 
 import lxml.html
 from markdown import markdown
 
+import settings
 from sync.image import sync_image
 from utils.fn import lmap, combine
 
@@ -80,6 +82,20 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
     return date, title
 
 
+def create_post(md, image_path_fn):
+    html = lxml.html.fromstring(md)
+
+    for img in html.cssselect('img'):
+        src = img.get('src')
+        path = image_path_fn(src)
+
+        if path:
+            img.set('src', path)
+            img.set('class', settings.album_html_img_class)
+            img.set('data-file', src)
+    return etree.tounicode(html)
+
+
 def images_from_html(md):
     if not md:
         return []

utils/Sync/sync/album.py

@@ -2,7 +2,7 @@ import os
 
 import settings
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \
-    synced_images_ids
+    synced_images_ids, synced_image_id
 from sync.core.album import SyncAlbum
 from sync.data import list_images
 from utils.fn import lmap, lmapfn, key_mapper
@@ -19,10 +19,11 @@ class ImageCreator:
     def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
         filehash = self.provider.hash(file)
         cached = self.cached(filehash)
-        return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
+        return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir,
+                            skip_processing=skip_processing) if not cached else cached
 
-    def cached(self, hash):
-        return self.cache[hash] if hash in self.cache else None
+    def cached(self, name):
+        return self.cache[name] if name in self.cache else None
 
 
 def get_manifest(provider, filepath):
@@ -121,6 +122,10 @@ def main(sync, do_update, do_delete):
         key_mapper('images', synced_images_ids),
         documents
     )
+    documents = lmap(
+        key_mapper('preview', synced_image_id),
+        documents
+    )
 
     # SYNC OBJECT WITH DB
     if do_update:

utils/Sync/sync/core/album.py

@@ -3,6 +3,7 @@ from lxml import etree
 
 import lxml.html
 import settings
+from sync import create_post
 from sync.core import Sync
 from sync.models.product import create_product
 from utils.fn import lmap, iterate_iter_over_fns, combine
@@ -10,20 +11,6 @@ from utils.hash import hash_file, hash_str
 from utils.text.transform import url_encode_text
 
 
-def create_post(md, image_path_fn):
-    html = lxml.html.fromstring(md)
-
-    for img in html.cssselect('img'):
-        src = img.get('src')
-        path = image_path_fn(src)
-
-        if path:
-            img.set('src', path)
-            img.set('class', settings.album_html_img_class)
-            img.set('data-file', src)
-    return etree.tounicode(html)
-
-
 def create_url(a, i, s, e):
     return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
         album=a,
@@ -120,6 +107,13 @@ class SyncAlbum(Sync):
         images = []
         post_html = create_post(doc['post'], process_post_image)
 
+        if 'preview' in doc:
+            preview = os.path.join(doc['folder'], doc['preview'])
+            doc['preview'] = self.create_image(preview, url_creator(
+                doc['id'],
+                image_id(doc['preview'])
+            ))
+
         doc['post'] = post_html
         doc['images'] = images
         return doc

utils/Sync/sync/page.py

@@ -2,16 +2,36 @@ import os
 
 import settings
 from sync.core import Sync
-from sync.data import list_images
+from sync.data import list_images, Provider
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
-    synced_images_ids, untouched, title_from_html
+    synced_images_ids, untouched, title_from_html, create_post
 from sync.core.post import SyncPost
 from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
 from utils.hash import hash_str
+from utils.image import create_image
 from utils.io import read_yaml_md
 from utils.text.transform import url_encode_text
 
 
+class ImageCreator:
+    def __init__(self, provider: Provider, sizes: list, output_dir: str, cache: dict):
+        super().__init__()
+        self.provider = provider
+        self.sizes = sizes
+        self.output_dir = output_dir
+        self.cache = cache if cache else set()
+
+    def create(self, file, name_fn):
+        filehash = self.provider.hash(file)
+        cached = self.cached(filehash)
+        if cached:
+            return cached
+        return create_image(self.provider.get_abs(file), self.sizes, name_fn, self.output_dir, skip_processing=False)
+
+    def cached(self, name):
+        return self.cache[name] if name in self.cache else None
+
+
 def create_document_from_folder(provider, path):
     document = {
         'file': os.path.basename(path),
@@ -79,9 +99,44 @@ def create_hash(provider, document):
     }
 
 
+def create_page(provider: Provider, img: ImageCreator, document):
+    image_path = lambda x: os.path.join(document['folder'], x)
+    image_id = lambda filename: url_encode_text(
+        os.path.splitext(filename)[0]
+    )
+
+    def process_image(src: str):
+        path = image_path(src)
+
+        if provider.exists(path):
+            url = lambda s, e: 'https://static.shburg.org/art/images/page-{page}-{id}-{size}{ext}'.format(
+                page=document['id'],
+                id=image_id(src),
+                size=s,
+                ext=e
+            )
+
+            image = img.create(path, url)
+            if image:
+                images.append(image)
+                return image['data']['big']['url']
+        return None
+
+    images = []
+    post_html = create_post(document['data'], process_image)
+
+    document['data'] = post_html
+    document['images'] = images
+    return document
+
+
 def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True):
+    img_sizes = settings.album_image_sizes
+    output_dir = settings.dir_static_images
+
     sync = Sync()
     sync.collection = collection
+    image_creator = ImageCreator(provider, img_sizes, output_dir, {})
 
     documents = list(filter(
         lambda x: provider.is_dir(x),
@@ -120,6 +175,11 @@ def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=Tr
     #     documents
     # )
 
+    documents = lmap(
+        lambda document: create_page(provider, image_creator, document),
+        documents
+    )
+
     # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
     documents = lmap(
         key_mapper('images', synced_images_ids),

utils/Sync/utils/fn.py

@@ -61,6 +61,7 @@ def map_item_key(item, key, map_fn):
     :param map_fn:
     :return:
     """
+    if key in item:
         item[key] = map_fn(item[key])
     return item