[sync] updated: + albums, kazimir rendering, other changes

utils/Sync/sync/core/album.py

@@ -0,0 +1,145 @@
+import os
+from lxml import etree
+
+import lxml.html
+import settings
+from sync.core import Sync
+from sync.models.product import create_product
+from utils.fn import lmap, iterate_iter_over_fns, combine
+from utils.hash import hash_file, hash_str
+from utils.text.transform import url_encode_text
+
+
+def create_post(md, image_path_fn):
+    html = lxml.html.fromstring(md)
+
+    for img in html.cssselect('img'):
+        src = img.get('src')
+        path = image_path_fn(src)
+
+        if path:
+            img.set('src', path)
+            img.set('class', settings.album_html_img_class)
+            img.set('data-file', src)
+    return etree.tounicode(html)
+
+
+def create_url(a, i, s, e):
+    return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
+        album=a,
+        id=i,
+        size=s,
+        ext=e
+    )
+
+
+def url_creator(aid, pid):
+    return lambda size, ext: create_url(aid, pid, size, ext.lower())
+
+
+class SyncAlbum(Sync):
+    @staticmethod
+    def id_from_file(i):
+        return url_encode_text(
+            os.path.splitext(i)[0]
+        )
+
+    def __init__(self, collection, provider, image_creator, images_dir):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+        self.image_creator = image_creator
+        self.skip_image_processing = False
+        self.images_dir = images_dir
+
+    def create_id(self, document):
+        if 'id' in document:
+            return document
+
+        y = str(document['date'].year)
+        document['id'] = url_encode_text('-'.join([y, document['title']]))
+        return document
+
+    def create_hash(self, document):
+        files = sorted(document['images'])
+        hashes = lmap(
+            self.provider.hash,
+            lmap(
+                lambda i: os.path.join(document['folder'], i),
+                files
+            )
+        )
+
+        document['hash'] = hash_str(
+            combine([hash_str(document)] + hashes)
+        )
+
+        return document
+
+    def create_image(self, file, url_fn):
+        ss = settings.album_image_sizes
+        od = self.images_dir
+        sp = self.skip_image_processing
+
+        return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
+
+    def create_album(self, doc):
+        # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
+
+        image_id = SyncAlbum.id_from_file
+        image_path = lambda i: os.path.join(doc['folder'], i)
+
+        if 'title' not in doc or not doc['title']:
+            return None
+
+        create_album_id = lambda date, title: '{date}-{title}'.format(
+            date=date.strftime('%Y'),
+            title=url_encode_text(title)
+        )
+
+        doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
+
+        if 'album' in doc:
+            doc['album'] = self.create_products(doc, image_path)
+
+        def process_post_image(src):
+            path = image_path(src)
+
+            if self.provider.exists(path):
+                url = url_creator(
+                    doc['id'],
+                    image_id(src)
+                )
+
+                image = self.create_image(path, url)
+                if image:
+                    images.append(image)
+                    return image['data']['big']['url']
+            return None
+
+        images = []
+        post_html = create_post(doc['post'], process_post_image)
+
+        doc['post'] = post_html
+        doc['images'] = images
+        return doc
+
+    def create_products(self, doc, image_file_fn):
+        image_id = SyncAlbum.id_from_file
+        iter_album = iterate_iter_over_fns([
+            lambda product: {
+                **product,
+                'image': self.create_image(
+                    image_file_fn(product['file']),
+                    url_creator(
+                        doc['id'],
+                        image_id(product['file'])
+                    )
+                )
+            }
+        ])
+
+        return lmap(
+            create_product,
+            iter_album(doc['album'])
+        )

utils/Sync/sync/core/person.py

@@ -1,6 +1,6 @@
 import settings
 from sync.core import Sync
-from utils.hash import hash_str, hash_file
+from utils.hash import hash_str
 from utils.image import create_image
 from utils.text.transform import url_encode_text
 
@@ -16,8 +16,9 @@ class SyncPerson(Sync):
         self.dir_local_images = dir_local_images
 
     def compile(self, profile):
+        image_path = self.provider.get_abs(profile['picture'])
         profile['picture'] = create_image(
-            profile['picture'],
+            image_path,
             sizes,
             lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
             self.dir_local_images

utils/Sync/sync/core/post.py

@@ -12,7 +12,7 @@ from utils.text.transform import url_encode_text, url_encode_file
 
 
 class SyncPost(Sync):
-    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes):
+    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
         super().__init__()
         self.collection = collection
         self.provider = provider
@@ -20,6 +20,7 @@ class SyncPost(Sync):
         self.type = document_type
         self.image_url_base = image_url_base
         self.dir_local_images = dir_local_images
+        self.origin = origin
 
     def create_id(self, document):
         new_id = url_encode_text(document['title'])
@@ -77,12 +78,14 @@ class SyncPost(Sync):
 
         document['post'] = lxml.html.tostring(post_html).decode('utf-8')
         document['images'] = images
+        document['origin'] = self.origin
         return document
 
     def create_remove_query(self, query):
         if self.type:
             query = {
                 **query,
-                'type': self.type
+                'type': self.type,
+                'orogin': self.origin
             }
         return query