[sync] post updated

utils/Sync/settings.py

@@ -96,6 +96,7 @@ date_formats_reverse = [
 
 collection_documents = 'documents'
 collection_schedules = 'schedules'
+collection_events = 'posts'
 
 sync_provider_type = env('SYNC_PROVIDER', 'fs')
 

utils/Sync/sync.py

@@ -2,8 +2,7 @@ import settings
 from db import db
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
-from sync.document import sync_documents
-from sync.schedule import sync_schedules
+from sync.post import sync_posts
 from utils.fn import lprint_json
 
 
@@ -19,17 +18,24 @@ def main(provider, fn, collection_name):
     if provider:
         collection = db()[collection_name]
 
+        print('SYNC: %s' % (collection_name.upper()))
         u, d = fn(provider, collection, update=settings.do_update, delete=settings.do_update)
+
         print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d))))
         lprint_json(d)
 
         print('UPDATE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_update else str(len(u))))
         lprint_json(u)
 
+        print('')
+
 
 if __name__ == '__main__':
     # client = get_provider(settings.sync_provider_type, settings.dir_documents)
     # main(client, sync_documents, settings.collection_documents)
 
-    client = get_provider(settings.sync_provider_type, settings.dir_schedules)
-    main(client, sync_schedules, settings.collection_schedules)
+    # client = get_provider(settings.sync_provider_type, settings.dir_schedules)
+    # main(client, sync_schedules, settings.collection_schedules)
+
+    client = get_provider(settings.sync_provider_type, settings.dir_events)
+    main(client, sync_posts, settings.collection_events)

utils/Sync/sync/__init__.py

@@ -10,14 +10,6 @@ from sync.image import sync_image
 from utils.fn import lmap, combine
 
 
-def list_images(dir):
-    image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
-    return combine(map(
-        lambda type: glob(os.path.join(dir, type)),
-        image_types
-    ))
-
-
 def create_date(date_str, date_formats=None):
     """
     2016

utils/Sync/sync/core/post.py

@@ -0,0 +1,87 @@
+import os
+from tempfile import mkstemp
+
+import lxml.html
+
+from sync.core import Sync
+from utils.fn import last_good, lmap, combine
+from utils.hash import hash_str, hash_file
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+class SyncPost(Sync):
+    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+        self.sizes = sizes
+        self.type = document_type
+        self.image_url_base = image_url_base
+        self.dir_local_images = dir_local_images
+
+    def create_id(self, document):
+        document['id'] = document['id'] if 'id' in document else url_encode_text(document['title'])
+        return document
+
+    def create_hash(self, document):
+        hash_images = lmap(
+            self.provider.hash,
+            lmap(
+                lambda i: os.path.join(document['folder'], i),
+                document['images']
+            )
+        )
+
+        document['hash'] = hash_str(
+            combine([hash_str(document)] + hash_images)
+        )
+
+        return document
+
+    def create(self, document):
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+        if self.type:
+            document['type'] = self.type
+
+        post_html = lxml.html.fromstring(document['post'])
+        images = []
+        for img in post_html.cssselect('img'):
+            src = img.get('src')
+            img_path = os.path.abspath(
+                os.path.join(document['folder'], src)
+            )
+
+            if os.path.exists(img_path):
+                img_id = url_encode_text(os.path.splitext(src)[0])
+                url_fn = lambda size, ext: self.image_url_base.format(
+                    id=document['id'],
+                    img=img_id,
+                    size=size,
+                    ext=ext.lower()
+                )
+
+                image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
+                if image:
+                    images.append(image)
+                    img.set('src', image['data']['big']['url'])
+                else:
+                    print('fail: ', document['folder'], src)
+
+        document['post'] = lxml.html.tostring(post_html).decode('utf-8')
+        document['images'] = images
+        return document
+
+    def create_remove_query(self, query):
+        if self.type:
+            query = {
+                **query,
+                'type': self.type
+            }
+        return query

utils/Sync/sync/data/__init__.py

@@ -69,3 +69,11 @@ def scan_subdirs(provider, ext, path='.'):
 def get_data(provider, path, transform):
     data = provider.read(path)
     return transform(data)
+
+
+def list_images(provider, path):
+    image_types = ['.jpg', '.JPG', '.png', '.PNG']
+    return combine(lmap(
+        lambda ext: provider.type_filter(path, ext),
+        image_types
+    ))

utils/Sync/sync/post.py

@@ -5,90 +5,22 @@ from glob import glob
 import lxml.html
 
 import settings
+from sync.data import list_images
 from db import db
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \
-    images_from_html, list_images, synced_images_ids, untouched
+from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
+    synced_images_ids, untouched
+from sync.core.post import SyncPost
 from sync.image import sync_image
-from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn
+from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn, constant
 from utils.hash import hash_file, hash_str
 from utils.image import create_image
 from utils.io import read_yaml_md
 from utils.text.transform import url_encode_text
 
 
-class SyncPost(Sync):
-    def __init__(self, collection_name, document_type, image_url_base, dir_local_images, sizes):
-        super().__init__()
-        self.collection = db()[collection_name]
-        self.sizes = sizes
-        self.type = document_type
-        self.image_url_base = image_url_base
-        self.dir_local_images = dir_local_images
-
-    def create_id(self, document):
-        document['id'] = document['id'] if 'id' in document else url_encode_text(document['title'])
-        return document
-
-    def create_hash(self, document):
-        hash_images = lmap(
-            hash_file,
-            lmap(
-                lambda i: os.path.join(document['folder'], i),
-                document['images']
-            )
-        )
-
-        document['hash'] = hash_str(
-            combine([hash_str(document)] + hash_images)
-        )
-
-        return document
-
-    def create(self, document):
-        if 'date' not in document or not document['date']:
-            return None
-
-        if 'title' not in document or not document['title']:
-            return None
-
-        if self.type:
-            document['type'] = self.type
-
-        post_html = lxml.html.fromstring(document['post'])
-        images = []
-        for img in post_html.cssselect('img'):
-            src = img.get('src')
-            img_path = os.path.abspath(
-                os.path.join(document['folder'], src)
-            )
-
-            if os.path.exists(img_path):
-                img_id = url_encode_text(os.path.splitext(src)[0])
-                url_fn = lambda size, ext: self.image_url_base.format(id=document['id'], img=img_id, size=size,
-                                                                      ext=ext.lower())
-
-                image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
-                if image:
-                    images.append(image)
-                    img.set('src', image['data']['big']['url'])
-                else:
-                    print('fail: ', document['folder'], src)
-
-        document['post'] = lxml.html.tostring(post_html).decode('utf-8')
-        document['images'] = images
-        return document
-
-    def create_remove_query(self, query):
-        if self.type:
-            query = {
-                **query,
-                'type': self.type
-            }
-        return query
-
-
-def get_manifest(path):
-    y, m = read_yaml_md(path)
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    y, m = read_yaml_md(data)
     y = y if y else {}
 
     if 'date' in y:
@@ -107,25 +39,23 @@ def get_manifest(path):
     }
 
 
-def md_from_folder(i):
-    return first(glob(i + '/*.md'))
+def md_from_folder(provider, i):
+    return first(provider.type_filter(i, '.md'))
 
 
 def get_folder_documents(sync, file_time_formats):
     # GET EVENT FOLDERS
     documents = lmap(
-        lambda i: i.path,
+        lambda i: i,
         filter(
-            lambda i: i.is_dir(),
-            os.scandir('.')
+            lambda i: sync.provider.is_dir(i),
+            sync.provider.scan('.')
         )
     )
 
     # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
     documents = lmap(
-        lambda i: (i,) + create_date_and_title_from_folder_name(
-            os.path.basename(i), file_time_formats
-        ),
+        lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats),
         documents
     )
     documents = lmap(
@@ -143,7 +73,7 @@ def get_folder_documents(sync, file_time_formats):
             **i,
             'images': lmap(
                 lambda path: os.path.relpath(path, i['folder']),
-                list_images(i['folder'])
+                list_images(sync.provider, i['folder'])
             )
         },
         documents
@@ -163,10 +93,10 @@ def get_folder_documents(sync, file_time_formats):
             **map_cases(
                 i,
                 [(
-                    lambda i: md_from_folder(i['folder']),
-                    lambda i: get_manifest(md_from_folder(i['folder'])),
+                    lambda doc: md_from_folder(sync.provider, doc['folder']),
+                    lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, doc['folder'])),
                 )],
-                lambda i: {}
+                constant({})
             )
         },
         documents
@@ -183,11 +113,9 @@ def get_folder_documents(sync, file_time_formats):
 
 # GET EVENT FILES
 def get_file_documents(sync, file_time_formats):
-    documents = glob('*.md')
+    documents = sync.provider.type_filter('.', '.md')
     documents = lmap(
-        lambda i: (i,) + create_date_and_title_from_folder_name(
-            os.path.basename(i), file_time_formats
-        ),
+        lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats),
         documents
     )
 
@@ -204,7 +132,7 @@ def get_file_documents(sync, file_time_formats):
     documents = lmap(
         lambda i: {
             **i,
-            **get_manifest(i['file'])
+            **get_manifest(sync.provider, i['file'])
         },
         documents
     )
@@ -218,14 +146,11 @@ def get_file_documents(sync, file_time_formats):
     return documents
 
 
-def main(dir_documents, sync, file_time_formats, update_documents=True, delete_documents=True):
-    # INIT
-    os.chdir(dir_documents)
-
+def main(sync, file_time_formats, update_documents=True, delete_documents=True):
     documents = get_folder_documents(sync, file_time_formats)
     documents += get_file_documents(sync, file_time_formats)
 
-    # CREATE SCOPE OF CURRENT SESSION
+    # # CREATE SCOPE OF CURRENT SESSION
     scope_documents_ids = lmapfn(documents)(
         lambda i: i['id']
     )
@@ -268,50 +193,22 @@ def main(dir_documents, sync, file_time_formats, update_documents=True, delete_d
     else:
         documents_to_delete = []
 
-    # SCOPE
-    print('SCOPE:')
-    lprint(scope_documents_ids)
-
-    # DELETE
-    print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete))))
-    lprint_json(lmap(
-        lambda i: i['id'],
-        documents_to_delete
-    ))
-
-    # DELETE
-    print('UPDATE DOCUMENTS:')
-    lprint_json(lmap(
-        lambda i: i['id'],
-        documents
-    ))
-    # lprint_json(documents)
-    print('[SYNC DOCUMENTS DONE]')
+    return documents, documents_to_delete
 
 
-if __name__ == '__main__':
+def sync_posts(provider, collection, update=True, delete=True):
     image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
-    main(
-        settings.dir_events,
+
+    return main(
         SyncPost(
-            'events',
+            collection,
+            provider,
             None,
             image_url_base,
             settings.dir_static_images,
             settings.event_image_sizes
         ),
-        file_time_formats=['%Y.%m.%d']
-    )
-
-    image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
-    main(
-        settings.dir_news,
-        SyncPost(
-            'timeline',
-            'post',
-            image_url_base,
-            settings.dir_static_images,
-            settings.event_image_sizes
-        ),
-        file_time_formats=settings.date_formats_reverse
+        file_time_formats=['%Y.%m.%d'],
+        update_documents=update,
+        delete_documents=delete
     )

utils/Sync/utils/io.py

@@ -27,9 +27,9 @@ def read_md(file):
         return markdown(data)
 
 
-def read_yaml_md(file):
-    yaml, content = parse_yaml_front_matter(read_file(file))
-    return yaml, markdown(content)
+def read_yaml_md(data):
+    yaml_front_matter, content = parse_yaml_front_matter(data)
+    return yaml_front_matter, markdown(content)
 
     # data = frontmatter.load(file)
     # return data.metadata, markdown(data.content)