[sync] posts (events, news) update

utils/Sync/settings.py

@@ -96,7 +96,8 @@ date_formats_reverse = [
 
 collection_documents = 'documents'
 collection_schedules = 'schedules'
-collection_events = 'posts'
+collection_events = 'events'
+collection_news = 'news'
 
 sync_provider_type = env('SYNC_PROVIDER', 'fs')
 

utils/Sync/sync.py

@@ -1,5 +1,3 @@
-from subprocess import call
-
 import settings
 from db import db
 from sync.data.fs import FSProvider
@@ -8,8 +6,6 @@ from sync.document import sync_documents
 from sync.post import sync_posts
 from sync.schedule import sync_schedules
 from utils.fn import lprint_json
-from utils.image import read_image
-from utils.image import thumbnail
 
 
 def get_provider(provider_type, root):
@@ -42,3 +38,4 @@ if __name__ == '__main__':
     main(io(settings.dir_documents), sync_documents, settings.collection_documents)
     main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
     main(io(settings.dir_events), sync_posts, settings.collection_events)
+    main(io(settings.dir_news), sync_posts, settings.collection_news)

utils/Sync/sync/core/post.py

@@ -22,7 +22,8 @@ class SyncPost(Sync):
         self.dir_local_images = dir_local_images
 
     def create_id(self, document):
-        document['id'] = document['id'] if 'id' in document else url_encode_text(document['title'])
+        new_id = url_encode_text(document['title'])
+        document['id'] = document['id'] if 'id' in document else new_id
         return document
 
     def create_hash(self, document):

utils/Sync/sync/post.py

@@ -9,6 +9,44 @@ from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
 from utils.io import read_yaml_md
 
 
+def create_document_from_folder(sync, path, file_time_formats):
+    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+    doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
+    document = {
+        'folder': path,
+        'date': doc_date,
+        'title': doc_title,
+    }
+
+    # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+    images = lmap(
+        lambda img: os.path.relpath(img, path),
+        list_images(sync.provider, path)
+    )
+    post = create_post_from_image_list(images)
+
+    # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+    params = map_cases(
+        document,
+        [(
+            lambda doc: md_from_folder(sync.provider, path),
+            lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)),
+        )],
+        constant({})
+    )
+
+    document = {
+        **document,
+        **params,
+        'images': images,
+        'post': post
+    }
+
+    document = sync.create_id(document)
+    document = sync.create_hash(document)
+    return document
+
+
 def get_manifest(provider, path):
     data = provider.read(path).read().decode('utf-8')
     y, m = read_yaml_md(data)
@@ -44,62 +82,10 @@ def get_folder_documents(sync, file_time_formats):
         )
     )
 
-    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
-    documents = lmap(
-        lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats),
+    return lmap(
+        lambda doc: create_document_from_folder(sync, doc, file_time_formats),
         documents
     )
-    documents = lmap(
-        lambda i: {
-            'folder': i[0],
-            'date': i[1],
-            'title': i[2],
-        },
-        documents
-    )
-
-    # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
-    documents = lmap(
-        lambda i: {
-            **i,
-            'images': lmap(
-                lambda path: os.path.relpath(path, i['folder']),
-                list_images(sync.provider, i['folder'])
-            )
-        },
-        documents
-    )
-    documents = lmap(
-        lambda i: {
-            **i,
-            'post': create_post_from_image_list(i['images'])
-        },
-        documents
-    )
-
-    # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
-    documents = lmap(
-        lambda i: {
-            **i,
-            **map_cases(
-                i,
-                [(
-                    lambda doc: md_from_folder(sync.provider, doc['folder']),
-                    lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, doc['folder'])),
-                )],
-                constant({})
-            )
-        },
-        documents
-    )
-
-    # CREATE DOCUMENT IDENTITY
-    documents = lmap(sync.create_id, documents)
-
-    # CREATE HASH OF DOCUMENT FILE
-    documents = lmap(sync.create_hash, documents)
-
-    return documents
 
 
 # GET EVENT FILES
@@ -149,20 +135,20 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
     # SKIP UNTOUCHED DOCUMENTS
     documents = untouched(documents, sync)
 
-    # MAP EVENT MANIFEST -> EVENT_OBJECT
-    documents = lmap(
-        sync.create,
-        documents
-    )
+    # # MAP EVENT MANIFEST -> EVENT_OBJECT
+    # documents = lmap(
+    #     sync.create,
+    #     documents
+    # )
 
     # FILTER INCORRECT EVENTS
     documents = list(filter(None, documents))
 
-    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
-    documents = lmap(
-        key_mapper('images', synced_images_ids),
-        documents
-    )
+    # # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
+    # documents = lmap(
+    #     key_mapper('images', synced_images_ids),
+    #     documents
+    # )
 
     # SYNC EVENT_OBJECT WITH DB
     if update_documents:
@@ -171,6 +157,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
             documents
         )
 
+    documents_to_delete = []
     if delete_documents:
         remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
         documents_to_delete = sync.query(remove_query)
@@ -181,8 +168,6 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
                 documents_to_delete
             )
         )
-    else:
-        documents_to_delete = []
 
     return documents, documents_to_delete