diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index ae1d1468..e3ef4eb3 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -96,7 +96,8 @@ date_formats_reverse = [ collection_documents = 'documents' collection_schedules = 'schedules' -collection_events = 'posts' +collection_events = 'events' +collection_news = 'news' sync_provider_type = env('SYNC_PROVIDER', 'fs') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 6e0d4652..05e327b8 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,5 +1,3 @@ -from subprocess import call - import settings from db import db from sync.data.fs import FSProvider @@ -8,8 +6,6 @@ from sync.document import sync_documents from sync.post import sync_posts from sync.schedule import sync_schedules from utils.fn import lprint_json -from utils.image import read_image -from utils.image import thumbnail def get_provider(provider_type, root): @@ -42,3 +38,4 @@ if __name__ == '__main__': main(io(settings.dir_documents), sync_documents, settings.collection_documents) main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) main(io(settings.dir_events), sync_posts, settings.collection_events) + main(io(settings.dir_news), sync_posts, settings.collection_news) diff --git a/utils/Sync/sync/core/post.py b/utils/Sync/sync/core/post.py index 92051095..2b0cc8c9 100644 --- a/utils/Sync/sync/core/post.py +++ b/utils/Sync/sync/core/post.py @@ -22,7 +22,8 @@ class SyncPost(Sync): self.dir_local_images = dir_local_images def create_id(self, document): - document['id'] = document['id'] if 'id' in document else url_encode_text(document['title']) + new_id = url_encode_text(document['title']) + document['id'] = document['id'] if 'id' in document else new_id return document def create_hash(self, document): diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 0750090b..348adef8 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -9,6 +9,44 @@ from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant from utils.io import read_yaml_md +def create_document_from_folder(sync, path, file_time_formats): + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats) + document = { + 'folder': path, + 'date': doc_date, + 'title': doc_title, + } + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + images = lmap( + lambda img: os.path.relpath(img, path), + list_images(sync.provider, path) + ) + post = create_post_from_image_list(images) + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + params = map_cases( + document, + [( + lambda doc: md_from_folder(sync.provider, path), + lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)), + )], + constant({}) + ) + + document = { + **document, + **params, + 'images': images, + 'post': post + } + + document = sync.create_id(document) + document = sync.create_hash(document) + return document + + def get_manifest(provider, path): data = provider.read(path).read().decode('utf-8') y, m = read_yaml_md(data) @@ -44,62 +82,10 @@ def get_folder_documents(sync, file_time_formats): ) ) - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - documents = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats), + return lmap( + lambda doc: create_document_from_folder(sync, doc, file_time_formats), documents ) - documents = lmap( - lambda i: { - 'folder': i[0], - 'date': i[1], - 'title': i[2], - }, - documents - ) - - # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - documents = lmap( - lambda i: { - **i, - 'images': lmap( - lambda path: os.path.relpath(path, i['folder']), - list_images(sync.provider, i['folder']) - ) - }, - documents - ) - documents = lmap( - lambda i: { - **i, - 'post': create_post_from_image_list(i['images']) - }, - documents - ) - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - documents = lmap( - lambda i: { - **i, - **map_cases( - i, - [( - lambda doc: md_from_folder(sync.provider, doc['folder']), - lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, doc['folder'])), - )], - constant({}) - ) - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) - - # CREATE HASH OF DOCUMENT FILE - documents = lmap(sync.create_hash, documents) - - return documents # GET EVENT FILES @@ -149,20 +135,20 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True): # SKIP UNTOUCHED DOCUMENTS documents = untouched(documents, sync) - # MAP EVENT MANIFEST -> EVENT_OBJECT - documents = lmap( - sync.create, - documents - ) + # # MAP EVENT MANIFEST -> EVENT_OBJECT + # documents = lmap( + # sync.create, + # documents + # ) # FILTER INCORRECT EVENTS documents = list(filter(None, documents)) - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) + # # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB + # documents = lmap( + # key_mapper('images', synced_images_ids), + # documents + # ) # SYNC EVENT_OBJECT WITH DB if update_documents: @@ -171,6 +157,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True): documents ) + documents_to_delete = [] if delete_documents: remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}}) documents_to_delete = sync.query(remove_query) @@ -181,8 +168,6 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True): documents_to_delete ) ) - else: - documents_to_delete = [] return documents, documents_to_delete