diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 44c98ac0..9e7c7c48 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -98,6 +98,7 @@ collection_documents = 'documents' collection_schedules = 'schedules' collection_events = 'events' collection_news = 'news' +collection_collective = 'collective' sync_provider_type = env('SYNC_PROVIDER', 'fs') @@ -111,6 +112,7 @@ dir_documents = f('Hudozka/Site/Documents') dir_schedules = f('Hudozka/Site/Schedules') dir_events = f('Hudozka/Site/Events') dir_news = f('Hudozka/Site/News') +dir_collective = f('Hudozka/Site/Collective') dir_static = env('SYNC_STATIC', '~/Hudozka Static') dir_static_uploads = absolute(dir_static + '/uploads') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 05e327b8..62dece00 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -3,6 +3,7 @@ from db import db from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider from sync.document import sync_documents +from sync.person import sync_persons from sync.post import sync_posts from sync.schedule import sync_schedules from utils.fn import lprint_json @@ -39,3 +40,4 @@ if __name__ == '__main__': main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) main(io(settings.dir_events), sync_posts, settings.collection_events) main(io(settings.dir_news), sync_posts, settings.collection_news) + main(io(settings.dir_collective), sync_persons, settings.collection_collective) diff --git a/utils/Sync/sync/core/person.py b/utils/Sync/sync/core/person.py new file mode 100644 index 00000000..b78b8af7 --- /dev/null +++ b/utils/Sync/sync/core/person.py @@ -0,0 +1,38 @@ +import settings +from sync.core import Sync +from utils.hash import hash_str, hash_file +from utils.image import create_image +from utils.text.transform import url_encode_text + +url_base = settings.teacher_image_url_base +sizes = settings.teacher_image_sizes + + +class SyncPerson(Sync): + def __init__(self, collection, provider, dir_local_images): + super().__init__() + self.collection = collection + self.provider = provider + self.dir_local_images = dir_local_images + + def compile(self, profile): + profile['picture'] = create_image( + profile['picture'], + sizes, + lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), + self.dir_local_images + ) + return profile + + def create_id(self, document): + if 'id' in document and document['id']: + return document + + document['id'] = url_encode_text(document['name']) + return document + + def create_hash(self, document): + document['hash'] = hash_str( + self.provider.hash(document['file']) + self.provider.hash(document['picture']) + ) + return document diff --git a/utils/Sync/sync/person.py b/utils/Sync/sync/person.py new file mode 100644 index 00000000..d0d2dc83 --- /dev/null +++ b/utils/Sync/sync/person.py @@ -0,0 +1,125 @@ +import os +from glob import glob + +import settings +from sync import untouched +from sync.core.person import SyncPerson +from sync.image import sync_image +from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn +from utils.io import read_yaml_md + +url_base = settings.teacher_image_url_base +dir_local_images = settings.teacher_image_output +sizes = settings.teacher_image_sizes + + +def sync_persons(provider, collection, update=True, delete=True): + """ + :param collection: + :param provider: + :param update: + :param delete: + :return: + """ + + return main( + SyncPerson( + collection, + provider, + dir_local_images + ), + update_documents=update, + delete_documents=delete + ) + + +def main(sync, update_documents=False, delete_documents=False): + # GET TEACHER YAML/MD MANIFEST FILES + documents = sync.provider.type_filter('', '.md') + + # READ TEACHER YAML/MD MANIFEST + documents = lmap( + lambda i: (i,) + get_manifest(sync.provider, i), + documents + ) + + # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY + documents = lmap( + lambda i: { + **i[1], + 'file': i[0], + 'biography': i[2] + }, + documents + ) + + # DEFINE PROFILE PICTURE + documents = lmap( + lambda i: { + **i, + 'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']), + }, + documents + ) + + # CREATE DOCUMENT IDENTITY + documents = lmapfn(documents)(sync.create_id) + + # CREATE HASH OF DOCUMENT FILES + documents = lmapfn(documents)(sync.create_hash) + + # CREATE SCOPE OF CURRENT SESSION + scope_documents_ids = lmap( + lambda i: i['id'], + documents + ) + + # SKIP UNTOUCHED DOCUMENTS + documents = untouched(documents, sync) + + # documents = lmap( + # lambda i: i[0], + # filter( + # lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), + # lmap( + # lambda document: (document, sync.open(document)), + # documents + # ) + # )) + + # DO HEAVY PROCESS WITH DOCUMENTS + documents = lmap( + sync.compile, + documents + ) + + # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB + sync_picture = lambda i: None if i is None else sync_image(i)['_id'] + documents = lmap( + key_mapper('picture', sync_picture), + documents + ) + + # SYNC TEACHER_PROFILE + if update_documents: + documents = lmap( + sync.update, + documents + ) + + documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}}) + if delete_documents: + documents_to_delete = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_delete + ) + ) + + return documents, documents_to_delete + + +def get_manifest(provider, path): + data = provider.read(path).read().decode('utf-8') + return read_yaml_md(data) \ No newline at end of file diff --git a/utils/Sync/sync/teacher.py b/utils/Sync/sync/teacher.py deleted file mode 100644 index 10cb2fe4..00000000 --- a/utils/Sync/sync/teacher.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -from glob import glob - -import settings -from db import db -from sync import Sync -from sync.image import sync_image -from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn -from utils.hash import hash_str, hash_file -from utils.image import create_image -from utils.io import read_yaml_md -from utils.text.transform import url_encode_text - -dir_teachers = '/Users/tmshv/Dropbox/Dev/Hud school/Collective' -url_base = settings.teacher_image_url_base -dir_local_images = settings.teacher_image_output -sizes = settings.teacher_image_sizes - - -class Teacher(Sync): - def __init__(self): - super().__init__() - self.collection = db().collective - - def compile(self, profile): - profile['picture'] = create_image( - profile['picture'], - sizes, - lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), - dir_local_images - ) - return profile - - def create_id(self, document): - if 'id' in document and document['id']: - return document - - document['id'] = url_encode_text(document['name']) - return document - - def create_hash(self, document): - document['hash'] = hash_str( - hash_file(document['file']) + hash_file(document['picture']) - ) - return document - - -def main(dir_teachers, sync): - os.chdir(dir_teachers) - - # GET TEACHER YAML/MD MANIFEST FILES - documents = glob('*.md') - - # READ TEACHER YAML/MD MANIFEST - documents = lmap( - lambda i: (i,) + read_yaml_md(i), - documents - ) - - # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY - documents = lmap( - lambda i: { - **i[1], - 'file': i[0], - 'biography': i[2] - }, - documents - ) - - # DEFINE PROFILE PICTURE - documents = lmap( - lambda i: { - **i, - 'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']), - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmapfn(documents)(sync.create_id) - - # CREATE HASH OF DOCUMENT FILES - documents = lmapfn(documents)(sync.create_hash) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = lmap( - lambda i: i[0], - filter( - lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), - lmap( - lambda document: (document, sync.open(document)), - documents - ) - )) - - # DO HEAVY PROCESS WITH DOCUMENTS - documents = lmap( - sync.compile, - documents - ) - - # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB - sync_picture = lambda i: None if i is None else sync_image(i)['_id'] - documents = lmap( - key_mapper('picture', sync_picture), - documents - ) - - # SYNC TEACHER_PROFILE - documents = lmap( - sync.update, - documents - ) - - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove - ) - ) - - # SCOPE - print('SCOPE:') - lprint(scope_documents_ids) - - # DELETE - print('DELETE DOCUMENTS:') - lprint_json(documents_to_remove) - - # DELETE - print('UPDATE DOCUMENTS:') - lprint_json(documents) - print('[SYNC DOCUMENTS DONE]') - - -if __name__ == '__main__': - main(dir_teachers, Teacher())