From 953ef0bc284658bd01ba8ad2a33a69080c8edaa1 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Tue, 12 Sep 2017 13:08:40 +0300 Subject: [PATCH] [Sync] + new Person sync --- utils/Sync/settings.py | 10 +-- utils/Sync/sync.py | 8 +- utils/Sync/sync/core/SyncPerson.py | 75 ++++++++++++++++++ utils/Sync/sync/core/person.py | 39 ---------- utils/Sync/sync/models/Person.py | 109 ++++++++++++++++++++++++++ utils/Sync/sync/models/__init__.py | 16 +++- utils/Sync/sync/person.py | 119 ----------------------------- utils/Sync/utils/fn.py | 4 +- 8 files changed, 210 insertions(+), 170 deletions(-) create mode 100644 utils/Sync/sync/core/SyncPerson.py delete mode 100644 utils/Sync/sync/core/person.py create mode 100644 utils/Sync/sync/models/Person.py delete mode 100644 utils/Sync/sync/person.py diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index e4de0d7d..3658a526 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -88,14 +88,6 @@ event_image_sizes = [ ('big', 1200, 800) ] -teacher_image_url_base = 'https://static.shlisselburg.org/art/images/teacher-{id}-{size}{ext}' -teacher_image_output = '/Users/tmshv/Desktop/Hudozka Static/images' -teacher_image_sizes = [ - ('big', 1200, 800), - ('medium', 640, 480), - ('preview', 50, 50), -] - album_image_sizes = [ ('big', 1200, 800), ('medium', 640, 480), @@ -156,6 +148,8 @@ image_url_upload = 'https://static.shlisselburg.org/upload/art/images/' image_url_base = 'https://static.shlisselburg.org/art/images/' image_name_format = '{type}-{id}-{img}-{size}{ext}' +person_image_url_template = 'https://static.shlisselburg.org/art/images/person-{id}-{size}{ext}' + yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN') update_enabled = value('update_enabled') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 96e67f62..771ef63b 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -3,6 +3,7 @@ from db import collection from sync.core import Sync from sync.core.SyncDocument import SyncDocument from sync.core.SyncPage import SyncPage +from sync.core.SyncPerson import SyncPerson from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider @@ -51,17 +52,22 @@ async def run(run_interval=0): collection=c(settings.collection_pages), ) + persons = SyncPerson( + provider=io(settings.dir_collective), + collection=c(settings.collection_collective), + ) + while True: # main(io(settings.dir_documents), sync_documents, settings.collection_documents) # main(io(settings.dir_awards), sync_documents, settings.collection_awards) # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) # main(io(settings.dir_articles), sync_posts, settings.collection_articles) - # main(io(settings.dir_collective), sync_persons, settings.collection_collective) # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) await asyncio.wait([ main(documents), main(pages), + main(persons), ]) if run_interval == 0: diff --git a/utils/Sync/sync/core/SyncPerson.py b/utils/Sync/sync/core/SyncPerson.py new file mode 100644 index 00000000..9b5d758b --- /dev/null +++ b/utils/Sync/sync/core/SyncPerson.py @@ -0,0 +1,75 @@ +import os + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import Provider +from sync.models.Person import Person + +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Person') + + +class SyncPerson(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Person.scan(self.provider) + + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build( + sizes=settings.image_sizes, + url_factory=get_image_url_fn(document.id), + ) + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() + + +def get_image_url_fn(page_id: str): + image_id = lambda filename: url_encode_text( + os.path.splitext(os.path.basename(filename))[0] + ) + + def img_url_fn(file, size, ext): + return settings.page_url_base.format( + page=page_id, + id=image_id(file), + size=size, + ext=ext + ) + + return img_url_fn diff --git a/utils/Sync/sync/core/person.py b/utils/Sync/sync/core/person.py deleted file mode 100644 index cf3e2728..00000000 --- a/utils/Sync/sync/core/person.py +++ /dev/null @@ -1,39 +0,0 @@ -import settings -from sync.core import Sync -from utils.hash import hash_str -from utils.image import create_image -from utils.text.transform import url_encode_text - -url_base = settings.teacher_image_url_base -sizes = settings.teacher_image_sizes - - -class SyncPerson(Sync): - def __init__(self, collection, provider, dir_local_images): - super().__init__() - self.collection = collection - self.provider = provider - self.dir_local_images = dir_local_images - - def compile(self, profile): - image_path = self.provider.get_abs(profile['picture']) - profile['picture'] = create_image( - image_path, - sizes, - lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), - self.dir_local_images - ) - return profile - - def create_id(self, document): - if 'id' in document and document['id']: - return document - - document['id'] = url_encode_text(document['name']) - return document - - def create_hash(self, document): - document['hash'] = hash_str( - self.provider.hash(document['file']) + self.provider.hash(document['picture']) - ) - return document diff --git a/utils/Sync/sync/models/Person.py b/utils/Sync/sync/models/Person.py new file mode 100644 index 00000000..099fe305 --- /dev/null +++ b/utils/Sync/sync/models/Person.py @@ -0,0 +1,109 @@ +import logging + +import settings +from db import collection +from sync.data import Provider +from sync.models import Model +from sync.models.Image import Image + +from utils.fn import swap_ext +from utils.hash import hash_str +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Person') +store = collection(settings.collection_collective) + + +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Person(Model): + @staticmethod + async def scan(provider): + documents = provider.type_filter('', '.md') + documents = [Person.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + params = get_manifest(provider, path) + + return Person( + provider, + path, + params=params, + ) + + def __init__(self, provider, file, params=None): + self.__hash_keys = ['id', 'url'] + + self.__id_template: str = '{category}-{file}' + self.__url_template: str = settings.document_url_template + self.__url_preview_template: str = settings.document_url_preview_template + + self.params = params if params else {} + self.picture = None + + super().__init__(provider, store, file) + + def init(self): + pass + self.__set_id() + self.__set_hash() + + async def build(self, **kwargs): + sizes = kwargs['sizes'] + image_path = self.provider.get_abs(self.params['picture']) + + self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id)) + + def bake(self): + return { + **self.params, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + 'picture': self.picture.id, + } + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + self.id = url_encode_text(self.params['name']) + + def __set_hash(self): + self.hash = hash_str( + self.provider.hash(self.file) + self.provider.hash(self.params['picture']) + ) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def get_manifest(provider: Provider, path: str) -> dict: + data = provider.read(path).read().decode('utf-8') + + manifest, body = read_yaml_md(data) + manifest = { + **manifest, + 'file': path, + 'biography': body, + 'picture': manifest['picture'] if 'picture' in manifest else swap_ext('.jpg')(path), + } + + return manifest + + +def url_factory(id): + def url(file, size, ext): + return settings.person_image_url_template.format(id=id, size=size, ext=ext) + + return url diff --git a/utils/Sync/sync/models/__init__.py b/utils/Sync/sync/models/__init__.py index df47f68d..7a7e9b2b 100644 --- a/utils/Sync/sync/models/__init__.py +++ b/utils/Sync/sync/models/__init__.py @@ -1,3 +1,6 @@ +import os + + def find(store, item_id: str): q = {'id': item_id} try: @@ -40,5 +43,16 @@ class Model: return not (self.hash == item['hash']) + def _filename(self): + return os.path.basename(self.file) + async def save(self): - pass + document = self.bake() + + query = {'id': document['id']} + try: + self.store.update_one(query, {'$set': document}, upsert=True) + return self + except ValueError: + pass + return None diff --git a/utils/Sync/sync/person.py b/utils/Sync/sync/person.py deleted file mode 100644 index f42e0b13..00000000 --- a/utils/Sync/sync/person.py +++ /dev/null @@ -1,119 +0,0 @@ -import os -from glob import glob - -import settings -from sync import untouched -from sync.core.person import SyncPerson -from sync.image import sync_image -from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn -from utils.io import read_yaml_md - -url_base = settings.teacher_image_url_base -dir_local_images = settings.teacher_image_output -sizes = settings.teacher_image_sizes - - -def sync_persons(provider, collection, update=True, delete=True): - """ - :param collection: - :param provider: - :param update: - :param delete: - :return: - """ - - return main( - SyncPerson( - collection, - provider, - dir_local_images - ), - update_documents=update, - delete_documents=delete - ) - - -def make_person(provider, file): - pass - - -def main(sync, update_documents=False, delete_documents=False): - # GET TEACHER YAML/MD MANIFEST FILES - documents = sync.provider.type_filter('', '.md') - - # READ TEACHER YAML/MD MANIFEST - documents = lmap( - lambda i: (i,) + get_manifest(sync.provider, i), - documents - ) - - # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY - documents = lmap( - lambda i: { - **i[1], - 'file': i[0], - 'biography': i[2] - }, - documents - ) - - # DEFINE PROFILE PICTURE - documents = lmap( - lambda i: { - **i, - 'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']), - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmapfn(documents)(sync.create_id) - - # CREATE HASH OF DOCUMENT FILES - documents = lmapfn(documents)(sync.create_hash) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # DO HEAVY PROCESS WITH DOCUMENTS - documents = lmap( - sync.compile, - documents - ) - - # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB - sync_picture = lambda i: None if i is None else sync_image(i)['_id'] - documents = lmap( - key_mapper('picture', sync_picture), - documents - ) - - # SYNC TEACHER_PROFILE - if update_documents: - documents = lmap( - sync.update, - documents - ) - - documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}}) - if delete_documents: - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - - return documents, documents_to_delete - - -def get_manifest(provider, path): - data = provider.read(path).read().decode('utf-8') - return read_yaml_md(data) \ No newline at end of file diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index 7cd0b53a..c1fd0ca6 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -49,8 +49,8 @@ lprint_json = lambda i: lprint(lmap(to_json, i)) first = lambda i: i[0] if len(i) > 0 else None -def ext(ext): - return lambda i: os.path.splitext(i)[0] + ext +def swap_ext(extension: str): + return lambda i: os.path.splitext(i)[0] + extension def map_item_key(item, key, map_fn):