From e54c4487046f09efcadf0c8bffc64428f3d84cd0 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Thu, 14 Sep 2017 21:33:40 +0300 Subject: [PATCH] [Sync] + new Schedule sync --- utils/Sync/sync.py | 8 +- utils/Sync/sync/core/SyncSchedule.py | 51 +++++++++++++ utils/Sync/sync/core/schedule.py | 38 ---------- utils/Sync/sync/models/Schedule.py | 109 +++++++++++++++++++++++++++ utils/Sync/sync/models/__init__.py | 3 + utils/Sync/sync/schedule.py | 97 ------------------------ 6 files changed, 170 insertions(+), 136 deletions(-) create mode 100644 utils/Sync/sync/core/SyncSchedule.py delete mode 100644 utils/Sync/sync/core/schedule.py create mode 100644 utils/Sync/sync/models/Schedule.py delete mode 100644 utils/Sync/sync/schedule.py diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 7e895382..2dcdeff5 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -5,6 +5,7 @@ from sync.core.SyncArticle import SyncArticle from sync.core.SyncDocument import SyncDocument from sync.core.SyncPage import SyncPage from sync.core.SyncPerson import SyncPerson +from sync.core.SyncSchedule import SyncSchedule from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider @@ -63,8 +64,12 @@ async def run(run_interval=0): collection=c(settings.collection_articles), ) + schedules = SyncSchedule( + provider=io(settings.dir_schedules), + collection=c(settings.collection_schedules), + ) + while True: - # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) await asyncio.wait([ @@ -72,6 +77,7 @@ async def run(run_interval=0): main(pages), main(persons), main(articles), + main(schedules), ]) if run_interval == 0: diff --git a/utils/Sync/sync/core/SyncSchedule.py b/utils/Sync/sync/core/SyncSchedule.py new file mode 100644 index 00000000..39579bb4 --- /dev/null +++ b/utils/Sync/sync/core/SyncSchedule.py @@ -0,0 +1,51 @@ +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.models.Schedule import Schedule + +logger = logging.getLogger(settings.name + '.Schedule') + + +class SyncSchedule(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Schedule.scan(self.provider) + + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build() + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() diff --git a/utils/Sync/sync/core/schedule.py b/utils/Sync/sync/core/schedule.py deleted file mode 100644 index dd402ce9..00000000 --- a/utils/Sync/sync/core/schedule.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -from tempfile import mkstemp - -from sync.core import Sync -from utils.fn import last_good, lmap -from utils.hash import hash_str, hash_file -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class SyncSchedule(Sync): - def __init__(self, collection, provider): - super().__init__() - self.collection = collection - self.provider = provider - - def create_id(self, document): - period = get_period(document['period']) - doc_id = document['id'] if 'id' in document and document['id'] else None - if not doc_id: - doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=document['semester']) - document['id'] = doc_id - return document - - def create_hash(self, document): - return { - **document, - 'hash': self.provider.hash(document['file']) - } - - -def get_period(value): - if isinstance(value, str): - return value - - if isinstance(value, list): - return '-'.join(lmap(str, value)) diff --git a/utils/Sync/sync/models/Schedule.py b/utils/Sync/sync/models/Schedule.py new file mode 100644 index 00000000..2bb6ad83 --- /dev/null +++ b/utils/Sync/sync/models/Schedule.py @@ -0,0 +1,109 @@ +import logging + +from toml import loads as toml_str +from yaml import load as yaml + +import settings +from db import collection +from sync.data import Provider, get_data +from sync.models import Model + +from utils.fn import constant + +logger = logging.getLogger(settings.name + '.Person') +store = collection(settings.collection_schedules) + + +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Schedule(Model): + @staticmethod + async def scan(provider): + documents = provider.type_filter('.', '.yaml') + documents += provider.type_filter('.', '.toml') + + documents = [Schedule.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + params = get_manifest(provider, path) + + return Schedule( + provider, + path, + params=params, + ) + + def __init__(self, provider, file, params=None): + self.params = params if params else {} + super().__init__(provider, store, file) + + def init(self): + pass + self.__set_id() + self.__set_hash() + + def bake(self): + return { + **self.params, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + } + + def __set_id(self): + doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None + if not doc_id: + period = self.__get_period() + doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester']) + self.id = doc_id + + def __set_hash(self): + self.hash = self.provider.hash(self.file) + + def __get_period(self): + value = self.params['period'] + if isinstance(value, str): + return value + + if isinstance(value, list): + return '-'.join(list(map(str, value))) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def combine(fns): + def call(value): + data = value.read() + for f in fns: + try: + return f(data) + except Exception as e: + pass + return None + + return call + + +def toml(bytes_value): + s = bytes_value.decode('utf-8') + return toml_str(s) + + +def get_manifest(provider: Provider, path: str) -> dict: + transform = combine([yaml, toml, constant({})]) + manifest = get_data(provider, path, transform) + + return { + 'file': path, + **manifest, + } diff --git a/utils/Sync/sync/models/__init__.py b/utils/Sync/sync/models/__init__.py index 7a7e9b2b..f1c87488 100644 --- a/utils/Sync/sync/models/__init__.py +++ b/utils/Sync/sync/models/__init__.py @@ -27,6 +27,9 @@ class Model: def init(self): pass + async def build(self, **kwargs): + pass + def bake(self): return None diff --git a/utils/Sync/sync/schedule.py b/utils/Sync/sync/schedule.py deleted file mode 100644 index 05737efb..00000000 --- a/utils/Sync/sync/schedule.py +++ /dev/null @@ -1,97 +0,0 @@ -from toml import loads as toml_str -from yaml import load as yaml - -from sync import untouched -from sync.core.schedule import SyncSchedule -from sync.data import get_data -from utils.fn import lmap, constant - -dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules' - - -def sync_schedules(provider, collection, update=True, delete=True): - """ - :param collection: - :param provider: - :param update: - :param delete: - :return: - """ - - return main( - SyncSchedule( - collection, - provider - ), - update_documents=update, - delete_documents=delete - ) - - -def combine(fns): - def call(value): - data = value.read() - for f in fns: - try: - return f(data) - except Exception as e: - pass - return None - return call - - -def toml(bytes_value): - s = bytes_value.decode('utf-8') - return toml_str(s) - - -def main(sync, update_documents=False, delete_documents=False): - # COLLECT YAML/MD/JPG/PDF MANIFEST FILES - documents = sync.provider.type_filter('.', '.yaml') - documents += sync.provider.type_filter('.', '.toml') - - # READ MANIFEST - documents = lmap( - lambda i: { - 'file': i, - **get_data(sync.provider, i, combine([yaml, toml, constant({})])) - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) - - # CREATE DOCUMENT HASH - documents = lmap(sync.create_hash, documents) - - # CREATE SCOPE OF CURRENT SESSION - # ALL DOCUMENTS IDS FOUNDED - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # DO HEAVY PROCESS WITH DOCUMENTS - - # SYNC SCHEDULE - if update_documents: - documents = lmap( - sync.update, - documents - ) - - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - if delete_documents: - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove - ) - ) - - return documents, documents_to_remove