From 35a488978cccfa7d8911b4cb0ecda7b15a532d3b Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Tue, 13 Sep 2016 23:29:23 +0300 Subject: [PATCH] [sync] schedules updated (new format), toml based schedules --- utils/Sync/settings.py | 2 + utils/Sync/sync.py | 7 +- utils/Sync/sync/core/schedule.py | 38 ++++++++++ utils/Sync/sync/data/__init__.py | 5 ++ utils/Sync/sync/schedule.py | 118 ++++++++++++++++--------------- 5 files changed, 112 insertions(+), 58 deletions(-) create mode 100644 utils/Sync/sync/core/schedule.py diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 187c2c43..417c2837 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -95,6 +95,7 @@ date_formats_reverse = [ ] collection_documents = 'documents' +collection_schedules = 'schedules' sync_provider_type = env('SYNC_PROVIDER', 'fs') @@ -105,6 +106,7 @@ _providers_roots = { f = _providers_roots[sync_provider_type] dir_documents = f('Hudozka/Site/Documents') +dir_schedules = f('Hudozka/Site/Schedules') dir_events = f('Hudozka/Site/Events') dir_news = f('Hudozka/Site/News') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 2bc46ba6..e6b23f13 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -3,6 +3,7 @@ from db import db from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider from sync.document import sync_documents +from sync.schedule import sync_schedules from utils.fn import lprint_json @@ -27,6 +28,8 @@ def main(provider, fn, collection_name): if __name__ == '__main__': - client = get_provider(settings.sync_provider_type, settings.dir_documents) + # client = get_provider(settings.sync_provider_type, settings.dir_documents) + # main(client, sync_documents, settings.collection_documents) - main(client, sync_documents, settings.collection_documents) + client = get_provider(settings.sync_provider_type, settings.dir_schedules) + main(client, sync_schedules, settings.collection_schedules) diff --git a/utils/Sync/sync/core/schedule.py b/utils/Sync/sync/core/schedule.py new file mode 100644 index 00000000..dd402ce9 --- /dev/null +++ b/utils/Sync/sync/core/schedule.py @@ -0,0 +1,38 @@ +import os +from tempfile import mkstemp + +from sync.core import Sync +from utils.fn import last_good, lmap +from utils.hash import hash_str, hash_file +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + + +class SyncSchedule(Sync): + def __init__(self, collection, provider): + super().__init__() + self.collection = collection + self.provider = provider + + def create_id(self, document): + period = get_period(document['period']) + doc_id = document['id'] if 'id' in document and document['id'] else None + if not doc_id: + doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=document['semester']) + document['id'] = doc_id + return document + + def create_hash(self, document): + return { + **document, + 'hash': self.provider.hash(document['file']) + } + + +def get_period(value): + if isinstance(value, str): + return value + + if isinstance(value, list): + return '-'.join(lmap(str, value)) diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index 398d9db1..4bec6475 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -64,3 +64,8 @@ def scan_subdirs(provider, ext, path='.'): items = combine(items) return items + + +def get_data(provider, path, transform): + data = provider.read(path) + return transform(data) diff --git a/utils/Sync/sync/schedule.py b/utils/Sync/sync/schedule.py index 2b83aa66..19e27e1c 100644 --- a/utils/Sync/sync/schedule.py +++ b/utils/Sync/sync/schedule.py @@ -1,95 +1,101 @@ -import os -from glob import glob +from yaml import load as yaml +from toml import loads as toml_str -from db import db -from sync import Sync, untouched -from utils.fn import lprint, lprint_json, lmap -from utils.hash import hash_file +from sync import untouched +from sync.core.schedule import SyncSchedule +from sync.data import scan_subdirs, get_data +from utils.fn import lmap from utils.io import read_yaml dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules' -class SyncSchedule(Sync): - def __init__(self): - super().__init__() - self.collection = db().schedules +def sync_schedules(provider, collection, update=True, delete=True): + """ + :param provider: + :param update: + :param delete: + :return: + """ - def create_id(self, document): - id = document['id'] if 'id' in document and document['id'] else None - if not id: - id = 'schedule-{period}-{semester}'.format(**document) - document['id'] = id - return document - - def create_hash(self, document): - return { - **document, - 'hash': hash_file(document['file']) - } + return main( + SyncSchedule( + collection, + provider + ), + update_documents=update, + delete_documents=delete + ) -if __name__ == '__main__': - # ALL DOCUMENTS IDS FOUNDED - scope_documents_ids = [] +def combine(fns): + def call(value): + data = value.read() + for f in fns: + try: + return f(data) + except Exception as e: + pass + return None + return call - # INIT - os.chdir(dir_documents) - sync = SyncSchedule() +def constant(value): + return lambda _: value + + +def toml(bytes_value): + s = bytes_value.decode('utf-8') + return toml_str(s) + + +def main(sync, update_documents=False, delete_documents=False): # COLLECT YAML/MD/JPG/PDF MANIFEST FILES - documents = glob('*.yaml') + documents = sync.provider.type_filter('.', '.yaml') + documents += sync.provider.type_filter('.', '.toml') # READ MANIFEST documents = lmap( lambda i: { 'file': i, - **read_yaml(i) + **get_data(sync.provider, i, combine([yaml, toml, constant({})])) }, documents ) - # # CREATE DOCUMENT IDENTITY + # CREATE DOCUMENT IDENTITY documents = lmap(sync.create_id, documents) # CREATE DOCUMENT HASH documents = lmap(sync.create_hash, documents) - # # CREATE SCOPE OF CURRENT SESSION + # CREATE SCOPE OF CURRENT SESSION + # ALL DOCUMENTS IDS FOUNDED scope_documents_ids = lmap( lambda i: i['id'], documents ) - # # SKIP UNTOUCHED DOCUMENTS + # SKIP UNTOUCHED DOCUMENTS documents = untouched(documents, sync) # DO HEAVY PROCESS WITH DOCUMENTS - # SYNC TEACHER_PROFILE - documents = lmap( - sync.update, - documents - ) + # SYNC SCHEDULE + if update_documents: + documents = lmap( + sync.update, + documents + ) documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove + if delete_documents: + documents_to_remove = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) ) - ) - # SCOPE - print('SCOPE:') - lprint(scope_documents_ids) - - # # DELETE - print('DELETE DOCUMENTS:') - lprint_json(documents_to_remove) - - # DELETE - print('UPDATE DOCUMENTS:') - lprint_json(documents) - print('[SYNC DOCUMENTS DONE]') + return documents, documents_to_remove