[sync] schedules updated (new format), toml based schedules

utils/Sync/settings.py

@@ -95,6 +95,7 @@ date_formats_reverse = [
 ]
 
 collection_documents = 'documents'
+collection_schedules = 'schedules'
 
 sync_provider_type = env('SYNC_PROVIDER', 'fs')
 
@@ -105,6 +106,7 @@ _providers_roots = {
 f = _providers_roots[sync_provider_type]
 
 dir_documents = f('Hudozka/Site/Documents')
+dir_schedules = f('Hudozka/Site/Schedules')
 dir_events = f('Hudozka/Site/Events')
 dir_news = f('Hudozka/Site/News')
 

utils/Sync/sync.py

@@ -3,6 +3,7 @@ from db import db
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
 from sync.document import sync_documents
+from sync.schedule import sync_schedules
 from utils.fn import lprint_json
 
 
@@ -27,6 +28,8 @@ def main(provider, fn, collection_name):
 
 
 if __name__ == '__main__':
-    client = get_provider(settings.sync_provider_type, settings.dir_documents)
+    # client = get_provider(settings.sync_provider_type, settings.dir_documents)
+    # main(client, sync_documents, settings.collection_documents)
 
-    main(client, sync_documents, settings.collection_documents)
+    client = get_provider(settings.sync_provider_type, settings.dir_schedules)
+    main(client, sync_schedules, settings.collection_schedules)

utils/Sync/sync/core/schedule.py

@@ -0,0 +1,38 @@
+import os
+from tempfile import mkstemp
+
+from sync.core import Sync
+from utils.fn import last_good, lmap
+from utils.hash import hash_str, hash_file
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+class SyncSchedule(Sync):
+    def __init__(self, collection, provider):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+
+    def create_id(self, document):
+        period = get_period(document['period'])
+        doc_id = document['id'] if 'id' in document and document['id'] else None
+        if not doc_id:
+            doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=document['semester'])
+        document['id'] = doc_id
+        return document
+
+    def create_hash(self, document):
+        return {
+            **document,
+            'hash': self.provider.hash(document['file'])
+        }
+
+
+def get_period(value):
+    if isinstance(value, str):
+        return value
+
+    if isinstance(value, list):
+        return '-'.join(lmap(str, value))

utils/Sync/sync/data/__init__.py

@@ -64,3 +64,8 @@ def scan_subdirs(provider, ext, path='.'):
 
     items = combine(items)
     return items
+
+
+def get_data(provider, path, transform):
+    data = provider.read(path)
+    return transform(data)

utils/Sync/sync/schedule.py

@@ -1,95 +1,101 @@
-import os
-from glob import glob
+from yaml import load as yaml
+from toml import loads as toml_str
 
-from db import db
-from sync import Sync, untouched
-from utils.fn import lprint, lprint_json, lmap
-from utils.hash import hash_file
+from sync import untouched
+from sync.core.schedule import SyncSchedule
+from sync.data import scan_subdirs, get_data
+from utils.fn import lmap
 from utils.io import read_yaml
 
 dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules'
 
 
-class SyncSchedule(Sync):
-    def __init__(self):
-        super().__init__()
-        self.collection = db().schedules
+def sync_schedules(provider, collection, update=True, delete=True):
+    """
+    :param provider:
+    :param update:
+    :param delete:
+    :return:
+    """
 
-    def create_id(self, document):
-        id = document['id'] if 'id' in document and document['id'] else None
-        if not id:
-            id = 'schedule-{period}-{semester}'.format(**document)
-        document['id'] = id
-        return document
-
-    def create_hash(self, document):
-        return {
-            **document,
-            'hash': hash_file(document['file'])
-        }
+    return main(
+        SyncSchedule(
+            collection,
+            provider
+        ),
+        update_documents=update,
+        delete_documents=delete
+    )
 
 
-if __name__ == '__main__':
-    # ALL DOCUMENTS IDS FOUNDED
-    scope_documents_ids = []
+def combine(fns):
+    def call(value):
+        data = value.read()
+        for f in fns:
+            try:
+                return f(data)
+            except Exception as e:
+                pass
+        return None
+    return call
 
-    # INIT
-    os.chdir(dir_documents)
-    sync = SyncSchedule()
 
+def constant(value):
+    return lambda _: value
+
+
+def toml(bytes_value):
+    s = bytes_value.decode('utf-8')
+    return toml_str(s)
+
+
+def main(sync, update_documents=False, delete_documents=False):
     # COLLECT YAML/MD/JPG/PDF MANIFEST FILES
-    documents = glob('*.yaml')
+    documents = sync.provider.type_filter('.', '.yaml')
+    documents += sync.provider.type_filter('.', '.toml')
 
     # READ MANIFEST
     documents = lmap(
         lambda i: {
             'file': i,
-            **read_yaml(i)
+            **get_data(sync.provider, i, combine([yaml, toml, constant({})]))
         },
         documents
     )
 
-    # # CREATE DOCUMENT IDENTITY
+    # CREATE DOCUMENT IDENTITY
     documents = lmap(sync.create_id, documents)
 
     # CREATE DOCUMENT HASH
     documents = lmap(sync.create_hash, documents)
 
-    # # CREATE SCOPE OF CURRENT SESSION
+    # CREATE SCOPE OF CURRENT SESSION
+    # ALL DOCUMENTS IDS FOUNDED
     scope_documents_ids = lmap(
         lambda i: i['id'],
         documents
     )
 
-    # # SKIP UNTOUCHED DOCUMENTS
+    # SKIP UNTOUCHED DOCUMENTS
     documents = untouched(documents, sync)
 
     # DO HEAVY PROCESS WITH DOCUMENTS
 
-    # SYNC TEACHER_PROFILE
-    documents = lmap(
-        sync.update,
-        documents
-    )
+    # SYNC SCHEDULE
+    if update_documents:
+        documents = lmap(
+            sync.update,
+            documents
+        )
 
     documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
-    documents_to_remove = lmap(
-        sync.delete,
-        map(
-            lambda i: {'_id': i['_id']},
-            documents_to_remove
+    if delete_documents:
+        documents_to_remove = lmap(
+            sync.delete,
+            map(
+                lambda i: {'_id': i['_id']},
+                documents_to_remove
+            )
         )
-    )
 
-    # SCOPE
-    print('SCOPE:')
-    lprint(scope_documents_ids)
-
-    # # DELETE
-    print('DELETE DOCUMENTS:')
-    lprint_json(documents_to_remove)
-
-    # DELETE
-    print('UPDATE DOCUMENTS:')
-    lprint_json(documents)
-    print('[SYNC DOCUMENTS DONE]')
+    return documents, documents_to_remove