FUCKING BIG SYNC UPDATE: caching, events, albums, documents, awards, news, schedules, teachers, other

utils/Sync/sync/__init__.py

@@ -1,9 +1,11 @@
 import re
-
 from datetime import datetime
 
 from markdown import markdown
 
+from db import db
+from utils.fn import lmap, key_mapper
+
 
 def create_date(date_str, date_formats=None):
     """
@@ -36,6 +38,18 @@ def create_date(date_str, date_formats=None):
 folder_name_pattern = re.compile('([\d.]+)(.*)')
 
 
+def untouched(documents, store):
+    return lmap(
+        lambda i: i[0],
+        filter(
+            lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
+            map(
+                lambda i: (i, store.read(i)),
+                documents
+            )
+        ))
+
+
 def create_date_and_title_from_folder_name(folder_name, date_formats=None):
     m = folder_name_pattern.findall(folder_name)
     if not m:
@@ -56,4 +70,49 @@ create_post_from_image_list = lambda images: markdown(
         lambda i: '![]({img})'.format(img=i),
         images
     ))
-)
+)
+
+
+class Sync:
+    def __init__(self):
+        super().__init__()
+
+        self.collection = None
+
+    def read(self, document):
+        if isinstance(document, list):
+            return lmap(self.read, document)
+
+        q = {'id': document['id']}
+        try:
+            return self.collection.find_one(q)
+        except ValueError:
+            pass
+        return None
+
+    def update(self, document):
+        if isinstance(document, list):
+            return lmap(self.update, document)
+
+        q = {'id': document['id']}
+        try:
+            self.collection.update_one(q, {'$set': document}, upsert=True)
+            return self.read(document)
+        except ValueError:
+            pass
+        return None
+
+    def compile(self, document):
+        return document
+
+    def query(self, q):
+        return self.collection.find(q)
+
+    def delete(self, q):
+        return self.collection.find_one_and_delete(q)
+
+    def create_id(self, document):
+        return None
+
+    def create_hash(self, document):
+        return None

utils/Sync/sync/album.py

@@ -1,5 +1,6 @@
 import os
 from glob import glob
+from lxml import etree
 
 import lxml.html
 
@@ -8,10 +9,25 @@ from db import db
 from models.product import create_product
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date
 from sync.image import sync_image
-from utils.fn import lmap, first, map_cases, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns
+from utils.fn import lmap, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns, first, map_cases
+from utils.hash import file_hash
 from utils.image import create_image
 from utils.io import read_yaml_md
-from utils.text.transform import url_encode_text, interpolate, interpolate_swift
+from utils.text.transform import url_encode_text
+
+
+class ImageCreator:
+    def __init__(self, cache):
+        super().__init__()
+        self.cache = cache if cache else set()
+
+    def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
+        hash = file_hash(file)
+        cached = self.cached(hash)
+        return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
+
+    def cached(self, hash):
+        return self.cache[hash] if hash in self.cache else None
 
 
 def sync_album(record):
@@ -24,7 +40,7 @@ def sync_album(record):
     q = {'id': record['id']}
     try:
         collection = db().events
-        update_result = collection.update_one(q, {'$set': record}, upsert=True)
+        collection.update_one(q, {'$set': record}, upsert=True)
         i = collection.find_one({'id': record['id']})
         return i
     except ValueError:
@@ -33,115 +49,150 @@ def sync_album(record):
     return None
 
 
-def create_album(doc):
-    sizes = settings.event_image_sizes
-    # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
-    images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
-    url = lambda a, i, s, e: 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
+def create_post(md, image_path_fn):
+    html = lxml.html.fromstring(md)
+
+    for img in html.cssselect('img'):
+        src = img.get('src')
+        path = image_path_fn(src)
+
+        if path:
+            img.set('src', path)
+            img.set('class', settings.album_html_img_class)
+            img.set('data-file', src)
+    return etree.tounicode(html)
+
+
+def create_url(a, i, s, e):
+    return 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
         album=a,
         id=i,
         size=s,
         ext=e
     )
-    create_url = lambda aid, pid: lambda size, ext: url(aid, pid, size, ext.lower())
-    image_id = lambda i: url_encode_text(
-        os.path.splitext(i)[0]
-    )
-    image_path = lambda i: os.path.join(doc['folder'], i)
 
-    if 'title' not in doc or not doc['title']:
-        return None
 
-    create_album_id = lambda date, title: '{date}-{title}'.format(
-        date=date.strftime('%Y'),
-        title=url_encode_text(title)
-    )
+def url_creator(aid, pid):
+    return lambda size, ext: create_url(aid, pid, size, ext.lower())
 
-    images = []
 
-    doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
+class Album:
+    @staticmethod
+    def id_from_file(i):
+        return url_encode_text(
+            os.path.splitext(i)[0]
+        )
 
-    if 'album' in doc:
-        # album_context =
+    def __init__(self, image_creator):
+        super().__init__()
+        self.image_creator = image_creator
+        self.skip_image_processing = False
+        self.images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
+
+    def create_image(self, file, url_fn):
+        ss = settings.album_image_sizes
+        od = self.images_dir
+        sp = self.skip_image_processing
+
+        return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
+
+    def create_album(self, doc):
+        # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
+
+        image_id = Album.id_from_file
+        image_path = lambda i: os.path.join(doc['folder'], i)
+
+        if 'title' not in doc or not doc['title']:
+            return None
+
+        create_album_id = lambda date, title: '{date}-{title}'.format(
+            date=date.strftime('%Y'),
+            title=url_encode_text(title)
+        )
+
+        doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
+
+        if 'album' in doc:
+            doc['album'] = self.create_products(doc, image_path)
+
+        def process_post_image(src):
+            path = image_path(src)
+
+            if os.path.exists(path):
+                url = url_creator(
+                    doc['id'],
+                    image_id(src)
+                )
+
+                image = self.create_image(path, url)
+                if image:
+                    images.append(image)
+                    return image['data']['big']['url']
+            return None
+
+        images = []
+        post_html = create_post(doc['post'], process_post_image)
+
+        doc['post'] = post_html
+        doc['images'] = images
+        return doc
+
+    def create_products(self, doc, image_file_fn):
+        image_id = Album.id_from_file
         iter_album = iterate_iter_over_fns([
             lambda product: {
                 **product,
-                'image': create_image(
-                    image_path(product['file']),
-                    sizes,
-                    create_url(
+                'image': self.create_image(
+                    image_file_fn(product['file']),
+                    url_creator(
                         doc['id'],
                         image_id(product['file'])
-                    ),
-                    images_dir,
-                    skip_processing=True
+                    )
                 )
             },
-            lambda product: {
-                **product,
-                'awards': lmap(
-                    lambda award: {
-                        **award,
-                        'file': os.path.join(doc['folder'], award['file']),
-                        'assigned': interpolate(award['assigned'], product, interpolate_swift),
-                        # 'awardFor': product
-                    },
-                    product['awards']
-                )
-            }
+            # lambda product: {
+            #     **product,
+            #     'awards': lmap(
+            #         lambda award: {
+            #             **award,
+            #             'file': os.path.join(doc['folder'], award['file']),
+            #             'assigned': interpolate(award['assigned'], product, interpolate_swift),
+            #             # 'awardFor': product
+            #         },
+            #         product['awards']
+            #     )
+            # }
         ])
 
-        doc['album'] = lmap(
+        return lmap(
             create_product,
             iter_album(doc['album'])
         )
 
-    post_html = lxml.html.fromstring(doc['post'])
-
-    for img in post_html.cssselect('img'):
-        src = img.get('src')
-        img_path = image_path(src)
-
-        if os.path.exists(img_path):
-            image = create_image(
-                img_path,
-                sizes,
-                create_url(
-                    doc['id'],
-                    image_id(src)
-                ),
-                images_dir,
-                skip_processing=True
-            )
-            images.append(image)
-
-            img.set('src', image['data']['big']['url'])
-            img.set('class', settings.album_html_img_class)
-            img.set('data-file', src)
-
-    doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
-    doc['images'] = images
-    return doc
-
 
 def get_manifest(filepath):
     y, m = read_yaml_md(filepath)
 
     data = {**y} if y else {}
     _ = lambda key, d=None: data[key] if key in data else d
-    date = create_date(_('date'), settings.date_formats_direct)
+
+    date = create_date(str(_('date')), settings.date_formats_direct)
     if date:
         data['date'] = date
 
-    id = _('id')
-    if id:
-        data['id'] = str(id)
+    album_id = _('id')
+    if album_id:
+        data['id'] = str(album_id)
 
     post = {'post': m} if m else {}
     return {**data, **post}
 
 
 if __name__ == '__main__':
+    image_creator = ImageCreator(cache=None)
+    alb = Album(image_creator=image_creator)
+    # alb.skip_image_processing = True
+
     albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
     os.chdir(albums_dir)
 
@@ -157,6 +208,7 @@ if __name__ == '__main__':
     # print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md'))
     # print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md'))
     # exit()
+    # albums = [albums[5]]
 
     # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
     albums = lmap(
@@ -200,13 +252,14 @@ if __name__ == '__main__':
 
     # MAP EVENT MANIFEST -> EVENT_OBJECT
     albums = lmap(
-        create_album,
+        alb.create_album,
         albums
     )
 
     # FILTER INCORRECT EVENTS
     albums = list(filter(None, albums))
 
+
     # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
     def sync_images(i):
         return lmap(
@@ -221,10 +274,10 @@ if __name__ == '__main__':
     )
 
     # SYNC EVENT_OBJECT WITH DB
-    # albums = lmap(
-    #     sync_album,
-    #     albums
-    # )
+    albums = lmap(
+        sync_album,
+        albums
+    )
 
     lprint_json(albums)
     print('SYNC ALBUMS DONE')

utils/Sync/sync/award.py

@@ -0,0 +1,196 @@
+import os
+from glob import glob
+from shutil import copyfile
+from tempfile import mkstemp
+
+import settings
+from db import db
+from sync.image import sync_image
+from utils.fn import lmap, lprint_json, key_mapper, lprint
+from utils.hash import hash_file, hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+prefix = 'award'
+dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Awards'
+dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
+dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
+url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
+url_base_document = 'https://static.shburg.org/art/uploads/{file}'
+
+
+def collection():
+    return db().awards
+
+
+def pdf_to_jpg(pdf):
+    cwd = os.getcwd()
+    abspdf = os.path.join(cwd, pdf)
+    _, temp = mkstemp('.jpg')
+    image_magick_pdf_to_img(abspdf, temp)
+    return temp
+
+
+def sync_document(record):
+    if isinstance(record, list):
+        return lmap(
+            sync_document,
+            record
+        )
+
+    q = {'id': record['id']}
+    try:
+        collection().update_one(q, {'$set': record}, upsert=True)
+        return collection().find_one(q)
+    except ValueError:
+        pass
+
+    return None
+
+
+def read_document(i, query_fn=None):
+    q = query_fn(i) if query_fn else {'id': i['id']}
+    try:
+        return collection().find_one(q)
+    except ValueError:
+        pass
+
+    return None
+
+
+def query_documents(q):
+    return collection().find(q)
+
+
+def delete_document(q):
+    return collection().find_one_and_delete(q)
+
+
+def create_preview(pdf, sizes, preview_dir):
+    temp_preview_path = pdf_to_jpg(pdf)
+
+    id = url_encode_text(pdf)
+    url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
+
+    img = create_image(temp_preview_path, sizes, url, preview_dir)
+    os.remove(temp_preview_path)
+    return img
+
+
+def create_document(doc):
+    sizes = settings.awards_image_sizes
+    file = doc['file']
+
+    doc['type'] = prefix
+    doc['url'] = url_base_document.format(file=url_encode_file(file))
+    doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
+    doc['file'] = {
+        'name': file,
+        'size': os.path.getsize((file))
+    }
+
+    return doc
+
+
+def create_id(doc):
+    return url_encode_text('{type}-{file}'.format(
+        type=prefix,
+        file=doc['file']
+    ))
+
+
+if __name__ == '__main__':
+    # ALL DOCUMENTS IDS FOUNDED
+    scope_documents_ids = []
+
+    # GET DOCUMENT YAML_MANIFEST FILES
+    os.chdir(dir_documents)
+    documents = glob('*.pdf')
+
+    # READ YAML_MANIFEST FILES
+    documents = lmap(
+        lambda i: {
+            'file': i
+        },
+        documents
+    )
+
+    # CREATE DOCUMENT IDENTITY
+    documents = lmap(
+        lambda i: {**i, 'id': create_id(i)},
+        documents
+    )
+
+    # CREATE HASH OF DOCUMENT FILE
+    documents = lmap(
+        lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
+        documents
+    )
+
+    # CREATE SCOPE OF CURRENT SESSION
+    scope_documents_ids = lmap(
+        lambda i: i['id'],
+        documents
+    )
+
+    # SKIP UNTOUCHED DOCUMENTS
+    documents = lmap(
+        lambda i: i[0],
+        filter(
+            lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
+            lmap(
+                lambda document: (document, read_document(document)),
+                documents
+            )
+        ))
+
+    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
+    documents = lmap(
+        create_document,
+        documents
+    )
+
+    # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
+    documents = lmap(
+        key_mapper('preview', lambda i: sync_image(i)['_id']),
+        documents
+    )
+
+    # COPY FILE -> STATIC_DIR/URL_FILENAME
+    out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
+    lmap(
+        lambda doc: copyfile(
+            doc['file']['name'],
+            out_path(doc)
+        ),
+        documents
+    )
+
+    # SYNC DOCUMENT_OBJECT WITH DB
+    documents = lmap(
+        sync_document,
+        documents
+    )
+
+    documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
+    documents_to_remove = lmap(
+        delete_document,
+        map(
+            lambda i: {'_id': i['_id']},
+            documents_to_remove
+        )
+    )
+
+    # SCOPE
+    print('SCOPE:')
+    lprint(scope_documents_ids)
+
+    # DELETE
+    print('DELETE DOCUMENTS:')
+    lprint_json(documents_to_remove)
+
+    # DELETE
+    print('UPDATE DOCUMENTS:')
+    lprint_json(documents)
+    print('[SYNC DOCUMENTS DONE]')

utils/Sync/sync/collective.py

@@ -9,6 +9,22 @@ from utils.image import create_image
 from utils.io import read_yaml_md
 
 
+def read_teacher_by_id(record):
+    if isinstance(record, list):
+        return lmap(
+            read_teacher_by_id,
+            record
+        )
+
+    q = {'id': record}
+    try:
+        return db().collective.find_one(q)
+    except ValueError:
+        pass
+
+    return None
+
+
 def sync_collective(record):
     if isinstance(record, list):
         return lmap(

utils/Sync/sync/document.py

@@ -6,12 +6,20 @@ from tempfile import mkstemp
 import settings
 from db import db
 from sync.image import sync_image
-from utils.fn import combine, lmap, lprint, key_mapper
+from utils.fn import combine, lmap, lprint_json, key_mapper, lprint
+from utils.hash import hash_file, hash_str
 from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
 from utils.io import read_yaml
 from utils.text.transform import url_encode_text, url_encode_file
 
+prefix = 'document'
+dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
+dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
+dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
+url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
+url_base_document = 'https://static.shburg.org/art/uploads/{file}'
+
 # Adds specified category to each file in files list
 unwrap_file_list = lambda files, title: map(
     lambda profile: {'category': title, **profile},
@@ -19,25 +27,6 @@ unwrap_file_list = lambda files, title: map(
 )
 
 
-def sync_document(record):
-    if isinstance(record, list):
-        return lmap(
-            sync_document,
-            record
-        )
-
-    q = {'url': record['url']}
-    try:
-        documents = db().documents
-        update_result = documents.update_one(q, {'$set': record}, upsert=True)
-        i = documents.find_one({'url': record['url']})
-        return i
-    except ValueError:
-        pass
-
-    return None
-
-
 def pdf_to_jpg(pdf):
     cwd = os.getcwd()
     abspdf = os.path.join(cwd, pdf)
@@ -46,12 +35,48 @@ def pdf_to_jpg(pdf):
     return temp
 
 
+def sync_document(record):
+    if isinstance(record, list):
+        return lmap(
+            sync_document,
+            record
+        )
+
+    q = {'id': record['id']}
+    try:
+        documents = db().documents
+        update_result = documents.update_one(q, {'$set': record}, upsert=True)
+        i = documents.find_one({'id': record['id']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+def read_document(i, query_fn=None):
+    q = query_fn(i) if query_fn else {'id': i['id']}
+    try:
+        return db().documents.find_one(q)
+    except ValueError:
+        pass
+
+    return None
+
+
+def query_documents(q):
+    return db().documents.find(q)
+
+
+def delete_document(q):
+    return db().documents.find_one_and_delete(q)
+
+
 def create_preview(pdf, sizes, preview_dir):
     temp_preview_path = pdf_to_jpg(pdf)
 
     id = url_encode_text(pdf)
-    url_base = 'https://static.shburg.org/art/images/document-{id}-{size}{ext}'
-    url = lambda size, ext: url_base.format(id=id, size=size, ext=ext)
+    url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
 
     img = create_image(temp_preview_path, sizes, url, preview_dir)
     os.remove(temp_preview_path)
@@ -60,12 +85,11 @@ def create_preview(pdf, sizes, preview_dir):
 
 def create_document(doc):
     sizes = settings.image_sizes
-    preview_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
     file = doc['file']
 
-    doc['type'] = 'document'
-    doc['url'] = 'https://static.shburg.org/art/uploads/' + url_encode_file(file)
-    doc['preview'] = create_preview(file, sizes=sizes, preview_dir=preview_dir)
+    doc['type'] = prefix
+    doc['url'] = url_base_document.format(file=url_encode_file(file))
+    doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
     doc['file'] = {
         'name': file,
         'size': os.path.getsize((file))
@@ -74,6 +98,14 @@ def create_document(doc):
     return doc
 
 
+def create_id(doc):
+    return url_encode_text('{type}-{category}-{file}'.format(
+        type=prefix,
+        category=doc['category'],
+        file=doc['file']
+    ))
+
+
 def unwrap_manifest(param):
     if isinstance(param, list):
         return combine(map(
@@ -94,21 +126,50 @@ def unwrap_manifest(param):
 
 
 if __name__ == '__main__':
-    dir = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
-
-    os.chdir(dir)
+    # ALL DOCUMENTS IDS FOUNDED
+    scope_documents_ids = []
 
     # GET DOCUMENT YAML_MANIFEST FILES
-    files = glob(dir + '/*.yaml')
+    os.chdir(dir_documents)
+    documents = glob(dir_documents + '/*.yaml')
 
     # READ YAML_MANIFEST FILES
-    files_data = lmap(
+    documents = lmap(
         read_yaml,
-        files
+        documents
     )
 
     # GET FLAT LIST OF DOCUMENTS
-    documents = unwrap_manifest(files_data)
+    documents = unwrap_manifest(documents)
+
+    # CREATE DOCUMENT IDENTITY
+    documents = lmap(
+        lambda i: {**i, 'id': create_id(i)},
+        documents
+    )
+
+    # CREATE HASH OF DOCUMENT FILE
+    documents = lmap(
+        lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
+        documents
+    )
+
+    # CREATE HASH OF DOCUMENT FILE
+    scope_documents_ids = lmap(
+        lambda i: i['id'],
+        documents
+    )
+
+    # SKIP UNTOUCHED DOCUMENTS
+    documents = lmap(
+        lambda i: i[0],
+        filter(
+            lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
+            lmap(
+                lambda document: (document, read_document(document)),
+                documents
+            )
+        ))
 
     # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
     documents = lmap(
@@ -117,15 +178,13 @@ if __name__ == '__main__':
     )
 
     # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
-    sync_preview = lambda i: sync_image(i)['_id']
     documents = lmap(
-        key_mapper('preview', sync_preview),
+        key_mapper('preview', lambda i: sync_image(i)['_id']),
         documents
     )
 
     # COPY FILE -> STATIC_DIR/URL_FILENAME
-    uploads = '/Users/tmshv/Dropbox/Dev/Hud school/Static/uploads'
-    out_path = lambda doc: os.path.join(uploads, os.path.basename(doc['url']))
+    out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
     lmap(
         lambda doc: copyfile(
             doc['file']['name'],
@@ -140,18 +199,44 @@ if __name__ == '__main__':
         documents
     )
 
-    lprint(documents)
-    print('SYNC DOCUMENTS DONE')
+    documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
+    documents_to_remove = lmap(
+        delete_document,
+        map(
+            lambda i: {'_id': i['_id']},
+            documents_to_remove
+        )
+    )
+
+    # SCOPE
+    print('SCOPE:')
+    lprint(scope_documents_ids)
+
+    # DELETE
+    print('DELETE DOCUMENTS:')
+    lprint_json(documents_to_remove)
+
+    # DELETE
+    print('UPDATE DOCUMENTS:')
+    lprint_json(documents)
+    print('[SYNC DOCUMENTS DONE]')
 
     # DOCUMENT_OBJECT SAMPLE
     # {
-    #     "_id": ObjectId("56d08411ace9573958e5e490"),
-    #     "url": "https://static.shburg.org/art/uploads/egryul.pdf",
-    #     "title": "Свидетельство о внесении в «ЕГРЮЛ»",
-    #     "preview": ObjectId("56d07d87ace9573958e5e488"),
-    #     "file": {
-    #       "name": ""ЕГРЮЛ.pdf",
-    #       "size": 100500
+    #     "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
+    #     "_id": {
+    #         "$oid": "56d08411ace9573958e5e497"
     #     },
-    #     "category": "Основные документы"
+    #     "type": "document",
+    #     "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
+    #     "category": "Для поступающих",
+    #     "file": {
+    #         "size": 53088,
+    #         "name": "Договор о пожертвовании.pdf"
+    #     },
+    #     "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
+    #     "preview": {
+    #         "$oid": "56d08007ace9573958e5e48f"
+    #     },
+    #     "title": "Договор о пожертвовании"
     # }

utils/Sync/sync/event.py

@@ -70,11 +70,15 @@ def create_event(doc):
     return doc
 
 
-def get_md_manifest(md_path):
+def get_manifest(md_path):
     y, m = read_yaml_md(md_path)
+    y = y if y else {}
 
-    y['date'] = datetime.strptime(y['date'], '%d.%m.%Y')
-    y['id'] = str(y['id'])
+    if 'date' in y:
+        y['date'] = datetime.strptime(y['date'], '%d.%m.%Y')
+
+    if 'id' in y:
+        y['id'] = str(y['id'])
 
     return {
         **y,
@@ -126,13 +130,13 @@ if __name__ == '__main__':
     # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
     md_glob = lambda i : first(glob(i + '/*.md'))
     events = lmap(
-        lambda i: {
-            **i,
+        lambda event: {
+            **event,
             **map_cases(
-                i,
+                event,
                 [(
                     lambda i: md_glob(i['folder']),
-                    lambda i: get_md_manifest(md_glob(i['folder'])),
+                    lambda i: get_manifest(md_glob(i['folder'])),
                 )],
 
                 lambda i: {}

utils/Sync/sync/news.py

@@ -0,0 +1,97 @@
+import os
+from glob import glob
+
+from db import db
+from sync import create_date
+from utils.fn import lmap, lprint_json
+from utils.io import read_yaml_md
+from utils.text.transform import url_encode_text
+
+
+def sync_news(record):
+    if isinstance(record, list):
+        return lmap(
+            sync_news,
+            record
+        )
+
+    q = {'id': record['id']}
+    try:
+        collection = db().timeline
+        update_result = collection.update_one(q, {'$set': record}, upsert=True)
+        i = collection.find_one({'id': record['id']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+def create_news(profile):
+    formats = [
+        '%d.%m.%Y',
+        '%d.%m.%Y %H:%M',
+        '%d.%m.%Y %H:%M:%S'
+    ]
+
+    if 'title' not in profile:
+        return None
+
+    if 'date' in profile:
+        profile['date'] = create_date(profile['date'], formats)
+
+    if 'until' in profile:
+        profile['until'] = create_date(profile['until'], formats)
+
+    if 'id' not in profile:
+        id = lambda date, title: '{date}-{title}'.format(
+            date=date.strftime('%Y-%m-%d'),
+            title=url_encode_text(title)
+        )
+        profile['id'] = id(profile['date'], profile['title'])
+
+    return profile
+
+
+if __name__ == '__main__':
+    news_dir = '/Users/tmshv/Dropbox/Dev/Hud school/News'
+    os.chdir(news_dir)
+
+    # GET NEWS FILES
+    file_glob = os.path.join(news_dir, '*.md')
+    news = glob(file_glob)
+
+    # READ MANIFEST FILES -> NEWS_MANIFEST
+    news = lmap(
+        lambda i: (os.path.basename(i),) + read_yaml_md(i),
+        news
+    )
+    news = lmap(
+        lambda i: {
+            **i[1],
+            'file': i[0],
+            'message': i[2]
+        },
+        news
+    )
+
+    # NEWS_MANIFEST -> NEWS_OBJECT
+    news = lmap(
+        create_news,
+        news
+    )
+
+    # FILTER BROKEN NEWS_OBJECT
+    news = list(filter(
+        None,
+        news
+    ))
+
+    # SYNC EVENT_OBJECT WITH DB
+    events = lmap(
+        sync_news,
+        news
+    )
+
+    lprint_json(news)
+    print('SYNC NEWS DONE')

utils/Sync/sync/schedule.py

@@ -0,0 +1,137 @@
+import os
+from glob import glob
+
+from db import db
+from sync import Sync, untouched
+from utils.fn import lprint, lprint_json, lmap
+from utils.hash import hash_file
+from utils.io import read_yaml
+
+dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules'
+
+
+class SyncSchedule(Sync):
+    def __init__(self):
+        super().__init__()
+        self.collection = db().schedules
+
+    # def sync_collective(self, record):
+    #     if isinstance(record, list):
+    #         return lmap(
+    #             self.sync_collective,
+    #             record
+    #         )
+    #
+    #     q = {'id': record['id']}
+    #     try:
+    #         collection = db().collective
+    #         update_result = collection.update_one(q, {'$set': record}, upsert=True)
+    #         i = collection.find_one({'id': record['id']})
+    #         return i
+    #     except ValueError:
+    #         pass
+    #
+    #     return None
+    #
+    # def compile(self, profile):
+    #     profile['picture'] = create_image(
+    #         profile['picture'],
+    #         sizes,
+    #         lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
+    #         dir_local_images
+    #     )
+    #     return profile
+    #
+    # def read_document(self, i, query_fn=None):
+    #     q = query_fn(i) if query_fn else {'id': i['id']}
+    #     try:
+    #         return self.collection().find_one(q)
+    #     except ValueError:
+    #         pass
+    #
+    #     return None
+    #
+    # def query_documents(self, q):
+    #     return self.collection().find(q)
+    #
+    # def delete_document(self, q):
+    #     return self.collection().find_one_and_delete(q)
+
+    def create_id(self, document):
+        id = document['id'] if 'id' in document and document['id'] else None
+        if not id:
+            id = 'schedule-{period}-{semester}'.format(**document)
+        document['id'] = id
+        return document
+
+    def create_hash(self, document):
+        return {
+            **document,
+            'hash': hash_file(document['file'])
+        }
+
+
+if __name__ == '__main__':
+    # ALL DOCUMENTS IDS FOUNDED
+    scope_documents_ids = []
+
+    # INIT
+    os.chdir(dir_documents)
+    sync = SyncSchedule()
+
+    # COLLECT YAML/MD/JPG/PDF MANIFEST FILES
+    documents = glob('*.yaml')
+
+    # READ MANIFEST
+    documents = lmap(
+        lambda i: {
+            'file': i,
+            **read_yaml(i)
+        },
+        documents
+    )
+
+    # # CREATE DOCUMENT IDENTITY
+    documents = lmap(sync.create_id, documents)
+
+    # CREATE DOCUMENT HASH
+    documents = lmap(sync.create_hash, documents)
+
+    # # CREATE SCOPE OF CURRENT SESSION
+    scope_documents_ids = lmap(
+        lambda i: i['id'],
+        documents
+    )
+
+    # # SKIP UNTOUCHED DOCUMENTS
+    documents = untouched(documents, sync)
+
+    # DO HEAVY PROCESS WITH DOCUMENTS
+
+    # SYNC TEACHER_PROFILE
+    documents = lmap(
+        sync.update,
+        documents
+    )
+
+    documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
+    documents_to_remove = lmap(
+        sync.delete,
+        map(
+            lambda i: {'_id': i['_id']},
+            documents_to_remove
+        )
+    )
+
+    # SCOPE
+    print('SCOPE:')
+    lprint(scope_documents_ids)
+
+    # # DELETE
+    print('DELETE DOCUMENTS:')
+    lprint_json(documents_to_remove)
+
+    # DELETE
+    print('UPDATE DOCUMENTS:')
+    lprint_json(documents)
+    print('[SYNC DOCUMENTS DONE]')

utils/Sync/sync/teacher.py

@@ -0,0 +1,193 @@
+import os
+from glob import glob
+
+import settings
+from db import db
+from sync.image import sync_image
+from utils.fn import lmap, lprint, key_mapper, lprint_json, ext
+from utils.hash import hash_str, hash_file
+from utils.image import create_image
+from utils.io import read_yaml_md
+from utils.text.transform import url_encode_text
+
+dir_teachers = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
+url_base = settings.teacher_image_url_base
+dir_local_images = settings.teacher_image_output
+sizes = settings.teacher_image_sizes
+
+class Teacher:
+    def collection(self):
+        return db().collective
+
+    def read_teacher_by_id(self, record):
+        if isinstance(record, list):
+            return lmap(
+                self.read_teacher_by_id,
+                record
+            )
+
+        q = {'id': record}
+        try:
+            return db().collective.find_one(q)
+        except ValueError:
+            pass
+
+        return None
+
+    def sync_collective(self, record):
+        if isinstance(record, list):
+            return lmap(
+                self.sync_collective,
+                record
+            )
+
+        q = {'id': record['id']}
+        try:
+            collection = db().collective
+            update_result = collection.update_one(q, {'$set': record}, upsert=True)
+            i = collection.find_one({'id': record['id']})
+            return i
+        except ValueError:
+            pass
+
+        return None
+
+    def compile(self, profile):
+        profile['picture'] = create_image(
+            profile['picture'],
+            sizes,
+            lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
+            dir_local_images
+        )
+        return profile
+
+    def read_document(self, i, query_fn=None):
+        q = query_fn(i) if query_fn else {'id': i['id']}
+        try:
+            return self.collection().find_one(q)
+        except ValueError:
+            pass
+
+        return None
+
+    def query_documents(self, q):
+        return self.collection().find(q)
+
+    def delete_document(self, q):
+        return self.collection().find_one_and_delete(q)
+
+    def create_id(self, document):
+        if 'id' in document and document['id']:
+            return document['id']
+
+        document['id'] = url_encode_text(document['name'])
+
+
+if __name__ == '__main__':
+    # ALL DOCUMENTS IDS FOUNDED
+    scope_documents_ids = []
+
+    os.chdir(dir_teachers)
+    sync = Teacher()
+
+    # GET TEACHER YAML/MD MANIFEST FILES
+    documents = glob('*.md')
+
+    # # READ TEACHER YAML/MD MANIFEST
+    documents = lmap(
+        lambda i: (i,) + read_yaml_md(i),
+        documents
+    )
+
+    # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
+    documents = lmap(
+        lambda i: {
+            **i[1],
+            'file': i[0],
+            'biography': i[2]
+        },
+        documents
+    )
+
+    # DEFINE PROFILE PICTURE
+    documents = lmap(
+        lambda i: {
+            **i,
+            'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
+        },
+        documents
+    )
+
+    # CREATE DOCUMENT IDENTITY
+    documents = lmap(
+        lambda i: {**i, 'id': sync.create_id(i)},
+        documents
+    )
+
+    # CREATE HASH OF DOCUMENT FILES
+    documents = lmap(
+        lambda i: {
+            **i, 'hash': hash_str(
+                hash_file(i['file']) + hash_file(i['picture'])
+            )
+        },
+        documents
+    )
+
+    # CREATE SCOPE OF CURRENT SESSION
+    scope_documents_ids = lmap(
+        lambda i: i['id'],
+        documents
+    )
+
+    # SKIP UNTOUCHED DOCUMENTS
+    documents = lmap(
+        lambda i: i[0],
+        filter(
+            lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
+            lmap(
+                lambda document: (document, sync.read_document(document)),
+                documents
+            )
+        ))
+
+    # DO HEAVY PROCESS WITH DOCUMENTS
+    documents = lmap(
+        sync.compile,
+        documents
+    )
+
+    # # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
+    sync_picture = lambda i: None if i is None else sync_image(i)['_id']
+    documents = lmap(
+        key_mapper('picture', sync_picture),
+        documents
+    )
+
+    # # SYNC TEACHER_PROFILE
+    documents = lmap(
+        sync.sync_collective,
+        documents
+    )
+
+    documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}})
+    documents_to_remove = lmap(
+        sync.delete_document,
+        map(
+            lambda i: {'_id': i['_id']},
+            documents_to_remove
+        )
+    )
+
+    # SCOPE
+    print('SCOPE:')
+    lprint(scope_documents_ids)
+
+    # DELETE
+    print('DELETE DOCUMENTS:')
+    lprint_json(documents_to_remove)
+
+    # DELETE
+    print('UPDATE DOCUMENTS:')
+    lprint_json(documents)
+    print('[SYNC DOCUMENTS DONE]')