[sync] updated: + albums, kazimir rendering, other changes

utils/Sync/README.md

@@ -9,6 +9,7 @@ Sync Hudozka
 ### Environment variables
 
 **YANDEX_DISK_ACCESS_TOKEN**
+
 For access to files using Yandex.Disk API
 
 [How to obtain token for testing](https://tech.yandex.ru/oauth/doc/dg/tasks/get-oauth-token-docpage/)
@@ -23,7 +24,8 @@ Authorize url. Replace with your app id
 https://oauth.yandex.ru/authorize?response_type=token&client_id=<идентификатор приложения>
 ```
 
-**SYNC_LOCAL_PATH**
+#### SYNC_LOCAL_PATH
+
 Path to scan files from
 
 Example:
@@ -31,7 +33,8 @@ Example:
 SYNC_LOCAL_PATH=/Users/tmshv/Yandex.Disk
 ```
 
-**SYNC_STATIC**
+#### SYNC_STATIC
+
 Path to local folder for compiled files (images, uploads)
 
 Example: 
@@ -39,7 +42,8 @@ Example:
 SYNC_STATIC=~/Desktop/Hudozka Static
 ```
 
-**SYNC_PROVIDER**
+#### SYNC_PROVIDER
+
 Type of file provider
 
 - *fs* for local access
@@ -50,10 +54,18 @@ Example:
 SYNC_PROVIDER=fs
 ```
 
-**MONGO_URI**
-URI for access to MongoDB
-Example: mongodb://localhost:27017/hudozka
+#### MONGO_URI
+
+URI for MongoDB access
+
+Example:
+ ```bash
+ mongodb://localhost:27017/hudozka
+ ```
+
+#### SYNC_ENV
+
+Environment like *NODE_ENV*
 
-**SYNC_ENV**
 - *production* default value
 - *development* don't update/delete documents from DB

utils/Sync/db.py

@@ -6,3 +6,7 @@ import settings
 def db():
     client = MongoClient(settings.database_uri)
     return client.hudozka
+
+
+def collection(name):
+    return db()[name]

utils/Sync/settings.py

@@ -20,14 +20,9 @@ def env(param, default=None):
     return default
 
 
-root_dir_gallery = './Gallery'
-root_dir_schedule = './Schedules'
-
-default_description_version = 1
-
 database_uri = env('MONGO_URI')
 
-teachers_names = {
+person_uri = {
     'Н.В.Андреева': 'nv-andreeva',
     'И.Н.Втюрина': 'in-vturina',
     'О.Д.Гоголева': 'od-gogoleva',
@@ -94,11 +89,14 @@ date_formats_reverse = [
     '%Y.%m.%d'
 ]
 
+origin = 'Yandex.Disk'
+
 collection_documents = 'documents'
 collection_schedules = 'schedules'
 collection_events = 'events'
-collection_news = 'news'
+collection_news = 'timeline'
 collection_collective = 'collective'
+collection_albums = 'albums'
 
 sync_provider_type = env('SYNC_PROVIDER', 'fs')
 
@@ -113,6 +111,7 @@ dir_schedules = f('Hudozka/Site/Schedules')
 dir_events = f('Hudozka/Site/Events')
 dir_news = f('Hudozka/Site/News')
 dir_collective = f('Hudozka/Site/Collective')
+dir_gallery = f('Hudozka/Site/Gallery')
 
 dir_static = env('SYNC_STATIC', '~/Hudozka Static')
 dir_static_uploads = absolute(dir_static + '/uploads')

utils/Sync/sync.py

@@ -1,5 +1,6 @@
 import settings
-from db import db
+from db import db, collection
+from sync.album import sync_albums
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
 from sync.document import sync_documents
@@ -7,6 +8,7 @@ from sync.person import sync_persons
 from sync.post import sync_posts
 from sync.schedule import sync_schedules
 from utils.fn import lprint_json
+from utils.image import thumbnail
 
 
 def get_provider(provider_type, root):
@@ -19,10 +21,8 @@ def get_provider(provider_type, root):
 
 def main(provider, fn, collection_name):
     if provider:
-        collection = db()[collection_name]
-
         print('SYNC: %s' % (collection_name.upper()))
-        u, d = fn(provider, collection, update=settings.do_update, delete=settings.do_update)
+        u, d = fn(provider, collection(collection_name), update=settings.do_update, delete=settings.do_update)
 
         print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d))))
         lprint_json(d)
@@ -41,3 +41,4 @@ if __name__ == '__main__':
     main(io(settings.dir_events), sync_posts, settings.collection_events)
     main(io(settings.dir_news), sync_posts, settings.collection_news)
     main(io(settings.dir_collective), sync_persons, settings.collection_collective)
+    main(io(settings.dir_gallery), sync_albums, settings.collection_albums)

utils/Sync/sync/album.py

@@ -1,182 +1,33 @@
 import os
-from glob import glob
-
-import lxml.html
-from lxml import etree
 
 import settings
-from db import db
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \
-    untouched, synced_images_ids
-from sync.models import create_product
-from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \
-    key_mapper, lprint
-from utils.hash import hash_file, hash_str
+from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \
+    synced_images_ids
+from sync.core.album import SyncAlbum
+from sync.data import list_images
+from utils.fn import lmap, lmapfn, key_mapper
 from utils.image import create_image
 from utils.io import read_yaml_md
-from utils.text.transform import url_encode_text
 
 
 class ImageCreator:
-    def __init__(self, cache):
+    def __init__(self, provider, cache):
         super().__init__()
+        self.provider = provider
         self.cache = cache if cache else set()
 
     def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
-        hash = hash_file(file)
-        cached = self.cached(hash)
-        return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
+        filehash = self.provider.hash(file)
+        cached = self.cached(filehash)
+        return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
 
     def cached(self, hash):
         return self.cache[hash] if hash in self.cache else None
 
 
-def create_post(md, image_path_fn):
-    html = lxml.html.fromstring(md)
-
-    for img in html.cssselect('img'):
-        src = img.get('src')
-        path = image_path_fn(src)
-
-        if path:
-            img.set('src', path)
-            img.set('class', settings.album_html_img_class)
-            img.set('data-file', src)
-    return etree.tounicode(html)
-
-
-def create_url(a, i, s, e):
-    return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
-        album=a,
-        id=i,
-        size=s,
-        ext=e
-    )
-
-
-def url_creator(aid, pid):
-    return lambda size, ext: create_url(aid, pid, size, ext.lower())
-
-
-class SyncAlbum(Sync):
-    @staticmethod
-    def id_from_file(i):
-        return url_encode_text(
-            os.path.splitext(i)[0]
-        )
-
-    def __init__(self, image_creator, images_dir):
-        super().__init__()
-        self.collection = db().albums
-        self.image_creator = image_creator
-        self.skip_image_processing = False
-        self.images_dir = images_dir
-
-    def create_id(self, document):
-        if 'id' in document:
-            return document
-
-        y = str(document['date'].year)
-        document['id'] = url_encode_text('-'.join([y, document['title']]))
-        return document
-
-    def create_hash(self, document):
-        hash_images = lmap(
-            hash_file,
-            lmap(
-                lambda i: os.path.join(document['folder'], i),
-                document['images']
-            )
-        )
-        document['hash'] = hash_str(
-            combine([hash_str(document)] + hash_images)
-        )
-
-        return document
-
-    def create_image(self, file, url_fn):
-        ss = settings.album_image_sizes
-        od = self.images_dir
-        sp = self.skip_image_processing
-
-        return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
-
-    def create_album(self, doc):
-        # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
-
-        image_id = SyncAlbum.id_from_file
-        image_path = lambda i: os.path.join(doc['folder'], i)
-
-        if 'title' not in doc or not doc['title']:
-            return None
-
-        create_album_id = lambda date, title: '{date}-{title}'.format(
-            date=date.strftime('%Y'),
-            title=url_encode_text(title)
-        )
-
-        doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
-
-        if 'album' in doc:
-            doc['album'] = self.create_products(doc, image_path)
-
-        def process_post_image(src):
-            path = image_path(src)
-
-            if os.path.exists(path):
-                url = url_creator(
-                    doc['id'],
-                    image_id(src)
-                )
-
-                image = self.create_image(path, url)
-                if image:
-                    images.append(image)
-                    return image['data']['big']['url']
-            return None
-
-        images = []
-        post_html = create_post(doc['post'], process_post_image)
-
-        doc['post'] = post_html
-        doc['images'] = images
-        return doc
-
-    def create_products(self, doc, image_file_fn):
-        image_id = SyncAlbum.id_from_file
-        iter_album = iterate_iter_over_fns([
-            lambda product: {
-                **product,
-                'image': self.create_image(
-                    image_file_fn(product['file']),
-                    url_creator(
-                        doc['id'],
-                        image_id(product['file'])
-                    )
-                )
-            },
-            # lambda product: {
-            #     **product,
-            #     'awards': lmap(
-            #         lambda award: {
-            #             **award,
-            #             'file': os.path.join(doc['folder'], award['file']),
-            #             'assigned': interpolate(award['assigned'], product, interpolate_swift),
-            #             # 'awardFor': product
-            #         },
-            #         product['awards']
-            #     )
-            # }
-        ])
-
-        return lmap(
-            create_product,
-            iter_album(doc['album'])
-        )
-
-
-def get_manifest(filepath):
-    y, m = read_yaml_md(filepath)
+def get_manifest(provider, filepath):
+    data = provider.read(filepath).read().decode('utf-8')
+    y, m = read_yaml_md(data)
 
     data = {**y} if y else {}
     _ = lambda key, d=None: data[key] if key in data else d
@@ -193,76 +44,60 @@ def get_manifest(filepath):
     return {**data, **post}
 
 
-def main(dir_documents, sync):
-    os.chdir(dir_documents)
-
-    # GET ALBUMS FOLDERS
-    documents = lmap(
-        lambda i: i.path,
-        filter(
-            lambda i: i.is_dir(),
-            os.scandir('.')
-        )
+def album_from_folder(sync, path):
+    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+    date, title = create_date_and_title_from_folder_name(
+        os.path.basename(path)
     )
 
-    # documents = [documents[3]]
-    # documents = documents[:15]
-
-    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
-    documents = lmap(
-        lambda i: (i,) + create_date_and_title_from_folder_name(
-            os.path.basename(i)
-        ),
-        documents
+    images = lmap(
+        lambda i: os.path.relpath(i, path),
+        list_images(sync.provider, path)
     )
 
     # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
-    glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
-    documents = lmap(
-        lambda i: {
-            'folder': i[0],
-            'date': i[1],
-            'title': i[2],
-            'post': create_post_from_image_list(lmap(
-                lambda path: os.path.relpath(path, i[0]),
-                glob_in(i[0])
-            ))
-        },
-        documents
-    )
+    document = {
+        'folder': path,
+        'date': date,
+        'title': title,
+        'post': create_post_from_image_list(images),
+        'images': images
+    }
 
     # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
-    md_glob = lambda i: first(glob(i + '/*.md'))
-    documents = lmap(
-        lambda album: {
-            **album,
-            **map_cases(
-                album,
-                [(
-                    lambda album: md_glob(album['folder']),
-                    lambda album: get_manifest(md_glob(album['folder'])),
-                )],
-                lambda i: {}
-            )
-        },
-        documents
-    )
-    documents = lmap(
-        lambda i: {
-            **i,
-            'images': lmap(
-                lambda path: os.path.relpath(path, i['folder']),
-                list_images(i['folder'])
-            )
-        },
-        documents
-    )
+    manifest = sync.provider.type_filter(path, '.md')
+    manifest = manifest[0] if len(manifest) else None
+    manifest = get_manifest(sync.provider, manifest) if manifest else {}
+    manifest = manifest if manifest else {}
+
+    document = {
+        **document,
+        **manifest
+    }
 
     # CREATE DOCUMENT IDENTITY
-    documents = lmap(sync.create_id, documents)
+    document = sync.create_id(document)
 
     # CREATE HASH OF DOCUMENT FILE
-    documents = lmap(sync.create_hash, documents)
+    document = sync.create_hash(document)
+
+    return document
+
+
+def main(sync, do_update, do_delete):
+    # GET ALBUMS FOLDERS
+    documents = lmap(
+        lambda i: i,
+        filter(
+            lambda i: sync.provider.is_dir(i),
+            sync.provider.scan('.')
+        )
+    )
+
+    documents = lmap(
+        lambda folder: album_from_folder(sync, folder),
+        documents
+    )
 
     # CREATE SCOPE OF CURRENT SESSION
     scope_documents_ids = lmapfn(documents)(
@@ -274,11 +109,11 @@ def main(dir_documents, sync):
 
     # MAP EVENT MANIFEST -> EVENT_OBJECT
     documents = lmap(
-        alb.create_album,
+        sync.create_album,
         documents
     )
 
-    # FILTER INCORRECT EVENTS
+    # # FILTER INCORRECT EVENTS
     documents = list(filter(None, documents))
 
     # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
@@ -288,63 +123,42 @@ def main(dir_documents, sync):
     )
 
     # SYNC OBJECT WITH DB
-    documents = lmap(
-        sync.update,
-        documents
-    )
+    if do_update:
+        documents = lmap(
+            sync.update,
+            documents
+        )
 
     documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
-    documents_to_remove = lmap(
-        sync.delete,
-        map(
-            lambda i: {'_id': i['_id']},
-            documents_to_remove
+
+    if do_delete:
+        documents_to_remove = lmap(
+            sync.delete,
+            map(
+                lambda i: {'_id': i['_id']},
+                documents_to_remove
+            )
         )
+
+    return documents, documents_to_remove
+
+
+def sync_albums(provider, collection, update=True, delete=True):
+    """
+    :param collection:
+    :param provider:
+    :param update:
+    :param delete:
+    :return:
+    """
+
+    return main(
+        SyncAlbum(
+            collection,
+            provider,
+            image_creator=ImageCreator(provider, cache=None),
+            images_dir=settings.dir_static_images
+        ),
+        do_update=update,
+        do_delete=delete
     )
-
-    # SCOPE
-    print('SCOPE:')
-    lprint(scope_documents_ids)
-
-    # DELETE
-    print('DELETE DOCUMENTS:')
-    lprint_json(lmap(
-        lambda i: i['id'],
-        documents_to_remove
-    ))
-
-    # DELETE
-    print('UPDATE DOCUMENTS:')
-    lprint_json(lmap(
-        lambda i: i['id'],
-        documents
-    ))
-    # lprint_json(documents)
-    print('[SYNC DOCUMENTS DONE]')
-
-
-if __name__ == '__main__':
-    image_creator = ImageCreator(cache=None)
-    alb = SyncAlbum(image_creator=image_creator, images_dir=settings.dir_static_images)
-    # alb.skip_image_processing = True
-
-    # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
-    # dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Gallery'
-    dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/Gallery'
-    main(dir_documents=dir_docs, sync=alb)
-
-    # dir_docs += '/2013 Валяние Зверушки'
-    # dir_docs += '/Валяние Зверушки.md'
-
-    # dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/News/2015 Выставка 70-летия победы.md'
-
-    # print(frontmatter.load(dir_docs).content)
-    # d = read_file(dir_docs)
-    # d = d.replace('\u0D0A', '\n')
-    # print(frontmatter.loads(d).metadata)
-    # print(parse_yaml_front_matter(d))
-    #
-    # print(len(re.split('^---', d, 2, re.MULTILINE)))
-    # print(re.split('^---', d, 2)[1])
-    #
-    # print(read_yaml_md(dir_docs))

utils/Sync/sync/core/album.py

@@ -0,0 +1,145 @@
+import os
+from lxml import etree
+
+import lxml.html
+import settings
+from sync.core import Sync
+from sync.models.product import create_product
+from utils.fn import lmap, iterate_iter_over_fns, combine
+from utils.hash import hash_file, hash_str
+from utils.text.transform import url_encode_text
+
+
+def create_post(md, image_path_fn):
+    html = lxml.html.fromstring(md)
+
+    for img in html.cssselect('img'):
+        src = img.get('src')
+        path = image_path_fn(src)
+
+        if path:
+            img.set('src', path)
+            img.set('class', settings.album_html_img_class)
+            img.set('data-file', src)
+    return etree.tounicode(html)
+
+
+def create_url(a, i, s, e):
+    return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
+        album=a,
+        id=i,
+        size=s,
+        ext=e
+    )
+
+
+def url_creator(aid, pid):
+    return lambda size, ext: create_url(aid, pid, size, ext.lower())
+
+
+class SyncAlbum(Sync):
+    @staticmethod
+    def id_from_file(i):
+        return url_encode_text(
+            os.path.splitext(i)[0]
+        )
+
+    def __init__(self, collection, provider, image_creator, images_dir):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+        self.image_creator = image_creator
+        self.skip_image_processing = False
+        self.images_dir = images_dir
+
+    def create_id(self, document):
+        if 'id' in document:
+            return document
+
+        y = str(document['date'].year)
+        document['id'] = url_encode_text('-'.join([y, document['title']]))
+        return document
+
+    def create_hash(self, document):
+        files = sorted(document['images'])
+        hashes = lmap(
+            self.provider.hash,
+            lmap(
+                lambda i: os.path.join(document['folder'], i),
+                files
+            )
+        )
+
+        document['hash'] = hash_str(
+            combine([hash_str(document)] + hashes)
+        )
+
+        return document
+
+    def create_image(self, file, url_fn):
+        ss = settings.album_image_sizes
+        od = self.images_dir
+        sp = self.skip_image_processing
+
+        return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
+
+    def create_album(self, doc):
+        # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
+
+        image_id = SyncAlbum.id_from_file
+        image_path = lambda i: os.path.join(doc['folder'], i)
+
+        if 'title' not in doc or not doc['title']:
+            return None
+
+        create_album_id = lambda date, title: '{date}-{title}'.format(
+            date=date.strftime('%Y'),
+            title=url_encode_text(title)
+        )
+
+        doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
+
+        if 'album' in doc:
+            doc['album'] = self.create_products(doc, image_path)
+
+        def process_post_image(src):
+            path = image_path(src)
+
+            if self.provider.exists(path):
+                url = url_creator(
+                    doc['id'],
+                    image_id(src)
+                )
+
+                image = self.create_image(path, url)
+                if image:
+                    images.append(image)
+                    return image['data']['big']['url']
+            return None
+
+        images = []
+        post_html = create_post(doc['post'], process_post_image)
+
+        doc['post'] = post_html
+        doc['images'] = images
+        return doc
+
+    def create_products(self, doc, image_file_fn):
+        image_id = SyncAlbum.id_from_file
+        iter_album = iterate_iter_over_fns([
+            lambda product: {
+                **product,
+                'image': self.create_image(
+                    image_file_fn(product['file']),
+                    url_creator(
+                        doc['id'],
+                        image_id(product['file'])
+                    )
+                )
+            }
+        ])
+
+        return lmap(
+            create_product,
+            iter_album(doc['album'])
+        )

utils/Sync/sync/core/person.py

@@ -1,6 +1,6 @@
 import settings
 from sync.core import Sync
-from utils.hash import hash_str, hash_file
+from utils.hash import hash_str
 from utils.image import create_image
 from utils.text.transform import url_encode_text
 
@@ -16,8 +16,9 @@ class SyncPerson(Sync):
         self.dir_local_images = dir_local_images
 
     def compile(self, profile):
+        image_path = self.provider.get_abs(profile['picture'])
         profile['picture'] = create_image(
-            profile['picture'],
+            image_path,
             sizes,
             lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
             self.dir_local_images

utils/Sync/sync/core/post.py

@@ -12,7 +12,7 @@ from utils.text.transform import url_encode_text, url_encode_file
 
 
 class SyncPost(Sync):
-    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes):
+    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
         super().__init__()
         self.collection = collection
         self.provider = provider
@@ -20,6 +20,7 @@ class SyncPost(Sync):
         self.type = document_type
         self.image_url_base = image_url_base
         self.dir_local_images = dir_local_images
+        self.origin = origin
 
     def create_id(self, document):
         new_id = url_encode_text(document['title'])
@@ -77,12 +78,14 @@ class SyncPost(Sync):
 
         document['post'] = lxml.html.tostring(post_html).decode('utf-8')
         document['images'] = images
+        document['origin'] = self.origin
         return document
 
     def create_remove_query(self, query):
         if self.type:
             query = {
                 **query,
-                'type': self.type
+                'type': self.type,
+                'orogin': self.origin
             }
         return query

utils/Sync/sync/data/__init__.py

@@ -30,6 +30,9 @@ class Provider:
     def size(self, path):
         return 0
 
+    def exists(self, path):
+        return True
+
     def get_abs(self, path):
         a = os.path.join(self.root, path)
         return os.path.normpath(a)

utils/Sync/sync/data/fs.py

@@ -41,6 +41,10 @@ class FSProvider(Provider):
         file = self.get_abs(path)
         return os.path.getsize(file)
 
+    def exists(self, path):
+        file = self.get_abs(path)
+        return os.path.exists(file)
+
     def is_dir(self, path):
         return os.path.isdir(self.get_abs(path))
 

utils/Sync/sync/document.py

@@ -1,5 +1,4 @@
 import os
-from glob import glob
 
 import settings
 from sync import synced_image_id, untouched
@@ -7,7 +6,6 @@ from sync.core.document import SyncDocument
 from sync.data import scan_subdirs
 from sync.models.document import Document
 from utils.fn import combine, lmap, key_mapper
-from utils.io import read_yaml
 
 
 def sync_documents(provider, collection, update=True, delete=True):
@@ -177,19 +175,3 @@ def unwrap_manifest(param):
         return None
 
     return param
-
-
-def documents_from_yaml(dirpath):
-    # GET DOCUMENT YAML_MANIFEST FILES
-    documents = glob(dirpath + '/*.yaml')
-
-    # READ YAML_MANIFEST FILES
-    documents = lmap(
-        read_yaml,
-        documents
-    )
-
-    # GET FLAT LIST OF DOCUMENTS
-    documents = unwrap_manifest(documents)
-
-    return documents

utils/Sync/sync/models/product.py

@@ -1,5 +1,7 @@
+import settings
+from db import collection
+from sync.core.person import SyncPerson
 from sync.image import sync_image
-from sync.teacher import Teacher
 from utils.fn import key_mapper
 from utils.text.transform import interpolate_swift
 
@@ -18,10 +20,13 @@ def resolve_value(param, store):
 unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
 
 
-
 def resolve_teacher(i):
-    sync = Teacher()
-    return resolve_value(i, lambda i: sync.read({'id': i}))
+    sync = SyncPerson(
+        collection(settings.collection_collective),
+        None,
+        ''
+    )
+    return resolve_value(i, lambda teacher_id: sync.read({'id': teacher_id}))
 
 
 def create_product(doc):
@@ -33,7 +38,7 @@ def create_product(doc):
     unpop_image(product)
 
     product['teacher'] = resolve_teacher(doc['teacher'])
-    if(product['teacher']):
+    if (product['teacher']):
         unpop_teacher(product)
 
     return product

utils/Sync/sync/person.py

@@ -33,6 +33,10 @@ def sync_persons(provider, collection, update=True, delete=True):
     )
 
 
+def make_person(provider, file):
+    pass
+
+
 def main(sync, update_documents=False, delete_documents=False):
     # GET TEACHER YAML/MD MANIFEST FILES
     documents = sync.provider.type_filter('', '.md')
@@ -75,17 +79,7 @@ def main(sync, update_documents=False, delete_documents=False):
     )
 
     # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # documents = lmap(
-    #     lambda i: i[0],
-    #     filter(
-    #         lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
-    #         lmap(
-    #             lambda document: (document, sync.open(document)),
-    #             documents
-    #         )
-    #     ))
+    # documents = untouched(documents, sync)
 
     # DO HEAVY PROCESS WITH DOCUMENTS
     documents = lmap(

utils/Sync/sync/post.py

@@ -36,10 +36,10 @@ def create_document_from_folder(sync, path, file_time_formats):
     )
 
     document = {
+        'images': images,
+        'post': post,
         **document,
         **params,
-        'images': images,
-        'post': post
     }
 
     document = sync.create_id(document)
@@ -135,20 +135,20 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
     # SKIP UNTOUCHED DOCUMENTS
     documents = untouched(documents, sync)
 
-    # # MAP EVENT MANIFEST -> EVENT_OBJECT
-    # documents = lmap(
-    #     sync.create,
-    #     documents
-    # )
+    # MAP EVENT MANIFEST -> EVENT_OBJECT
+    documents = lmap(
+        sync.create,
+        documents
+    )
 
     # FILTER INCORRECT EVENTS
     documents = list(filter(None, documents))
 
-    # # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
-    # documents = lmap(
-    #     key_mapper('images', synced_images_ids),
-    #     documents
-    # )
+    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
+    documents = lmap(
+        key_mapper('images', synced_images_ids),
+        documents
+    )
 
     # SYNC EVENT_OBJECT WITH DB
     if update_documents:
@@ -179,10 +179,11 @@ def sync_posts(provider, collection, update=True, delete=True):
         SyncPost(
             collection,
             provider,
-            None,
+            'post',
             image_url_base,
             settings.dir_static_images,
-            settings.event_image_sizes
+            settings.event_image_sizes,
+            origin=settings.origin
         ),
         file_time_formats=['%Y.%m.%d'],
         update_documents=update,

utils/Sync/sync/schedule.py

@@ -1,17 +1,17 @@
-from yaml import load as yaml
 from toml import loads as toml_str
+from yaml import load as yaml
 
 from sync import untouched
 from sync.core.schedule import SyncSchedule
-from sync.data import scan_subdirs, get_data
+from sync.data import get_data
 from utils.fn import lmap, constant
-from utils.io import read_yaml
 
 dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules'
 
 
 def sync_schedules(provider, collection, update=True, delete=True):
     """
+    :param collection:
     :param provider:
     :param update:
     :param delete:

utils/Sync/utils/image/resize.py

@@ -10,13 +10,13 @@ import utils.image
 
 
 def thumbnail(src, dest, size, ext=None):
-    # image_magick_resize(src, dest, size, quality=85)
-    # return utils.image.read_image(dest)
+    image_magick_resize(src, dest, size, quality=85)
+    return utils.image.read_image(dest)
 
-    image = utils.image.read_image(src)
-    image.thumbnail(size)
-    image.save(override_extension(dest, ext))
-    return image
+    # image = utils.image.read_image(src)
+    # image.thumbnail(size)
+    # image.save(override_extension(dest, ext))
+    # return image
 
 
 def optimize(src, dest, quality=85, ext=None):
@@ -69,9 +69,9 @@ def orient(image):
     return image
 
 
-def image_magick_resize(input_file, output_file, size, quality=85):
+def image_magick_resize(ifile, ofile, size, quality=85):
     s = '%dx%d' % size
-    call(['convert', input_file, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), output_file])
+    call(['/usr/local/bin/convert', ifile, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), ofile])
 
 
 # def image_magick_resize(input_file, output_file, size, quality=85):

utils/Sync/utils/io.py

@@ -1,54 +1,14 @@
 import re
-
-import frontmatter
 import yaml
-from markdown import markdown
 
-from utils.text.split import split, split_with
+from kazimir import kazimir_to_html
 
-# FM_BOUNDARY = re.compile(r'^-{3,}$', re.MULTILINE)
 FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
 
-def read_file(file):
-    with open(file, 'rb') as f:
-        data = f.read().decode('utf-8')
-        return data
-
-
-def read_yaml(file):
-    with open(file, 'rb') as f:
-        data = f.read().decode('utf-8')
-        return yaml.load(data)
-
-
-def read_md(file):
-    with open(file, 'rb') as f:
-        data = f.read().decode('utf-8')
-        return markdown(data)
-
 
 def read_yaml_md(data):
     yaml_front_matter, content = parse_yaml_front_matter(data)
-    return yaml_front_matter, markdown(content)
-
-    # data = frontmatter.load(file)
-    # return data.metadata, markdown(data.content)
-
-
-def read_md_yaml(file):
-    data = read_file(file)
-    yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data)
-
-    if not yaml_data and not md_data:
-        return (None, read_md(file))
-
-    try:
-        y = yaml.load(yaml_data)
-    except:
-        y = None
-
-    m = markdown(md_data) if md_data else None
-    return y, m
+    return yaml_front_matter, kazimir_to_html(content)
 
 
 def parse_yaml_front_matter(text, **defaults):
@@ -75,4 +35,4 @@ def parse_yaml_front_matter(text, **defaults):
     if isinstance(fm, dict):
         metadata.update(fm)
 
-    return metadata, content.strip()
+    return metadata, content.strip()

utils/Sync/utils/validation.py

@@ -9,10 +9,3 @@ __check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings.
 
 def check_description_filename(filename):
     return __check_description_filename_exp.match(filename)
-
-
-def suggest_description_filename(filename):
-    bn = os.path.basename(filename)
-    bn = os.path.splitext(bn)[0]
-    v = settings.default_description_version
-    return '%s [%s].md' % (bn, v)