merge branch 'feature/async-sync' into dev

utils/Sync/config.yaml

@@ -0,0 +1,23 @@
+---
+database_uri: 'mongodb://localhost:27017/hudozka'
+
+env: 'production'
+origin: 'Yandex.Disk'
+yandex_disk_access_token: '<YANDEX_DISK_TOKEN>'
+interval: 0
+
+skip_unchanged: True
+
+image_processing_enabled: True
+
+upload_enabled: True
+upload_auth:
+  login: '<LOGIN>'
+  password: '<PASSWORD>'
+
+provider:
+  name: 'fs'
+  root: '<FS_ROOT>'
+
+update_enabled: True
+delete_enabled: True

utils/Sync/kazimir/__init__.py

@@ -16,7 +16,7 @@ def markdown_to_html(text):
 
 
 def kazimir_to_html(text):
-    html = markdown_to_html(text)
+    html = make(text)
     return typo(html)
 
 
@@ -41,15 +41,209 @@ def typo(html):
     return html
 
 
+def make(text):
+    tokens = text.split('\n')
+    tokens = markup_tokens(tokens)
+    tokens = join_tokens(tokens)
+    tokens = merge_tokens(tokens)
+
+    return compile_tokens(tokens)
+    # return tokens
+
+
+def join_tokens(tokens):
+    buffer_tag = None
+    buffer = []
+    ts = []
+    for token in tokens:
+        tag = token['tag']
+        changed = buffer_tag is not None and tag != buffer_tag
+
+        flush = is_tag_split(tag)
+        if changed:
+            flush = True
+
+        if not is_tag_joinable(tag):
+            flush = True
+
+        if flush:
+            if buffer:
+                ts.append(buffer)
+            if is_tag_split(tag):
+                buffer = []
+                buffer_tag = None
+            else:
+                buffer = [token]
+                buffer_tag = tag
+        else:
+            buffer_tag = tag
+            buffer.append(token)
+
+    if len(buffer):
+        ts.append(buffer)
+    return ts
+
+
+def merge_tokens(tokens: list):
+    tokens_out = []
+    for ts in tokens:
+        tag = ts[0]['tag']
+        fn = None
+        if tag == 'image':
+            fn = merge_image
+        elif tag == 'text':
+            fn = merge_text
+        else:
+            fn = merge_any
+        tokens_out.append(fn(ts))
+    return tokens_out
+
+
+def compile_tokens(tokens: list):
+    compiled = []
+    for ts in tokens:
+        tag = ts['tag']
+        fn = None
+        if tag == 'text':
+            fn = compile_text
+        elif tag == 'image_collection':
+            fn = compile_image_collection
+        elif tag == 'image':
+            fn = compile_image
+        else:
+            fn = compile_any
+        compiled.append(fn(ts))
+    return '\n'.join(compiled)
+
+
+def markup_tokens(tokens: list):
+    marked_tokens = []
+    for token in tokens:
+        tag = 'text'
+        if is_url(token):
+            tag = 'url'
+        elif is_image(token):
+            tag = 'image'
+        elif not token:
+            tag = 'split'
+            token = None
+
+        marked_tokens.append({
+            'tag': tag,
+            'data': token,
+        })
+    return marked_tokens
+
+
+def is_tag_split(tag):
+    return tag == 'split'
+
+
+def is_tag_joinable(tag: str):
+    return tag not in ['url']
+
+
+def is_url(sample: str):
+    from urllib.parse import urlparse
+    url = urlparse(sample)
+
+    return url.scheme and url.netloc
+
+
+def is_image(sample: str):
+    return sample[:2] == '![' and sample[-1:] == ')'
+
+
+def merge_any(tokens):
+    return tokens[0]
+
+
+def merge_text(tokens):
+    data = ''
+
+    for t in tokens:
+        data += t['data']
+        data += '\n'
+
+    return {
+        'tag': 'text',
+        'data': data.strip(),
+    }
+
+
+def merge_image(tokens):
+    if len(tokens) == 1:
+        return tokens[0]
+    else:
+        data = [t['data'] for t in tokens]
+        return {
+            'tag': 'image_collection',
+            'data': data,
+        }
+
+
+def compile_text(token):
+    return markdown_to_html(token['data'])
+
+
+def compile_image_collection(token):
+    file = lambda i: i[4:][:-1]
+    imgs = ['<img src="{}">'.format(file(i)) for i in token['data']]
+    return '''
+        <div class="kazimir__image-collection">
+            <div class="fotorama"
+                 data-width="100%"
+                 data-ratio="800/600"
+            >
+                {}
+            </div>
+        </div>
+    '''.format('\n'.join(imgs))
+
+
+def compile_image(token):
+    img = markdown_to_html(token['data'])
+    return '''
+        <div class="kazimir__image">
+            {}
+        </div>
+    '''.format(img)
+
+
+def compile_any(token):
+    return markdown_to_html(token['data'])
+
+
 if __name__ == '__main__':
     sample_text = '''
 Hello!
 
 https://www.instagram.com/p/BUelE04l1kU
+ku!
 
 https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka
+https://www.youtube.com/watch?v=qnguOqWpraI
+![](1.jpg)
+![](2.jpg)
+![](3.jpg)
+
+LOL NOOB
+![](12.jpg)
+![](22.jpg)
+WTF OMG
 
 Bye!
+Fye!
+
+![](1.jpg)
+
+More text
     '''
 
-    print(markdown_to_html(sample_text))
+    ts = make(sample_text)
+    if isinstance(ts, list):
+        for i in ts:
+            print(i)
+    else:
+        print(ts)
+

utils/Sync/kazimir/instagram.py

@@ -55,7 +55,7 @@ def embed_instagram(media):
         i = etree.fromstring(html)
 
         obj = etree.Element('div')
-        obj.set('class', "kazimir-embed")
+        obj.set('class', "kazimir__embed")
         obj.append(i)
         return obj
     else:

utils/Sync/kazimir/video.py

@@ -33,6 +33,6 @@ def flex_video(url):
     iframe.set('allowfullscreen', '')
 
     obj = etree.Element('div')
-    obj.set('class', "kazimir-video")
+    obj.set('class', "kazimir__video")
     obj.append(iframe)
     return obj

utils/Sync/settings.py

@@ -1,7 +1,22 @@
 import os
 
+import yaml
+import argparse
+
 __author__ = 'Roman Timashev'
 
+name = 'HudozkaSync'
+
+_parser = argparse.ArgumentParser(description='Hudozka Sync Daemon')
+_parser.add_argument('--config', dest='config', default='config.yaml', help='path to config.yaml')
+_args = _parser.parse_args()
+
+config = yaml.load(open(_args.config, 'r'))
+
+
+def value(key, default=None):
+    return config[key] if key in config else default
+
 
 def absolute(path, ensure=True):
     a = os.path.expanduser(path)
@@ -20,9 +35,11 @@ def env(param, default=None):
     return default
 
 
-database_uri = env('MONGO_URI')
+interval = value('interval', 0)
 
-skip_unchanged = False
+database_uri = value('database_uri')  # env('MONGO_URI')
+
+skip_unchanged = value('skip_unchanged')
 
 person_uri = {
     'Н.В.Андреева': 'nv-andreeva',
@@ -33,18 +50,35 @@ person_uri = {
     'А.С.Тимашева': 'as-timasheva',
 }
 
-image_processing = True
+upload_enabled = value('upload_enabled')
+upload_url_image = 'https://static.shlisselburg.org/upload/art/images/{}'
+upload_auth = (
+    value('upload_auth')['login'],
+    value('upload_auth')['password'],
+)
+
+image_processing_enabled = value('image_processing')
 
 album_html_img_class = 'hudozka-product'
 
 image_ext = '.jpg'
 image_sizes = [
     ('original', None, None),
+
     ('big', 1500, 667),
+    ('big@2', 3000, 1334),
+
     ('medium', 400, 300),
+    ('medium@2', 800, 600),
+
     ('small', 250, 175),
-    ('little', 100, 100),
-    ('preview', 50, 50),
+    ('small@2', 500, 350),
+
+    ('og', 968, 504),
+    ('og@2', 1936, 1008),
+
+    ('fb', 1200, 630),
+    ('fb@2', 2400, 1260),
 ]
 
 awards_image_sizes = [
@@ -57,14 +91,6 @@ event_image_sizes = [
     ('big', 1200, 800)
 ]
 
-teacher_image_url_base = 'https://static.shlisselburg.org/art/images/teacher-{id}-{size}{ext}'
-teacher_image_output = '/Users/tmshv/Desktop/Hudozka Static/images'
-teacher_image_sizes = [
-    ('big', 1200, 800),
-    ('medium', 640, 480),
-    ('preview', 50, 50),
-]
-
 album_image_sizes = [
     ('big', 1200, 800),
     ('medium', 640, 480),
@@ -91,8 +117,9 @@ date_formats_reverse = [
     '%Y.%m.%d'
 ]
 
-origin = 'Yandex.Disk'
+origin = value('origin')
 
+collection_images = 'images'
 collection_documents = 'documents'
 collection_awards = 'awards'
 collection_schedules = 'schedules'
@@ -101,34 +128,32 @@ collection_collective = 'collective'
 collection_albums = 'albums'
 collection_pages = 'pages'
 
-sync_provider_type = env('SYNC_PROVIDER', 'fs')
+provider_name = value('provider')['name']  # env('SYNC_PROVIDER', 'fs')
+provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
 
-_providers_roots = {
-    'fs': abs_fn(env('SYNC_LOCAL_PATH')),
-    'yd': abs_fn('/'),
-}
-f = _providers_roots[sync_provider_type]
+f = abs_fn(provider_root)
 
-dir_documents = f('Hudozka/Site/Documents')
-dir_awards = f('Hudozka/Site/Awards')
-dir_schedules = f('Hudozka/Site/Schedules')
-dir_articles = f('Hudozka/Site/Articles')
-dir_collective = f('Hudozka/Site/Collective')
-dir_gallery = f('Hudozka/Site/Gallery')
-dir_pages = f('Hudozka/Site/Pages')
+dir_documents = f('Documents')
+dir_awards = f('Awards')
+dir_schedules = f('Schedules')
+dir_articles = f('Articles')
+dir_collective = f('Collective')
+dir_albums = f('Gallery')
+dir_pages = f('Pages')
 
-dir_static = env('SYNC_STATIC', '~/Hudozka Static')
-dir_static_uploads = absolute(dir_static + '/uploads')
-dir_static_images = absolute(dir_static + '/images')
+page_url_base = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}'
 
-url_page_base_preview = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}'
-url_base_preview = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
-url_base_document = 'https://static.shlisselburg.org/art/uploads/{file}'
+document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
+document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploads/{file}'
+document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
 
-image_base_url = 'https://static.shlisselburg.org/art/images/'
+image_url_upload = 'https://static.shlisselburg.org/upload/art/images/'
+image_url_base = 'https://static.shlisselburg.org/art/images/'
 image_name_format = '{type}-{id}-{img}-{size}{ext}'
 
-yandex_disk_access_token = env('YANDEX_DISK_ACCESS_TOKEN')
+person_image_url_template = 'https://static.shlisselburg.org/art/images/person-{id}-{size}{ext}'
 
-do_update = env('SYNC_ENV', 'production') == 'production'
-do_delete = env('SYNC_ENV', 'production') == 'production'
+yandex_disk_access_token = value('yandex_disk_access_token')  # env('YANDEX_DISK_ACCESS_TOKEN')
+
+update_enabled = value('update_enabled')
+delete_enabled = value('delete_enabled')

utils/Sync/sync.py

@@ -1,15 +1,93 @@
 import settings
-from db import db, collection
-from sync.album import sync_albums
+from db import collection
+from sync.core import Sync
+from sync.core.SyncAlbum import SyncAlbum
+from sync.core.SyncArticle import SyncArticle
+from sync.core.SyncDocument import SyncDocument
+from sync.core.SyncPage import SyncPage
+from sync.core.SyncPerson import SyncPerson
+from sync.core.SyncSchedule import SyncSchedule
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
-from sync.document import sync_documents
-from sync.page import sync_pages
-from sync.person import sync_persons
-from sync.post import sync_posts
-from sync.schedule import sync_schedules
-from utils.fn import lprint_json
-from utils.image import thumbnail
+
+import asyncio
+import logging
+
+# create logger with 'spam_application'
+logger = logging.getLogger(settings.name)
+logger.setLevel(logging.DEBUG)
+
+# create file handler which logs even debug messages
+fh = logging.FileHandler('hudozka.log')
+fh.setLevel(logging.DEBUG)
+
+# create console handler with a higher log level
+ch = logging.StreamHandler()
+# ch.setLevel(logging.ERROR)
+ch.setLevel(logging.DEBUG)
+
+# create formatter and add it to the handlers
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+fh.setFormatter(formatter)
+ch.setFormatter(formatter)
+
+# add the handlers to the logger
+logger.addHandler(fh)
+logger.addHandler(ch)
+
+
+async def main(sync: Sync):
+    await sync.run()
+
+
+async def run(run_interval=0):
+    io = lambda root: get_provider(settings.provider_name, root)
+    c = collection
+
+    documents = SyncDocument(
+        provider=io(settings.dir_documents),
+        collection=c(settings.collection_documents),
+        sizes=settings.image_sizes,
+    )
+
+    pages = SyncPage(
+        provider=io(settings.dir_pages),
+        collection=c(settings.collection_pages),
+    )
+
+    persons = SyncPerson(
+        provider=io(settings.dir_collective),
+        collection=c(settings.collection_collective),
+    )
+
+    articles = SyncArticle(
+        provider=io(settings.dir_articles),
+        collection=c(settings.collection_articles),
+    )
+
+    schedules = SyncSchedule(
+        provider=io(settings.dir_schedules),
+        collection=c(settings.collection_schedules),
+    )
+    albums = SyncAlbum(
+        provider=io(settings.dir_albums),
+        collection=c(settings.collection_albums),
+    )
+
+    while True:
+        await asyncio.wait([
+            main(documents),
+            main(pages),
+            main(persons),
+            main(articles),
+            main(schedules),
+            main(albums),
+        ])
+
+        if run_interval == 0:
+            break
+        else:
+            await asyncio.sleep(run_interval)
 
 
 def get_provider(provider_type, root):
@@ -20,27 +98,12 @@ def get_provider(provider_type, root):
     return None
 
 
-def main(provider, fn, collection_name):
-    if provider:
-        print('SYNC: %s' % (collection_name.upper()))
-        u, d = fn(provider, collection(collection_name), update=settings.do_update, delete=settings.do_update)
-
-        print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d))))
-        lprint_json(d)
-
-        print('UPDATE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_update else str(len(u))))
-        lprint_json(u)
-
-        print('')
-
-
 if __name__ == '__main__':
-    io = lambda root: get_provider(settings.sync_provider_type, root)
+    interval = settings.interval
 
-    main(io(settings.dir_documents), sync_documents, settings.collection_documents)
-    main(io(settings.dir_awards), sync_documents, settings.collection_awards)
-    main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
-    main(io(settings.dir_articles), sync_posts, settings.collection_articles)
-    main(io(settings.dir_collective), sync_persons, settings.collection_collective)
-    main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
-    main(io(settings.dir_pages), sync_pages, settings.collection_pages)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run(interval))
+    loop.close()
+
+    # Да, 968×504 пискеля это меньше минимально рекомендованных Фейсбуком 1200×630.
+    # Зато при таком размере и ратио картинку нигде не кропят, и выглядит она отлично.

utils/Sync/sync/__init__.py

@@ -9,6 +9,7 @@ from markdown import markdown
 
 import settings
 from sync.image import sync_image
+from sync.models import Model
 from utils.fn import lmap, combine
 
 
@@ -43,26 +44,30 @@ def create_date(date_str, date_formats=None):
 folder_name_pattern = re.compile('([\d.]+)(.*)')
 
 
-def untouched(documents, store):
+async def untouched(items: [Model]) -> [Model]:
     if settings.skip_unchanged:
-        return documents
-    return lmap(
-        lambda i: i[0],
-        filter(
-            lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
-            map(
-                lambda i: (i, store.read(i)),
-                documents
-            )
-        ))
+        return items
+
+    result = []
+    for item in items:
+        status = await item.is_changed()
+        if status:
+            result.append(item)
+    return result
+
+    # def is_equals(e, n):
+    #     return (n is None) or ('hash' not in n) or (e.hash != n['hash'])
+    #
+    # doc = lambda i: {'id': i.id, 'hash': i.hash}
+    #
+    # stored_documents = [(i, store.read(doc(i))) for i in documents]
+    # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)]
+
+    # return filtered_documents
 
 
 def synced_images_ids(images):
-    ids = lmap(
-        lambda image: image['_id'],
-        sync_image(images)
-    )
-    return ids
+    return [i['_id'] for i in sync_image(images)]
 
 
 def synced_image_id(image):
@@ -84,12 +89,12 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
     return date, title
 
 
-def create_post(md, image_path_fn):
+async def create_post(md, image_path_fn):
     html = lxml.html.fromstring(md)
 
     for img in html.cssselect('img'):
         src = img.get('src')
-        path = image_path_fn(src)
+        path = await image_path_fn(src)
 
         if path:
             img.set('src', path)

utils/Sync/sync/album.py

@@ -1,169 +0,0 @@
-import os
-
-import settings
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \
-    synced_images_ids, synced_image_id
-from sync.core.album import SyncAlbum
-from sync.data import list_images
-from utils.fn import lmap, lmapfn, key_mapper
-from utils.image import create_image
-from utils.io import read_yaml_md
-
-
-class ImageCreator:
-    def __init__(self, provider, cache):
-        super().__init__()
-        self.provider = provider
-        self.cache = cache if cache else set()
-
-    def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
-        filehash = self.provider.hash(file)
-        cached = self.cached(filehash)
-        return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir,
-                            skip_processing=skip_processing) if not cached else cached
-
-    def cached(self, name):
-        return self.cache[name] if name in self.cache else None
-
-
-def get_manifest(provider, filepath):
-    data = provider.read(filepath).read().decode('utf-8')
-    y, m = read_yaml_md(data)
-
-    data = {**y} if y else {}
-    _ = lambda key, d=None: data[key] if key in data else d
-
-    date = create_date(str(_('date')), settings.date_formats_direct)
-    if date:
-        data['date'] = date
-
-    album_id = _('id')
-    if album_id:
-        data['id'] = str(album_id)
-
-    post = {'post': m} if m else {}
-    return {**data, **post}
-
-
-def album_from_folder(sync, path):
-    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
-    date, title = create_date_and_title_from_folder_name(
-        os.path.basename(path)
-    )
-
-    images = lmap(
-        lambda i: os.path.relpath(i, path),
-        list_images(sync.provider, path)
-    )
-
-    # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
-    document = {
-        'folder': path,
-        'date': date,
-        'title': title,
-        'post': create_post_from_image_list(images),
-        'images': images
-    }
-
-    # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
-    manifest = sync.provider.type_filter(path, '.md')
-    manifest = manifest[0] if len(manifest) else None
-    manifest = get_manifest(sync.provider, manifest) if manifest else {}
-    manifest = manifest if manifest else {}
-
-    document = {
-        **document,
-        **manifest
-    }
-
-    # CREATE DOCUMENT IDENTITY
-    document = sync.create_id(document)
-
-    # CREATE HASH OF DOCUMENT FILE
-    document = sync.create_hash(document)
-
-    return document
-
-
-def main(sync, do_update, do_delete):
-    # GET ALBUMS FOLDERS
-    documents = lmap(
-        lambda i: i,
-        filter(
-            lambda i: sync.provider.is_dir(i),
-            sync.provider.scan('.')
-        )
-    )
-
-    documents = lmap(
-        lambda folder: album_from_folder(sync, folder),
-        documents
-    )
-
-    # CREATE SCOPE OF CURRENT SESSION
-    scope_documents_ids = lmapfn(documents)(
-        lambda i: i['id']
-    )
-
-    # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # MAP EVENT MANIFEST -> EVENT_OBJECT
-    documents = lmap(
-        sync.create_album,
-        documents
-    )
-
-    # # FILTER INCORRECT EVENTS
-    documents = list(filter(None, documents))
-
-    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
-    documents = lmap(
-        key_mapper('images', synced_images_ids),
-        documents
-    )
-    documents = lmap(
-        key_mapper('preview', synced_image_id),
-        documents
-    )
-
-    # SYNC OBJECT WITH DB
-    if do_update:
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
-
-    if do_delete:
-        documents_to_remove = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_remove
-            )
-        )
-
-    return documents, documents_to_remove
-
-
-def sync_albums(provider, collection, update=True, delete=True):
-    """
-    :param collection:
-    :param provider:
-    :param update:
-    :param delete:
-    :return:
-    """
-
-    return main(
-        SyncAlbum(
-            collection,
-            provider,
-            image_creator=ImageCreator(provider, cache=None),
-            images_dir=settings.dir_static_images
-        ),
-        do_update=update,
-        do_delete=delete
-    )

utils/Sync/sync/compile.py

@@ -1,36 +0,0 @@
-import os
-from tempfile import mkstemp
-
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text
-
-
-class Preview:
-    def __init__(self, provider, url_base, output, sizes):
-        super().__init__()
-        self.provider = provider
-        self.url_base = url_base
-        self.output = output
-        self.sizes = sizes
-
-    def create(self, file):
-        def url(size, ext):
-            return self.url_base.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = self.pdf_to_jpg(file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, self.sizes, url, self.output)
-        os.remove(temp_preview_path)
-        return img
-
-    def pdf_to_jpg(self, pdf):
-        _, temp_in = mkstemp(suffix='.pdf')
-        _, temp_out = mkstemp(suffix='.jpg')
-
-        abspdf = self.provider.copy(pdf, temp_in)
-        image_magick_pdf_to_img(abspdf, temp_out)
-
-        os.remove(temp_in)
-        return temp_out

utils/Sync/sync/core/SyncAlbum.py

@@ -0,0 +1,51 @@
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.models.Album import Album
+
+logger = logging.getLogger(settings.name + '.Album')
+
+
+class SyncAlbum(Sync):
+    async def run(self):
+        logger.info('Checking for update')
+
+        documents = await Album.scan(self.provider)
+
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Item(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        documents = await untouched(documents)
+        logger.info('Changed {} Items(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Items to update')
+
+            for document in documents:
+                logger.info('Updating Item {}'.format(document.id))
+
+                await document.build()
+                await document.save()
+
+                logger.info('Updated Item {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Items to delete')
+            else:
+                for document in documents_delete:
+                    logger.info('Deleting Item {}'.format(document['id']))
+
+                    q = {'_id': document['_id']}
+                    self.delete(q)
+
+        # await self.clean()

utils/Sync/sync/core/SyncArticle.py

@@ -0,0 +1,58 @@
+import os
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Article import Article
+from sync.models.Person import Person
+
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Article')
+
+
+class SyncArticle(Sync):
+    async def run(self):
+        logger.info('Checking for update')
+
+        documents = await Article.scan(self.provider)
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Item(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        documents = await untouched(documents)
+        logger.info('Changed {} Items(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Items to update')
+
+            for document in documents:
+                logger.info('Updating Item {}'.format(document.id))
+
+                await document.build(
+                    sizes=settings.image_sizes,
+                )
+                await document.save()
+
+                logger.info('Updated Item {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Items to delete')
+            else:
+                for document in documents_delete:
+                    logger.info('Deleting Item {}'.format(document['id']))
+
+                    q = {'_id': document['_id']}
+                    self.delete(q)
+
+        # await self.clean()

utils/Sync/sync/core/SyncDocument.py

@@ -0,0 +1,186 @@
+import os
+from tempfile import mkstemp
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import scan_subdirs, Provider
+from sync.models.Document import Document
+
+from sync.models import Model
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+logger = logging.getLogger(settings.name + '.Document')
+
+
+class SyncDocument(Sync):
+    def __init__(self, provider: Provider, collection, sizes):
+        super().__init__(provider, collection)
+        self.sizes = sizes
+        self.dir_static_previews = ''
+
+        self.temp_dir = '/Users/tmshv/HudozkaSyncTemp'
+
+    async def clean(self):
+        pass
+
+    async def run(self):
+        logger.info('Checking documents for update')
+
+        documents = await Document.scan(self.provider)
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Document(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        # documents = untouched(documents, self)
+        documents = await untouched(documents)
+        logger.info('Changed {} Document(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Documents to update')
+
+            for document in documents:
+                logger.info('Updating Document {}'.format(document.id))
+
+                await document.upload()
+                await document.create_preview(self.sizes)
+                await document.save()
+
+                logger.info('Updated Document {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Documents to delete')
+
+            for document in documents_delete:
+                logger.info('Deleting {}'.format(document['id']))
+
+                q = {'_id': document['_id']}
+                self.delete(q)
+
+        await self.clean()
+
+
+async def get_assets(self, document) -> []:
+    return []
+
+
+def validate(self, documents):
+    files = [doc['file'] for doc in documents]
+    file_names = [os.path.basename(f) for f in files]
+    file_names = set(file_names)
+
+    if len(file_names) != len(documents):
+        raise Exception('File names should be unique')
+
+
+def create(self, document):
+    file = document['file']
+    filename = os.path.basename(file)
+
+    document['type'] = document_type(document)
+    document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
+    document['title'] = last_good([
+        document['title'],
+        get_pdf_title(self.provider, document['file'])
+    ])
+    document['file'] = {
+        'name': filename,
+        'size': self.provider.size(file)
+    }
+
+    return document
+
+
+def create_id(self, document: dict) -> dict:
+    filepath = document['file']
+    filename = os.path.basename(filepath)
+
+    return {
+        **document,
+        'id': self.id_template.format(
+            category=url_encode_text(document['category']),
+            file=url_encode_text(filename)
+        ),
+    }
+
+
+def create_url(self, document: dict) -> dict:
+    filename = os.path.basename(document['file'])
+
+    return {
+        **document,
+        'url': self.url_template.format(
+            file=url_encode_file(filename)
+        ),
+    }
+
+
+def create_hash(self, document: dict) -> dict:
+    hash_doc = [document[key] for key in self.hash_keys]
+
+    return {
+        **document,
+        'hash': hash_str(
+            hash_str(hash_doc) + self.provider.hash(document['file'])
+        ),
+    }
+
+
+def create_preview(self, file, sizes, preview_dir):
+    def url(size, ext):
+        return self.url_base_preview.format(id=file, size=size, ext=ext)
+
+    temp_preview_path = pdf_to_jpg(self.provider, file)
+    file = os.path.basename(file)
+    file = url_encode_text(file)
+    img = create_image(temp_preview_path, sizes, url, preview_dir)
+    os.remove(temp_preview_path)
+    return img
+
+
+def document_type(document):
+    if document['category'] == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/core/SyncPage.py

@@ -0,0 +1,77 @@
+import os
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Page import Page
+
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Page')
+
+
+class SyncPage(Sync):
+    def __init__(self, provider: Provider, collection):
+        super().__init__(provider, collection)
+
+    async def clean(self):
+        pass
+
+    async def run(self):
+        logger.info('Checking for update')
+
+        documents = await Page.scan(self.provider)
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Page(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        documents = await untouched(documents)
+        logger.info('Changed {} Pages(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Pages to update')
+
+            for document in documents:
+                logger.info('Updating Page {}'.format(document.id))
+
+                await document.setup_images(settings.image_sizes, get_image_url_fn(document.id))
+                await document.save()
+
+                logger.info('Updated Page {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Pages to delete')
+            else:
+                for document in documents_delete:
+                    logger.info('Deleting Page {}'.format(document['id']))
+
+                    q = {'_id': document['_id']}
+                    self.delete(q)
+
+        await self.clean()
+
+
+def get_image_url_fn(page_id: str):
+    image_id = lambda filename: url_encode_text(
+        os.path.splitext(os.path.basename(filename))[0]
+    )
+
+    def img_url_fn(file, size, ext):
+        return settings.page_url_base.format(
+            page=page_id,
+            id=image_id(file),
+            size=size,
+            ext=ext
+        )
+
+    return img_url_fn

utils/Sync/sync/core/SyncPerson.py

@@ -0,0 +1,75 @@
+import os
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Person import Person
+
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+
+
+class SyncPerson(Sync):
+    async def run(self):
+        logger.info('Checking for update')
+
+        documents = await Person.scan(self.provider)
+
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Item(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        documents = await untouched(documents)
+        logger.info('Changed {} Items(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Items to update')
+
+            for document in documents:
+                logger.info('Updating Item {}'.format(document.id))
+
+                await document.build(
+                    sizes=settings.image_sizes,
+                    url_factory=get_image_url_fn(document.id),
+                )
+                await document.save()
+
+                logger.info('Updated Item {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Items to delete')
+            else:
+                for document in documents_delete:
+                    logger.info('Deleting Item {}'.format(document['id']))
+
+                    q = {'_id': document['_id']}
+                    self.delete(q)
+
+        # await self.clean()
+
+
+def get_image_url_fn(page_id: str):
+    image_id = lambda filename: url_encode_text(
+        os.path.splitext(os.path.basename(filename))[0]
+    )
+
+    def img_url_fn(file, size, ext):
+        return settings.page_url_base.format(
+            page=page_id,
+            id=image_id(file),
+            size=size,
+            ext=ext
+        )
+
+    return img_url_fn

utils/Sync/sync/core/SyncSchedule.py

@@ -0,0 +1,51 @@
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.models.Schedule import Schedule
+
+logger = logging.getLogger(settings.name + '.Schedule')
+
+
+class SyncSchedule(Sync):
+    async def run(self):
+        logger.info('Checking for update')
+
+        documents = await Schedule.scan(self.provider)
+
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Item(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        documents = await untouched(documents)
+        logger.info('Changed {} Items(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Items to update')
+
+            for document in documents:
+                logger.info('Updating Item {}'.format(document.id))
+
+                await document.build()
+                await document.save()
+
+                logger.info('Updated Item {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Items to delete')
+            else:
+                for document in documents_delete:
+                    logger.info('Deleting Item {}'.format(document['id']))
+
+                    q = {'_id': document['_id']}
+                    self.delete(q)
+
+        # await self.clean()

utils/Sync/sync/core/__init__.py

@@ -1,21 +1,43 @@
+from sync.data import Provider
 from sync.models import Model
 from utils.fn import lmap
 
 
+def get_id(item):
+    try:
+        return item.id
+    except AttributeError:
+        return item['id']
+
+
 class Sync:
     @staticmethod
     def compile(document):
         return document.bake()
 
-    def __init__(self):
+    def __init__(self, provider: Provider, collection):
         super().__init__()
 
-        self.collection = None
+        self.provider = provider
+        self.collection = collection
 
-    def read(self, document):
-        if isinstance(document, list):
-            return lmap(self.read, document)
+    async def run(self):
+        """
+        # Get scope files
+        # Validate these files. Raise an error
+        # calc_hashes
+        # make diff with previous run
+        # compile objects
+        # upload
 
+        :return:
+        """
+        return None, None
+
+    async def upload(self):
+        pass
+
+    def read(self, document: dict):
         q = {'id': document['id']}
         try:
             return self.collection.find_one(q)
@@ -23,16 +45,10 @@ class Sync:
             pass
         return None
 
-    def update(self, document):
-        if isinstance(document, list):
-            return lmap(self.update, document)
-
-        if isinstance(document, Model):
-            return self.update(self.compile(document))
-
-        q = {'id': document['id']}
+    def update(self, document: dict):
+        query = {'id': document['id']}
         try:
-            self.collection.update_one(q, {'$set': document}, upsert=True)
+            self.collection.update_one(query, {'$set': document}, upsert=True)
             return self.read(document)
         except ValueError:
             pass

utils/Sync/sync/core/album.py

@@ -1,139 +0,0 @@
-import os
-from lxml import etree
-
-import lxml.html
-import settings
-from sync import create_post
-from sync.core import Sync
-from sync.models.product import create_product
-from utils.fn import lmap, iterate_iter_over_fns, combine
-from utils.hash import hash_file, hash_str
-from utils.text.transform import url_encode_text
-
-
-def create_url(a, i, s, e):
-    return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
-        album=a,
-        id=i,
-        size=s,
-        ext=e
-    )
-
-
-def url_creator(aid, pid):
-    return lambda size, ext: create_url(aid, pid, size, ext.lower())
-
-
-class SyncAlbum(Sync):
-    @staticmethod
-    def id_from_file(i):
-        return url_encode_text(
-            os.path.splitext(i)[0]
-        )
-
-    def __init__(self, collection, provider, image_creator, images_dir):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.image_creator = image_creator
-        self.skip_image_processing = False
-        self.images_dir = images_dir
-
-    def create_id(self, document):
-        if 'id' in document:
-            return document
-
-        y = str(document['date'].year)
-        document['id'] = url_encode_text('-'.join([y, document['title']]))
-        return document
-
-    def create_hash(self, document):
-        files = sorted(document['images'])
-        hashes = lmap(
-            self.provider.hash,
-            lmap(
-                lambda i: os.path.join(document['folder'], i),
-                files
-            )
-        )
-
-        document['hash'] = hash_str(
-            combine([hash_str(document)] + hashes)
-        )
-
-        return document
-
-    def create_image(self, file, url_fn):
-        ss = settings.album_image_sizes
-        od = self.images_dir
-        sp = self.skip_image_processing
-
-        return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
-
-    def create_album(self, doc):
-        # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
-
-        image_id = SyncAlbum.id_from_file
-        image_path = lambda i: os.path.join(doc['folder'], i)
-
-        if 'title' not in doc or not doc['title']:
-            return None
-
-        create_album_id = lambda date, title: '{date}-{title}'.format(
-            date=date.strftime('%Y'),
-            title=url_encode_text(title)
-        )
-
-        doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
-
-        if 'album' in doc:
-            doc['album'] = self.create_products(doc, image_path)
-
-        def process_post_image(src):
-            path = image_path(src)
-
-            if self.provider.exists(path):
-                url = url_creator(
-                    doc['id'],
-                    image_id(src)
-                )
-
-                image = self.create_image(path, url)
-                if image:
-                    images.append(image)
-                    return image['data']['big']['url']
-            return None
-
-        images = []
-        post_html = create_post(doc['post'], process_post_image)
-
-        if 'preview' in doc:
-            preview = os.path.join(doc['folder'], doc['preview'])
-            doc['preview'] = self.create_image(preview, url_creator(
-                doc['id'],
-                image_id(doc['preview'])
-            ))
-
-        doc['post'] = post_html
-        doc['images'] = images
-        return doc
-
-    def create_products(self, doc, image_file_fn):
-        image_id = SyncAlbum.id_from_file
-        iter_album = iterate_iter_over_fns([
-            lambda product: {
-                **product,
-                'image': self.create_image(
-                    image_file_fn(product['file']),
-                    url_creator(
-                        doc['id'],
-                        image_id(product['file'])
-                    )
-                )
-            }
-        ])
-
-        return lmap(
-            create_product,
-            iter_album(doc['album'])
-        )

utils/Sync/sync/core/document.py

@@ -1,107 +0,0 @@
-import os
-from tempfile import mkstemp
-
-from sync.core import Sync
-from utils.fn import last_good, lmap
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class SyncDocument(Sync):
-    def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.sizes = sizes
-        self.dir_static_previews = dir_static_previews
-        self.url_base_preview = url_base_preview
-        self.url_base_document = url_base_document
-
-    def create(self, document):
-        file = document['file']
-        filename = os.path.basename(file)
-
-        document['type'] = document_type(document)
-        document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
-        document['title'] = last_good([
-            document['title'],
-            get_pdf_title(self.provider, document['file'])
-        ])
-        document['file'] = {
-            'name': filename,
-            'size': self.provider.size(file)
-        }
-
-        return document
-
-    def create_id(self, document):
-        bn = os.path.basename(document['file'])
-        document['id'] = url_encode_text('{category}-{file}'.format(
-            category=document['category'],
-            file=bn
-        ))
-        return document
-
-    def create_url(self, document):
-        filename = os.path.basename(document['file'])
-        document['url'] = self.url_base_document.format(file=url_encode_file(filename))
-        return document
-
-    def create_hash(self, document):
-        hash_keys = ['id', 'url']
-        hash_doc = lmap(
-            lambda key: document[key],
-            hash_keys
-        )
-        document['hash'] = hash_str(
-            hash_str(hash_doc) + self.provider.hash(document['file'])
-        )
-        return document
-
-    def create_preview(self, file, sizes, preview_dir):
-        def url(size, ext):
-            return self.url_base_preview.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = pdf_to_jpg(self.provider, file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, sizes, url, preview_dir)
-        os.remove(temp_preview_path)
-        return img
-
-
-def document_type(document):
-    if document['category'] == 'Награды':
-        return 'award'
-    return 'document'
-
-
-def pdf_to_jpg(provider, pdf):
-    _, temp_in = mkstemp(suffix='.pdf')
-    _, temp_out = mkstemp(suffix='.jpg')
-
-    abspdf = provider.copy(pdf, temp_in)
-    image_magick_pdf_to_img(abspdf, temp_out)
-
-    os.remove(temp_in)
-    return temp_out
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None

utils/Sync/sync/core/person.py

@@ -1,39 +0,0 @@
-import settings
-from sync.core import Sync
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.text.transform import url_encode_text
-
-url_base = settings.teacher_image_url_base
-sizes = settings.teacher_image_sizes
-
-
-class SyncPerson(Sync):
-    def __init__(self, collection, provider, dir_local_images):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.dir_local_images = dir_local_images
-
-    def compile(self, profile):
-        image_path = self.provider.get_abs(profile['picture'])
-        profile['picture'] = create_image(
-            image_path,
-            sizes,
-            lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
-            self.dir_local_images
-        )
-        return profile
-
-    def create_id(self, document):
-        if 'id' in document and document['id']:
-            return document
-
-        document['id'] = url_encode_text(document['name'])
-        return document
-
-    def create_hash(self, document):
-        document['hash'] = hash_str(
-            self.provider.hash(document['file']) + self.provider.hash(document['picture'])
-        )
-        return document

utils/Sync/sync/core/post.py

@@ -1,92 +0,0 @@
-import os
-from tempfile import mkstemp
-
-import lxml.html
-
-import kazimir
-from sync.core import Sync
-from utils.fn import last_good, lmap, combine
-from utils.hash import hash_str, hash_file
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class SyncPost(Sync):
-    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.sizes = sizes
-        self.type = document_type
-        self.image_url_base = image_url_base
-        self.dir_local_images = dir_local_images
-        self.origin = origin
-
-    def create_id(self, document):
-        new_id = url_encode_text(document['title'])
-        document['id'] = document['id'] if 'id' in document else new_id
-        return document
-
-    def create_hash(self, document):
-        files = sorted(document['images'])
-        hashes = lmap(
-            self.provider.hash,
-            lmap(
-                lambda i: os.path.join(document['folder'], i),
-                files
-            )
-        )
-
-        document['hash'] = hash_str(
-            combine([hash_str(document)] + hashes)
-        )
-
-        return document
-
-    def create(self, document):
-        if 'date' not in document or not document['date']:
-            return None
-
-        if 'title' not in document or not document['title']:
-            return None
-
-        if self.type:
-            document['type'] = self.type
-
-        post_html = lxml.html.fromstring(document['post'])
-        images = []
-        for img in post_html.cssselect('img'):
-            src = img.get('src')
-            relative_image_path = os.path.join(document['folder'], src)
-
-            img_path = self.provider.get_local(relative_image_path)
-            if os.path.exists(img_path):
-                img_id = url_encode_text(os.path.splitext(src)[0])
-                url_fn = lambda size, ext: self.image_url_base.format(
-                    id=document['id'],
-                    img=img_id,
-                    size=size,
-                    ext=ext.lower()
-                )
-
-                image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
-                if image:
-                    images.append(image)
-                    img.set('src', image['data']['big']['url'])
-                else:
-                    print('fail: ', document['folder'], src)
-
-        document['post'] = kazimir.html_from_tree(post_html)
-        document['images'] = images
-        document['origin'] = self.origin
-        return document
-
-    def create_remove_query(self, query):
-        if self.type:
-            query = {
-                **query,
-                'type': self.type,
-                'origin': self.origin
-            }
-        return query

utils/Sync/sync/core/schedule.py

@@ -1,38 +0,0 @@
-import os
-from tempfile import mkstemp
-
-from sync.core import Sync
-from utils.fn import last_good, lmap
-from utils.hash import hash_str, hash_file
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class SyncSchedule(Sync):
-    def __init__(self, collection, provider):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-
-    def create_id(self, document):
-        period = get_period(document['period'])
-        doc_id = document['id'] if 'id' in document and document['id'] else None
-        if not doc_id:
-            doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=document['semester'])
-        document['id'] = doc_id
-        return document
-
-    def create_hash(self, document):
-        return {
-            **document,
-            'hash': self.provider.hash(document['file'])
-        }
-
-
-def get_period(value):
-    if isinstance(value, str):
-        return value
-
-    if isinstance(value, list):
-        return '-'.join(lmap(str, value))

utils/Sync/sync/data/__init__.py

@@ -3,6 +3,7 @@ from io import BytesIO
 
 import requests
 
+import settings
 from utils.fn import combine, lmap
 
 
@@ -85,23 +86,3 @@ def list_images(provider, path):
         lambda ext: provider.type_filter(path, ext),
         image_types
     ))
-
-
-def upload(file_path, relative_path):
-    """
-    /path/to/file, /url/path -> https://static.shlisselburg.org/art/url/path
-    :param file_path:
-    :param relative_path:
-    :return:
-    """
-    files = {'file': open(file_path, 'rb')}
-    url = 'https://static.shlisselburg.org/upload/art{}'.format(relative_path)
-    r = requests.put(
-        url=url,
-        files=files,
-        auth=('uploader', '1207690962638465')
-    )
-    if r.status_code == 200:
-        return url
-    else:
-        return None

utils/Sync/sync/data/fs.py

@@ -13,13 +13,9 @@ class FSProvider(Provider):
         return open(self.get_abs(path), 'rb')
 
     def scan(self, path):
-        return lmap(
-            self.get_rel,
-            map(
-                lambda i: i.path,
-                os.scandir(self.get_abs(path))
-            )
-        )
+        paths = [i.path for i in os.scandir(self.get_abs(path))]
+
+        return [self.get_rel(path) for path in paths]
 
     def glob(self, pattr):
         return lmap(self.get_rel, glob(self.get_abs(pattr)))

utils/Sync/sync/data/request.py

@@ -0,0 +1,41 @@
+import aiohttp
+import requests
+
+import settings
+
+
+async def upload(url, path):
+    """
+    :param url:
+    :param path:
+    :return:
+    """
+
+    if not settings.upload_enabled:
+        return url
+
+    # urlf = 'https://static.shlisselburg.org/upload/art{}'
+
+    files = {'file': open(path, 'rb')}
+    r = requests.put(
+        url=url,
+        files=files,
+        # auth=settings.upload_auth
+        auth=('uploader', '1207690962638465')
+    )
+    if r.status_code == 200:
+        return url
+    else:
+        return None
+
+
+async def get(url):
+    async with aiohttp.ClientSession as session:
+        res = session.get(url)
+        return await res.text()
+
+
+async def put(url, data):
+    async with aiohttp.ClientSession as session:
+        res = session.put(url, data=data)
+        return await res.text()

utils/Sync/sync/document.py

@@ -1,206 +0,0 @@
-import os
-
-import requests
-
-import settings
-from sync import synced_image_id, untouched
-from sync.core.document import SyncDocument
-from sync.data import scan_subdirs
-from sync.models.document import Document
-from utils.fn import combine, lmap, key_mapper
-
-
-def sync_documents(provider, collection, update=True, delete=True):
-    """
-    DOCUMENT_OBJECT SAMPLE
-    {
-        "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
-        "_id": {
-            "$oid": "56d08411ace9573958e5e497"
-        },
-        "type": "document",
-        "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
-        "category": "Для поступающих",
-        "file": {
-            "size": 53088,
-            "name": "Договор о пожертвовании.pdf"
-        },
-        "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
-        "preview": {
-            "$oid": "56d08007ace9573958e5e48f"
-        },
-        "title": "Договор о пожертвовании"
-    }
-
-    :param provider:
-    :param update:
-    :param delete:
-    :return:
-    """
-
-    static_path = static_path_fn(settings.dir_static_uploads)
-
-    return main(
-        SyncDocument(
-            collection,
-            provider,
-            sizes=settings.image_sizes,
-            dir_static_previews=settings.dir_static_images,
-            url_base_preview=settings.url_base_preview,
-            url_base_document=settings.url_base_document,
-        ),
-        static_path,
-        update_documents=update,
-        delete_documents=delete
-    )
-
-
-def upload(provider, document):
-    f = document['file']
-    p = provider.get_local(f)
-    n = os.path.basename(document['url'])
-
-    files = {'file': open(p, 'rb')}
-    # values = {'DB': 'photcat', 'OUT': 'csv', 'SHORT': 'short'}
-
-    r = requests.put(
-        url='https://static.shlisselburg.org/upload/art/uploads/{}'.format(n),
-        files=files,
-        auth=('uploader', '1207690962638465')
-    )
-    if r.status_code == 200:
-        print('Document {} uploaded'.format(document['url']))
-    else:
-        print('Failed to upload document {}'.format(document['url']))
-        print('https://static.shlisselburg.org/upload/art/uploads/{}'.format(n))
-        print(r.request.headers)
-        print(r.headers)
-        print(r.text)
-        print()
-
-
-def main(sync, static_path, update_documents, delete_documents):
-    documents = scan_subdirs(sync.provider, '.pdf')
-    documents = lmap(
-        lambda document: read_document(sync, document),
-        documents
-    )
-    documents_id = lmap(
-        lambda document: document['id'],
-        documents
-    )
-
-    # CHECKING
-    file_names = set(lmap(
-        lambda document: document['file'],
-        documents
-    ))
-    if len(file_names) != len(documents):
-        raise Exception('File names should be unique')
-
-    # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # COPY FILE -> STATIC_DIR/URL_FILENAME
-    for i in documents:
-        sync.provider.copy(
-            i['file'],
-            static_path(i)
-        )
-
-    for i in documents:
-        upload(sync.provider, i)
-
-    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
-    documents = lmap(
-        sync.create,
-        documents
-    )
-
-    if update_documents:
-        # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
-        documents = lmap(
-            key_mapper('preview', lambda img: synced_image_id(img)),
-            documents
-        )
-
-        # SYNC DOCUMENT_OBJECT WITH DB
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    # DELETING
-    documents_delete = sync.query({'id': {'$nin': documents_id}})
-    if delete_documents:
-        documents_delete = lmap(
-            sync.delete,
-            map(
-                lambda document: {'_id': document['_id']},
-                documents_delete
-            )
-        )
-
-    return documents, documents_delete
-
-
-def read_document(sync, path):
-    name = os.path.basename(path)
-
-    document = {
-        'file': path,
-        'category': os.path.dirname(path),
-        'title': os.path.splitext(name)[0]
-    }
-
-    document = sync.create_id(document)
-    document = sync.create_url(document)
-    document = sync.create_hash(document)
-
-    return document
-
-
-def create_document(provider, m):
-    # dir_static_previews = settings.dir_static_images,
-    # url_base_preview = settings.url_base_preview,
-    # url_base_document = settings.url_base_document
-    return Document(provider, m, {})
-
-
-def static_path_fn(dir_path):
-    return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
-
-
-def categorize_files_list(files, title):
-    """
-    Adds specified category to each file in files list
-    :param files:
-    :param title:
-    :return:
-    """
-    return map(
-        lambda profile: {
-            **profile,
-            'category': title
-        },
-        files
-    )
-
-
-def unwrap_manifest(param):
-    if isinstance(param, list):
-        return combine(map(
-            unwrap_manifest,
-            param
-        ))
-
-    if 'files' in param:
-        return lmap(
-            unwrap_manifest,
-            categorize_files_list(param['files'], param['title'])
-        )
-
-    if 'file' not in param:
-        return None
-
-    return param

utils/Sync/sync/image.py

@@ -1,48 +1,7 @@
 from db import db
-from settings import image_name_format, image_base_url
 from utils.fn import lmap
 
 
-# class SyncImage(Sync):
-#     def __init__(self, sizes, save_path, create_url_fn):
-#         super().__init__()
-#         self.sizes = sizes
-#         self.save_path = save_path
-#         self.create_url_fn = create_url_fn
-#         self.collection = db().images
-#
-#     def create_id(self, document):
-#         id = document['id'] if 'id' in document and document['id'] else None
-#         if not id:
-#             id = 'schedule-{period}-{semester}'.format(**document)
-#         document['id'] = id
-#         return document
-#
-#     def read(self, document):
-#         return super().read(document)
-#
-#     def create_hash(self, document):
-#         return {
-#             **document,
-#             'hash': hash_file(document)
-#         }
-#
-#     def create(self, document):
-#         image = create_image(document, self.sizes, self.create_url_fn, self.save_path)
-#
-#         return {
-#             'file': document,
-#             'hash': self.create_hash(document),
-#             'data': image['data']
-#         }
-#
-#     def find_or_create(self, document):
-#         i = self.read(document)
-#         if not i:
-#             return self.create(document)
-#         return i
-
-
 def sync_image(record):
     if isinstance(record, list):
         return lmap(
@@ -59,8 +18,3 @@ def sync_image(record):
         pass
 
     return None
-
-
-def image_create_url(file):
-    url = image_base_url + image_name_format
-    return url.format(file)

utils/Sync/sync/models/Album.py

@@ -1,21 +1,251 @@
-__author__ = 'Roman Timashev'
+import logging
+
+import os
+
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from kazimir import kazimir_to_html
+from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import Provider, list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Album')
+store = collection(settings.collection_albums)
 
 
-class Album:
-    def __init__(self, type, title, uri, course, course_uri, date, teacher, comment):
-        super().__init__()
-        self.products = []
-        self.type = type
-        self.title = title
-        self.uri = uri
-        self.course = course
-        self.course_uri = course_uri
-        self.date = date
-        self.teacher = teacher
-        self.comment = comment
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
 
-    def add_product(self, product):
-        self.products.append(product)
 
-    def __repr__(self, *args, **kwargs):
-        return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products))
+class Album(Model):
+    @staticmethod
+    async def scan(provider):
+        documents = [i for i in provider.scan('.') if provider.is_dir(i)]
+        documents = [Album.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        date, title = create_date_and_title_from_folder_name(
+            os.path.basename(path)
+        )
+
+        images = list_images(provider, path)
+        images = [os.path.relpath(i, path) for i in images]
+
+        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+        document = {
+            'folder': path,
+            'date': date,
+            'title': title,
+            'post': create_post_from_image_list(images),
+            'images': images
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = provider.type_filter(path, '.md')
+        manifest_file = manifest_file[0] if len(manifest_file) else None
+        manifest = get_manifest(provider, manifest_file) if manifest_file else {}
+        manifest = manifest if manifest else {}
+
+        file = manifest_file if manifest_file else path
+        return Album(
+            provider,
+            file,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        # self.__id_template: str = '{category}-{file}'
+        # self.__url_template: str = settings.document_url_template
+        # self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.sizes = settings.image_sizes
+        self.version = '1'
+
+        self.origin = settings.origin
+        self.post = None
+        self.images = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        self.__set_id()
+        self.__set_hash()
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        params = self.params
+        image_id = id_from_file
+        images = []
+
+        create_album_id = lambda date, title: '{date}-{title}'.format(
+            date=date.strftime('%Y'),
+            title=url_encode_text(title)
+        )
+
+        # params['id'] = params['id'] if 'id' in params else create_album_id(params['date'], params['title'])
+
+        # if 'album' in params:
+        #     params['album'] = self.create_products(params, image_path)
+
+        async def process_post_image(src):
+            url = url_creator(
+                self.id,
+                id_from_file(src)
+            )
+
+            image_path = os.path.join(params['folder'], src)
+            image = await Image.new(self.provider, image_path, self.sizes, url)
+            if image:
+                images.append(image)
+                return image.get_size('big')['url']
+            return None
+
+        post_html = await create_post(params['post'], process_post_image)
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            preview = await Image.new(self.provider, preview, sizes=self.sizes)
+        else:
+            # preview = 'https://art.shlisselburg.org/entrance.jpg'
+            preview = None
+
+        self.post = post_html
+        self.images = images
+        self.preview = preview
+
+        # sizes = kwargs['sizes']
+        # post, images = await create_post(self.provider, self.id, self.params, sizes)
+        # self.post = post
+        # self.images = images
+        #
+        # post, images = await create_post(self.provider, self.id, self.params, sizes)
+        # self.post = kazimir.html_from_tree(post)
+        # self.images = images
+
+    def create_products(self, doc, image_file_fn):
+        pass
+        # image_id = id_from_file
+        # iter_album = iterate_iter_over_fns([
+        #     lambda product: {
+        #         **product,
+        #         'image': self.create_image(
+        #             image_file_fn(product['file']),
+        #             url_creator(
+        #                 doc['id'],
+        #                 image_id(product['file'])
+        #             )
+        #         )
+        #     }
+        # ])
+        #
+        # return lmap(
+        #     create_product,
+        #     iter_album(doc['album'])
+        # )
+
+    def bake(self):
+        preview = None if not self.preview else self.preview.id
+
+        return {
+            **self.params,
+            'images': [i.id for i in self.images],
+            'preview': preview,
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            year = str(self.params['date'].year)
+            title = self.params['title']
+            self.id = url_encode_text('%s-%s' % (year, title))
+
+    def __set_hash(self):
+        files = self.get_param('images')
+        files = [self._get_relpath(i) for i in files]
+
+        if not self.provider.is_dir(self.file):
+            files.append(self.file)
+
+        files = sorted(files)
+        files = [self.provider.hash(i) for i in files]
+
+        self.hash = hash_str(files)
+        self.hash = hash_str(
+            combine([hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return '<Album hash={} id={} file={}>'.format(self.hash, self.id, self.file)
+
+
+def get_manifest(provider, filepath):
+    data = provider.read(filepath).read().decode('utf-8')
+    y, m = read_yaml_md(data)
+
+    data = {**y} if y else {}
+    _ = lambda key, d=None: data[key] if key in data else d
+
+    date = create_date(str(_('date')), settings.date_formats_direct)
+    if date:
+        data['date'] = date
+
+    album_id = _('id')
+    if album_id:
+        data['id'] = str(album_id)
+
+    post = {'post': m} if m else {}
+    return {**data, **post}
+
+
+def create_url(a, i, s, e):
+    return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
+        album=a,
+        id=i,
+        size=s,
+        ext=e
+    )
+
+
+def url_creator(aid, pid):
+    return lambda file, size, ext: create_url(aid, pid, size, ext.lower())
+
+
+def id_from_file(i):
+    return url_encode_text(
+        os.path.splitext(i)[0]
+    )

utils/Sync/sync/models/Article.py

@@ -0,0 +1,202 @@
+import logging
+
+import os
+
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from kazimir import kazimir_to_html
+from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date
+from sync.data import Provider, list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext, first, map_cases, constant, combine
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_articles)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Article(Model):
+    @staticmethod
+    async def scan(provider):
+        documents = [i for i in provider.scan('.') if provider.is_dir(i)]
+        documents = [Article.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        file_time_formats = ['%Y.%m.%d']
+
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
+        document = {
+            'folder': path,
+            'date': doc_date,
+            'title': doc_title,
+        }
+
+        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+        images = list_images(provider, path)
+        images = [os.path.relpath(i, path) for i in images]
+        document['images'] = images
+        document['post'] = create_post_from_image_list(images)
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = md_from_folder(provider, path)
+        if manifest_file:
+            manifest = get_manifest(provider, manifest_file)
+        else:
+            manifest = {}
+
+        return Article(
+            provider,
+            path,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        # self.__id_template: str = '{category}-{file}'
+        # self.__url_template: str = settings.document_url_template
+        # self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.origin = settings.origin
+        self.version = '2'
+        self.post = None
+        self.images = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        self.__set_id()
+        self.__set_hash()
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        post, images = await create_post(self.provider, self.id, self.params, sizes)
+        self.post = post
+        self.images = images
+
+        post, images = await create_post(self.provider, self.id, self.params, sizes)
+        self.post = kazimir.html_from_tree(post)
+        self.images = images
+
+        if 'preview' in self.params:
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            preview = await Image.new(self.provider, preview, sizes)
+            self.preview = preview
+        elif len(self.images):
+            self.preview = self.images[0]
+        else:
+            self.preview = 'https://art.shlisselburg.org/entrance.jpg'
+
+    def bake(self):
+        return {
+            **self.params,
+            'images': [i.id for i in self.images],
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+            'preview': self.preview if isinstance(self.preview, str) else self.preview.id,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['title'])
+
+    def __set_hash(self):
+        f = self.params['folder']
+        images = sorted(self.params['images'])
+        images = [os.path.join(f, i) for i in images]
+        images = [self.provider.hash(i) for i in images]
+
+        self.hash = hash_str(
+            combine([hash_str(self.params)] + images)
+        )
+
+    def __str__(self):
+        return '<Article hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def md_from_folder(provider, i):
+    return first(provider.type_filter(i, '.md'))
+
+
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    html = kazimir_to_html(manifest['content'])
+    return {
+        **manifest,
+        'post': html,
+        'images': images_from_html(html)  # Add images for hashing
+    }
+
+
+async def create_post(provider, doc_id, document, sizes):
+    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
+
+    post_html = lxml.html.fromstring(document['post'])
+    images = []
+    for img in post_html.cssselect('img'):
+        src = img.get('src')
+        relative_image_path = os.path.join(document['folder'], src)
+
+        img_path = provider.get_local(relative_image_path)
+        if os.path.exists(img_path):
+            img_id = url_encode_text(os.path.splitext(src)[0])
+            url_fn = lambda file, size, ext: image_url_base.format(
+                id=doc_id,
+                img=img_id,
+                size=size,
+                ext=ext.lower()
+            )
+
+            image = await Image.new(provider, img_path, sizes, url_fn)
+            if image:
+                url = image.get_size('big')['url']
+                images.append(image)
+                img.set('src', url)
+            else:
+                logger.error('Fail to get Image', document['folder'], src)
+    return post_html, images

utils/Sync/sync/models/Document.py

@@ -0,0 +1,188 @@
+import os
+from tempfile import mkstemp
+
+import requests
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from sync.models.Image import Image
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Document(Model):
+    @staticmethod
+    async def scan(provider):
+        paths = scan_subdirs(provider, '.pdf')
+        return [Document.read_path(provider, path) for path in paths]
+
+    @staticmethod
+    def read_path(provider, path):
+        name = os.path.basename(path)
+
+        document = {
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return Document(
+            provider,
+            path,
+            **document
+        )
+
+    def __init__(self, provider, file, category=None, title=None):
+        store = collection(settings.collection_documents)
+
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.category = category
+        self.title = title
+        self.type = None
+        self.preview = None
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            # return self.read(document)
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def upload(self):
+        filepath = self.provider.get_local(self.file)
+        name = os.path.basename(self.url)
+        url = settings.document_url_upload_template.format(file=name)
+
+        await request.upload(url, filepath)
+
+    def bake(self):
+        file = self.file
+        filename = os.path.basename(file)
+
+        return {
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'type': document_type(self.type),
+            'file': {
+                'name': filename,
+                'size': self.provider.size(file)
+            },
+            'preview': self.preview,
+            'title': last_good([
+                self.title,
+                get_pdf_title(self.provider, self.file)
+            ]),
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        self.id = self.__id_template.format(
+            category=url_encode_text(self.category),
+            file=url_encode_text(self.__filename())
+        )
+
+    def __set_url(self):
+        self.url = self.__url_template.format(
+            file=url_encode_file(self.__filename())
+        )
+
+    def __set_hash(self):
+        doc = [getattr(self, key) for key in self.__hash_keys]
+
+        self.hash = hash_str(
+            hash_str(doc) + self.provider.hash(self.file)
+        )
+
+    async def create_preview(self, sizes):
+        def url(_, size, ext):
+            file = url_encode_text(self.__filename())
+            return self.__url_preview_template.format(id=file, size=size, ext=ext)
+
+        temp_preview_path = pdf_to_jpg(self.provider, self.file)
+        preview = await Image.new(self.provider, temp_preview_path, sizes, url)
+
+        if preview:
+            self.preview = preview.id
+
+        os.remove(temp_preview_path)
+
+    def __str__(self):
+        return '<Document hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/models/Image.py

@@ -0,0 +1,231 @@
+import tempfile
+from tempfile import mkstemp
+from typing import Optional, List
+
+from db import collection
+from sync.data import request
+from sync.models import Model
+from utils.hash import md5
+from utils.image.resize import image_magick_pdf_to_img
+import os
+
+import logging
+
+import settings
+from utils.image.resize import optimize, thumbnail
+
+logger = logging.getLogger(settings.name + '.Image')
+
+store = collection(settings.collection_images)
+
+
+def get_upload_url(url):
+    return os.path.join(settings.image_url_upload, os.path.basename(url))
+
+
+def default_url_factory(file, size, ext):
+    url = settings.image_url_base + '{}-{}{}'
+    return url.format(md5(file), size, ext)
+
+
+class Image(Model):
+    @staticmethod
+    async def find_one(query):
+        document = store.find_one(query)
+        return document
+
+    @staticmethod
+    async def new(provider, file, sizes, url_factory=None):
+        url_factory = url_factory if url_factory else default_url_factory
+
+        img = Image(provider, file, data=None, url_factory=url_factory)
+        changed = await img.is_changed()
+        if changed:
+            await img.compile(sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await Image.find_one({'file': file})
+            img = Image(provider, file, doc['data'], url_factory)
+            img.__set_id(doc['_id'])
+
+        return img
+
+    def __init__(self, provider, file, data, url_factory):
+        self.data = data
+        self.url_factory = url_factory
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.hash = self.__get_hash()
+
+    async def is_changed(self):
+        i = await Image.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def compile(self, sizes):
+        img_in = self.provider.get_abs(self.file)
+        img_out = tempfile.mkdtemp()
+
+        images = create_image(img_in, sizes, self.url_factory, img_out)
+        data = {}
+        if images:
+            for i in images:
+                size_name = i['size']
+                data[size_name] = {
+                    **i,
+                }
+        self.data = data
+
+    async def upload(self):
+        logger.info('Uploading Image {}'.format(self.hash))
+
+        if self.data:
+            for i in self.data.values():
+                url = get_upload_url(i['url'])
+                file = i['file']
+
+                await request.upload(url, file)
+
+    async def save(self):
+        c = sync_image(self.bake())
+        self.__set_id(c['_id'])
+
+    def bake(self):
+        return {
+            'file': self.file,
+            'hash': self.hash,
+            'data': self.data,
+        }
+
+    def get_size(self, size):
+        if size in self.data:
+            return self.data[size]
+        return None
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self, value):
+        self.id = value
+
+    def __get_hash(self):
+        return self.provider.hash(self.file)
+
+    def __str__(self):
+        return '<Image hash={} file={}>'.format(self.hash, self.file)
+
+
+def sync_image(record):
+    from db import db
+
+    q = {'hash': record['hash']}
+    try:
+        db().images.update_one(q, {'$set': record}, upsert=True)
+        i = db().images.find_one({'hash': record['hash']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None
+
+
+def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
+    """
+
+    :param file: Image path to process
+    :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
+    :param url_fn: function (size_name, ext) for creating image url
+    :param output_dir: path to local folder for thumbs storing
+    :return:
+    """
+    if not os.path.exists(file):
+        return None
+
+    _, ext = os.path.splitext(file)
+
+    def fn(size):
+        size_name, width, height = size
+
+        image_url = url_fn(file, size_name, ext)
+        image_filename = os.path.basename(image_url)
+        local_image_path = os.path.join(output_dir, image_filename)
+
+        if settings.image_processing_enabled:
+            image = process_image(file, local_image_path, size)
+            if not image:
+                logger.warning('Failed to process image {}'.format(file))
+                return None
+            width, height = image.size
+
+        return {
+            'file': local_image_path,
+            'url': image_url,
+            'size': size_name,
+            'width': width,
+            'height': height
+        }
+
+    return [fn(s) for s in sizes]
+
+
+def process_image(input_file, output_file, size):
+    size_name, width, height = size
+
+    if size_name == 'original':
+        return optimize(input_file, output_file, quality=90)
+    else:
+        image = read_image(input_file)
+        if not image:
+            return None
+        image_width, image_height = image.size
+        if image_height > image_width:
+            width, height = height, width
+        return thumbnail(input_file, output_file, (width, height))
+
+
+def read_image(src):
+    from PIL import Image
+
+    try:
+        i = Image.open(src)
+        return i
+    except Exception as e:
+        print(src, e)
+        return None

utils/Sync/sync/models/Page.py

@@ -0,0 +1,203 @@
+import os
+
+import logging
+
+from lxml import etree
+import lxml.html
+
+import settings
+from db import collection
+from sync.data import request
+from sync.models import Model
+
+from sync import create_date, images_from_html, title_from_html
+from sync.models.Image import Image
+from utils.hash import hash_str
+from utils.io import read_yaml_md
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Page')
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Page(Model):
+    @staticmethod
+    async def scan(provider):
+        documents = [i for i in provider.scan('.') if provider.is_dir(i)]
+        documents = [provider.type_filter(i, '.md') for i in documents]
+        documents = [i[0] for i in documents if len(i)]
+        documents = [Page.read_path(provider, i) for i in documents]
+        documents = [i for i in documents if i]
+
+        return documents
+
+    @staticmethod
+    def read_path(provider, path):
+        params = {
+            'file': os.path.basename(path),
+            'folder': os.path.dirname(path),
+            'title': os.path.basename(path),
+        }
+
+        manifest = get_manifest(provider, path)
+        params = {
+            **params,
+            **manifest,
+        }
+
+        if 'url' not in params:
+            logger.warning('URL not specified for Page {}'.format(path))
+            return None
+
+        return Page(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        store = collection(settings.collection_pages)
+
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.params = params if params else {}
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def setup_images(self, sizes, url_factory):
+        folder = self.params['folder']
+        images = []
+        for i in self.params['images']:
+            img_file = os.path.join(folder, i)
+
+            img = await Image.new(self.provider, img_file, sizes, url_factory)
+            if img:
+                images.append(img)
+        self.images = images
+
+    def bake(self):
+        images = [image.id for image in self.images]
+
+        data = create_post(self.params['data'], lambda src: self.__image_url_by_src(src))
+
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'file': self.file,
+            'data': data,
+            'images': images,
+        }
+
+    def __image_url_by_src(self, src):
+        for img in self.images:
+            if img.file.endswith(src):
+                s = img.get_size('big')
+                if s:
+                    return s['url']
+                else:
+                    return None
+
+        return None
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['url'])
+
+    def __set_url(self):
+        self.url = self.params['url']
+
+    def __set_hash(self):
+        images = [os.path.join(self.params['folder'], i) for i in self.params['images']]
+        files = sorted([self.file] + images)
+        hashes = [self.provider.hash(i) for i in files]
+
+        self.hash = hash_str(''.join(hashes))
+
+    def __str__(self):
+        return '<Page hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def create_post(md, image_path_fn):
+    html = lxml.html.fromstring(md)
+
+    for img in html.cssselect('img'):
+        src = img.get('src')
+        path = image_path_fn(src)
+
+        if path:
+            img.set('src', path)
+            img.set('class', settings.album_html_img_class)
+            img.set('data-file', src)
+    return etree.tounicode(html)
+
+
+def get_manifest(provider, path):
+    dirname = os.path.dirname(path)
+    data = provider.read(path).read().decode('utf-8')
+    manifest, body = read_yaml_md(data)
+    manifest = manifest if manifest else {}
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    if 'content' in manifest:
+        body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8')
+        del manifest['content']
+
+    return {
+        **manifest,
+        'data': body,
+        'title': title_from_html(body),
+        'images': images_from_html(body)
+    }

utils/Sync/sync/models/Person.py

@@ -0,0 +1,109 @@
+import logging
+
+import settings
+from db import collection
+from sync.data import Provider
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext
+from utils.hash import hash_str
+from utils.io import read_yaml_md
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_collective)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Person(Model):
+    @staticmethod
+    async def scan(provider):
+        documents = provider.type_filter('', '.md')
+        documents = [Person.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Person(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.params = params if params else {}
+        self.picture = None
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        image_path = self.provider.get_abs(self.params['picture'])
+
+        self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id))
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'picture': self.picture.id,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['name'])
+
+    def __set_hash(self):
+        self.hash = hash_str(
+            self.provider.hash(self.file) + self.provider.hash(self.params['picture'])
+        )
+
+    def __str__(self):
+        return '<Person hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    data = provider.read(path).read().decode('utf-8')
+
+    manifest, body = read_yaml_md(data)
+    manifest = {
+        **manifest,
+        'file': path,
+        'biography': body,
+        'picture': manifest['picture'] if 'picture' in manifest else swap_ext('.jpg')(path),
+    }
+
+    return manifest
+
+
+def url_factory(id):
+    def url(file, size, ext):
+        return settings.person_image_url_template.format(id=id, size=size, ext=ext)
+
+    return url

utils/Sync/sync/models/Schedule.py

@@ -0,0 +1,109 @@
+import logging
+
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_schedules)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Schedule(Model):
+    @staticmethod
+    async def scan(provider):
+        documents = provider.type_filter('.', '.yaml')
+        documents += provider.type_filter('.', '.toml')
+
+        documents = [Schedule.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Schedule(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.params = params if params else {}
+        super().__init__(provider, store, file)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None
+        if not doc_id:
+            period = self.__get_period()
+            doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester'])
+        self.id = doc_id
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __get_period(self):
+        value = self.params['period']
+        if isinstance(value, str):
+            return value
+
+        if isinstance(value, list):
+            return '-'.join(list(map(str, value)))
+
+    def __str__(self):
+        return '<Schedule hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def combine(fns):
+    def call(value):
+        data = value.read()
+        for f in fns:
+            try:
+                return f(data)
+            except Exception as e:
+                pass
+        return None
+
+    return call
+
+
+def toml(bytes_value):
+    s = bytes_value.decode('utf-8')
+    return toml_str(s)
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    transform = combine([yaml, toml, constant({})])
+    manifest = get_data(provider, path, transform)
+
+    return {
+        'file': path,
+        **manifest,
+    }

utils/Sync/sync/models/__init__.py

@@ -1,6 +1,78 @@
+import os
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
 class Model:
-    def __init__(self):
+    def __init__(self, provider, store, file, params=None):
         super().__init__()
 
+        self.provider = provider
+        self.store = store
+        self.file = file
+
+        self.id = None
+        self.url = None
+        self.hash = None
+        self.params = params if params else {}
+
+        self.init()
+
+    def init(self):
+        pass
+
+    async def build(self, **kwargs):
+        pass
+
     def bake(self):
         return None
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        return self._is_changed_hash(i)
+
+    def has_param(self, name):
+        if not self.params:
+            return False
+        return name in self.params
+
+    def get_param(self, name, default_value=None):
+        if not self.params:
+            return default_value
+        if not (name in self.params):
+            return default_value
+        return self.params[name]
+
+    def _is_changed_hash(self, item):
+        if not item:
+            return True
+
+        if 'hash' not in item:
+            return True
+
+        return not (self.hash == item['hash'])
+
+    def _filename(self):
+        return os.path.basename(self.file)
+
+    def _get_relpath(self, path):
+        folder = self.get_param('folder', self.file)
+        return os.path.join(folder, path)
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            return self
+        except ValueError:
+            pass
+        return None

utils/Sync/sync/models/document.py

@@ -1,124 +0,0 @@
-import os
-from tempfile import mkstemp
-
-import settings
-from sync.models import Model
-from utils.fn import last_good, lmap
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class Document(Model):
-    @staticmethod
-    def read(path):
-        name = os.path.basename(path)
-
-        document = {
-            'file': path,
-            'category': os.path.dirname(path),
-            'title': os.path.splitext(name)[0]
-        }
-
-        return document
-
-    def __init__(self, provider, path, url_base, preview_compiler):
-        super().__init__()
-
-        self.path = path
-
-        self.category = None
-        self.file = None
-        self.id = None
-        self.type = None
-        self.url = None
-        self.title = None
-        self.preview = None
-
-        for k, v in m.items():
-            setattr(self, k, v)
-
-        self.provider = provider
-
-    def bake(self):
-        file = self.file
-        filename = os.path.basename(file)
-
-        self.type = document_type(self.type)
-        self.preview = self.create_preview(file)
-        self.title = last_good([
-            self.title,
-            get_pdf_title(self.provider, self.file)
-        ])
-        self.file = {
-            'name': filename,
-            'size': self.provider.size(file)
-        }
-
-    def __set_id(self):
-        bn = os.path.basename(self.file)
-        self.id = url_encode_text('{category}-{file}'.format(
-            category=self.category,
-            file=bn
-        ))
-
-    def __set_url(self):
-        filename = os.path.basename(self.file)
-        self.url = self.url_base_document.format(file=url_encode_file(filename))
-
-    def __set_hash(self):
-        hash_keys = ['id', 'url']
-        hash_doc = lmap(
-            lambda key: getattr(self, key),
-            hash_keys
-        )
-        self.hash = hash_str(
-            hash_str(hash_doc) + self.provider.hash(self.file)
-        )
-
-    def create_preview(self, file):
-        def url(size, ext):
-            return self.url_base_preview.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = pdf_to_jpg(self.provider, file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images)
-        os.remove(temp_preview_path)
-        return img
-
-
-def document_type(doctype):
-    if doctype == 'Награды':
-        return 'award'
-    return 'document'
-
-
-def pdf_to_jpg(provider, pdf):
-    _, temp_in = mkstemp(suffix='.pdf')
-    _, temp_out = mkstemp(suffix='.jpg')
-
-    abspdf = provider.copy(pdf, temp_in)
-    image_magick_pdf_to_img(abspdf, temp_out)
-
-    os.remove(temp_in)
-    return temp_out
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None

utils/Sync/sync/page.py

@@ -1,213 +0,0 @@
-import os
-
-import settings
-from sync.core import Sync
-from sync.data import list_images, Provider
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
-    synced_images_ids, untouched, title_from_html, create_post
-from sync.core.post import SyncPost
-from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.io import read_yaml_md
-from utils.text.transform import url_encode_text
-
-
-class ImageCreator:
-    def __init__(self, provider: Provider, sizes: list, output_dir: str, cache: dict):
-        super().__init__()
-        self.provider = provider
-        self.sizes = sizes
-        self.output_dir = output_dir
-        self.cache = cache if cache else set()
-
-    def create(self, file, name_fn):
-        filehash = self.provider.hash(file)
-        cached = self.cached(filehash)
-        if cached:
-            return cached
-        return create_image(self.provider.get_abs(file), self.sizes, name_fn, self.output_dir, skip_processing=False)
-
-    def cached(self, name):
-        return self.cache[name] if name in self.cache else None
-
-
-def create_document_from_folder(provider, path):
-    document = {
-        'file': os.path.basename(path),
-        'folder': os.path.dirname(path),
-        'title': os.path.basename(path),
-    }
-
-    manifest = get_manifest(provider, path)
-    document = {
-        **document,
-        **manifest,
-    }
-
-    if 'url' not in document:
-        return None
-
-    document = create_id(document)
-    document = create_hash(provider, document)
-    return document
-
-
-def get_manifest(provider, path):
-    dirname = os.path.dirname(path)
-    data = provider.read(path).read().decode('utf-8')
-    manifest, body = read_yaml_md(data)
-    manifest = manifest if manifest else {}
-
-    if 'date' in manifest:
-        manifest['date'] = create_date(manifest['date'], settings.date_formats)
-
-    if 'until' in manifest:
-        manifest['until'] = create_date(manifest['until'], settings.date_formats)
-
-    if 'id' in manifest:
-        manifest['id'] = str(manifest['id'])
-
-    if 'content' in manifest:
-        body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8')
-        del manifest['content']
-
-    return {
-        **manifest,
-        'data': body,
-        'title': title_from_html(body),
-        'images': images_from_html(body)
-    }
-
-
-def create_id(document):
-    new_id = url_encode_text(document['url'])
-    return {
-        **document,
-        'id': document['id'] if 'id' in document else new_id
-    }
-
-
-def create_hash(provider, document):
-    files = sorted([document['file']] + document['images'])
-    hashes = lmap(
-        provider.hash,
-        lmap(
-            lambda x: os.path.join(document['folder'], x),
-            files
-        )
-    )
-
-    return {
-        **document,
-        'hash': hash_str(''.join(hashes))
-    }
-
-
-def create_page(provider: Provider, img: ImageCreator, document):
-    image_path = lambda x: os.path.join(document['folder'], x)
-    image_id = lambda filename: url_encode_text(
-        os.path.splitext(filename)[0]
-    )
-
-    def process_image(src: str):
-        path = image_path(src)
-
-        if provider.exists(path):
-            url = lambda s, e: settings.url_page_base_preview.format(
-                page=document['id'],
-                id=image_id(src),
-                size=s,
-                ext=e
-            )
-
-            image = img.create(path, url)
-            if image:
-                images.append(image)
-                return image['data']['big']['url']
-        return None
-
-    images = []
-    post_html = create_post(document['data'], process_image)
-
-    document['data'] = post_html
-    document['images'] = images
-    return document
-
-
-def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True):
-    img_sizes = settings.album_image_sizes
-    output_dir = settings.dir_static_images
-
-    sync = Sync()
-    sync.collection = collection
-    image_creator = ImageCreator(provider, img_sizes, output_dir, {})
-
-    documents = list(filter(
-        lambda x: provider.is_dir(x),
-        provider.scan('.')
-    ))
-
-    documents = lmap(
-        lambda x: provider.type_filter(x, '.md'),
-        documents
-    )
-
-    documents = list(filter(
-        lambda x: len(x),
-        documents
-    ))
-
-    documents = lmap(
-        lambda x: create_document_from_folder(provider, x[0]),
-        documents
-    )
-
-    documents = list(filter(None, documents))
-
-    # CREATE SCOPE OF CURRENT SESSION
-    documents_ids = lmapfn(documents)(
-        lambda i: i['id']
-    )
-
-    # SKIP UNTOUCHED DOCUMENTS
-    if skip_unchanged:
-        documents = untouched(documents, sync)
-
-    # # MAP EVENT MANIFEST -> EVENT_OBJECT
-    # documents = lmap(
-    #     sync.create,
-    #     documents
-    # )
-
-    documents = lmap(
-        lambda document: create_page(provider, image_creator, document),
-        documents
-    )
-
-    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
-    documents = lmap(
-        key_mapper('images', synced_images_ids),
-        documents
-    )
-
-    # SYNC EVENT_OBJECT WITH DB
-    if update:
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    documents_to_delete = []
-    if delete:
-        remove_query = sync.create_remove_query({'id': {'$nin': documents_ids}})
-        documents_to_delete = sync.query(remove_query)
-        documents_to_delete = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_delete
-            )
-        )
-
-    return documents, documents_to_delete

utils/Sync/sync/person.py

@@ -1,119 +0,0 @@
-import os
-from glob import glob
-
-import settings
-from sync import untouched
-from sync.core.person import SyncPerson
-from sync.image import sync_image
-from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
-from utils.io import read_yaml_md
-
-url_base = settings.teacher_image_url_base
-dir_local_images = settings.teacher_image_output
-sizes = settings.teacher_image_sizes
-
-
-def sync_persons(provider, collection, update=True, delete=True):
-    """
-    :param collection:
-    :param provider:
-    :param update:
-    :param delete:
-    :return:
-    """
-
-    return main(
-        SyncPerson(
-            collection,
-            provider,
-            dir_local_images
-        ),
-        update_documents=update,
-        delete_documents=delete
-    )
-
-
-def make_person(provider, file):
-    pass
-
-
-def main(sync, update_documents=False, delete_documents=False):
-    # GET TEACHER YAML/MD MANIFEST FILES
-    documents = sync.provider.type_filter('', '.md')
-
-    # READ TEACHER YAML/MD MANIFEST
-    documents = lmap(
-        lambda i: (i,) + get_manifest(sync.provider, i),
-        documents
-    )
-
-    # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
-    documents = lmap(
-        lambda i: {
-            **i[1],
-            'file': i[0],
-            'biography': i[2]
-        },
-        documents
-    )
-
-    # DEFINE PROFILE PICTURE
-    documents = lmap(
-        lambda i: {
-            **i,
-            'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
-        },
-        documents
-    )
-
-    # CREATE DOCUMENT IDENTITY
-    documents = lmapfn(documents)(sync.create_id)
-
-    # CREATE HASH OF DOCUMENT FILES
-    documents = lmapfn(documents)(sync.create_hash)
-
-    # CREATE SCOPE OF CURRENT SESSION
-    scope_documents_ids = lmap(
-        lambda i: i['id'],
-        documents
-    )
-
-    # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # DO HEAVY PROCESS WITH DOCUMENTS
-    documents = lmap(
-        sync.compile,
-        documents
-    )
-
-    # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
-    sync_picture = lambda i: None if i is None else sync_image(i)['_id']
-    documents = lmap(
-        key_mapper('picture', sync_picture),
-        documents
-    )
-
-    # SYNC TEACHER_PROFILE
-    if update_documents:
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
-    if delete_documents:
-        documents_to_delete = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_delete
-            )
-        )
-
-    return documents, documents_to_delete
-
-
-def get_manifest(provider, path):
-    data = provider.read(path).read().decode('utf-8')
-    return read_yaml_md(data)

utils/Sync/sync/post.py

@@ -1,192 +0,0 @@
-import os
-
-import settings
-from kazimir import markdown_to_html
-from sync.data import list_images
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
-    synced_images_ids, untouched
-from sync.core.post import SyncPost
-from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
-from utils.io import read_yaml_md, parse_yaml_front_matter
-
-
-def create_document_from_folder(sync, path, file_time_formats):
-    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
-    doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
-    document = {
-        'folder': path,
-        'date': doc_date,
-        'title': doc_title,
-    }
-
-    # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
-    images = lmap(
-        lambda img: os.path.relpath(img, path),
-        list_images(sync.provider, path)
-    )
-    post = create_post_from_image_list(images)
-
-    # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
-    params = map_cases(
-        document,
-        [(
-            lambda doc: md_from_folder(sync.provider, path),
-            lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)),
-        )],
-        constant({})
-    )
-
-    document = {
-        'images': images,
-        'post': post,
-        **document,
-        **params,
-    }
-
-    document = sync.create_id(document)
-    document = sync.create_hash(document)
-    return document
-
-
-def get_manifest(provider, path):
-    data = provider.read(path).read().decode('utf-8')
-    manifest = parse_yaml_front_matter(data)
-
-    if 'date' in manifest:
-        manifest['date'] = create_date(manifest['date'], settings.date_formats)
-
-    if 'until' in manifest:
-        manifest['until'] = create_date(manifest['until'], settings.date_formats)
-
-    if 'id' in manifest:
-        manifest['id'] = str(manifest['id'])
-
-    html = markdown_to_html(manifest['content'])
-    return {
-        **manifest,
-        'post': html,
-        'images': images_from_html(html)  # Add images for hashing
-    }
-
-
-def md_from_folder(provider, i):
-    return first(provider.type_filter(i, '.md'))
-
-
-def get_folder_documents(sync, file_time_formats):
-    # GET EVENT FOLDERS
-    documents = lmap(
-        lambda i: i,
-        filter(
-            lambda i: sync.provider.is_dir(i),
-            sync.provider.scan('.')
-        )
-    )
-
-    return lmap(
-        lambda doc: create_document_from_folder(sync, doc, file_time_formats),
-        documents
-    )
-
-
-# GET EVENT FILES
-def get_file_documents(sync, file_time_formats):
-    documents = sync.provider.type_filter('.', '.md')
-    documents = lmap(
-        lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats),
-        documents
-    )
-
-    documents = lmap(
-        lambda i: {
-            'file': i[0],
-            'date': i[1],
-            'title': i[2],
-        },
-        documents
-    )
-
-    # OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST
-    documents = lmap(
-        lambda i: {
-            **i,
-            **get_manifest(sync.provider, i['file'])
-        },
-        documents
-    )
-
-    # CREATE DOCUMENT IDENTITY
-    documents = lmap(sync.create_id, documents)
-
-    # CREATE HASH OF DOCUMENT FILE
-    documents = lmap(sync.create_hash, documents)
-
-    return documents
-
-
-def main(sync, file_time_formats, update_documents=True, delete_documents=True):
-    documents = get_folder_documents(sync, file_time_formats)
-    documents += get_file_documents(sync, file_time_formats)
-
-    # CREATE SCOPE OF CURRENT SESSION
-    scope_documents_ids = lmapfn(documents)(
-        lambda i: i['id']
-    )
-
-    # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # MAP EVENT MANIFEST -> EVENT_OBJECT
-    documents = lmap(
-        sync.create,
-        documents
-    )
-
-    # FILTER INCORRECT EVENTS
-    documents = list(filter(None, documents))
-
-    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
-    documents = lmap(
-        key_mapper('images', synced_images_ids),
-        documents
-    )
-
-    # SYNC EVENT_OBJECT WITH DB
-    if update_documents:
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    documents_to_delete = []
-    if delete_documents:
-        remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
-        documents_to_delete = sync.query(remove_query)
-        documents_to_delete = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_delete
-            )
-        )
-
-    return documents, documents_to_delete
-
-
-def sync_posts(provider, collection, update=True, delete=True):
-    image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
-
-    return main(
-        SyncPost(
-            collection,
-            provider,
-            'post',
-            image_url_base,
-            settings.dir_static_images,
-            settings.event_image_sizes,
-            origin=settings.origin
-        ),
-        file_time_formats=['%Y.%m.%d'],
-        update_documents=update,
-        delete_documents=delete
-    )

utils/Sync/sync/schedule.py

@@ -1,97 +0,0 @@
-from toml import loads as toml_str
-from yaml import load as yaml
-
-from sync import untouched
-from sync.core.schedule import SyncSchedule
-from sync.data import get_data
-from utils.fn import lmap, constant
-
-dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules'
-
-
-def sync_schedules(provider, collection, update=True, delete=True):
-    """
-    :param collection:
-    :param provider:
-    :param update:
-    :param delete:
-    :return:
-    """
-
-    return main(
-        SyncSchedule(
-            collection,
-            provider
-        ),
-        update_documents=update,
-        delete_documents=delete
-    )
-
-
-def combine(fns):
-    def call(value):
-        data = value.read()
-        for f in fns:
-            try:
-                return f(data)
-            except Exception as e:
-                pass
-        return None
-    return call
-
-
-def toml(bytes_value):
-    s = bytes_value.decode('utf-8')
-    return toml_str(s)
-
-
-def main(sync, update_documents=False, delete_documents=False):
-    # COLLECT YAML/MD/JPG/PDF MANIFEST FILES
-    documents = sync.provider.type_filter('.', '.yaml')
-    documents += sync.provider.type_filter('.', '.toml')
-
-    # READ MANIFEST
-    documents = lmap(
-        lambda i: {
-            'file': i,
-            **get_data(sync.provider, i, combine([yaml, toml, constant({})]))
-        },
-        documents
-    )
-
-    # CREATE DOCUMENT IDENTITY
-    documents = lmap(sync.create_id, documents)
-
-    # CREATE DOCUMENT HASH
-    documents = lmap(sync.create_hash, documents)
-
-    # CREATE SCOPE OF CURRENT SESSION
-    # ALL DOCUMENTS IDS FOUNDED
-    scope_documents_ids = lmap(
-        lambda i: i['id'],
-        documents
-    )
-
-    # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # DO HEAVY PROCESS WITH DOCUMENTS
-
-    # SYNC SCHEDULE
-    if update_documents:
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
-    if delete_documents:
-        documents_to_remove = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_remove
-            )
-        )
-
-    return documents, documents_to_remove

utils/Sync/test/sync/test_document.py

@@ -3,8 +3,7 @@ import unittest
 import settings
 from sync.data import scan_subdirs
 from sync.data.fs import FSProvider
-from sync.document import create_document
-from sync.models.document import Document
+from sync.models.Document import Document
 from utils.fn import lmap
 
 

utils/Sync/utils/fn.py

@@ -49,8 +49,8 @@ lprint_json = lambda i: lprint(lmap(to_json, i))
 first = lambda i: i[0] if len(i) > 0 else None
 
 
-def ext(ext):
-    return lambda i: os.path.splitext(i)[0] + ext
+def swap_ext(extension: str):
+    return lambda i: os.path.splitext(i)[0] + extension
 
 
 def map_item_key(item, key, map_fn):

utils/Sync/utils/hash.py

@@ -26,3 +26,8 @@ def hash_str(data):
         data = to_json(data)
 
     return hash_data(str(data).encode())
+
+
+def md5(data: str) -> str:
+    i = hashlib.md5(data.encode('utf-8'))
+    return i.hexdigest()

utils/Sync/utils/image/__init__.py

@@ -1,12 +1,17 @@
 import os
 
+import logging
+from typing import Optional
+
 from PIL import Image
 
-from sync.data import upload
+import settings
 from utils.hash import hash_file
 # from utils.image import resize
 from utils.image.resize import optimize, thumbnail, orient
 
+logger = logging.getLogger(settings.name)
+
 
 def read_image(src):
     try:
@@ -17,19 +22,15 @@ def read_image(src):
         return None
 
 
-def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
+def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[dict]:
     """
 
     :param file: Image path to process
     :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
     :param url_fn: function (size_name, ext) for creating image url
     :param output_dir: path to local folder for thumbs storing
-    :param skip_processing: True/False to skip thumbs generation
     :return:
     """
-    # image = read_image(file)
-    # if not image:
-    #     return None
     if not os.path.exists(file):
         return None
 
@@ -38,18 +39,20 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
     for size in sizes:
         size_name, width, height = size
 
-        image_url = url_fn(size_name, ext)
+        image_url = url_fn(file, size_name, ext)
         image_filename = os.path.basename(image_url)
         local_image_path = os.path.join(output_dir, image_filename)
 
-        if not skip_processing:
+        if settings.image_processing_enabled:
             image = process_image(file, local_image_path, size)
             if not image:
+                logger.warning('Failed to process image {}'.format(file))
                 continue
+
             width, height = image.size
-            upload_url = upload(local_image_path, '/images/{}'.format(image_filename))
-            if not upload_url:
-                print('Image {} not uploaded'.format(file))
+            # upload_url = upload(local_image_path, '/images/{}'.format(image_filename))
+            # if not upload_url:
+            #     print('Image {} not uploaded'.format(file))
 
         result[size_name] = {
             'url': image_url,
@@ -58,11 +61,7 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
             'height': height
         }
 
-    return {
-        'file': os.path.basename(file),
-        'hash': hash_file(file),
-        'data': result
-    }
+    return result
 
 
 def process_image(input_file, output_file, size):