[sync] updated: config based; async loop; Documents adopted to new system

utils/Sync/config.yaml

@@ -0,0 +1,21 @@
+---
+database_uri: 'mongodb://localhost:27017/hudozka'
+
+env: 'production'
+
+skip_unchanged: True
+image_processing: True
+upload_enabled: True
+origin: 'Yandex.Disk'
+upload_auth:
+  login: '<LOGIN>'
+  password: '<PASSWORD>'
+
+provider:
+  name: 'fs'
+  root: '<FS_ROOT>'
+
+yandex_disk_access_token: '<YANDEX_DISK_TOKEN>'
+
+update_enabled: True
+delete_enabled: True

utils/Sync/kazimir/__init__.py

@@ -14,7 +14,7 @@ def markdown_to_html(text):
 
 
 def kazimir_to_html(text):
-    html = markdown_to_html(text)
+    html = make(text)
     return typo(html)
 
 
@@ -39,15 +39,209 @@ def typo(html):
     return html
 
 
+def make(text):
+    tokens = text.split('\n')
+    tokens = markup_tokens(tokens)
+    tokens = join_tokens(tokens)
+    tokens = merge_tokens(tokens)
+
+    return compile_tokens(tokens)
+    # return tokens
+
+
+def join_tokens(tokens):
+    buffer_tag = None
+    buffer = []
+    ts = []
+    for token in tokens:
+        tag = token['tag']
+        changed = buffer_tag is not None and tag != buffer_tag
+
+        flush = is_tag_split(tag)
+        if changed:
+            flush = True
+
+        if not is_tag_joinable(tag):
+            flush = True
+
+        if flush:
+            if buffer:
+                ts.append(buffer)
+            if is_tag_split(tag):
+                buffer = []
+                buffer_tag = None
+            else:
+                buffer = [token]
+                buffer_tag = tag
+        else:
+            buffer_tag = tag
+            buffer.append(token)
+
+    if len(buffer):
+        ts.append(buffer)
+    return ts
+
+
+def merge_tokens(tokens: list):
+    tokens_out = []
+    for ts in tokens:
+        tag = ts[0]['tag']
+        fn = None
+        if tag == 'image':
+            fn = merge_image
+        elif tag == 'text':
+            fn = merge_text
+        else:
+            fn = merge_any
+        tokens_out.append(fn(ts))
+    return tokens_out
+
+
+def compile_tokens(tokens: list):
+    compiled = []
+    for ts in tokens:
+        tag = ts['tag']
+        fn = None
+        if tag == 'text':
+            fn = compile_text
+        elif tag == 'image_collection':
+            fn = compile_image_collection
+        elif tag == 'image':
+            fn = compile_image
+        else:
+            fn = compile_any
+        compiled.append(fn(ts))
+    return '\n'.join(compiled)
+
+
+def markup_tokens(tokens: list):
+    marked_tokens = []
+    for token in tokens:
+        tag = 'text'
+        if is_url(token):
+            tag = 'url'
+        elif is_image(token):
+            tag = 'image'
+        elif not token:
+            tag = 'split'
+            token = None
+
+        marked_tokens.append({
+            'tag': tag,
+            'data': token,
+        })
+    return marked_tokens
+
+
+def is_tag_split(tag):
+    return tag == 'split'
+
+
+def is_tag_joinable(tag: str):
+    return tag not in ['url']
+
+
+def is_url(sample: str):
+    from urllib.parse import urlparse
+    url = urlparse(sample)
+
+    return url.scheme and url.netloc
+
+
+def is_image(sample: str):
+    return sample[:2] == '![' and sample[-1:] == ')'
+
+
+def merge_any(tokens):
+    return tokens[0]
+
+
+def merge_text(tokens):
+    data = ''
+
+    for t in tokens:
+        data += t['data']
+        data += '\n'
+
+    return {
+        'tag': 'text',
+        'data': data.strip(),
+    }
+
+
+def merge_image(tokens):
+    if len(tokens) == 1:
+        return tokens[0]
+    else:
+        data = [t['data'] for t in tokens]
+        return {
+            'tag': 'image_collection',
+            'data': data,
+        }
+
+
+def compile_text(token):
+    return markdown_to_html(token['data'])
+
+
+def compile_image_collection(token):
+    file = lambda i: i[4:][:-1]
+    imgs = ['<img src="{}">'.format(file(i)) for i in token['data']]
+    return '''
+        <div class="kazimir__image-collection">
+            <div class="fotorama"
+                 data-width="100%"
+                 data-ratio="800/600"
+            >
+                {}
+            </div>
+        </div>
+    '''.format('\n'.join(imgs))
+
+
+def compile_image(token):
+    img = markdown_to_html(token['data'])
+    return '''
+        <div class="kazimir__image">
+            {}
+        </div>
+    '''.format(img)
+
+
+def compile_any(token):
+    return markdown_to_html(token['data'])
+
+
 if __name__ == '__main__':
     sample_text = '''
 Hello!
 
 https://www.instagram.com/p/BUelE04l1kU
+ku!
 
 https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka
+https://www.youtube.com/watch?v=qnguOqWpraI
+![](1.jpg)
+![](2.jpg)
+![](3.jpg)
+
+LOL NOOB
+![](12.jpg)
+![](22.jpg)
+WTF OMG
 
 Bye!
+Fye!
+
+![](1.jpg)
+
+More text
     '''
 
-    print(markdown_to_html(sample_text))
+    ts = make(sample_text)
+    if isinstance(ts, list):
+        for i in ts:
+            print(i)
+    else:
+        print(ts)
+

utils/Sync/kazimir/instagram.py

@@ -55,7 +55,7 @@ def embed_instagram(media):
         i = etree.fromstring(html)
 
         obj = etree.Element('div')
-        obj.set('class', "kazimir-embed")
+        obj.set('class', "kazimir__embed")
         obj.append(i)
         return obj
     else:

utils/Sync/kazimir/video.py

@@ -33,6 +33,6 @@ def flex_video(url):
     iframe.set('allowfullscreen', '')
 
     obj = etree.Element('div')
-    obj.set('class', "kazimir-video")
+    obj.set('class', "kazimir__video")
     obj.append(iframe)
     return obj

utils/Sync/settings.py

@@ -1,7 +1,13 @@
 import os
 
+import yaml
+
 __author__ = 'Roman Timashev'
 
+name = 'HudozkaSync'
+
+config = yaml.load(open('config.yaml', 'r'))
+
 
 def absolute(path, ensure=True):
     a = os.path.expanduser(path)
@@ -20,9 +26,9 @@ def env(param, default=None):
     return default
 
 
-database_uri = env('MONGO_URI')
+database_uri = config['database_uri']  # env('MONGO_URI')
 
-skip_unchanged = False
+skip_unchanged = config['skip_unchanged']
 
 person_uri = {
     'Н.В.Андреева': 'nv-andreeva',
@@ -33,18 +39,32 @@ person_uri = {
     'А.С.Тимашева': 'as-timasheva',
 }
 
-image_processing = True
+upload_enabled = config['upload_enabled']
+upload_url_image = 'https://static.shlisselburg.org/upload/art/images/{}'
+upload_auth = (
+    config['upload_auth']['login'],
+    config['upload_auth']['password'],
+)
+
+image_processing_enabled = config['image_processing']
 
 album_html_img_class = 'hudozka-product'
 
 image_ext = '.jpg'
 image_sizes = [
     ('original', None, None),
+
     ('big', 1500, 667),
+    ('big@2', 3000, 1334),
+
     ('medium', 400, 300),
+    ('medium@2', 800, 600),
+
     ('small', 250, 175),
+    ('small@2', 500, 350),
+
     ('little', 100, 100),
-    ('preview', 50, 50),
+    ('little@2', 200, 200),
 ]
 
 awards_image_sizes = [
@@ -91,8 +111,9 @@ date_formats_reverse = [
     '%Y.%m.%d'
 ]
 
-origin = 'Yandex.Disk'
+origin = config['origin']
 
+collection_images = 'images'
 collection_documents = 'documents'
 collection_awards = 'awards'
 collection_schedules = 'schedules'
@@ -101,34 +122,37 @@ collection_collective = 'collective'
 collection_albums = 'albums'
 collection_pages = 'pages'
 
-sync_provider_type = env('SYNC_PROVIDER', 'fs')
+provider_name = config['provider']['name']  # env('SYNC_PROVIDER', 'fs')
+provider_root = config['provider']['root']  # env('SYNC_PROVIDER', 'fs')
 
-_providers_roots = {
+# _providers_roots = {
-    'fs': abs_fn(env('SYNC_LOCAL_PATH')),
+#     'fs': abs_fn(env('SYNC_LOCAL_PATH')),
-    'yd': abs_fn('/'),
+#     'yd': abs_fn('/'),
-}
+# }
-f = _providers_roots[sync_provider_type]
+# f = _providers_roots[provider_name]
+f = abs_fn(provider_root)
 
-dir_documents = f('Hudozka/Site/Documents')
+dir_documents = f('Documents')
-dir_awards = f('Hudozka/Site/Awards')
+dir_awards = f('Awards')
-dir_schedules = f('Hudozka/Site/Schedules')
+dir_schedules = f('Schedules')
-dir_articles = f('Hudozka/Site/Articles')
+dir_articles = f('Articles')
-dir_collective = f('Hudozka/Site/Collective')
+dir_collective = f('Collective')
-dir_gallery = f('Hudozka/Site/Gallery')
+dir_gallery = f('Gallery')
-dir_pages = f('Hudozka/Site/Pages')
+dir_pages = f('Pages')
-
-dir_static = env('SYNC_STATIC', '~/Hudozka Static')
-dir_static_uploads = absolute(dir_static + '/uploads')
-dir_static_images = absolute(dir_static + '/images')
 
 url_page_base_preview = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}'
-url_base_preview = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
-url_base_document = 'https://static.shlisselburg.org/art/uploads/{file}'
 
-image_base_url = 'https://static.shlisselburg.org/art/images/'
+document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
+document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploads/{file}'
+document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
+
+image_url_upload = 'https://static.shlisselburg.org/upload/art/images/'
+image_url_base = 'https://static.shlisselburg.org/art/images/'
 image_name_format = '{type}-{id}-{img}-{size}{ext}'
 
-yandex_disk_access_token = env('YANDEX_DISK_ACCESS_TOKEN')
+yandex_disk_access_token = config['yandex_disk_access_token']  # env('YANDEX_DISK_ACCESS_TOKEN')
 
-do_update = env('SYNC_ENV', 'production') == 'production'
+# update_enabled = env('SYNC_ENV', 'production') == 'production'
-do_delete = env('SYNC_ENV', 'production') == 'production'
+# delete_enabled = env('SYNC_ENV', 'production') == 'production'
+update_enabled = config['update_enabled']
+delete_enabled = config['delete_enabled']

utils/Sync/sync.py

@@ -1,6 +1,9 @@
 import settings
 from db import db, collection
 from sync.album import sync_albums
+from sync.core import Sync
+from sync.core.SyncDocument import SyncDocument
+from sync.data import Provider
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
 from sync.document import sync_documents
@@ -11,6 +14,65 @@ from sync.schedule import sync_schedules
 from utils.fn import lprint_json
 from utils.image import thumbnail
 
+import asyncio
+import datetime
+import logging
+
+# create logger with 'spam_application'
+logger = logging.getLogger(settings.name)
+logger.setLevel(logging.DEBUG)
+
+# create file handler which logs even debug messages
+fh = logging.FileHandler('hudozka.log')
+fh.setLevel(logging.DEBUG)
+
+# create console handler with a higher log level
+ch = logging.StreamHandler()
+# ch.setLevel(logging.ERROR)
+ch.setLevel(logging.DEBUG)
+
+# create formatter and add it to the handlers
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+fh.setFormatter(formatter)
+ch.setFormatter(formatter)
+
+# add the handlers to the logger
+logger.addHandler(fh)
+logger.addHandler(ch)
+
+
+async def main(sync: Sync):
+    await sync.run()
+
+
+async def run(run_interval=0):
+    io = lambda root: get_provider(settings.provider_name, root)
+    c = collection
+
+    documents = SyncDocument(
+        provider=io(settings.dir_documents),
+        collection=c(settings.collection_documents),
+        sizes=settings.image_sizes,
+    )
+
+    while True:
+        # main(io(settings.dir_documents), sync_documents, settings.collection_documents)
+        # main(io(settings.dir_awards), sync_documents, settings.collection_awards)
+        # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
+        # main(io(settings.dir_articles), sync_posts, settings.collection_articles)
+        # main(io(settings.dir_collective), sync_persons, settings.collection_collective)
+        # main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
+        # main(io(settings.dir_pages), sync_pages, settings.collection_pages)
+
+        await asyncio.wait([
+            main(documents),
+        ])
+
+        if run_interval == 0:
+            break
+        else:
+            await asyncio.sleep(run_interval)
+
 
 def get_provider(provider_type, root):
     if provider_type == 'fs':
@@ -20,27 +82,13 @@ def get_provider(provider_type, root):
     return None
 
 
-def main(provider, fn, collection_name):
-    if provider:
-        print('SYNC: %s' % (collection_name.upper()))
-        u, d = fn(provider, collection(collection_name), update=settings.do_update, delete=settings.do_update)
-
-        print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d))))
-        lprint_json(d)
-
-        print('UPDATE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_update else str(len(u))))
-        lprint_json(u)
-
-        print('')
-
-
 if __name__ == '__main__':
-    io = lambda root: get_provider(settings.sync_provider_type, root)
+    interval = 0
+    # interval = 5
 
-    main(io(settings.dir_documents), sync_documents, settings.collection_documents)
+    loop = asyncio.get_event_loop()
-    main(io(settings.dir_awards), sync_documents, settings.collection_awards)
+    loop.run_until_complete(run(interval))
-    main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
+    loop.close()
-    main(io(settings.dir_articles), sync_posts, settings.collection_articles)
+
-    main(io(settings.dir_collective), sync_persons, settings.collection_collective)
+    # Да, 968×504 пискеля это меньше минимально рекомендованных Фейсбуком 1200×630.
-    main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
+    # Зато при таком размере и ратио картинку нигде не кропят, и выглядит она отлично.
-    main(io(settings.dir_pages), sync_pages, settings.collection_pages)

utils/Sync/sync/__init__.py

@@ -9,6 +9,7 @@ from markdown import markdown
 
 import settings
 from sync.image import sync_image
+from sync.models import Model
 from utils.fn import lmap, combine
 
 
@@ -43,26 +44,30 @@ def create_date(date_str, date_formats=None):
 folder_name_pattern = re.compile('([\d.]+)(.*)')
 
 
-def untouched(documents, store):
+async def untouched(items: [Model]) -> [Model]:
     if settings.skip_unchanged:
-        return documents
+        return items
-    return lmap(
+
-        lambda i: i[0],
+    result = []
-        filter(
+    for item in items:
-            lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
+        status = await item.is_changed()
-            map(
+        if status:
-                lambda i: (i, store.read(i)),
+            result.append(item)
-                documents
+    return result
-            )
+
-        ))
+    # def is_equals(e, n):
+    #     return (n is None) or ('hash' not in n) or (e.hash != n['hash'])
+    #
+    # doc = lambda i: {'id': i.id, 'hash': i.hash}
+    #
+    # stored_documents = [(i, store.read(doc(i))) for i in documents]
+    # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)]
+
+    # return filtered_documents
 
 
 def synced_images_ids(images):
-    ids = lmap(
+    return [i['_id'] for i in sync_image(images)]
-        lambda image: image['_id'],
-        sync_image(images)
-    )
-    return ids
 
 
 def synced_image_id(image):

utils/Sync/sync/core/SyncDocument.py

@@ -0,0 +1,188 @@
+import os
+from tempfile import mkstemp
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import scan_subdirs, Provider
+from sync.models.Document import Document
+
+from sync.models import Model
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+logger = logging.getLogger(settings.name + '.Document')
+
+
+class SyncDocument(Sync):
+    def __init__(self, provider: Provider, collection, sizes):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+        self.sizes = sizes
+        self.dir_static_previews = ''
+
+        self.temp_dir = '/Users/tmshv/HudozkaSyncTemp'
+
+    async def clean(self):
+        pass
+
+    async def run(self) -> ([Model], [Model]):
+        logger.info('Checking documents for update')
+
+        documents = await Document.scan(self.provider)
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Document(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        # documents = untouched(documents, self)
+        documents = await untouched(documents)
+        logger.info('Changed {} Document(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Documents to update')
+
+            for document in documents:
+                logger.info('Updating Document {}'.format(document.id))
+
+                await document.upload()
+                await document.create_preview(self.sizes)
+                await document.save()
+
+                logger.info('Updated Document {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Documents to delete')
+
+            for document in documents_delete:
+                logger.info('Deleting {}'.format(document['id']))
+
+                q = {'_id': document['_id']}
+                self.delete(q)
+
+        await self.clean()
+
+
+async def get_assets(self, document) -> []:
+    return []
+
+
+def validate(self, documents):
+    files = [doc['file'] for doc in documents]
+    file_names = [os.path.basename(f) for f in files]
+    file_names = set(file_names)
+
+    if len(file_names) != len(documents):
+        raise Exception('File names should be unique')
+
+
+def create(self, document):
+    file = document['file']
+    filename = os.path.basename(file)
+
+    document['type'] = document_type(document)
+    document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
+    document['title'] = last_good([
+        document['title'],
+        get_pdf_title(self.provider, document['file'])
+    ])
+    document['file'] = {
+        'name': filename,
+        'size': self.provider.size(file)
+    }
+
+    return document
+
+
+def create_id(self, document: dict) -> dict:
+    filepath = document['file']
+    filename = os.path.basename(filepath)
+
+    return {
+        **document,
+        'id': self.id_template.format(
+            category=url_encode_text(document['category']),
+            file=url_encode_text(filename)
+        ),
+    }
+
+
+def create_url(self, document: dict) -> dict:
+    filename = os.path.basename(document['file'])
+
+    return {
+        **document,
+        'url': self.url_template.format(
+            file=url_encode_file(filename)
+        ),
+    }
+
+
+def create_hash(self, document: dict) -> dict:
+    hash_doc = [document[key] for key in self.hash_keys]
+
+    return {
+        **document,
+        'hash': hash_str(
+            hash_str(hash_doc) + self.provider.hash(document['file'])
+        ),
+    }
+
+
+def create_preview(self, file, sizes, preview_dir):
+    def url(size, ext):
+        return self.url_base_preview.format(id=file, size=size, ext=ext)
+
+    temp_preview_path = pdf_to_jpg(self.provider, file)
+    file = os.path.basename(file)
+    file = url_encode_text(file)
+    img = create_image(temp_preview_path, sizes, url, preview_dir)
+    os.remove(temp_preview_path)
+    return img
+
+
+def document_type(document):
+    if document['category'] == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/core/__init__.py

@@ -2,6 +2,13 @@ from sync.models import Model
 from utils.fn import lmap
 
 
+def get_id(item):
+    try:
+        return item.id
+    except AttributeError:
+        return item['id']
+
+
 class Sync:
     @staticmethod
     def compile(document):
@@ -12,10 +19,23 @@ class Sync:
 
         self.collection = None
 
-    def read(self, document):
+    async def run(self) -> ([Model], [Model]):
-        if isinstance(document, list):
+        """
-            return lmap(self.read, document)
+        # Get scope files
+        # Validate these files. Raise an error
+        # calc_hashes
+        # make diff with previous run
+        # compile objects
+        # upload
 
+        :return:
+        """
+        return None, None
+
+    async def upload(self):
+        pass
+
+    def read(self, document: dict):
         q = {'id': document['id']}
         try:
             return self.collection.find_one(q)
@@ -23,16 +43,10 @@ class Sync:
             pass
         return None
 
-    def update(self, document):
+    def update(self, document: dict):
-        if isinstance(document, list):
+        query = {'id': document['id']}
-            return lmap(self.update, document)
-
-        if isinstance(document, Model):
-            return self.update(self.compile(document))
-
-        q = {'id': document['id']}
         try:
-            self.collection.update_one(q, {'$set': document}, upsert=True)
+            self.collection.update_one(query, {'$set': document}, upsert=True)
             return self.read(document)
         except ValueError:
             pass

utils/Sync/sync/core/document.py

@@ -1,107 +0,0 @@
-import os
-from tempfile import mkstemp
-
-from sync.core import Sync
-from utils.fn import last_good, lmap
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class SyncDocument(Sync):
-    def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.sizes = sizes
-        self.dir_static_previews = dir_static_previews
-        self.url_base_preview = url_base_preview
-        self.url_base_document = url_base_document
-
-    def create(self, document):
-        file = document['file']
-        filename = os.path.basename(file)
-
-        document['type'] = document_type(document)
-        document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
-        document['title'] = last_good([
-            document['title'],
-            get_pdf_title(self.provider, document['file'])
-        ])
-        document['file'] = {
-            'name': filename,
-            'size': self.provider.size(file)
-        }
-
-        return document
-
-    def create_id(self, document):
-        bn = os.path.basename(document['file'])
-        document['id'] = url_encode_text('{category}-{file}'.format(
-            category=document['category'],
-            file=bn
-        ))
-        return document
-
-    def create_url(self, document):
-        filename = os.path.basename(document['file'])
-        document['url'] = self.url_base_document.format(file=url_encode_file(filename))
-        return document
-
-    def create_hash(self, document):
-        hash_keys = ['id', 'url']
-        hash_doc = lmap(
-            lambda key: document[key],
-            hash_keys
-        )
-        document['hash'] = hash_str(
-            hash_str(hash_doc) + self.provider.hash(document['file'])
-        )
-        return document
-
-    def create_preview(self, file, sizes, preview_dir):
-        def url(size, ext):
-            return self.url_base_preview.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = pdf_to_jpg(self.provider, file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, sizes, url, preview_dir)
-        os.remove(temp_preview_path)
-        return img
-
-
-def document_type(document):
-    if document['category'] == 'Награды':
-        return 'award'
-    return 'document'
-
-
-def pdf_to_jpg(provider, pdf):
-    _, temp_in = mkstemp(suffix='.pdf')
-    _, temp_out = mkstemp(suffix='.jpg')
-
-    abspdf = provider.copy(pdf, temp_in)
-    image_magick_pdf_to_img(abspdf, temp_out)
-
-    os.remove(temp_in)
-    return temp_out
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None

utils/Sync/sync/data/__init__.py

@@ -3,6 +3,7 @@ from io import BytesIO
 
 import requests
 
+import settings
 from utils.fn import combine, lmap
 
 
@@ -96,6 +97,10 @@ def upload(file_path, relative_path):
     """
     files = {'file': open(file_path, 'rb')}
     url = 'https://static.shlisselburg.org/upload/art{}'.format(relative_path)
+
+    if not settings.upload_enabled:
+        return url
+
     r = requests.put(
         url=url,
         files=files,

utils/Sync/sync/data/fs.py

@@ -13,13 +13,9 @@ class FSProvider(Provider):
         return open(self.get_abs(path), 'rb')
 
     def scan(self, path):
-        return lmap(
+        paths = [i.path for i in os.scandir(self.get_abs(path))]
-            self.get_rel,
+
-            map(
+        return [self.get_rel(path) for path in paths]
-                lambda i: i.path,
-                os.scandir(self.get_abs(path))
-            )
-        )
 
     def glob(self, pattr):
         return lmap(self.get_rel, glob(self.get_abs(pattr)))

utils/Sync/sync/data/request.py

@@ -0,0 +1,41 @@
+import aiohttp
+import requests
+
+import settings
+
+
+async def upload(url, path):
+    """
+    :param url:
+    :param path:
+    :return:
+    """
+
+    if not settings.upload_enabled:
+        return url
+
+    # urlf = 'https://static.shlisselburg.org/upload/art{}'
+
+    files = {'file': open(path, 'rb')}
+    r = requests.put(
+        url=url,
+        files=files,
+        # auth=settings.upload_auth
+        auth=('uploader', '1207690962638465')
+    )
+    if r.status_code == 200:
+        return url
+    else:
+        return None
+
+
+async def get(url):
+    async with aiohttp.ClientSession as session:
+        res = session.get(url)
+        return await res.text()
+
+
+async def put(url, data):
+    async with aiohttp.ClientSession as session:
+        res = session.put(url, data=data)
+        return await res.text()

utils/Sync/sync/document.py

@@ -4,9 +4,9 @@ import requests
 
 import settings
 from sync import synced_image_id, untouched
-from sync.core.document import SyncDocument
+from sync.core.SyncDocument import SyncDocument
 from sync.data import scan_subdirs
-from sync.models.document import Document
+from sync.models.Document import Document
 from utils.fn import combine, lmap, key_mapper
 
 
@@ -47,7 +47,7 @@ def sync_documents(provider, collection, update=True, delete=True):
             sizes=settings.image_sizes,
             dir_static_previews=settings.dir_static_images,
             url_base_preview=settings.url_base_preview,
-            url_base_document=settings.url_base_document,
+            url_base_document=settings.document_url_template,
         ),
         static_path,
         update_documents=update,

utils/Sync/sync/image.py

@@ -1,5 +1,5 @@
 from db import db
-from settings import image_name_format, image_base_url
+from settings import image_name_format, image_url_base
 from utils.fn import lmap
 
 
@@ -62,5 +62,5 @@ def sync_image(record):
 
 
 def image_create_url(file):
-    url = image_base_url + image_name_format
+    url = image_url_base + image_name_format
     return url.format(file)

utils/Sync/sync/models/Document.py

@@ -0,0 +1,188 @@
+import os
+from tempfile import mkstemp
+
+import requests
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from sync.models.Image import Image
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Document(Model):
+    @staticmethod
+    async def scan(provider):
+        paths = scan_subdirs(provider, '.pdf')
+        return [Document.read_path(provider, path) for path in paths]
+
+    @staticmethod
+    def read_path(provider, path):
+        name = os.path.basename(path)
+
+        document = {
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return Document(
+            provider,
+            path,
+            **document
+        )
+
+    def __init__(self, provider, file, category=None, title=None):
+        store = collection(settings.collection_documents)
+
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.category = category
+        self.title = title
+        self.type = None
+        self.preview = None
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            # return self.read(document)
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def upload(self):
+        filepath = self.provider.get_local(self.file)
+        name = os.path.basename(self.url)
+        url = settings.document_url_upload_template.format(file=name)
+
+        await request.upload(url, filepath)
+
+    def bake(self):
+        file = self.file
+        filename = os.path.basename(file)
+
+        return {
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'type': document_type(self.type),
+            'file': {
+                'name': filename,
+                'size': self.provider.size(file)
+            },
+            'preview': self.preview,
+            'title': last_good([
+                self.title,
+                get_pdf_title(self.provider, self.file)
+            ]),
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        self.id = self.__id_template.format(
+            category=url_encode_text(self.category),
+            file=url_encode_text(self.__filename())
+        )
+
+    def __set_url(self):
+        self.url = self.__url_template.format(
+            file=url_encode_file(self.__filename())
+        )
+
+    def __set_hash(self):
+        doc = [getattr(self, key) for key in self.__hash_keys]
+
+        self.hash = hash_str(
+            hash_str(doc) + self.provider.hash(self.file)
+        )
+
+    async def create_preview(self, sizes):
+        def url(_, size, ext):
+            file = url_encode_text(self.__filename())
+            return self.__url_preview_template.format(id=file, size=size, ext=ext)
+
+        temp_preview_path = pdf_to_jpg(self.provider, self.file)
+        preview = await Image.new(self.provider, temp_preview_path, sizes, url)
+
+        if preview:
+            self.preview = preview.id
+
+        os.remove(temp_preview_path)
+
+    def __str__(self):
+        return '<Document hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/models/Image.py

@@ -0,0 +1,207 @@
+import os
+import tempfile
+from tempfile import mkstemp
+from typing import Optional, List
+
+import requests
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+import os
+
+import logging
+
+import settings
+from sync.data import upload
+from utils.hash import hash_file
+from utils.image.resize import optimize, thumbnail, orient
+
+logger = logging.getLogger(settings.name)
+
+
+def get_upload_url(url):
+    return os.path.join(settings.image_url_upload, os.path.basename(url))
+
+
+class Image(Model):
+    @staticmethod
+    async def new(provider, file, sizes, url_factory):
+        img = Image(provider, file, url_factory)
+        await img.compile(sizes)
+        await img.upload()
+        await img.sync()
+        return img
+
+    def __init__(self, provider, file, url_factory):
+        store = collection(settings.collection_images)
+        self.url_factory = url_factory
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.hash = self.__get_hash()
+
+    async def compile(self, sizes):
+        img_out = tempfile.mkdtemp()
+        self.image = create_image(self.file, sizes, self.url_factory, img_out)
+
+    async def upload(self):
+        if self.image:
+            for i in self.image:
+                url = get_upload_url(i['url'])
+                file = i['file']
+
+                logger.info('Uploading image {}'.format(file))
+
+                await request.upload(url, file)
+
+    async def sync(self):
+        c = sync_image(self.bake())
+        self.id = c['_id']
+
+    def bake(self):
+        data = {}
+        if self.image:
+            for i in self.image:
+                size_name = i['size']
+                data[size_name] = {
+                    'url': i['url'],
+                    'size': size_name,
+                    'width': i['width'],
+                    'height': i['height'],
+                }
+
+        return {
+            'file': self.file,
+            'hash': self.hash,
+            'data': data,
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __get_hash(self):
+        return self.provider.hash(self.file)
+
+    def __str__(self):
+        return '<Image hash={} file={}>'.format(self.hash, self.file)
+
+
+def sync_image(record):
+    from db import db
+
+    q = {'hash': record['hash']}
+    try:
+        db().images.update_one(q, {'$set': record}, upsert=True)
+        i = db().images.find_one({'hash': record['hash']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None
+
+
+def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
+    """
+
+    :param file: Image path to process
+    :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
+    :param url_fn: function (size_name, ext) for creating image url
+    :param output_dir: path to local folder for thumbs storing
+    :return:
+    """
+    if not os.path.exists(file):
+        return None
+
+    _, ext = os.path.splitext(file)
+
+    def fn(size):
+        size_name, width, height = size
+
+        image_url = url_fn(file, size_name, ext)
+        image_filename = os.path.basename(image_url)
+        local_image_path = os.path.join(output_dir, image_filename)
+
+        if settings.image_processing_enabled:
+            image = process_image(file, local_image_path, size)
+            if not image:
+                logger.warning('Failed to process image {}'.format(file))
+                return None
+            width, height = image.size
+
+        return {
+            'file': local_image_path,
+            'url': image_url,
+            'size': size_name,
+            'width': width,
+            'height': height
+        }
+
+    return [fn(s) for s in sizes]
+
+
+def process_image(input_file, output_file, size):
+    size_name, width, height = size
+
+    if size_name == 'original':
+        return optimize(input_file, output_file, quality=90)
+    else:
+        image = read_image(input_file)
+        if not image:
+            return None
+        image_width, image_height = image.size
+        if image_height > image_width:
+            width, height = height, width
+        return thumbnail(input_file, output_file, (width, height))
+
+
+def read_image(src):
+    from PIL import Image
+
+    try:
+        i = Image.open(src)
+        return i
+    except Exception as e:
+        print(src, e)
+        return None

utils/Sync/sync/models/__init__.py

@@ -1,6 +1,25 @@
 class Model:
-    def __init__(self):
+    def __init__(self, provider, store, file):
         super().__init__()
 
+        self.provider = provider
+        self.store = store
+        self.file = file
+
+        self.id = None
+        self.url = None
+        self.hash = None
+
+        self.init()
+
+    def init(self):
+        pass
+
     def bake(self):
         return None
+
+    async def is_changed(self):
+        return False
+
+    async def save(self):
+        pass

utils/Sync/sync/models/document.py

@@ -1,124 +0,0 @@
-import os
-from tempfile import mkstemp
-
-import settings
-from sync.models import Model
-from utils.fn import last_good, lmap
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class Document(Model):
-    @staticmethod
-    def read(path):
-        name = os.path.basename(path)
-
-        document = {
-            'file': path,
-            'category': os.path.dirname(path),
-            'title': os.path.splitext(name)[0]
-        }
-
-        return document
-
-    def __init__(self, provider, path, url_base, preview_compiler):
-        super().__init__()
-
-        self.path = path
-
-        self.category = None
-        self.file = None
-        self.id = None
-        self.type = None
-        self.url = None
-        self.title = None
-        self.preview = None
-
-        for k, v in m.items():
-            setattr(self, k, v)
-
-        self.provider = provider
-
-    def bake(self):
-        file = self.file
-        filename = os.path.basename(file)
-
-        self.type = document_type(self.type)
-        self.preview = self.create_preview(file)
-        self.title = last_good([
-            self.title,
-            get_pdf_title(self.provider, self.file)
-        ])
-        self.file = {
-            'name': filename,
-            'size': self.provider.size(file)
-        }
-
-    def __set_id(self):
-        bn = os.path.basename(self.file)
-        self.id = url_encode_text('{category}-{file}'.format(
-            category=self.category,
-            file=bn
-        ))
-
-    def __set_url(self):
-        filename = os.path.basename(self.file)
-        self.url = self.url_base_document.format(file=url_encode_file(filename))
-
-    def __set_hash(self):
-        hash_keys = ['id', 'url']
-        hash_doc = lmap(
-            lambda key: getattr(self, key),
-            hash_keys
-        )
-        self.hash = hash_str(
-            hash_str(hash_doc) + self.provider.hash(self.file)
-        )
-
-    def create_preview(self, file):
-        def url(size, ext):
-            return self.url_base_preview.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = pdf_to_jpg(self.provider, file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images)
-        os.remove(temp_preview_path)
-        return img
-
-
-def document_type(doctype):
-    if doctype == 'Награды':
-        return 'award'
-    return 'document'
-
-
-def pdf_to_jpg(provider, pdf):
-    _, temp_in = mkstemp(suffix='.pdf')
-    _, temp_out = mkstemp(suffix='.jpg')
-
-    abspdf = provider.copy(pdf, temp_in)
-    image_magick_pdf_to_img(abspdf, temp_out)
-
-    os.remove(temp_in)
-    return temp_out
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None

utils/Sync/sync/page.py

@@ -135,6 +135,17 @@ def create_page(provider: Provider, img: ImageCreator, document):
     return document
 
 
+def check_url(documents):
+    urls = list(map(
+        lambda i: i['url'],
+        documents
+    ))
+
+    unique_urls = set(urls)
+    if len(urls) != len(unique_urls):
+        raise Exception('URL of pages should be unique')
+
+
 def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True):
     img_sizes = settings.album_image_sizes
     output_dir = settings.dir_static_images
@@ -170,6 +181,8 @@ def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=Tr
         lambda i: i['id']
     )
 
+    check_url(documents)
+
     # SKIP UNTOUCHED DOCUMENTS
     if skip_unchanged:
         documents = untouched(documents, sync)

utils/Sync/sync/post.py

@@ -1,7 +1,7 @@
 import os
 
 import settings
-from kazimir import markdown_to_html
+from kazimir import kazimir_to_html
 from sync.data import list_images
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
     synced_images_ids, untouched
@@ -61,7 +61,7 @@ def get_manifest(provider, path):
     if 'id' in manifest:
         manifest['id'] = str(manifest['id'])
 
-    html = markdown_to_html(manifest['content'])
+    html = kazimir_to_html(manifest['content'])
     return {
         **manifest,
         'post': html,
@@ -134,7 +134,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
     )
 
     # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
+    # documents = untouched(documents, sync)
 
     # MAP EVENT MANIFEST -> EVENT_OBJECT
     documents = lmap(
@@ -174,7 +174,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
 
 
 def sync_posts(provider, collection, update=True, delete=True):
-    image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
+    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
 
     return main(
         SyncPost(

utils/Sync/test/sync/test_document.py

@@ -4,7 +4,7 @@ import settings
 from sync.data import scan_subdirs
 from sync.data.fs import FSProvider
 from sync.document import create_document
-from sync.models.document import Document
+from sync.models.Document import Document
 from utils.fn import lmap
 
 

utils/Sync/utils/image/__init__.py

@@ -1,12 +1,18 @@
 import os
 
+import logging
+from typing import Optional
+
 from PIL import Image
 
+import settings
 from sync.data import upload
 from utils.hash import hash_file
 # from utils.image import resize
 from utils.image.resize import optimize, thumbnail, orient
 
+logger = logging.getLogger(settings.name)
+
 
 def read_image(src):
     try:
@@ -17,19 +23,15 @@ def read_image(src):
         return None
 
 
-def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
+def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[dict]:
     """
 
     :param file: Image path to process
     :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
     :param url_fn: function (size_name, ext) for creating image url
     :param output_dir: path to local folder for thumbs storing
-    :param skip_processing: True/False to skip thumbs generation
     :return:
     """
-    # image = read_image(file)
-    # if not image:
-    #     return None
     if not os.path.exists(file):
         return None
 
@@ -38,18 +40,20 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
     for size in sizes:
         size_name, width, height = size
 
-        image_url = url_fn(size_name, ext)
+        image_url = url_fn(file, size_name, ext)
         image_filename = os.path.basename(image_url)
         local_image_path = os.path.join(output_dir, image_filename)
 
-        if not skip_processing:
+        if settings.image_processing_enabled:
             image = process_image(file, local_image_path, size)
             if not image:
+                logger.warning('Failed to process image {}'.format(file))
                 continue
+
             width, height = image.size
-            upload_url = upload(local_image_path, '/images/{}'.format(image_filename))
+            # upload_url = upload(local_image_path, '/images/{}'.format(image_filename))
-            if not upload_url:
+            # if not upload_url:
-                print('Image {} not uploaded'.format(file))
+            #     print('Image {} not uploaded'.format(file))
 
         result[size_name] = {
             'url': image_url,
@@ -58,11 +62,7 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
             'height': height
         }
 
-    return {
+    return result
-        'file': os.path.basename(file),
-        'hash': hash_file(file),
-        'data': result
-    }
 
 
 def process_image(input_file, output_file, size):