sync moved to modules

modules/sync/sync/models/Album.py

@@ -0,0 +1,199 @@
+import logging
+
+import os
+from lxml import etree
+
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import Provider, list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext, first, map_cases, constant, combine
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Album')
+store = collection(settings.collection_albums)
+
+
+class Album(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_albums
+        documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)]
+        documents = [Album.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        date, title = create_date_and_title_from_folder_name(
+            os.path.basename(path)
+        )
+
+        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+        document = {
+            'folder': path,
+            'date': date,
+            'title': title,
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = provider.type_filter(path, '.md')
+        manifest_file = manifest_file[0] if len(manifest_file) else None
+        manifest = get_manifest(provider, manifest_file) if manifest_file else {}
+        manifest = manifest if manifest else {}
+
+        file = manifest_file if manifest_file else path
+        return Album(
+            provider,
+            file,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_albums
+
+        self.version = '1'
+
+        self.origin = settings.origin
+        self.post = None
+        self.images = []
+        self.documents = []
+        self.preview = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        if self.has_param('content'):
+            images, documents = kazimir.extract_files(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+            documents = []
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_hash()
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        folder = self.get_param('folder')
+
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images, documents = await create_post(self.provider, folder, markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            preview = await Image.new(self.provider, preview, sizes=sizes)
+            self.preview = preview
+
+    def create_products(self, doc, image_file_fn):
+        pass
+        # image_id = id_from_file
+        # iter_album = iterate_iter_over_fns([
+        #     lambda product: {
+        #         **product,
+        #         'image': self.create_image(
+        #             image_file_fn(product['file']),
+        #             url_creator(
+        #                 doc['id'],
+        #                 image_id(product['file'])
+        #             )
+        #         )
+        #     }
+        # ])
+        #
+        # return lmap(
+        #     create_product,
+        #     iter_album(doc['album'])
+        # )
+
+    def bake(self):
+        preview = None if not self.preview else self.preview.ref
+
+        return {
+            **self.params,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'preview': preview,
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            year = str(self.params['date'].year)
+            title = self.params['title']
+            self.id = url_encode_text('%s-%s' % (year, title))
+
+    def __set_hash(self):
+        files = self.get_param('images')
+        files = [self._get_relpath(i) for i in files]
+
+        if not self.provider.is_dir(self.file):
+            files.append(self.file)
+
+        files = sorted(files)
+        files = [self.provider.hash(i) for i in files]
+
+        self.hash = hash_str(files)
+        self.hash = hash_str(
+            combine([self.__hash_salt] + [hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return '<Album id={} file={}>'.format(self.id, self.file)
+
+
+def get_manifest(provider, filepath):
+    data = provider.read(filepath).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(str(manifest['date']), settings.date_formats_direct)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    return manifest

modules/sync/sync/models/Article.py

@@ -0,0 +1,174 @@
+import logging
+
+import os
+
+import kazimir
+import settings
+from db import collection
+from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import first, combine
+from utils.hash import hash_str
+from utils.io import parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Article')
+store = collection(settings.collection_articles)
+
+
+class Article(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_articles
+        documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)]
+        documents = [Article.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        file_time_formats = ['%Y.%m.%d']
+
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
+        document = {
+            'folder': path,
+            'date': doc_date,
+            'title': doc_title,
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = md_from_folder(provider, path)
+        if manifest_file:
+            manifest = read_manifest(provider, manifest_file)
+        else:
+            manifest = {}
+
+        return Article(
+            provider,
+            path,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.hash_salt = settings.hash_salt_articles
+        self.version = '2'
+        self.post = None
+        self.preview = None
+        self.images = None
+        self.documents = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        if self.has_param('content'):
+            images, documents = kazimir.extract_files(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+            documents = []
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_hash()
+        self.origin = settings.origin
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        return {
+            **self.params,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['title'])
+
+    def __set_hash(self):
+        files = []
+        files += sorted(self.get_param('images'))
+        files += sorted(self.get_param('documents'))
+        if self.has_param('preview'):
+            files.append(self.get_param('preview'))
+
+        folder = self.get_param('folder')
+        files = [os.path.join(folder, x) for x in files]
+        files = [self.provider.hash(x) for x in files]
+
+        self.hash = hash_str(
+            combine([self.hash_salt] + [hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return f'<Article file={self.file} id={self.id}>'
+
+
+def md_from_folder(provider, i):
+    return first(provider.type_filter(i, '.md'))
+
+
+def read_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    return {
+        **manifest,
+    }

modules/sync/sync/models/Document.py

@@ -0,0 +1,194 @@
+import os
+from tempfile import mkstemp
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from sync.models.DocumentPreviewImage import DocumentPreviewImage
+from utils.hash import hash_str, md5
+from utils.image import image_magick_pdf_to_img
+
+store = collection(settings.collection_documents)
+
+
+class Document(Model):
+    @staticmethod
+    async def find_one(query):
+        return store.find_one(query)
+
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_documents
+        paths = scan_subdirs(provider, '.pdf', scan_dir)
+        return [Document.read_path(provider, path) for path in paths]
+
+    @staticmethod
+    def read_path(provider, path):
+        name = os.path.basename(path)
+
+        document = {
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return Document(
+            provider,
+            path,
+            document
+        )
+
+    @staticmethod
+    async def new(provider, file, sizes):
+        document = Document.read_path(provider, file)
+        changed = await document.is_changed()
+        if changed:
+            document.hide()
+            await document.build(sizes=sizes)
+            await document.upload()
+            await document.save()
+        else:
+            doc = await Document.find_one({'file': file})
+            document = Document(provider, file, doc)
+            document.ref = doc['_id']
+            await document.create_preview(sizes)
+
+        return document
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_documents
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = 'https://static.shlisselburg.org/art/uploads/{file}'
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.title = None
+        self.file_name = None
+        self.file_size = None
+        self.preview = None
+
+        self.type = 'document'  # 'award'
+        self.hidden = False
+
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+        self._set_origin()
+
+        filename = os.path.basename(self.file)
+        self.file_name = filename
+        self.file_size = self.provider.size(self.file)
+
+        if self.has_param('title'):
+            self.title = self.get_param('title')
+
+        if self.has_param('hidden'):
+            self.hidden = self.get_param('hidden')
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        await self.create_preview(sizes)
+
+        pdf_title = await get_pdf_title(self.provider, self.file)
+        if pdf_title:
+            self.title = pdf_title
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+
+            doc = await Document.find_one(query)
+            self.ref = doc['_id']
+
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def upload(self):
+        filepath = self.provider.get_local(self.file)
+        name = os.path.basename(self.url)
+
+        await request.s3_put(f'uploads/{name}', filepath)
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'file': self.file,
+            'hash': self.hash,
+            'url': self.url,
+            'type': self.type,
+            'title': self.title,
+            'origin': self.origin,
+            'hidden': self.hidden,
+            'preview': self.preview.ref,
+            'fileInfo': {
+                'name': self.file_name,
+                'size': self.file_size,
+            },
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        self.id = md5(self.file)
+
+    def __set_url(self):
+        s = os.path.splitext(self.file)
+        file = f'{self.id}{s[1]}'
+
+        self.url = self.__url_template.format(
+            file=file
+        )
+
+    def __set_hash(self):
+        doc = [getattr(self, key) for key in self.__hash_keys]
+
+        self.hash = hash_str(
+            self.__hash_salt + hash_str(doc) + self.provider.hash(self.file)
+        )
+
+    async def create_preview(self, sizes):
+        self.preview = await DocumentPreviewImage.new(self.provider, self.file, sizes)
+
+    def hide(self):
+        self.hidden = True
+        return self
+
+    def __str__(self):
+        return f'<Document file={self.file} id={self.id}>'
+
+
+async def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

modules/sync/sync/models/DocumentPreviewImage.py

@@ -0,0 +1,64 @@
+import os
+from tempfile import mkstemp
+
+from sync.models.Image import Image, default_url_factory
+from utils.image import image_magick_pdf_to_img
+
+
+class DocumentPreviewImage(Image):
+    @staticmethod
+    async def find_one(query):
+        return await Image.find_one(query)
+
+    @staticmethod
+    async def find(query):
+        return await Image.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return await Image.delete(query)
+
+    @staticmethod
+    async def new(provider, file, sizes, **kwargs):
+        img = DocumentPreviewImage(provider, file, data=None)
+        changed = await img.is_changed()
+        if changed:
+            await img.build(sizes=sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await DocumentPreviewImage.find_one({'file': file})
+            img = DocumentPreviewImage(provider, file, doc['data'])
+            img.ref = doc['_id']
+
+        return img
+
+    def __init__(self, provider, file, data):
+        super().__init__(provider, file, data, url_factory=default_url_factory)
+        self.__hash_salt = ''
+        self.temp_image_path = None
+
+    async def is_changed(self):
+        i = await DocumentPreviewImage.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def build(self, sizes):
+        self.temp_image_path = await pdf_to_jpg(self.provider, self.file)
+        await super().build(sizes)
+
+    def _get_image_file(self):
+        return self.temp_image_path
+
+    def __get_hash(self):
+        return self.provider.hash(self.file)
+
+
+async def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    await image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out

modules/sync/sync/models/Image.py

@@ -0,0 +1,253 @@
+import logging
+import os
+import tempfile
+import asyncio
+
+from typing import Optional, List
+
+import settings
+from db import collection
+from sync.data import request, list_images
+from sync.models import Model
+from utils.hash import md5, hash_str
+from utils.image import image_magick_resize
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Image')
+
+store = collection(settings.collection_images)
+
+
+def default_url_factory(file, size, ext):
+    url = settings.image_url_base + '{}-{}{}'
+    return url.format(md5(file), size, ext.lower())
+
+
+class Image(Model):
+    @staticmethod
+    async def find_one(query):
+        document = store.find_one(query)
+        return document
+
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def new(provider, file, sizes, url_factory=None):
+        url_factory = url_factory if url_factory else default_url_factory
+
+        img = Image(provider, file, data=None, url_factory=url_factory)
+        changed = await img.is_changed()
+        if changed:
+            await img.build(sizes=sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await Image.find_one({'file': file})
+            img = Image(provider, file, doc['data'], url_factory)
+            img.ref = doc['_id']
+
+        return img
+
+    @staticmethod
+    async def scan(provider):
+        scan_path = settings.dir_images
+        documents = list_images(provider, scan_path)
+
+        return [Image.read_path(provider, i) for i in documents]
+
+    @staticmethod
+    def read_path(provider, path):
+        return Image(
+            provider,
+            path,
+            data=None,
+            url_factory=default_url_factory
+        )
+
+    def __init__(self, provider, file, data, url_factory):
+        self.__hash_salt = settings.hash_salt_images
+
+        self.origin = None
+
+        self.data = data
+        self.url_factory = url_factory
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.id = self.__get_id()
+        self.hash = self.__get_hash()
+
+        self.origin = f"{settings.origin}:{os.path.dirname(self.file)}"
+
+    async def is_changed(self):
+        i = await Image.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def build(self, sizes):
+        logger.info(f'Building Image {self._get_abs(self.file)}')
+
+        img_in = self._get_image_file()
+        img_out = tempfile.mkdtemp()
+
+        images = await create_image(img_in, sizes, self.url_factory, img_out)
+        if not images:
+            raise Exception('Failed to create Image {}'.format(img_in))
+        data = {}
+        for i in images:
+            if i:
+                size_name = i['size']
+                data[size_name] = {
+                    **i,
+                }
+        self.data = data
+
+    async def upload(self):
+        if self.data:
+            async def fn(i):
+                file = i['file']
+                filename = os.path.basename(file)
+
+                await request.s3_put(f'images/{filename}', file)
+
+            await asyncio.wait([fn(i) for i in self.data.values()])
+
+    async def save(self):
+        c = sync_image(self.bake())
+        self.ref = c['_id']
+
+    def bake(self):
+        return {
+            'id': self.id,
+            'file': self.file,
+            'origin': self.origin,
+            'hash': self.hash,
+            'data': self.data,
+        }
+
+    def get_size(self, size):
+        if size in self.data:
+            return self.data[size]
+        return None
+
+    def _get_image_file(self):
+        return self._get_abs(self.file)
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __get_id(self):
+        return md5(self.file)
+
+    def __get_hash(self):
+        file_hash = self.provider.hash(self.file)
+        return hash_str(self.__hash_salt + file_hash)
+
+    def __str__(self):
+        return '<Image hash={} file={}>'.format(self.hash, self.file)
+
+
+def sync_image(record):
+    from db import db
+
+    q = {'file': record['file']}
+    try:
+        db().images.update_one(q, {'$set': record}, upsert=True)
+        i = db().images.find_one({'file': record['file']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
+    """
+
+    :param file: Image path to process
+    :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
+    :param url_fn: function (size_name, ext) for creating image url
+    :param output_dir: path to local folder for thumbs storing
+    :return:
+    """
+    if not os.path.exists(file):
+        return None
+
+    _, ext = os.path.splitext(file)
+    images = []
+
+    if not settings.image_processing_enabled:
+        logger.warning(
+            f'Image processing disabled {file}. Size of image will be set to original'
+        )
+
+    async def fn(size):
+        size_name, width, height = size
+
+        image_url = url_fn(file, size_name, ext)
+        image_filename = os.path.basename(image_url)
+        local_image_path = os.path.join(output_dir, image_filename)
+
+        if settings.image_processing_enabled:
+            image = await process_image(file, local_image_path, size)
+        else:
+            image = await read_image(file)
+        if not image:
+            logger.warning(f'Failed to process image {file}')
+            return None
+        width, height = image.size
+
+        images.append({
+            'file': local_image_path,
+            'url': image_url,
+            'size': size_name,
+            'width': width,
+            'height': height
+        })
+
+    await asyncio.wait([fn(s) for s in sizes])
+
+    return images
+
+
+async def process_image(input_file, output_file, size):
+    size_name, width, height = size
+
+    if size_name == 'original':
+        return await optimize(input_file, output_file, quality=90)
+    else:
+        image = await read_image(input_file)
+        if not image:
+            return None
+        image_width, image_height = image.size
+        if image_height > image_width:
+            width, height = height, width
+        return await thumbnail(input_file, output_file, (width, height), quality=90)
+
+
+async def read_image(src):
+    import aiofiles
+    from PIL import Image
+    import io
+
+    async with aiofiles.open(src, mode='rb') as f:
+        image_data = await f.read()
+        image = Image.open(io.BytesIO(image_data))
+        return image
+
+
+async def thumbnail(src, dest, size, quality):
+    await image_magick_resize(src, dest, size, quality)
+    return await read_image(dest)
+
+
+async def optimize(src, dest, quality):
+    image = await read_image(src)
+    image.save(dest, quality=quality)
+    return image

modules/sync/sync/models/Page.py

@@ -0,0 +1,204 @@
+import os
+
+import logging
+
+from lxml import etree
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from sync.data import request
+from sync.models import Model
+
+from sync import create_date, images_from_html, title_from_html, create_post
+from sync.models.Image import Image
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Page')
+store = collection(settings.collection_pages)
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Page(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_path = settings.dir_pages
+        items = [
+            Page.read(provider, path)
+            for path in provider.scan(scan_path)
+            if provider.is_dir(path)
+        ]
+        items = [i for i in items if i]
+
+        return items
+
+    @staticmethod
+    def read(provider, path):
+        """
+        :param provider:
+        :param path: path to page manifest file. For example: /Pages/<Some_Page>/<Some_Page>.md
+        :return:
+        """
+        dir_name = os.path.basename(path)
+
+        manifest_file = lambda ext: os.path.join(path, dir_name + ext)
+        manifest_path = manifest_file('.md')
+
+        if not provider.exists(manifest_path):
+            return None
+
+        params = {
+            'file': os.path.basename(manifest_path),
+            'folder': os.path.dirname(manifest_path),
+            'title': os.path.basename(manifest_path),
+        }
+
+        manifest = get_manifest(provider, manifest_path)
+        params = {
+            **params,
+            **manifest,
+        }
+
+        if 'url' not in params:
+            logger.warning('URL not specified for Page {}'.format(manifest_path))
+            return None
+
+        return Page(
+            provider,
+            manifest_path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_pages
+        self.__hash_keys = ['id', 'url']
+        self.images = []
+        self.documents = []
+        self.title = None
+        self.data = None
+        self.preview = None
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        images, documents = kazimir.extract_files(self.get_param('content'))
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+
+        markdown_post = self.get_param('content')
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.title = title_from_html(post)
+        self.data = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        images = [x.ref for x in self.images]
+        documents = [x.ref for x in self.documents]
+
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'file': self.file,
+            'data': self.data,
+            'images': images,
+            'documents': documents,
+            'title': self.title,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.get_param('id')
+        else:
+            self.id = url_encode_text(self.get_param('url'))
+
+    def __set_url(self):
+        self.url = self.params['url']
+
+    def __set_hash(self):
+        files = []
+        files += self.get_param('images')
+        files += self.get_param('documents')
+        if self.has_param('contentFile'):
+            files += [self.get_param('contentFile')]
+
+        folder = self.get_param('folder')
+        files = [os.path.join(folder, x) for x in files]
+        files = sorted([self.file] + files)
+
+        hashes = [self.provider.hash(x) for x in files]
+        self.hash = hash_str(self.__hash_salt + ''.join(hashes))
+
+    def __str__(self):
+        return '<Page file={} id={}>'.format(self.file, self.id)
+
+
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    if 'contentFile' in manifest:
+        dirname = os.path.dirname(path)
+        manifest['contentHtml'] = provider.read(os.path.join(dirname, manifest['contentFile'])).read().decode('utf-8')
+
+    return manifest

modules/sync/sync/models/Person.py

@@ -0,0 +1,130 @@
+import logging
+
+import os
+
+import settings
+from db import collection
+from sync import create_post
+from sync.data import Provider
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_collective)
+
+
+class Person(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_persons
+        documents = provider.type_filter(scan_dir, '.md')
+        documents = [Person.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Person(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_person
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.picture = None
+        self.preview = None
+        self.post = None
+        self.images = None
+        self.documents = None
+
+        super().__init__(provider, store, file, params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        image_path = self._get_relpath(self.get_param('picture'))
+
+        self.picture = await Image.new(self.provider, image_path, sizes)
+
+        markdown_post = self.get_param('content')
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        return {
+            **self.params,
+            'post': self.post,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'picture': self.picture.ref,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['name'])
+
+    def __set_hash(self):
+        files = [self.file]
+        files.append(self._get_relpath(self.get_param('picture')))
+
+        if self.has_param('preview'):
+            files.append(self._get_relpath(self.get_param('preview')))
+
+        hashes = [self.provider.hash(x) for x in files]
+        self.hash = hash_str(
+            self.__hash_salt + ''.join(hashes)
+        )
+
+    def __str__(self):
+        return f'<Person file={self.file} id={self.id}>'
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    data = provider.read(path).read().decode('utf-8')
+
+    manifest = parse_yaml_front_matter(data)
+    return {
+        **manifest,
+        'picture': manifest['picture'] if 'picture' in manifest else os.path.basename(swap_ext('.jpg')(path)),
+        'preview': manifest['preview'] if 'preview' in manifest else os.path.basename(swap_ext('.jpg')(path)),
+        'folder': os.path.dirname(path),
+        'hidden': manifest['hidden'] if 'hidden' in manifest else False,
+    }

modules/sync/sync/models/Schedule.py

@@ -0,0 +1,114 @@
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+store = collection(settings.collection_schedules)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Schedule(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_schedules
+        documents = provider.type_filter(scan_dir, '.yaml')
+        documents += provider.type_filter(scan_dir, '.toml')
+
+        documents = [Schedule.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Schedule(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None
+        if not doc_id:
+            period = self.__get_period()
+            doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester'])
+        self.id = doc_id
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __get_period(self):
+        value = self.params['period']
+        if isinstance(value, str):
+            return value
+
+        if isinstance(value, list):
+            return '-'.join(list(map(str, value)))
+
+    def __str__(self):
+        return '<Schedule hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def combine(fns):
+    def call(value):
+        data = value.read()
+        for f in fns:
+            try:
+                return f(data)
+            except Exception as e:
+                pass
+        return None
+
+    return call
+
+
+def toml(bytes_value):
+    s = bytes_value.decode('utf-8')
+    return toml_str(s)
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    transform = combine([yaml, toml, constant({})])
+    manifest = get_data(provider, path, transform)
+
+    return {
+        'file': path,
+        **manifest,
+    }

modules/sync/sync/models/Settings.py

@@ -0,0 +1,64 @@
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+store = collection(settings.collection_settings)
+
+
+class Settings(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_settings
+        documents = provider.type_filter(scan_dir, '.yaml')
+
+        documents = [Settings.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = yaml(provider.read(path))
+
+        return Settings(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        self.id = 'settings'
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __str__(self):
+        return f'<Settings file={self.file} id={self.id}>'

modules/sync/sync/models/__init__.py

@@ -0,0 +1,91 @@
+import os
+
+import settings
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Model:
+    def __init__(self, provider, store, file, params=None):
+        super().__init__()
+
+        self.provider = provider
+        self.store = store
+        self.file = file
+
+        self.origin = None
+        self.id = None
+        self.url = None
+        self.hash = None
+        self.ref = None  # Data base reference
+        self.params = params if params else {}
+
+        self.init()
+
+    def init(self):
+        pass
+
+    async def upload(self, **kwargs):
+        pass
+
+    async def build(self, **kwargs):
+        pass
+
+    def bake(self):
+        return None
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        return self._is_changed_hash(i)
+
+    def has_param(self, name):
+        if not self.params:
+            return False
+        return name in self.params
+
+    def get_param(self, name, default_value=None):
+        if not self.params:
+            return default_value
+        if not (name in self.params):
+            return default_value
+        return self.params[name]
+
+    def _set_origin(self):
+        self.origin = f'{settings.origin}:{os.path.dirname(self.file)}'
+
+    def _is_changed_hash(self, item):
+        if not item:
+            return True
+
+        if 'hash' not in item:
+            return True
+
+        return not (self.hash == item['hash'])
+
+    def _filename(self):
+        return os.path.basename(self.file)
+
+    def _get_relpath(self, path):
+        folder = self.get_param('folder', self.file)
+        return os.path.join(folder, path)
+
+    def _get_abs(self, path):
+        return self.provider.get_abs(path)
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            return self
+        except ValueError:
+            pass
+        return None

modules/sync/sync/models/product.py

@@ -0,0 +1,40 @@
+import settings
+from db import collection
+from sync.core.person import SyncPerson
+from utils.text.transform import interpolate_swift
+
+
+def resolve_value(param, store):
+    """
+
+    :param param: \(nv-andreeva)
+    :param store: function should return value by key nv-andreeva
+    :return:
+    """
+    i = interpolate_swift(param)
+    return store(i)
+
+
+def resolve_teacher(i):
+    sync = SyncPerson(
+        collection(settings.collection_collective),
+        None,
+        ''
+    )
+    return resolve_value(i, lambda teacher_id: sync.read({'id': teacher_id}))
+
+
+def create_product(doc):
+    # author, image, title, awards, teacher, date
+
+    # unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
+    # unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
+
+    product = {**doc}
+    # unpop_image(product)
+
+    # product['teacher'] = resolve_teacher(doc['teacher'])
+    # if (product['teacher']):
+        # unpop_teacher(product)
+
+    return product