sync moved to modules

modules/sync/sync/__init__.py

@@ -0,0 +1,191 @@
+import os
+import re
+from datetime import datetime
+
+import lxml.html
+from markdown import markdown
+
+import kazimir
+import settings
+from kazimir.CSVToken import CSVToken
+from kazimir.DocumentToken import DocumentToken
+from kazimir.YoutubeToken import YoutubeToken
+from kazimir.InstagramToken import InstagramToken
+from kazimir.Marker import Marker
+from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken
+from sync.data import Provider
+from sync.models import Model
+from sync.models.Document import Document
+from sync.models.Image import Image
+from utils.text.file import get_size
+from utils.text.file import get_ext
+
+
+def create_date(date_str, date_formats=None):
+    """
+    2016
+    2016.10
+    2016.10.10
+    -> datetime
+
+    :param date_formats:
+    :param date_str:
+    :return:
+    """
+    if not date_formats:
+        date_formats = [
+            '%Y',
+            '%Y.%m',
+            '%Y.%m.%d'
+        ]
+
+    for date_format in date_formats:
+        try:
+            return datetime.strptime(date_str, date_format)
+        except:
+            continue
+    return None
+
+
+# 2016 На крыльях бабочек
+# 2016.10 На крыльях бабочек
+# 2016.10.10 На крыльях бабочек
+folder_name_pattern = re.compile('([\d.]+)(.*)')
+
+
+async def untouched(items: [Model]) -> [Model]:
+    if settings.skip_unchanged:
+        return items
+
+    result = []
+    for item in items:
+        status = await item.is_changed()
+        if status:
+            result.append(item)
+    return result
+
+    # def is_equals(e, n):
+    #     return (n is None) or ('hash' not in n) or (e.hash != n['hash'])
+    #
+    # doc = lambda i: {'id': i.id, 'hash': i.hash}
+    #
+    # stored_documents = [(i, store.read(doc(i))) for i in documents]
+    # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)]
+
+    # return filtered_documents
+
+
+def create_date_and_title_from_folder_name(folder_name, date_formats=None):
+    m = folder_name_pattern.findall(folder_name)
+    if not m:
+        return None, None
+
+    date_str, title = m[0]
+    title = os.path.splitext(title.strip())[0]
+    date = create_date(date_str, date_formats)
+
+    if not date:
+        return None, title
+
+    return date, title
+
+
+async def create_post(provider: Provider, folder: str, md: str, sizes):
+    from kazimir.fix_links_quotes import fix_links_quotes
+
+    images = []
+    documents = []
+
+    def rel(file: str) -> str:
+        return os.path.join(folder, file)
+
+    async def build_document(data):
+        filepath = os.path.join(folder, data['file'])
+        document = await Document.new(provider, filepath, settings.image_sizes)
+
+        if not document:
+            raise Exception(f'Failed to build Document {filepath}')
+
+        text = data['caption'] if data['caption'] else data['file']
+        url = f'/document/{document.id}'
+
+        image_url = document.preview.get_size('small')['url']
+
+        return {
+            'url': url,
+            'image_url': image_url,
+            'file_url': document.url,
+            'title': document.title,
+            'file_size': get_size(document.file_size, 1),
+            'file_format': get_ext(data['file']),
+        }
+
+    async def build_image(data):
+        src = data['file']
+        relative_image_path = os.path.join(folder, src)
+
+        image_path = provider.get_local(relative_image_path)
+        if os.path.exists(image_path):
+            image = await Image.new(provider, relative_image_path, sizes)
+            if image:
+                url = image.get_size('big')['url']
+                images.append(image)
+
+                return {
+                    'src': url,
+                    'alt': data['caption'],
+                    'caption': data['caption'],
+                }
+        raise Exception('Fail to get Image', relative_image_path)
+
+    async def read_file(data):
+        filepath = data
+        filepath = rel(filepath)
+
+        return provider.read(filepath).read().decode('utf-8')
+
+    m = Marker()
+    m.add_token_factory(TokenFactory(SplitToken))
+    m.add_token_factory(TokenFactory(YoutubeToken))
+    m.add_token_factory(TokenFactory(InstagramToken))
+    m.add_token_factory(TokenFactory(UrlToken))
+    m.add_token_factory(BuildTokenFactory(ImageToken, build=build_image))
+    m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
+    m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file))
+    # m.add_token_factory(TokenFactory(FileToken))
+    m.add_token_factory(TokenFactory(TextToken))
+    m.add_tree_middleware(fix_links_quotes)
+    tree = await m.create_tree(md)
+
+    post = kazimir.html_from_tree(tree)
+    post = await kazimir.typo(post)
+
+    return post, images, documents
+
+
+def images_from_html(md):
+    if not md:
+        return []
+
+    post_html = lxml.html.fromstring(md)
+
+    images = []
+    for img in post_html.cssselect('img'):
+        src = img.get('src')
+        images.append(src)
+    return images
+
+
+def title_from_html(md):
+    html = lxml.html.fromstring(md)
+    titles = html.cssselect('h1')
+    if len(titles):
+        return titles[0].text
+
+
+create_post_from_image_list = lambda images: markdown(
+    '\n'.join(map(
+        lambda i: '![]({img})'.format(img=i),
+        images
+    ))
+)

modules/sync/sync/core/SyncDocument.py

@@ -0,0 +1,14 @@
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Document import Document
+
+
+class SyncDocument(Sync):
+    def __init__(self, provider: Provider, sizes):
+        super().__init__(provider, Document)
+        self.sizes = sizes
+
+    def _get_build_args(self):
+        return {
+            'sizes': self.sizes,
+        }

modules/sync/sync/core/__init__.py

@@ -0,0 +1,144 @@
+import logging
+
+import settings
+from sync import untouched
+from sync.data import Provider
+
+
+def get_id(item):
+    try:
+        return item.id
+    except AttributeError:
+        return item['id']
+
+
+class Sync:
+    def __init__(self, provider: Provider, model):
+        super().__init__()
+
+        self.validate_urls = True
+        self.strict_origin = False
+        self.__default_build_args = {
+            'sizes': settings.image_sizes,
+        }
+        self.__delete_query = lambda item: {'_id': item['_id']}
+
+        name = model.__name__
+        self.logger = logging.getLogger('%s.%s' % (settings.name, name))
+
+        self.provider = provider
+        self.model = model
+        self.filter_item_fn = lambda x: True
+
+        self.skip_unchanged = True
+        self.update_enabled = True
+        self.delete_enabled = True
+
+    def set_options(self, su: bool, u: bool, d: bool):
+        self.skip_unchanged = su
+        self.update_enabled = u
+        self.delete_enabled = d
+        return self
+
+    def set_item_filter(self, fn):
+        self.filter_item_fn = fn
+        return self
+
+    async def clean(self):
+        pass
+
+    async def build(self, item):
+        args = self._get_build_args()
+        await item.build(**args)
+        return item
+
+    async def update(self, item):
+        await item.upload()
+        await item.save()
+        return item
+
+    def _get_build_args(self):
+        return {
+            **self.__default_build_args
+        }
+
+    def __items_to_delete_query(self, items):
+        query = {
+            'id': {'$nin': [doc.id for doc in items]},
+        }
+
+        if self.strict_origin:
+            items_origin = list(set([doc.origin for doc in items]))
+
+            query = {
+                **query,
+                'origin': {'$in': items_origin},
+            }
+
+        return query
+
+    async def run(self):
+        """
+        # Get scope files
+        # Validate these files. Raise an error
+        # calc_hashes
+        # make diff with previous run
+        # compile objects
+        # upload
+
+        :return:
+        """
+        self.logger.info('Checking for update')
+
+        items = await self.model.scan(self.provider)
+        self.logger.info('Found {} Item(s)'.format(len(items)))
+        all_items = items
+
+        if self.validate_urls:
+            self.check_urls_uniqueness(all_items)
+
+        # SKIP UNTOUCHED DOCUMENTS
+        if self.skip_unchanged:
+            items = await untouched(all_items)
+
+        # Optionally trim list of models by custom filter
+        items = [
+            x
+            for x in items
+            if self.filter_item_fn(x)
+        ]
+
+        self.logger.info(f'Ready to process {len(items)} item(s)')
+
+        # BUILDING
+        for item in items:
+            self.logger.info(f'Build Item {item}')
+            await self.build(item)
+
+        # UPDATING
+        if self.update_enabled:
+            for item in items:
+                await self.update(item)
+                self.logger.info('Updated Item {}'.format(item))
+
+        # DELETING
+        if self.delete_enabled:
+            obsolete_items = await self.model.find(self.__items_to_delete_query(all_items))
+            obsolete_items = list(obsolete_items)
+
+            for item in obsolete_items:
+                self.logger.info('Deleting Item {}'.format(item['id']))
+                await self.model.delete(self.__delete_query(item))
+
+        await self.clean()
+
+    def check_urls_uniqueness(self, items):
+        urls = [x.url for x in items]
+        unique_urls = set(urls)
+
+        if len(urls) != len(unique_urls):
+            self.logger.error('URLs is not unique')
+            raise Exception('URLs is not unique')
+
+    async def upload(self):
+        pass

modules/sync/sync/data/__init__.py

@@ -0,0 +1,78 @@
+import os
+from io import BytesIO
+
+from utils.fn import combine
+
+
+class Provider:
+    def __init__(self, root):
+        super().__init__()
+        self.root = root
+
+    def read(self, path):
+        return BytesIO(bytes())
+
+    def scan(self, path):
+        return []
+
+    def glob(self, pattr):
+        return []
+
+    def type_filter(self, path, ext):
+        return []
+
+    def copy(self, path, out):
+        pass
+
+    def hash(self, path):
+        return None
+
+    def size(self, path):
+        return 0
+
+    def exists(self, path):
+        return True
+
+    def get_abs(self, path):
+        a = os.path.join(self.root, path)
+        return os.path.normpath(a)
+
+    def get_rel(self, path):
+        rel = os.path.relpath(path, self.root)
+        return rel
+
+    def get_local(self, path):
+        return path
+
+    def is_dir(self, path):
+        return False
+
+
+def scan_subdirs(provider, ext, path='.'):
+    """
+    Creates flat list of files with specific ext
+    It iterates only one level of sub directories
+    :param provider: instance of Provider
+    :param ext: file extension like .jpg of .pdf for file filtering
+    :param path: optional path prefix
+    :return: list of files relative to provider.root
+    """
+    items = provider.scan(path)
+    items = filter(
+        lambda i: provider.is_dir(i),
+        items
+    )
+    items = [provider.type_filter(folder, ext) for folder in items]
+    items = combine(items)
+    return items
+
+
+def get_data(provider, path, transform):
+    data = provider.read(path)
+    return transform(data)
+
+
+def list_images(provider, path):
+    image_types = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG']
+    images = [provider.type_filter(path, ext) for ext in image_types]
+    return combine(images)

modules/sync/sync/data/fs.py

@@ -0,0 +1,48 @@
+import os
+from glob import glob
+
+from shutil import copyfile
+
+from sync.data import Provider
+from utils.hash import hash_file
+
+
+class FSProvider(Provider):
+    def read(self, path):
+        return open(self.get_abs(path), 'rb')
+
+    def scan(self, path):
+        paths = [i.path for i in os.scandir(self.get_abs(path))]
+
+        return [self.get_rel(path) for path in paths]
+
+    def glob(self, pattr):
+        items = glob(self.get_abs(pattr))
+        return [self.get_rel(i) for i in items]
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
+
+    def hash(self, path):
+        return hash_file(self.get_abs(path))
+
+    def copy(self, path, out):
+        file = self.get_abs(path)
+        return copyfile(file, out)
+
+    def size(self, path):
+        file = self.get_abs(path)
+        return os.path.getsize(file)
+
+    def exists(self, path):
+        file = self.get_abs(path)
+        return os.path.exists(file)
+
+    def is_dir(self, path):
+        return os.path.isdir(self.get_abs(path))
+
+    def get_local(self, path):
+        return self.get_abs(path)

modules/sync/sync/data/request.py

@@ -0,0 +1,40 @@
+import aiohttp
+from boto3 import session
+from mimetypes import MimeTypes
+import logging
+import settings
+
+logger = logging.getLogger(settings.name + '.S3')
+
+
+async def s3_put(obj, path):
+    """
+    :param obj: uploads/file.pdf
+    :param path: to/to/local/file.pdf
+    :return:
+    """
+    logger.info('Put {}'.format(obj))
+
+    s = session.Session()
+    client = s.client('s3', **settings.auth_s3)
+
+    mime = MimeTypes()
+    mime_type, enc = mime.guess_type(path)
+
+    # Upload a file to your Space
+    client.upload_file(path, 'hudozka', obj, ExtraArgs={
+        'ACL': 'public-read',
+        'ContentType': mime_type,
+    })
+
+
+async def get(url):
+    async with aiohttp.ClientSession as aio_session:
+        res = aio_session.get(url)
+        return await res.text()
+
+
+async def post(url, data, params={}, headers={}):
+    async with aiohttp.ClientSession() as aio_session:
+        async with aio_session.post(url, data=data, params=params, headers=headers) as resp:
+            return await resp.text()

modules/sync/sync/data/yandexdisk.py

@@ -0,0 +1,69 @@
+import os
+
+import re
+
+import io
+from tempfile import mkstemp
+
+import requests
+
+from sync.data import Provider
+from yandexdisk import YDClient
+
+
+class YDProvider(Provider):
+    def __init__(self, root, access_token):
+        super().__init__(root)
+        self.api = YDClient(access_token)
+
+    def read(self, path):
+        remote = self.api.get_download_link(self.get_abs(path))
+        url = remote['href']
+        req = requests.get(url)
+        data = req.content
+        return io.BytesIO(data)
+
+    def scan(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        files = []
+        for item in res.children:
+            files.append(item.path)
+
+        files = map(
+            lambda fp: re.sub('disk:', '', fp),
+            files
+        )
+        return [self.get_rel(i) for i in files]
+
+    def glob(self, pattr):
+        return []
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
+
+    def hash(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res.sha256
+
+    def copy(self, path, out):
+        data = self.read(path)
+        f = open(out, 'wb')
+        f.write(data.read())
+        f.close()
+        return out
+
+    def size(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['size']
+
+    def is_dir(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res.type == 'dir'
+
+    def get_local(self, path):
+        ext = os.path.splitext(path)[1]
+        _, temp_in = mkstemp(suffix=ext)
+        return self.copy(path, temp_in)

modules/sync/sync/models/Album.py

@@ -0,0 +1,199 @@
+import logging
+
+import os
+from lxml import etree
+
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import Provider, list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext, first, map_cases, constant, combine
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Album')
+store = collection(settings.collection_albums)
+
+
+class Album(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_albums
+        documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)]
+        documents = [Album.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        date, title = create_date_and_title_from_folder_name(
+            os.path.basename(path)
+        )
+
+        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+        document = {
+            'folder': path,
+            'date': date,
+            'title': title,
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = provider.type_filter(path, '.md')
+        manifest_file = manifest_file[0] if len(manifest_file) else None
+        manifest = get_manifest(provider, manifest_file) if manifest_file else {}
+        manifest = manifest if manifest else {}
+
+        file = manifest_file if manifest_file else path
+        return Album(
+            provider,
+            file,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_albums
+
+        self.version = '1'
+
+        self.origin = settings.origin
+        self.post = None
+        self.images = []
+        self.documents = []
+        self.preview = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        if self.has_param('content'):
+            images, documents = kazimir.extract_files(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+            documents = []
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_hash()
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        folder = self.get_param('folder')
+
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images, documents = await create_post(self.provider, folder, markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            preview = await Image.new(self.provider, preview, sizes=sizes)
+            self.preview = preview
+
+    def create_products(self, doc, image_file_fn):
+        pass
+        # image_id = id_from_file
+        # iter_album = iterate_iter_over_fns([
+        #     lambda product: {
+        #         **product,
+        #         'image': self.create_image(
+        #             image_file_fn(product['file']),
+        #             url_creator(
+        #                 doc['id'],
+        #                 image_id(product['file'])
+        #             )
+        #         )
+        #     }
+        # ])
+        #
+        # return lmap(
+        #     create_product,
+        #     iter_album(doc['album'])
+        # )
+
+    def bake(self):
+        preview = None if not self.preview else self.preview.ref
+
+        return {
+            **self.params,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'preview': preview,
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            year = str(self.params['date'].year)
+            title = self.params['title']
+            self.id = url_encode_text('%s-%s' % (year, title))
+
+    def __set_hash(self):
+        files = self.get_param('images')
+        files = [self._get_relpath(i) for i in files]
+
+        if not self.provider.is_dir(self.file):
+            files.append(self.file)
+
+        files = sorted(files)
+        files = [self.provider.hash(i) for i in files]
+
+        self.hash = hash_str(files)
+        self.hash = hash_str(
+            combine([self.__hash_salt] + [hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return '<Album id={} file={}>'.format(self.id, self.file)
+
+
+def get_manifest(provider, filepath):
+    data = provider.read(filepath).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(str(manifest['date']), settings.date_formats_direct)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    return manifest

modules/sync/sync/models/Article.py

@@ -0,0 +1,174 @@
+import logging
+
+import os
+
+import kazimir
+import settings
+from db import collection
+from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import first, combine
+from utils.hash import hash_str
+from utils.io import parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Article')
+store = collection(settings.collection_articles)
+
+
+class Article(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_articles
+        documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)]
+        documents = [Article.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        file_time_formats = ['%Y.%m.%d']
+
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
+        document = {
+            'folder': path,
+            'date': doc_date,
+            'title': doc_title,
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = md_from_folder(provider, path)
+        if manifest_file:
+            manifest = read_manifest(provider, manifest_file)
+        else:
+            manifest = {}
+
+        return Article(
+            provider,
+            path,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.hash_salt = settings.hash_salt_articles
+        self.version = '2'
+        self.post = None
+        self.preview = None
+        self.images = None
+        self.documents = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        if self.has_param('content'):
+            images, documents = kazimir.extract_files(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+            documents = []
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_hash()
+        self.origin = settings.origin
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        return {
+            **self.params,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['title'])
+
+    def __set_hash(self):
+        files = []
+        files += sorted(self.get_param('images'))
+        files += sorted(self.get_param('documents'))
+        if self.has_param('preview'):
+            files.append(self.get_param('preview'))
+
+        folder = self.get_param('folder')
+        files = [os.path.join(folder, x) for x in files]
+        files = [self.provider.hash(x) for x in files]
+
+        self.hash = hash_str(
+            combine([self.hash_salt] + [hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return f'<Article file={self.file} id={self.id}>'
+
+
+def md_from_folder(provider, i):
+    return first(provider.type_filter(i, '.md'))
+
+
+def read_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    return {
+        **manifest,
+    }

modules/sync/sync/models/Document.py

@@ -0,0 +1,194 @@
+import os
+from tempfile import mkstemp
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from sync.models.DocumentPreviewImage import DocumentPreviewImage
+from utils.hash import hash_str, md5
+from utils.image import image_magick_pdf_to_img
+
+store = collection(settings.collection_documents)
+
+
+class Document(Model):
+    @staticmethod
+    async def find_one(query):
+        return store.find_one(query)
+
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_documents
+        paths = scan_subdirs(provider, '.pdf', scan_dir)
+        return [Document.read_path(provider, path) for path in paths]
+
+    @staticmethod
+    def read_path(provider, path):
+        name = os.path.basename(path)
+
+        document = {
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return Document(
+            provider,
+            path,
+            document
+        )
+
+    @staticmethod
+    async def new(provider, file, sizes):
+        document = Document.read_path(provider, file)
+        changed = await document.is_changed()
+        if changed:
+            document.hide()
+            await document.build(sizes=sizes)
+            await document.upload()
+            await document.save()
+        else:
+            doc = await Document.find_one({'file': file})
+            document = Document(provider, file, doc)
+            document.ref = doc['_id']
+            await document.create_preview(sizes)
+
+        return document
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_documents
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = 'https://static.shlisselburg.org/art/uploads/{file}'
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.title = None
+        self.file_name = None
+        self.file_size = None
+        self.preview = None
+
+        self.type = 'document'  # 'award'
+        self.hidden = False
+
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+        self._set_origin()
+
+        filename = os.path.basename(self.file)
+        self.file_name = filename
+        self.file_size = self.provider.size(self.file)
+
+        if self.has_param('title'):
+            self.title = self.get_param('title')
+
+        if self.has_param('hidden'):
+            self.hidden = self.get_param('hidden')
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        await self.create_preview(sizes)
+
+        pdf_title = await get_pdf_title(self.provider, self.file)
+        if pdf_title:
+            self.title = pdf_title
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+
+            doc = await Document.find_one(query)
+            self.ref = doc['_id']
+
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def upload(self):
+        filepath = self.provider.get_local(self.file)
+        name = os.path.basename(self.url)
+
+        await request.s3_put(f'uploads/{name}', filepath)
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'file': self.file,
+            'hash': self.hash,
+            'url': self.url,
+            'type': self.type,
+            'title': self.title,
+            'origin': self.origin,
+            'hidden': self.hidden,
+            'preview': self.preview.ref,
+            'fileInfo': {
+                'name': self.file_name,
+                'size': self.file_size,
+            },
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        self.id = md5(self.file)
+
+    def __set_url(self):
+        s = os.path.splitext(self.file)
+        file = f'{self.id}{s[1]}'
+
+        self.url = self.__url_template.format(
+            file=file
+        )
+
+    def __set_hash(self):
+        doc = [getattr(self, key) for key in self.__hash_keys]
+
+        self.hash = hash_str(
+            self.__hash_salt + hash_str(doc) + self.provider.hash(self.file)
+        )
+
+    async def create_preview(self, sizes):
+        self.preview = await DocumentPreviewImage.new(self.provider, self.file, sizes)
+
+    def hide(self):
+        self.hidden = True
+        return self
+
+    def __str__(self):
+        return f'<Document file={self.file} id={self.id}>'
+
+
+async def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

modules/sync/sync/models/DocumentPreviewImage.py

@@ -0,0 +1,64 @@
+import os
+from tempfile import mkstemp
+
+from sync.models.Image import Image, default_url_factory
+from utils.image import image_magick_pdf_to_img
+
+
+class DocumentPreviewImage(Image):
+    @staticmethod
+    async def find_one(query):
+        return await Image.find_one(query)
+
+    @staticmethod
+    async def find(query):
+        return await Image.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return await Image.delete(query)
+
+    @staticmethod
+    async def new(provider, file, sizes, **kwargs):
+        img = DocumentPreviewImage(provider, file, data=None)
+        changed = await img.is_changed()
+        if changed:
+            await img.build(sizes=sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await DocumentPreviewImage.find_one({'file': file})
+            img = DocumentPreviewImage(provider, file, doc['data'])
+            img.ref = doc['_id']
+
+        return img
+
+    def __init__(self, provider, file, data):
+        super().__init__(provider, file, data, url_factory=default_url_factory)
+        self.__hash_salt = ''
+        self.temp_image_path = None
+
+    async def is_changed(self):
+        i = await DocumentPreviewImage.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def build(self, sizes):
+        self.temp_image_path = await pdf_to_jpg(self.provider, self.file)
+        await super().build(sizes)
+
+    def _get_image_file(self):
+        return self.temp_image_path
+
+    def __get_hash(self):
+        return self.provider.hash(self.file)
+
+
+async def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    await image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out

modules/sync/sync/models/Image.py

@@ -0,0 +1,253 @@
+import logging
+import os
+import tempfile
+import asyncio
+
+from typing import Optional, List
+
+import settings
+from db import collection
+from sync.data import request, list_images
+from sync.models import Model
+from utils.hash import md5, hash_str
+from utils.image import image_magick_resize
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Image')
+
+store = collection(settings.collection_images)
+
+
+def default_url_factory(file, size, ext):
+    url = settings.image_url_base + '{}-{}{}'
+    return url.format(md5(file), size, ext.lower())
+
+
+class Image(Model):
+    @staticmethod
+    async def find_one(query):
+        document = store.find_one(query)
+        return document
+
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def new(provider, file, sizes, url_factory=None):
+        url_factory = url_factory if url_factory else default_url_factory
+
+        img = Image(provider, file, data=None, url_factory=url_factory)
+        changed = await img.is_changed()
+        if changed:
+            await img.build(sizes=sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await Image.find_one({'file': file})
+            img = Image(provider, file, doc['data'], url_factory)
+            img.ref = doc['_id']
+
+        return img
+
+    @staticmethod
+    async def scan(provider):
+        scan_path = settings.dir_images
+        documents = list_images(provider, scan_path)
+
+        return [Image.read_path(provider, i) for i in documents]
+
+    @staticmethod
+    def read_path(provider, path):
+        return Image(
+            provider,
+            path,
+            data=None,
+            url_factory=default_url_factory
+        )
+
+    def __init__(self, provider, file, data, url_factory):
+        self.__hash_salt = settings.hash_salt_images
+
+        self.origin = None
+
+        self.data = data
+        self.url_factory = url_factory
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.id = self.__get_id()
+        self.hash = self.__get_hash()
+
+        self.origin = f"{settings.origin}:{os.path.dirname(self.file)}"
+
+    async def is_changed(self):
+        i = await Image.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def build(self, sizes):
+        logger.info(f'Building Image {self._get_abs(self.file)}')
+
+        img_in = self._get_image_file()
+        img_out = tempfile.mkdtemp()
+
+        images = await create_image(img_in, sizes, self.url_factory, img_out)
+        if not images:
+            raise Exception('Failed to create Image {}'.format(img_in))
+        data = {}
+        for i in images:
+            if i:
+                size_name = i['size']
+                data[size_name] = {
+                    **i,
+                }
+        self.data = data
+
+    async def upload(self):
+        if self.data:
+            async def fn(i):
+                file = i['file']
+                filename = os.path.basename(file)
+
+                await request.s3_put(f'images/{filename}', file)
+
+            await asyncio.wait([fn(i) for i in self.data.values()])
+
+    async def save(self):
+        c = sync_image(self.bake())
+        self.ref = c['_id']
+
+    def bake(self):
+        return {
+            'id': self.id,
+            'file': self.file,
+            'origin': self.origin,
+            'hash': self.hash,
+            'data': self.data,
+        }
+
+    def get_size(self, size):
+        if size in self.data:
+            return self.data[size]
+        return None
+
+    def _get_image_file(self):
+        return self._get_abs(self.file)
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __get_id(self):
+        return md5(self.file)
+
+    def __get_hash(self):
+        file_hash = self.provider.hash(self.file)
+        return hash_str(self.__hash_salt + file_hash)
+
+    def __str__(self):
+        return '<Image hash={} file={}>'.format(self.hash, self.file)
+
+
+def sync_image(record):
+    from db import db
+
+    q = {'file': record['file']}
+    try:
+        db().images.update_one(q, {'$set': record}, upsert=True)
+        i = db().images.find_one({'file': record['file']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
+    """
+
+    :param file: Image path to process
+    :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
+    :param url_fn: function (size_name, ext) for creating image url
+    :param output_dir: path to local folder for thumbs storing
+    :return:
+    """
+    if not os.path.exists(file):
+        return None
+
+    _, ext = os.path.splitext(file)
+    images = []
+
+    if not settings.image_processing_enabled:
+        logger.warning(
+            f'Image processing disabled {file}. Size of image will be set to original'
+        )
+
+    async def fn(size):
+        size_name, width, height = size
+
+        image_url = url_fn(file, size_name, ext)
+        image_filename = os.path.basename(image_url)
+        local_image_path = os.path.join(output_dir, image_filename)
+
+        if settings.image_processing_enabled:
+            image = await process_image(file, local_image_path, size)
+        else:
+            image = await read_image(file)
+        if not image:
+            logger.warning(f'Failed to process image {file}')
+            return None
+        width, height = image.size
+
+        images.append({
+            'file': local_image_path,
+            'url': image_url,
+            'size': size_name,
+            'width': width,
+            'height': height
+        })
+
+    await asyncio.wait([fn(s) for s in sizes])
+
+    return images
+
+
+async def process_image(input_file, output_file, size):
+    size_name, width, height = size
+
+    if size_name == 'original':
+        return await optimize(input_file, output_file, quality=90)
+    else:
+        image = await read_image(input_file)
+        if not image:
+            return None
+        image_width, image_height = image.size
+        if image_height > image_width:
+            width, height = height, width
+        return await thumbnail(input_file, output_file, (width, height), quality=90)
+
+
+async def read_image(src):
+    import aiofiles
+    from PIL import Image
+    import io
+
+    async with aiofiles.open(src, mode='rb') as f:
+        image_data = await f.read()
+        image = Image.open(io.BytesIO(image_data))
+        return image
+
+
+async def thumbnail(src, dest, size, quality):
+    await image_magick_resize(src, dest, size, quality)
+    return await read_image(dest)
+
+
+async def optimize(src, dest, quality):
+    image = await read_image(src)
+    image.save(dest, quality=quality)
+    return image

modules/sync/sync/models/Page.py

@@ -0,0 +1,204 @@
+import os
+
+import logging
+
+from lxml import etree
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from sync.data import request
+from sync.models import Model
+
+from sync import create_date, images_from_html, title_from_html, create_post
+from sync.models.Image import Image
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Page')
+store = collection(settings.collection_pages)
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Page(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_path = settings.dir_pages
+        items = [
+            Page.read(provider, path)
+            for path in provider.scan(scan_path)
+            if provider.is_dir(path)
+        ]
+        items = [i for i in items if i]
+
+        return items
+
+    @staticmethod
+    def read(provider, path):
+        """
+        :param provider:
+        :param path: path to page manifest file. For example: /Pages/<Some_Page>/<Some_Page>.md
+        :return:
+        """
+        dir_name = os.path.basename(path)
+
+        manifest_file = lambda ext: os.path.join(path, dir_name + ext)
+        manifest_path = manifest_file('.md')
+
+        if not provider.exists(manifest_path):
+            return None
+
+        params = {
+            'file': os.path.basename(manifest_path),
+            'folder': os.path.dirname(manifest_path),
+            'title': os.path.basename(manifest_path),
+        }
+
+        manifest = get_manifest(provider, manifest_path)
+        params = {
+            **params,
+            **manifest,
+        }
+
+        if 'url' not in params:
+            logger.warning('URL not specified for Page {}'.format(manifest_path))
+            return None
+
+        return Page(
+            provider,
+            manifest_path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_pages
+        self.__hash_keys = ['id', 'url']
+        self.images = []
+        self.documents = []
+        self.title = None
+        self.data = None
+        self.preview = None
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        images, documents = kazimir.extract_files(self.get_param('content'))
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+
+        markdown_post = self.get_param('content')
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.title = title_from_html(post)
+        self.data = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        images = [x.ref for x in self.images]
+        documents = [x.ref for x in self.documents]
+
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'file': self.file,
+            'data': self.data,
+            'images': images,
+            'documents': documents,
+            'title': self.title,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.get_param('id')
+        else:
+            self.id = url_encode_text(self.get_param('url'))
+
+    def __set_url(self):
+        self.url = self.params['url']
+
+    def __set_hash(self):
+        files = []
+        files += self.get_param('images')
+        files += self.get_param('documents')
+        if self.has_param('contentFile'):
+            files += [self.get_param('contentFile')]
+
+        folder = self.get_param('folder')
+        files = [os.path.join(folder, x) for x in files]
+        files = sorted([self.file] + files)
+
+        hashes = [self.provider.hash(x) for x in files]
+        self.hash = hash_str(self.__hash_salt + ''.join(hashes))
+
+    def __str__(self):
+        return '<Page file={} id={}>'.format(self.file, self.id)
+
+
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    if 'contentFile' in manifest:
+        dirname = os.path.dirname(path)
+        manifest['contentHtml'] = provider.read(os.path.join(dirname, manifest['contentFile'])).read().decode('utf-8')
+
+    return manifest

modules/sync/sync/models/Person.py

@@ -0,0 +1,130 @@
+import logging
+
+import os
+
+import settings
+from db import collection
+from sync import create_post
+from sync.data import Provider
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_collective)
+
+
+class Person(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_persons
+        documents = provider.type_filter(scan_dir, '.md')
+        documents = [Person.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Person(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_person
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.picture = None
+        self.preview = None
+        self.post = None
+        self.images = None
+        self.documents = None
+
+        super().__init__(provider, store, file, params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        image_path = self._get_relpath(self.get_param('picture'))
+
+        self.picture = await Image.new(self.provider, image_path, sizes)
+
+        markdown_post = self.get_param('content')
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        return {
+            **self.params,
+            'post': self.post,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'picture': self.picture.ref,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['name'])
+
+    def __set_hash(self):
+        files = [self.file]
+        files.append(self._get_relpath(self.get_param('picture')))
+
+        if self.has_param('preview'):
+            files.append(self._get_relpath(self.get_param('preview')))
+
+        hashes = [self.provider.hash(x) for x in files]
+        self.hash = hash_str(
+            self.__hash_salt + ''.join(hashes)
+        )
+
+    def __str__(self):
+        return f'<Person file={self.file} id={self.id}>'
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    data = provider.read(path).read().decode('utf-8')
+
+    manifest = parse_yaml_front_matter(data)
+    return {
+        **manifest,
+        'picture': manifest['picture'] if 'picture' in manifest else os.path.basename(swap_ext('.jpg')(path)),
+        'preview': manifest['preview'] if 'preview' in manifest else os.path.basename(swap_ext('.jpg')(path)),
+        'folder': os.path.dirname(path),
+        'hidden': manifest['hidden'] if 'hidden' in manifest else False,
+    }

modules/sync/sync/models/Schedule.py

@@ -0,0 +1,114 @@
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+store = collection(settings.collection_schedules)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Schedule(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_schedules
+        documents = provider.type_filter(scan_dir, '.yaml')
+        documents += provider.type_filter(scan_dir, '.toml')
+
+        documents = [Schedule.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Schedule(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None
+        if not doc_id:
+            period = self.__get_period()
+            doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester'])
+        self.id = doc_id
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __get_period(self):
+        value = self.params['period']
+        if isinstance(value, str):
+            return value
+
+        if isinstance(value, list):
+            return '-'.join(list(map(str, value)))
+
+    def __str__(self):
+        return '<Schedule hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def combine(fns):
+    def call(value):
+        data = value.read()
+        for f in fns:
+            try:
+                return f(data)
+            except Exception as e:
+                pass
+        return None
+
+    return call
+
+
+def toml(bytes_value):
+    s = bytes_value.decode('utf-8')
+    return toml_str(s)
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    transform = combine([yaml, toml, constant({})])
+    manifest = get_data(provider, path, transform)
+
+    return {
+        'file': path,
+        **manifest,
+    }

modules/sync/sync/models/Settings.py

@@ -0,0 +1,64 @@
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+store = collection(settings.collection_settings)
+
+
+class Settings(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_settings
+        documents = provider.type_filter(scan_dir, '.yaml')
+
+        documents = [Settings.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = yaml(provider.read(path))
+
+        return Settings(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        self.id = 'settings'
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __str__(self):
+        return f'<Settings file={self.file} id={self.id}>'

modules/sync/sync/models/__init__.py

@@ -0,0 +1,91 @@
+import os
+
+import settings
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Model:
+    def __init__(self, provider, store, file, params=None):
+        super().__init__()
+
+        self.provider = provider
+        self.store = store
+        self.file = file
+
+        self.origin = None
+        self.id = None
+        self.url = None
+        self.hash = None
+        self.ref = None  # Data base reference
+        self.params = params if params else {}
+
+        self.init()
+
+    def init(self):
+        pass
+
+    async def upload(self, **kwargs):
+        pass
+
+    async def build(self, **kwargs):
+        pass
+
+    def bake(self):
+        return None
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        return self._is_changed_hash(i)
+
+    def has_param(self, name):
+        if not self.params:
+            return False
+        return name in self.params
+
+    def get_param(self, name, default_value=None):
+        if not self.params:
+            return default_value
+        if not (name in self.params):
+            return default_value
+        return self.params[name]
+
+    def _set_origin(self):
+        self.origin = f'{settings.origin}:{os.path.dirname(self.file)}'
+
+    def _is_changed_hash(self, item):
+        if not item:
+            return True
+
+        if 'hash' not in item:
+            return True
+
+        return not (self.hash == item['hash'])
+
+    def _filename(self):
+        return os.path.basename(self.file)
+
+    def _get_relpath(self, path):
+        folder = self.get_param('folder', self.file)
+        return os.path.join(folder, path)
+
+    def _get_abs(self, path):
+        return self.provider.get_abs(path)
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            return self
+        except ValueError:
+            pass
+        return None

modules/sync/sync/models/product.py

@@ -0,0 +1,40 @@
+import settings
+from db import collection
+from sync.core.person import SyncPerson
+from utils.text.transform import interpolate_swift
+
+
+def resolve_value(param, store):
+    """
+
+    :param param: \(nv-andreeva)
+    :param store: function should return value by key nv-andreeva
+    :return:
+    """
+    i = interpolate_swift(param)
+    return store(i)
+
+
+def resolve_teacher(i):
+    sync = SyncPerson(
+        collection(settings.collection_collective),
+        None,
+        ''
+    )
+    return resolve_value(i, lambda teacher_id: sync.read({'id': teacher_id}))
+
+
+def create_product(doc):
+    # author, image, title, awards, teacher, date
+
+    # unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
+    # unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
+
+    product = {**doc}
+    # unpop_image(product)
+
+    # product['teacher'] = resolve_teacher(doc['teacher'])
+    # if (product['teacher']):
+        # unpop_teacher(product)
+
+    return product