sync moved to modules

modules/sync/README.md

@@ -0,0 +1,71 @@
+## Usage
+
+Sync Hudozka
+```bash
+<ENV> python3 sync.py
+```
+
+
+### Environment variables
+
+**YANDEX_DISK_ACCESS_TOKEN**
+
+For access to files using Yandex.Disk API
+
+[How to obtain token for testing](https://tech.yandex.ru/oauth/doc/dg/tasks/get-oauth-token-docpage/)
+
+Example: 
+```bash
+YANDEX_DISK_ACCESS_TOKEN=<Your_Access_Token>
+```
+
+Authorize url. Replace with your app id
+```
+https://oauth.yandex.ru/authorize?response_type=token&client_id=<идентификатор приложения>
+```
+
+#### SYNC_LOCAL_PATH
+
+Path to scan files from
+
+Example:
+```bash
+SYNC_LOCAL_PATH=/Users/tmshv/Yandex.Disk
+```
+
+#### SYNC_STATIC
+
+Path to local folder for compiled files (images, uploads)
+
+Example: 
+```bash
+SYNC_STATIC=~/Desktop/Hudozka Static
+```
+
+#### SYNC_PROVIDER
+
+Type of file provider
+
+- *fs* for local access
+- *yd* for access using Yandex.Disk API (YANDEX_DISK_ACCESS_TOKEN is required)
+
+Example: 
+```bash
+SYNC_PROVIDER=fs
+```
+
+#### MONGO_URI
+
+URI for MongoDB access
+
+Example:
+ ```bash
+ mongodb://localhost:27017/hudozka
+ ```
+
+#### SYNC_ENV
+
+Environment like *NODE_ENV*
+
+- *production* default value
+- *development* don't update/delete documents from DB

modules/sync/config.yaml

@@ -0,0 +1,23 @@
+---
+database_uri: 'mongodb://localhost:27017/hudozka'
+
+env: 'production'
+origin: 'Yandex.Disk'
+yandex_disk_access_token: '<YANDEX_DISK_TOKEN>'
+interval: 0
+
+skip_unchanged: True
+
+image_processing_enabled: True
+
+upload_enabled: True
+upload_auth:
+  login: '<LOGIN>'
+  password: '<PASSWORD>'
+
+provider:
+  name: 'fs'
+  root: '<FS_ROOT>'
+
+update_enabled: True
+delete_enabled: True

modules/sync/db.py

@@ -0,0 +1,12 @@
+from pymongo import MongoClient
+
+import settings
+
+
+def db():
+    client = MongoClient(settings.database_uri)
+    return client.hudozka
+
+
+def collection(name):
+    return db()[name]

modules/sync/kazimir/CSVToken.py

@@ -0,0 +1,46 @@
+import csv
+import io
+
+from kazimir.Token import FileToken, is_file
+
+extensions = ['.csv']
+
+
+class CSVToken(FileToken):
+    @staticmethod
+    def test(data: str):
+        return is_file(data, extensions)
+
+    def __init__(self, data) -> None:
+        super().__init__(data=data)
+        self.name = 'csv'
+        self.joinable = False
+        self.build = None
+
+    async def compile(self):
+        data = await self.build(self.data)
+
+        f = io.StringIO(data)
+        reader = csv.reader(f, delimiter=',', quotechar='|')
+
+        headers = next(reader, None)
+
+        row = lambda xs: '\n'.join([
+            f'<td>{x}</td>' for x in xs
+        ])
+
+        head = '\n'.join([
+            f'<th>{x}</th>' for x in headers
+        ])
+        body = '\n'.join([
+            f'<tr>{row(x)}</tr>' for x in reader
+        ])
+
+        return f'''
+            <table class="MarkerTable">
+                <thead><tr>
+                    {head}
+                </tr></thead>
+                <tbody>{body}</tbody>
+            </table>
+        '''.strip()

modules/sync/kazimir/DocumentToken.py

@@ -0,0 +1,42 @@
+from kazimir.Token import parse_file, is_document, Token
+
+
+class DocumentToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_document(data)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='file', data=data)
+        self.joinable = False
+        self.build = None
+
+    async def compile(self):
+        data = parse_file(self.data)
+        document = await self.build(data)
+
+        return create_document(document)
+
+
+def create_document(document):
+    return f'''
+    <div>
+        <div class="document-row">
+            <a href="{document['url']}" class="invisible">
+                <div class="document-row__image">
+                    <img src="{document['image_url']}" alt="{document['title']}">
+                </div>
+            </a>
+
+            <div class="document-row__file">
+                <a href="{document['url']}">{document['title']}</a>
+            </div>
+
+            <div class="document-row__file-info">
+                <a href="{document['file_url']}" target="_blank">
+                    {document['file_format']} ({document['file_size']})
+                </a>
+            </div>
+        </div>
+    </div>
+    '''.strip()

modules/sync/kazimir/DocumentUrlToken.py

@@ -0,0 +1,37 @@
+from kazimir.Token import UrlToken, parse_link
+
+
+class DocumentUrlToken(UrlToken):
+    @staticmethod
+    def test(data: str):
+        link = parse_link(data)
+        if not link:
+            return False
+        url = link['url']
+        host = url.netloc
+        path = url.path
+        return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org')
+
+    def __init__(self, data) -> None:
+        super().__init__(data)
+        self.build = None
+
+    async def compile(self):
+        from kazimir.DocumentToken import create_document
+
+        link = parse_link(self.data)
+        url = link['url']
+        doc_id = url.path.replace('/document/', '')
+        doc = await self.build(doc_id)
+
+        if not doc:
+            raise Exception(f'Document {doc_id} not found')
+
+        # return f'''DOCUMENT: {# doc['title']}'''
+        return create_document(doc)
+
+    def merge(self, token):
+        return self
+
+    def __repr__(self) -> str:
+        return f'[Token {self.name} {self.data}]'

modules/sync/kazimir/InstagramToken.py

@@ -0,0 +1,72 @@
+from urllib.parse import urlparse
+
+import re
+import requests
+from markdown.util import etree
+
+from kazimir.Token import UrlToken, is_url
+
+pattr = re.compile(r'https?://(www\.)?instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$')
+
+
+class InstagramToken(UrlToken):
+    @staticmethod
+    def test(data: str):
+        if not is_url(data):
+            return False
+
+        url = urlparse(data)
+        return url.netloc in ['www.instagram.com', 'instagram.com']
+
+    async def compile(self):
+        match = pattr.match(self.data)
+        if match:
+            m = match.group('media')
+            return await embed_instagram(m)
+
+
+async def embed_instagram(media):
+    """
+    <blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
+        <div style="padding:8px;">
+            <div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
+                <div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
+                </div>
+            </div>
+
+            <p style=" margin:8px 0 0 0; padding:0 4px;">
+                <a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
+                    Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
+                </a>
+            </p>
+
+            <p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
+                A post shared by Шлиссельбургская ДХШ (@hudozka) on
+
+                <time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
+                    May 24, 2017 at 7:05am PDT
+                </time>
+            </p>
+        </div>
+    </blockquote>
+    """
+
+    url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
+    req = requests.get(url)
+    if req.status_code == 200:
+        html = req.json()['html']
+        html = fix_html(html)
+        i = etree.fromstring(html)
+
+        obj = etree.Element('div')
+        obj.set('class', "kazimir__embed")
+        obj.append(i)
+        return etree.tostring(obj).decode()
+    else:
+        raise Exception(
+            'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
+        )
+
+
+def fix_html(html: str):
+    return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')

modules/sync/kazimir/Marker.py

@@ -0,0 +1,118 @@
+import lxml.html
+from markdown import markdown
+
+from kazimir.Token import SplitToken
+
+
+class Marker:
+    def __init__(self) -> None:
+        super().__init__()
+        self.tokens = []
+        self.tree_middlewares = []
+
+    def add_token_factory(self, token):
+        self.tokens.append(token)
+
+    def add_tree_middleware(self, callback):
+        self.tree_middlewares.append(callback)
+
+    async def create_tree(self, text: str):
+        tokens = text.split('\n')
+        tokens = self.markup_tokens(tokens)
+        tokens = join_tokens(tokens)
+        tokens = merge_tokens(tokens)
+
+        compiled = await compile_tokens(tokens)
+        tree = lxml.html.fromstring(compiled)
+        return await self.process_tree(tree)
+
+    async def process_tree(self, tree):
+        for fn in self.tree_middlewares:
+            tree = await fn(tree)
+        return tree
+
+    def markup_tokens(self, tokens: list):
+        marked_tokens = []
+
+        for data in tokens:
+            t = self.create_token(data)
+            if t:
+                marked_tokens.append(t)
+            else:
+                raise Exception(f'Token not recognized for data {data}')
+        return marked_tokens
+
+    def create_token(self, data):
+        for f in self.tokens:
+            if f.token.test(data):
+                return f.create(data=data)
+        return None
+
+
+def join_tokens(tokens):
+    is_split = lambda x: isinstance(x, SplitToken)
+
+    buffer_tag = None
+    buffer = []
+    ts = []
+    for token in tokens:
+        tag = token.name
+        changed = buffer_tag is not None and tag != buffer_tag
+
+        flush = is_split(token)
+        if changed:
+            flush = True
+
+        if not token.joinable:
+            flush = True
+
+        if flush:
+            if buffer:
+                ts.append(buffer)
+            if is_split(token):
+                buffer = []
+                buffer_tag = None
+            else:
+                buffer = [token]
+                buffer_tag = tag
+        else:
+            buffer_tag = tag
+            buffer.append(token)
+
+    if len(buffer):
+        ts.append(buffer)
+    return ts
+
+
+def merge_tokens(tokens: list):
+    tokens_out = []
+    for ts in tokens:
+        token = ts[0]
+        for x in ts[1:]:
+            token = token.merge(x)
+        tokens_out.append(token)
+    return tokens_out
+
+
+async def compile_tokens(tokens: list):
+    result = []
+    for token in tokens:
+        data = await token.compile()
+        result.append(data)
+    return '\n'.join(result)
+
+
+def extract_files(text: str) -> [str]:
+    html = markdown(text)
+    post_html = lxml.html.fromstring(html)
+
+    images = post_html.cssselect('img')
+    images = [img.get('src') for img in images]
+
+    return images, []
+
+
+def html_from_tree(tree):
+    html = lxml.html.tostring(tree, encoding='unicode')
+
+    return html

modules/sync/kazimir/MarkerRun.py

@@ -0,0 +1,113 @@
+import asyncio
+
+from kazimir.DocumentUrlToken import DocumentUrlToken
+from kazimir.DocumentToken import DocumentToken
+from kazimir.CSVToken import CSVToken
+from kazimir.InstagramToken import InstagramToken
+from kazimir.Marker import Marker, html_from_tree
+from kazimir.YoutubeToken import YoutubeToken
+from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \
+    BuildTokenFactory
+from sync import Document
+
+if __name__ == '__main__':
+    sample_text = '''
+Hello!
+
+https://www.instagram1.com/p/BUelE04l1kU
+
+https://www.youtube.com/watch?v=qnguOqWpraI
+https://www.youtube.com/watch?v=qnguOqWpraI
+![](1.jpg)
+![](2.jpg)
+
+LOL NOOB
+![](12.jpg)
+![](22.jpg)
+WTF OMG
+
+![](/document/documents-arkhiv-ustav-2013-pdf)
+
+![File](file.pdf)
+hello.pdf
+file.doc
+sheet.xlsx
+
+print.docx
+image.jpg 1
+image.png 2
+
+image.jpeg Hi
+
+Bye!
+Fye!
+
+![](1.jpg)
+
+["Ya"](http://ya.ru)
+
+More text
+
+file.csv
+    '''
+
+
+    async def build_document(data):
+        text = data['caption'] if data['caption'] else data['file']
+        # document = await self.build()
+        document = {
+            'url': data['file'],
+            'image_url': data['file'],
+            'file_url': data['file'],
+            'title': text,
+            'file_size': '0 KB',
+            'file_format': 'PDF',
+        }
+        return document
+
+    async def read_csv(data):
+        return '\n'.join([
+            'N,Name',
+            f'0,{data}',
+            '1,Pop',
+            '2,Top',
+        ])
+
+    async def read_document(data):
+        doc = await Document.find_one({'id': data})
+        return {
+            'url': doc['url'],
+            'image_url': doc['url'],
+            'file_url': doc['url'],
+            'title': doc['title'],
+            'file_size': '0 KB',
+            'file_format': 'PDF',
+        }
+
+
+    async def run():
+        from kazimir.fix_links_quotes import fix_links_quotes
+
+        m = Marker()
+        m.add_token_factory(TokenFactory(SplitToken))
+        m.add_token_factory(TokenFactory(YoutubeToken))
+        m.add_token_factory(TokenFactory(InstagramToken))
+        m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document))
+        m.add_token_factory(TokenFactory(UrlToken))
+        # m.add_token_factory(TokenFactory(ImageToken))
+        m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
+        m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv))
+        m.add_token_factory(TokenFactory(FileToken))
+        m.add_token_factory(TokenFactory(TextToken))
+        m.add_tree_middleware(fix_links_quotes)
+        sample_tree = await m.create_tree(sample_text)
+
+        # for x in sample_tree:
+        #     print(x)
+
+        print(html_from_tree(sample_tree))
+
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run())
+    loop.close()

modules/sync/kazimir/Token.py

@@ -0,0 +1,244 @@
+import os
+import re
+from markdown import markdown
+from urllib.parse import urlparse
+
+image_extensions = ['.jpg', '.jpeg', '.png']
+document_extensions = ['.doc', '.docx', '.pdf', '.xls', '.xlsx']
+
+
+class TokenFactory:
+    def __init__(self, token) -> None:
+        super().__init__()
+        self.token = token
+
+    def create(self, data):
+        return self.token(data=data)
+
+
+class BuildTokenFactory(TokenFactory):
+    def __init__(self, token, build) -> None:
+        super().__init__(token)
+        self.build = build
+
+    def create(self, data):
+        x = self.token(data=data)
+        x.build = self.build
+        return x
+
+
+class Token:
+    @staticmethod
+    def test(data: str):
+        return True
+
+    def __init__(self, name, data) -> None:
+        super().__init__()
+        self.name = name
+        self.data = data
+        self.joinable = True
+
+    async def compile(self):
+        return str(self.data)
+
+    def merge(self, token):
+        return self
+
+    def __repr__(self) -> str:
+        return f'[Token {self.name} {self.data}]'
+
+
+class SplitToken(Token):
+    @staticmethod
+    def test(data: str):
+        data = data.strip()
+        return data == ''
+
+    def __init__(self, data) -> None:
+        super().__init__(name='split', data=None)
+
+
+class TextToken(Token):
+    @staticmethod
+    def test(data: str):
+        return True
+
+    def __init__(self, data) -> None:
+        super().__init__(name='text', data=data)
+
+    def merge(self, token: Token):
+        data = self.data
+        data += '\n'
+        data += token.data
+        return TextToken(data.strip())
+
+    async def compile(self):
+        return markdown_to_html(self.data)
+
+
+class FileToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_file(data, document_extensions + image_extensions)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='file', data=data)
+        self.joinable = False
+
+    async def compile(self):
+        data = parse_file(self.data)
+        url = data['file']
+        text = data['caption'] if data['caption'] else url
+        return f'<a href="{url}">{text}</a>'
+
+
+class ImageToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_image(data)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='image', data=data)
+        self.joinable = True
+        self.build = None
+
+    def merge(self, token: Token):
+        merged_token = ImageCollectionToken([self.data, token.data])
+        merged_token.build = self.build
+        return merged_token
+
+    async def compile(self):
+        img = self.parse_data()
+        img = await self.build(img)
+        alt = img['alt']
+        caption = markdown_to_html(img['caption'])
+        src = img['src']
+
+        tpl = f'''
+            <div class="kazimir__image">
+                <figure>
+                    <img alt="{alt}" src="{src}">
+                    <figcaption>{caption}</figcaption>
+                </figure>
+            </div>
+        '''
+        return tpl.format(img).strip()
+
+    def parse_data(self):
+        return parse_file(self.data)
+
+
+class ImageCollectionToken(ImageToken):
+    def __init__(self, data) -> None:
+        super().__init__(data)
+        self.name = 'image_collection'
+        self.build = None
+
+    def merge(self, token: Token):
+        if isinstance(token.data, list):
+            data = token.data
+        else:
+            data = [token.data]
+        merged_token = ImageCollectionToken([*self.data, *data])
+        merged_token.build = self.build
+        return merged_token
+
+    async def compile(self):
+        data = self.parse_data()
+        images = []
+        for x in data:
+            img = await self.build(x)
+            images.append(img)
+
+        images = [x['src'] for x in images]
+        images = [f'<img src="{x}">' for x in images]
+        images = '\n'.join(images)
+
+        return f'''
+            <div class="kazimir__image-collection">
+                <div class="fotorama"
+                     data-width="100%"
+                     data-ratio="800/600"
+                >
+                    {images}
+                </div>
+            </div>
+        '''
+
+    def parse_data(self):
+        return [parse_file(x) for x in self.data]
+
+
+class UrlToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_url(data)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='url', data=data)
+        self.joinable = False
+
+
+def markdown_to_html(text):
+    return markdown(text)
+
+
+def is_url(sample: str):
+    url = urlparse(sample)
+    return url.scheme and url.netloc
+
+
+def is_image(sample: str):
+    file = parse_file(sample)['file']
+    return is_file(file, image_extensions)
+
+
+def is_document(sample: str):
+    file = parse_file(sample)['file']
+    return is_file(file, document_extensions)
+
+
+def is_file(sample: str, extensions):
+    name, ext = os.path.splitext(sample)
+    ext = ext.lower()
+    return ext in extensions
+
+
+def parse_file(sample: str):
+    """
+        ![Caption](file.jpg) -> {file, caption}
+        file.jpg Caption -> {file, caption}
+
+        :param sample:
+        :return:
+        """
+    p = re.match(r'!\[(.*)\]\((.*)\)', sample)
+    if p:
+        caption = p.group(1)
+        file = p.group(2)
+    else:
+        sample = sample.split(' ')
+        file = sample[0]
+        caption = ' '.join(sample[1:])
+    return {
+        'file': file,
+        'caption': caption,
+    }
+
+
+def parse_link(sample: str):
+    """
+    ![Alt](href) -> {href, alt}
+
+    :param sample:
+    :return:
+    """
+    p = re.match(r'!\[(.*)\]\((.*)\)', sample)
+    if not p:
+        return None
+
+    url = p.group(2)
+    return {
+        'url': urlparse(url),
+        'alt': p.group(1),
+    }

modules/sync/kazimir/YoutubeToken.py

@@ -0,0 +1,32 @@
+from urllib.parse import urlparse, parse_qs
+from kazimir.Token import UrlToken, is_url
+
+
+class YoutubeToken(UrlToken):
+    @staticmethod
+    def test(data: str):
+        if not is_url(data):
+            return False
+
+        url = urlparse(data)
+        return url.netloc in ['www.youtube.com', 'youtube.com']
+
+    async def compile(self):
+        """
+        <div class="kazimir__video">
+            <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
+        </div>
+        """
+
+        url = urlparse(self.data)
+        query = parse_qs(url.query)
+
+        if 'v' not in query:
+            return ''
+
+        video_id = query['v'][0]
+
+        url = f'//www.youtube.com/embed/{video_id}'
+        return f'''
+            <div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>
+        '''.strip()

modules/sync/kazimir/__init__.py

@@ -0,0 +1,320 @@
+import lxml.html
+import re
+from markdown import markdown
+
+from kazimir.figure import FigureExtension
+from kazimir.instagram import InstagramExtension
+from sync.data import request
+
+quote_pairs = [
+    ('"', '"'),
+    ('«', '»'),
+]
+
+
+def markdown_to_html(text):
+    return markdown(text, extensions=[
+        FigureExtension(),
+        InstagramExtension(),
+    ])
+
+
+def extract_files(text: str) -> [str]:
+    html = markdown(text)
+    post_html = lxml.html.fromstring(html)
+
+    images = post_html.cssselect('img')
+    images = [img.get('src') for img in images]
+
+    return images, []
+
+
+def create_tree(text: str) -> str:
+    tokens = text.split('\n')
+    tokens = markup_tokens(tokens)
+    tokens = join_tokens(tokens)
+    tokens = merge_tokens(tokens)
+
+    kz = compile_tokens(tokens)
+    tree = lxml.html.fromstring(kz)
+    return clean_tree(tree)
+
+
+def clean_tree(tree):
+    return fix_links_quotes(tree)
+
+
+def fix_links_quotes(tree):
+    safe = lambda x: x if x else ''
+
+    for link in tree.iterlinks():
+        a = link[0]
+        if a.tag != 'a':
+            continue
+
+        quote = get_quote_pair(a.text)
+        if not quote:
+            continue
+
+        # New text of A without quotes
+        a.text = a.text[1:-1]
+
+        # Move first quote out of A to the prefix
+        a.getparent().text = safe(a.getparent().text) + quote[0]
+
+        # Move last quote out of A to the suffix
+        a.tail = quote[1] + safe(a.tail)
+
+    return tree
+
+
+def get_quote_pair(text: str):
+    """
+    sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
+        ? (s, t)
+        : None
+
+    :param text:
+    :return:
+    """
+    if len(text) < 2:
+        return None
+
+    first = text[0]
+    last = text[-1]
+
+    for pair_first, pair_last in quote_pairs:
+        if pair_first == first and pair_last == last:
+            return first, last
+
+    return None
+
+
+def html_from_tree(tree):
+    html = lxml.html.tostring(tree, encoding='unicode')
+
+    return html
+
+
+async def typo(text):
+    from kazimir import artlebedev
+    return await artlebedev.typograf(text)
+    # url = 'http://www.typograf.ru/webservice/'
+    # params = dict(text=text)
+    # return await request.post(url, params, {'chr': 'utf-8'})
+
+
+def join_tokens(tokens):
+    buffer_tag = None
+    buffer = []
+    ts = []
+    for token in tokens:
+        tag = token['tag']
+        changed = buffer_tag is not None and tag != buffer_tag
+
+        flush = is_tag_split(tag)
+        if changed:
+            flush = True
+
+        if not is_tag_joinable(tag):
+            flush = True
+
+        if flush:
+            if buffer:
+                ts.append(buffer)
+            if is_tag_split(tag):
+                buffer = []
+                buffer_tag = None
+            else:
+                buffer = [token]
+                buffer_tag = tag
+        else:
+            buffer_tag = tag
+            buffer.append(token)
+
+    if len(buffer):
+        ts.append(buffer)
+    return ts
+
+
+def merge_tokens(tokens: list):
+    tokens_out = []
+    for ts in tokens:
+        tag = ts[0]['tag']
+        fn = None
+        if tag == 'image':
+            fn = merge_image
+        elif tag == 'text':
+            fn = merge_text
+        else:
+            fn = merge_any
+        tokens_out.append(fn(ts))
+    return tokens_out
+
+
+def compile_tokens(tokens: list):
+    compiled = []
+    for ts in tokens:
+        tag = ts['tag']
+        fn = None
+        if tag == 'text':
+            fn = compile_text
+        elif tag == 'image_collection':
+            fn = compile_image_collection
+        elif tag == 'image':
+            fn = compile_image
+        elif tag == 'url':
+            fn = compile_url
+        else:
+            fn = compile_any
+        compiled.append(fn(ts))
+    return '\n'.join(compiled)
+
+
+def markup_tokens(tokens: list):
+    marked_tokens = []
+    for token in tokens:
+        tag = 'text'
+        if is_url(token):
+            tag = 'url'
+        elif is_image(token):
+            tag = 'image'
+        elif not token:
+            tag = 'split'
+            token = None
+
+        marked_tokens.append({
+            'tag': tag,
+            'data': token,
+        })
+    return marked_tokens
+
+
+def is_tag_split(tag):
+    return tag == 'split'
+
+
+def is_tag_joinable(tag: str):
+    return tag not in ['url']
+
+
+def is_url(sample: str):
+    from urllib.parse import urlparse
+    url = urlparse(sample)
+
+    return url.scheme and url.netloc
+
+
+def is_image(sample: str):
+    return sample[:2] == '![' and sample[-1:] == ')'
+
+
+def merge_any(tokens):
+    return tokens[0]
+
+
+def merge_text(tokens):
+    data = ''
+
+    for t in tokens:
+        data += t['data']
+        data += '\n'
+
+    return {
+        'tag': 'text',
+        'data': data.strip(),
+    }
+
+
+def merge_image(tokens):
+    if len(tokens) == 1:
+        return tokens[0]
+    else:
+        data = [t['data'] for t in tokens]
+        return {
+            'tag': 'image_collection',
+            'data': data,
+        }
+
+
+def compile_text(token):
+    return markdown_to_html(token['data'])
+
+
+def compile_image_collection(token):
+    file = lambda i: i[4:][:-1]
+    imgs = ['<img src="{}">'.format(file(i)) for i in token['data']]
+    return '''
+        <div class="kazimir__image-collection">
+            <div class="fotorama"
+                 data-width="100%"
+                 data-ratio="800/600"
+            >
+                {}
+            </div>
+        </div>
+    '''.format('\n'.join(imgs))
+
+
+def compile_image(token):
+    img = markdown_to_html(token['data'])
+    tpl = '''<div class="kazimir__image">{}</div>'''
+    return tpl.format(img)
+
+
+def compile_url(token):
+    return compile_youtube(token)
+
+
+def compile_youtube(token):
+    """
+    <div class="kazimir__video">
+     <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
+   </div>
+    """
+
+    youtube = r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P<youtubeargs>[A-Za-z0-9_&=-]+)\S*'
+    youtube_pattr = re.compile(youtube)
+    match = youtube_pattr.match(token['data'])
+    if match:
+        video_id = match.group(2)
+        url = f'//www.youtube.com/embed/{video_id}'
+        tpl = '''<div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>'''
+
+        return tpl.format(url=url)
+    else:
+        return ''
+
+
+def compile_any(token):
+    return markdown_to_html(token['data'])
+
+
+if __name__ == '__main__':
+    sample_text = '''
+Hello!
+
+https://www.instagram.com/p/BUelE04l1kU
+ku!
+
+https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka
+https://www.youtube.com/watch?v=qnguOqWpraI
+![](1.jpg)
+![](2.jpg)
+![](3.jpg)
+
+LOL NOOB
+![](12.jpg)
+![](22.jpg)
+WTF OMG
+
+Bye!
+Fye!
+
+![](1.jpg)
+
+More text
+    '''
+
+    sample_tree = create_tree(sample_text)
+    print(html_from_tree(sample_tree))

modules/sync/kazimir/artlebedev.py

@@ -0,0 +1,45 @@
+from sync.data import request
+from lxml import etree
+
+
+async def typograf(text, options={}):
+    text = text.replace('&', '&')
+    text = text.replace('<', '&lt;')
+    text = text.replace('>', '&gt;')
+
+    body = '''
+    <?xml version="1.0" encoding="UTF-8"?>
+    <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
+    <soap:Body>
+     <ProcessText xmlns="http://typograf.artlebedev.ru/webservices/">
+      <text>{text}</text>
+      <entityType>1</entityType>
+      <useBr>0</useBr>
+      <useP>1</useP>
+      <maxNobr>3</maxNobr>
+      </ProcessText>
+     </soap:Body>
+    </soap:Envelope>
+    '''.strip().format(text=text)
+
+    url = 'http://typograf.artlebedev.ru/webservices/typograf.asmx'
+    resp = await request.post(url, body, {}, {
+        'Content-Type': 'text/xml',
+        'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText',
+    })
+    resp_text = resp.encode()
+
+    try:
+        # create xml tree from response
+        response_tree = etree.fromstring(resp_text)
+
+        # look for ProcessTextResult
+        typograf_response = response_tree.getchildren()[0].getchildren()[0].getchildren()[0].text
+    except Exception as response_tree:
+        print(response_tree)
+        return text
+
+    typograf_response = typograf_response.replace('&amp;', '&')
+    typograf_response = typograf_response.replace('&lt;', '<')
+    typograf_response = typograf_response.replace('&gt;', '>')
+    return typograf_response

modules/sync/kazimir/figure.py

@@ -0,0 +1,44 @@
+from markdown import Extension
+from markdown.inlinepatterns import IMAGE_LINK_RE, IMAGE_REFERENCE_RE
+from markdown.blockprocessors import BlockProcessor
+from markdown.util import etree
+import re
+
+FIGURES = [
+    u'^\s*' + IMAGE_LINK_RE + u'\s*$',
+    u'^\s*' + IMAGE_REFERENCE_RE + u'\s*$',
+]
+
+TAG_FIGURE = 'figure'
+TAG_FIGCAPTION = 'figcaption'
+NAME = 'FigureAltCaption'
+
+
+class FigureProcessor(BlockProcessor):
+    FIGURES_RE = re.compile('|'.join(f for f in FIGURES))
+
+    def test(self, parent, block):
+        is_image = bool(self.FIGURES_RE.search(block))
+        is_single_list = (len(block.splitlines()) == 1)
+        is_in_figure = parent.tag == TAG_FIGURE
+
+        return is_image and is_single_list and not is_in_figure
+
+    def run(self, parent, blocks):
+        raw_block = blocks.pop(0)
+        caption_text = self.FIGURES_RE.search(raw_block).group(1)
+
+        figure = etree.SubElement(parent, TAG_FIGURE)
+        figure.text = raw_block
+
+        figcaption_elem = etree.SubElement(figure, TAG_FIGCAPTION)
+        figcaption_elem.text = caption_text
+
+
+class FigureExtension(Extension):
+    def extendMarkdown(self, md, md_globals):
+        md.parser.blockprocessors.add(
+            NAME,
+            FigureProcessor(md.parser),
+            '<ulist'
+        )

modules/sync/kazimir/fix_links_quotes.py

@@ -0,0 +1,50 @@
+quote_pairs = [
+    ('"', '"'),
+    ('«', '»'),
+]
+
+
+async def fix_links_quotes(tree):
+    safe = lambda x: x if x else ''
+
+    for link in tree.iterlinks():
+        a = link[0]
+        if a.tag != 'a':
+            continue
+
+        quote = get_quote_pair(a.text)
+        if not quote:
+            continue
+
+        # New text of A without quotes
+        a.text = a.text[1:-1]
+
+        # Move first quote out of A to the prefix
+        a.getparent().text = safe(a.getparent().text) + quote[0]
+
+        # Move last quote out of A to the suffix
+        a.tail = quote[1] + safe(a.tail)
+
+    return tree
+
+
+def get_quote_pair(text: str):
+    """
+    sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
+        ? (s, t)
+        : None
+
+    :param text:
+    :return:
+    """
+    if len(text) < 2:
+        return None
+
+    first = text[0]
+    last = text[-1]
+
+    for pair_first, pair_last in quote_pairs:
+        if pair_first == first and pair_last == last:
+            return first, last
+
+    return None

modules/sync/kazimir/instagram.py

@@ -0,0 +1,68 @@
+import requests
+from markdown import Extension
+from markdown.inlinepatterns import Pattern
+from markdown.util import etree
+
+
+class InstagramExtension(Extension):
+    def add_inline(self, md, name, klass, re):
+        pattern = klass(re)
+        pattern.md = md
+        pattern.ext = self
+        md.inlinePatterns.add(name, pattern, "<reference")
+
+    def extendMarkdown(self, md, md_globals):
+        match_url = r'https?://www\.instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$'
+        self.add_inline(md, 'instagram', Instagram, match_url)
+
+
+class Instagram(Pattern):
+    def handleMatch(self, m):
+        return embed_instagram(m.group('media'))
+
+
+def embed_instagram(media):
+    """
+    <blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
+        <div style="padding:8px;">
+            <div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
+                <div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
+                </div>
+            </div>
+
+            <p style=" margin:8px 0 0 0; padding:0 4px;">
+                <a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
+                    Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
+                </a>
+            </p>
+
+            <p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
+                A post shared by Шлиссельбургская ДХШ (@hudozka) on
+
+                <time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
+                    May 24, 2017 at 7:05am PDT
+                </time>
+            </p>
+        </div>
+    </blockquote>
+    """
+
+    url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
+    req = requests.get(url)
+    if req.status_code == 200:
+        html = req.json()['html']
+        html = fix_html(html)
+        i = etree.fromstring(html)
+
+        obj = etree.Element('div')
+        obj.set('class', "kazimir__embed")
+        obj.append(i)
+        return obj
+    else:
+        raise Exception(
+            'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
+        )
+
+
+def fix_html(html: str):
+    return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')

modules/sync/requirements.txt

@@ -0,0 +1,13 @@
+cssselect==0.9.2
+lxml==3.6.1
+Markdown==2.6.6
+Pillow==6.2.0
+pymongo==3.3.0
+PyPDF2==1.26.0
+python-frontmatter==0.3.1
+PyYAML==5.1
+requests==2.20.0
+six==1.10.0
+smartcrop==0.1
+toml==0.9.2
+Unidecode==0.4.19

modules/sync/settings.py

@@ -0,0 +1,144 @@
+import os
+
+import yaml
+import argparse
+
+__author__ = 'Roman Timashev'
+
+name = 'HudozkaSync'
+
+_parser = argparse.ArgumentParser(description='Hudozka Sync Daemon')
+_parser.add_argument('--config', dest='config', default='config.yaml', help='path to config.yaml')
+_args = _parser.parse_args()
+
+config = yaml.load(open(_args.config, 'r'))
+
+
+def value(key, default=None):
+    return config[key] if key in config else default
+
+
+def absolute(path, ensure=True):
+    a = os.path.expanduser(path)
+    if ensure:
+        os.makedirs(a, exist_ok=True)
+    return a
+
+
+def abs_fn(root):
+    return lambda path: os.path.join(root, path)
+
+
+def env(param, default=None):
+    if param in os.environ:
+        return os.environ[param]
+    return default
+
+
+interval = value('interval', 0)
+
+database_uri = value('database_uri')  # env('MONGO_URI')
+
+skip_unchanged = value('skip_unchanged')
+
+person_uri = {
+    'Н.В.Андреева': 'nv-andreeva',
+    'И.Н.Втюрина': 'in-vturina',
+    'О.Д.Гоголева': 'od-gogoleva',
+    'В.А.Саржин': 'va-sarzhin',
+    'Р.К.Тимашев': 'rk-timashev',
+    'А.С.Тимашева': 'as-timasheva',
+}
+
+image_processing_enabled = value('image_processing')
+
+album_html_img_class = 'hudozka-product'
+
+image_ext = '.jpg'
+image_sizes = [
+    ('original', None, None),
+
+    ('large', 2500, 1000),
+    ('large@2', 5000, 2000),
+
+    ('big', 1500, 667),
+    ('big@2', 3000, 1334),
+
+    ('medium', 400, 300),
+    ('medium@2', 800, 600),
+
+    ('small', 250, 175),
+    ('small@2', 500, 350),
+
+    ('og', 968, 504),
+    ('og@2', 1936, 1008),
+
+    ('fb', 1200, 630),
+    ('fb@2', 2400, 1260),
+]
+
+date_formats = [
+    '%Y',
+    '%m.%Y',
+    '%d.%m.%Y',
+    '%d.%m.%Y %H:%M',
+    '%d.%m.%Y %H:%M:%S'
+]
+
+date_formats_direct = [
+    '%Y',
+    '%m.%Y',
+    '%d.%m.%Y',
+]
+
+date_formats_reverse = [
+    '%Y',
+    '%Y.%m',
+    '%Y.%m.%d'
+]
+
+origin = value('origin')
+
+collection_settings = 'settings'
+collection_images = 'images'
+collection_documents = 'documents'
+collection_schedules = 'schedules'
+collection_articles = 'articles'
+collection_collective = 'collective'
+collection_albums = 'albums'
+collection_pages = 'pages'
+
+hash_salt_articles = '6'
+hash_salt_pages = '4'
+hash_salt_albums = '1'
+hash_salt_documents = '4'
+hash_salt_images = '5'
+hash_salt_person = '4'
+
+provider_name = value('provider')['name']  # env('SYNC_PROVIDER', 'fs')
+provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
+
+f = abs_fn(provider_root)
+
+dir_settings = './Settings'
+dir_documents = './Documents'
+dir_schedules = './Schedules'
+dir_articles = './Articles'
+dir_persons = './Collective'
+dir_albums = './Gallery'
+dir_pages = './Pages'
+dir_images = './Images'
+
+document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
+document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
+
+image_url_base = 'https://static.shlisselburg.org/art/images/'
+
+yandex_disk_access_token = value('yandex_disk_access_token')  # env('YANDEX_DISK_ACCESS_TOKEN')
+
+auth_s3 = {
+    'region_name': 'ams3',
+    'endpoint_url': 'https://dir.ams3.digitaloceanspaces.com',
+    'aws_access_key_id': value('s3')['client_id'],
+    'aws_secret_access_key': value('s3')['client_secret'],
+}

modules/sync/sync-smart.py

@@ -0,0 +1,50 @@
+import settings
+from sync.data.fs import FSProvider
+from sync.data.yandexdisk import YDProvider
+
+import asyncio
+
+
+async def run_check(run_interval=0):
+    run_interval = 1
+
+    from yandexdisk import YDClient
+    client = YDClient(settings.yandex_disk_access_token)
+
+    async def need_to_update():
+        res = client.last_uploaded()
+        items = res['items']
+        paths = [i['path'] for i in items]
+
+        for i in paths:
+            if i.startswith('disk:/Hudozka/Site'):
+                return True
+
+        return False
+
+    while True:
+        status = await need_to_update()
+        if status:
+            print('DO SYNC')
+            # await run(0)
+
+        if run_interval == 0:
+            break
+        else:
+            await asyncio.sleep(run_interval)
+
+
+def get_provider(provider_type, root):
+    if provider_type == 'fs':
+        return FSProvider(root)
+    elif provider_type == 'yd':
+        return YDProvider(root, settings.yandex_disk_access_token)
+    return None
+
+
+if __name__ == '__main__':
+    interval = settings.interval
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run_check(interval))
+    loop.close()

modules/sync/sync.py

@@ -0,0 +1,137 @@
+import settings
+from sync.core import Sync
+from sync.core.SyncDocument import SyncDocument
+from sync.data.fs import FSProvider
+from sync.data.yandexdisk import YDProvider
+
+import asyncio
+import logging
+
+from sync.models.Image import Image
+from sync.models.Album import Album
+from sync.models.Article import Article
+from sync.models.Page import Page
+from sync.models.Person import Person
+from sync.models.Schedule import Schedule
+from sync.models.Settings import Settings
+
+
+def init_logger(name: str, file: str):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+
+    # create file handler which logs even debug messages
+    fh = logging.FileHandler(file)
+    fh.setLevel(logging.DEBUG)
+
+    # create console handler with a higher log level
+    ch = logging.StreamHandler()
+    # ch.setLevel(logging.ERROR)
+    ch.setLevel(logging.DEBUG)
+
+    # create formatter and add it to the handlers
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    fh.setFormatter(formatter)
+    ch.setFormatter(formatter)
+
+    # add the handlers to the logger
+    logger.addHandler(fh)
+    logger.addHandler(ch)
+
+
+async def run_sync(sync: Sync, **kwargs):
+    await sync.run(**kwargs)
+
+
+async def run(run_interval=0):
+    io = lambda root: get_provider(settings.provider_name, root)
+
+    sync_documents = SyncDocument(
+        provider=io(settings.provider_root),
+        sizes=settings.image_sizes,
+    )
+    sync_documents.strict_origin = True
+
+    sync_pages = Sync(
+        provider=io(settings.provider_root),
+        model=Page,
+    )
+    sync_pages.set_options(True, True, True)
+
+    sync_persons = Sync(
+        provider=io(settings.provider_root),
+        model=Person,
+    )
+    sync_persons.set_options(True, True, True)
+    sync_persons.validate_urls = False
+
+    sync_articles = Sync(
+        provider=io(settings.provider_root),
+        model=Article,
+    )
+    sync_articles.set_options(True, True, True)
+    sync_articles.validate_urls = False
+
+    sync_schedules = Sync(
+        provider=io(settings.provider_root),
+        model=Schedule,
+    )
+    sync_schedules.set_options(True, True, True)
+    sync_schedules.validate_urls = False
+
+    sync_albums = Sync(
+        provider=io(settings.provider_root),
+        model=Album,
+    )
+    sync_albums.set_options(True, True, True)
+    sync_albums.validate_urls = False
+
+    sync_settings = Sync(
+        provider=io(settings.provider_root),
+        model=Settings,
+    )
+    sync_settings.set_options(True, True, True)
+    sync_settings.validate_urls = False
+
+    sync_images = Sync(
+        provider=io(settings.provider_root),
+        model=Image,
+    )
+    sync_images.set_options(True, True, True)
+    sync_images.strict_origin = True
+    sync_images.validate_urls = False
+
+    while True:
+        await asyncio.wait([
+            run_sync(sync_documents),
+            run_sync(sync_pages),
+            run_sync(sync_persons),
+            run_sync(sync_articles),
+            run_sync(sync_schedules),
+            run_sync(sync_albums),
+            run_sync(sync_settings),
+            run_sync(sync_images),
+        ])
+
+        if run_interval == 0:
+            break
+        else:
+            await asyncio.sleep(run_interval)
+
+
+def get_provider(provider_type, root):
+    if provider_type == 'fs':
+        return FSProvider(root)
+    elif provider_type == 'yd':
+        return YDProvider(root, settings.yandex_disk_access_token)
+    return None
+
+
+if __name__ == '__main__':
+    init_logger(settings.name, 'hudozka.log')
+
+    interval = settings.interval
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run(interval))
+    loop.close()

modules/sync/sync/__init__.py

@@ -0,0 +1,191 @@
+import os
+import re
+from datetime import datetime
+
+import lxml.html
+from markdown import markdown
+
+import kazimir
+import settings
+from kazimir.CSVToken import CSVToken
+from kazimir.DocumentToken import DocumentToken
+from kazimir.YoutubeToken import YoutubeToken
+from kazimir.InstagramToken import InstagramToken
+from kazimir.Marker import Marker
+from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken
+from sync.data import Provider
+from sync.models import Model
+from sync.models.Document import Document
+from sync.models.Image import Image
+from utils.text.file import get_size
+from utils.text.file import get_ext
+
+
+def create_date(date_str, date_formats=None):
+    """
+    2016
+    2016.10
+    2016.10.10
+    -> datetime
+
+    :param date_formats:
+    :param date_str:
+    :return:
+    """
+    if not date_formats:
+        date_formats = [
+            '%Y',
+            '%Y.%m',
+            '%Y.%m.%d'
+        ]
+
+    for date_format in date_formats:
+        try:
+            return datetime.strptime(date_str, date_format)
+        except:
+            continue
+    return None
+
+
+# 2016 На крыльях бабочек
+# 2016.10 На крыльях бабочек
+# 2016.10.10 На крыльях бабочек
+folder_name_pattern = re.compile('([\d.]+)(.*)')
+
+
+async def untouched(items: [Model]) -> [Model]:
+    if settings.skip_unchanged:
+        return items
+
+    result = []
+    for item in items:
+        status = await item.is_changed()
+        if status:
+            result.append(item)
+    return result
+
+    # def is_equals(e, n):
+    #     return (n is None) or ('hash' not in n) or (e.hash != n['hash'])
+    #
+    # doc = lambda i: {'id': i.id, 'hash': i.hash}
+    #
+    # stored_documents = [(i, store.read(doc(i))) for i in documents]
+    # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)]
+
+    # return filtered_documents
+
+
+def create_date_and_title_from_folder_name(folder_name, date_formats=None):
+    m = folder_name_pattern.findall(folder_name)
+    if not m:
+        return None, None
+
+    date_str, title = m[0]
+    title = os.path.splitext(title.strip())[0]
+    date = create_date(date_str, date_formats)
+
+    if not date:
+        return None, title
+
+    return date, title
+
+
+async def create_post(provider: Provider, folder: str, md: str, sizes):
+    from kazimir.fix_links_quotes import fix_links_quotes
+
+    images = []
+    documents = []
+
+    def rel(file: str) -> str:
+        return os.path.join(folder, file)
+
+    async def build_document(data):
+        filepath = os.path.join(folder, data['file'])
+        document = await Document.new(provider, filepath, settings.image_sizes)
+
+        if not document:
+            raise Exception(f'Failed to build Document {filepath}')
+
+        text = data['caption'] if data['caption'] else data['file']
+        url = f'/document/{document.id}'
+
+        image_url = document.preview.get_size('small')['url']
+
+        return {
+            'url': url,
+            'image_url': image_url,
+            'file_url': document.url,
+            'title': document.title,
+            'file_size': get_size(document.file_size, 1),
+            'file_format': get_ext(data['file']),
+        }
+
+    async def build_image(data):
+        src = data['file']
+        relative_image_path = os.path.join(folder, src)
+
+        image_path = provider.get_local(relative_image_path)
+        if os.path.exists(image_path):
+            image = await Image.new(provider, relative_image_path, sizes)
+            if image:
+                url = image.get_size('big')['url']
+                images.append(image)
+
+                return {
+                    'src': url,
+                    'alt': data['caption'],
+                    'caption': data['caption'],
+                }
+        raise Exception('Fail to get Image', relative_image_path)
+
+    async def read_file(data):
+        filepath = data
+        filepath = rel(filepath)
+
+        return provider.read(filepath).read().decode('utf-8')
+
+    m = Marker()
+    m.add_token_factory(TokenFactory(SplitToken))
+    m.add_token_factory(TokenFactory(YoutubeToken))
+    m.add_token_factory(TokenFactory(InstagramToken))
+    m.add_token_factory(TokenFactory(UrlToken))
+    m.add_token_factory(BuildTokenFactory(ImageToken, build=build_image))
+    m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
+    m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file))
+    # m.add_token_factory(TokenFactory(FileToken))
+    m.add_token_factory(TokenFactory(TextToken))
+    m.add_tree_middleware(fix_links_quotes)
+    tree = await m.create_tree(md)
+
+    post = kazimir.html_from_tree(tree)
+    post = await kazimir.typo(post)
+
+    return post, images, documents
+
+
+def images_from_html(md):
+    if not md:
+        return []
+
+    post_html = lxml.html.fromstring(md)
+
+    images = []
+    for img in post_html.cssselect('img'):
+        src = img.get('src')
+        images.append(src)
+    return images
+
+
+def title_from_html(md):
+    html = lxml.html.fromstring(md)
+    titles = html.cssselect('h1')
+    if len(titles):
+        return titles[0].text
+
+
+create_post_from_image_list = lambda images: markdown(
+    '\n'.join(map(
+        lambda i: '![]({img})'.format(img=i),
+        images
+    ))
+)

modules/sync/sync/core/SyncDocument.py

@@ -0,0 +1,14 @@
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Document import Document
+
+
+class SyncDocument(Sync):
+    def __init__(self, provider: Provider, sizes):
+        super().__init__(provider, Document)
+        self.sizes = sizes
+
+    def _get_build_args(self):
+        return {
+            'sizes': self.sizes,
+        }

modules/sync/sync/core/__init__.py

@@ -0,0 +1,144 @@
+import logging
+
+import settings
+from sync import untouched
+from sync.data import Provider
+
+
+def get_id(item):
+    try:
+        return item.id
+    except AttributeError:
+        return item['id']
+
+
+class Sync:
+    def __init__(self, provider: Provider, model):
+        super().__init__()
+
+        self.validate_urls = True
+        self.strict_origin = False
+        self.__default_build_args = {
+            'sizes': settings.image_sizes,
+        }
+        self.__delete_query = lambda item: {'_id': item['_id']}
+
+        name = model.__name__
+        self.logger = logging.getLogger('%s.%s' % (settings.name, name))
+
+        self.provider = provider
+        self.model = model
+        self.filter_item_fn = lambda x: True
+
+        self.skip_unchanged = True
+        self.update_enabled = True
+        self.delete_enabled = True
+
+    def set_options(self, su: bool, u: bool, d: bool):
+        self.skip_unchanged = su
+        self.update_enabled = u
+        self.delete_enabled = d
+        return self
+
+    def set_item_filter(self, fn):
+        self.filter_item_fn = fn
+        return self
+
+    async def clean(self):
+        pass
+
+    async def build(self, item):
+        args = self._get_build_args()
+        await item.build(**args)
+        return item
+
+    async def update(self, item):
+        await item.upload()
+        await item.save()
+        return item
+
+    def _get_build_args(self):
+        return {
+            **self.__default_build_args
+        }
+
+    def __items_to_delete_query(self, items):
+        query = {
+            'id': {'$nin': [doc.id for doc in items]},
+        }
+
+        if self.strict_origin:
+            items_origin = list(set([doc.origin for doc in items]))
+
+            query = {
+                **query,
+                'origin': {'$in': items_origin},
+            }
+
+        return query
+
+    async def run(self):
+        """
+        # Get scope files
+        # Validate these files. Raise an error
+        # calc_hashes
+        # make diff with previous run
+        # compile objects
+        # upload
+
+        :return:
+        """
+        self.logger.info('Checking for update')
+
+        items = await self.model.scan(self.provider)
+        self.logger.info('Found {} Item(s)'.format(len(items)))
+        all_items = items
+
+        if self.validate_urls:
+            self.check_urls_uniqueness(all_items)
+
+        # SKIP UNTOUCHED DOCUMENTS
+        if self.skip_unchanged:
+            items = await untouched(all_items)
+
+        # Optionally trim list of models by custom filter
+        items = [
+            x
+            for x in items
+            if self.filter_item_fn(x)
+        ]
+
+        self.logger.info(f'Ready to process {len(items)} item(s)')
+
+        # BUILDING
+        for item in items:
+            self.logger.info(f'Build Item {item}')
+            await self.build(item)
+
+        # UPDATING
+        if self.update_enabled:
+            for item in items:
+                await self.update(item)
+                self.logger.info('Updated Item {}'.format(item))
+
+        # DELETING
+        if self.delete_enabled:
+            obsolete_items = await self.model.find(self.__items_to_delete_query(all_items))
+            obsolete_items = list(obsolete_items)
+
+            for item in obsolete_items:
+                self.logger.info('Deleting Item {}'.format(item['id']))
+                await self.model.delete(self.__delete_query(item))
+
+        await self.clean()
+
+    def check_urls_uniqueness(self, items):
+        urls = [x.url for x in items]
+        unique_urls = set(urls)
+
+        if len(urls) != len(unique_urls):
+            self.logger.error('URLs is not unique')
+            raise Exception('URLs is not unique')
+
+    async def upload(self):
+        pass

modules/sync/sync/data/__init__.py

@@ -0,0 +1,78 @@
+import os
+from io import BytesIO
+
+from utils.fn import combine
+
+
+class Provider:
+    def __init__(self, root):
+        super().__init__()
+        self.root = root
+
+    def read(self, path):
+        return BytesIO(bytes())
+
+    def scan(self, path):
+        return []
+
+    def glob(self, pattr):
+        return []
+
+    def type_filter(self, path, ext):
+        return []
+
+    def copy(self, path, out):
+        pass
+
+    def hash(self, path):
+        return None
+
+    def size(self, path):
+        return 0
+
+    def exists(self, path):
+        return True
+
+    def get_abs(self, path):
+        a = os.path.join(self.root, path)
+        return os.path.normpath(a)
+
+    def get_rel(self, path):
+        rel = os.path.relpath(path, self.root)
+        return rel
+
+    def get_local(self, path):
+        return path
+
+    def is_dir(self, path):
+        return False
+
+
+def scan_subdirs(provider, ext, path='.'):
+    """
+    Creates flat list of files with specific ext
+    It iterates only one level of sub directories
+    :param provider: instance of Provider
+    :param ext: file extension like .jpg of .pdf for file filtering
+    :param path: optional path prefix
+    :return: list of files relative to provider.root
+    """
+    items = provider.scan(path)
+    items = filter(
+        lambda i: provider.is_dir(i),
+        items
+    )
+    items = [provider.type_filter(folder, ext) for folder in items]
+    items = combine(items)
+    return items
+
+
+def get_data(provider, path, transform):
+    data = provider.read(path)
+    return transform(data)
+
+
+def list_images(provider, path):
+    image_types = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG']
+    images = [provider.type_filter(path, ext) for ext in image_types]
+    return combine(images)

modules/sync/sync/data/fs.py

@@ -0,0 +1,48 @@
+import os
+from glob import glob
+
+from shutil import copyfile
+
+from sync.data import Provider
+from utils.hash import hash_file
+
+
+class FSProvider(Provider):
+    def read(self, path):
+        return open(self.get_abs(path), 'rb')
+
+    def scan(self, path):
+        paths = [i.path for i in os.scandir(self.get_abs(path))]
+
+        return [self.get_rel(path) for path in paths]
+
+    def glob(self, pattr):
+        items = glob(self.get_abs(pattr))
+        return [self.get_rel(i) for i in items]
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
+
+    def hash(self, path):
+        return hash_file(self.get_abs(path))
+
+    def copy(self, path, out):
+        file = self.get_abs(path)
+        return copyfile(file, out)
+
+    def size(self, path):
+        file = self.get_abs(path)
+        return os.path.getsize(file)
+
+    def exists(self, path):
+        file = self.get_abs(path)
+        return os.path.exists(file)
+
+    def is_dir(self, path):
+        return os.path.isdir(self.get_abs(path))
+
+    def get_local(self, path):
+        return self.get_abs(path)

modules/sync/sync/data/request.py

@@ -0,0 +1,40 @@
+import aiohttp
+from boto3 import session
+from mimetypes import MimeTypes
+import logging
+import settings
+
+logger = logging.getLogger(settings.name + '.S3')
+
+
+async def s3_put(obj, path):
+    """
+    :param obj: uploads/file.pdf
+    :param path: to/to/local/file.pdf
+    :return:
+    """
+    logger.info('Put {}'.format(obj))
+
+    s = session.Session()
+    client = s.client('s3', **settings.auth_s3)
+
+    mime = MimeTypes()
+    mime_type, enc = mime.guess_type(path)
+
+    # Upload a file to your Space
+    client.upload_file(path, 'hudozka', obj, ExtraArgs={
+        'ACL': 'public-read',
+        'ContentType': mime_type,
+    })
+
+
+async def get(url):
+    async with aiohttp.ClientSession as aio_session:
+        res = aio_session.get(url)
+        return await res.text()
+
+
+async def post(url, data, params={}, headers={}):
+    async with aiohttp.ClientSession() as aio_session:
+        async with aio_session.post(url, data=data, params=params, headers=headers) as resp:
+            return await resp.text()

modules/sync/sync/data/yandexdisk.py

@@ -0,0 +1,69 @@
+import os
+
+import re
+
+import io
+from tempfile import mkstemp
+
+import requests
+
+from sync.data import Provider
+from yandexdisk import YDClient
+
+
+class YDProvider(Provider):
+    def __init__(self, root, access_token):
+        super().__init__(root)
+        self.api = YDClient(access_token)
+
+    def read(self, path):
+        remote = self.api.get_download_link(self.get_abs(path))
+        url = remote['href']
+        req = requests.get(url)
+        data = req.content
+        return io.BytesIO(data)
+
+    def scan(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        files = []
+        for item in res.children:
+            files.append(item.path)
+
+        files = map(
+            lambda fp: re.sub('disk:', '', fp),
+            files
+        )
+        return [self.get_rel(i) for i in files]
+
+    def glob(self, pattr):
+        return []
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
+
+    def hash(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res.sha256
+
+    def copy(self, path, out):
+        data = self.read(path)
+        f = open(out, 'wb')
+        f.write(data.read())
+        f.close()
+        return out
+
+    def size(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['size']
+
+    def is_dir(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res.type == 'dir'
+
+    def get_local(self, path):
+        ext = os.path.splitext(path)[1]
+        _, temp_in = mkstemp(suffix=ext)
+        return self.copy(path, temp_in)

modules/sync/sync/models/Album.py

@@ -0,0 +1,199 @@
+import logging
+
+import os
+from lxml import etree
+
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import Provider, list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext, first, map_cases, constant, combine
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Album')
+store = collection(settings.collection_albums)
+
+
+class Album(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_albums
+        documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)]
+        documents = [Album.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        date, title = create_date_and_title_from_folder_name(
+            os.path.basename(path)
+        )
+
+        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+        document = {
+            'folder': path,
+            'date': date,
+            'title': title,
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = provider.type_filter(path, '.md')
+        manifest_file = manifest_file[0] if len(manifest_file) else None
+        manifest = get_manifest(provider, manifest_file) if manifest_file else {}
+        manifest = manifest if manifest else {}
+
+        file = manifest_file if manifest_file else path
+        return Album(
+            provider,
+            file,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_albums
+
+        self.version = '1'
+
+        self.origin = settings.origin
+        self.post = None
+        self.images = []
+        self.documents = []
+        self.preview = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        if self.has_param('content'):
+            images, documents = kazimir.extract_files(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+            documents = []
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_hash()
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        folder = self.get_param('folder')
+
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images, documents = await create_post(self.provider, folder, markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            preview = await Image.new(self.provider, preview, sizes=sizes)
+            self.preview = preview
+
+    def create_products(self, doc, image_file_fn):
+        pass
+        # image_id = id_from_file
+        # iter_album = iterate_iter_over_fns([
+        #     lambda product: {
+        #         **product,
+        #         'image': self.create_image(
+        #             image_file_fn(product['file']),
+        #             url_creator(
+        #                 doc['id'],
+        #                 image_id(product['file'])
+        #             )
+        #         )
+        #     }
+        # ])
+        #
+        # return lmap(
+        #     create_product,
+        #     iter_album(doc['album'])
+        # )
+
+    def bake(self):
+        preview = None if not self.preview else self.preview.ref
+
+        return {
+            **self.params,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'preview': preview,
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            year = str(self.params['date'].year)
+            title = self.params['title']
+            self.id = url_encode_text('%s-%s' % (year, title))
+
+    def __set_hash(self):
+        files = self.get_param('images')
+        files = [self._get_relpath(i) for i in files]
+
+        if not self.provider.is_dir(self.file):
+            files.append(self.file)
+
+        files = sorted(files)
+        files = [self.provider.hash(i) for i in files]
+
+        self.hash = hash_str(files)
+        self.hash = hash_str(
+            combine([self.__hash_salt] + [hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return '<Album id={} file={}>'.format(self.id, self.file)
+
+
+def get_manifest(provider, filepath):
+    data = provider.read(filepath).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(str(manifest['date']), settings.date_formats_direct)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    return manifest

modules/sync/sync/models/Article.py

@@ -0,0 +1,174 @@
+import logging
+
+import os
+
+import kazimir
+import settings
+from db import collection
+from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, \
+    create_post
+from sync.data import list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import first, combine
+from utils.hash import hash_str
+from utils.io import parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Article')
+store = collection(settings.collection_articles)
+
+
+class Article(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_articles
+        documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)]
+        documents = [Article.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        file_time_formats = ['%Y.%m.%d']
+
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
+        document = {
+            'folder': path,
+            'date': doc_date,
+            'title': doc_title,
+        }
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = md_from_folder(provider, path)
+        if manifest_file:
+            manifest = read_manifest(provider, manifest_file)
+        else:
+            manifest = {}
+
+        return Article(
+            provider,
+            path,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.hash_salt = settings.hash_salt_articles
+        self.version = '2'
+        self.post = None
+        self.preview = None
+        self.images = None
+        self.documents = None
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        if self.has_param('content'):
+            images, documents = kazimir.extract_files(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+            documents = []
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_hash()
+        self.origin = settings.origin
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        return {
+            **self.params,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['title'])
+
+    def __set_hash(self):
+        files = []
+        files += sorted(self.get_param('images'))
+        files += sorted(self.get_param('documents'))
+        if self.has_param('preview'):
+            files.append(self.get_param('preview'))
+
+        folder = self.get_param('folder')
+        files = [os.path.join(folder, x) for x in files]
+        files = [self.provider.hash(x) for x in files]
+
+        self.hash = hash_str(
+            combine([self.hash_salt] + [hash_str(self.params)] + files)
+        )
+
+    def __str__(self):
+        return f'<Article file={self.file} id={self.id}>'
+
+
+def md_from_folder(provider, i):
+    return first(provider.type_filter(i, '.md'))
+
+
+def read_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    return {
+        **manifest,
+    }

modules/sync/sync/models/Document.py

@@ -0,0 +1,194 @@
+import os
+from tempfile import mkstemp
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from sync.models.DocumentPreviewImage import DocumentPreviewImage
+from utils.hash import hash_str, md5
+from utils.image import image_magick_pdf_to_img
+
+store = collection(settings.collection_documents)
+
+
+class Document(Model):
+    @staticmethod
+    async def find_one(query):
+        return store.find_one(query)
+
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_documents
+        paths = scan_subdirs(provider, '.pdf', scan_dir)
+        return [Document.read_path(provider, path) for path in paths]
+
+    @staticmethod
+    def read_path(provider, path):
+        name = os.path.basename(path)
+
+        document = {
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return Document(
+            provider,
+            path,
+            document
+        )
+
+    @staticmethod
+    async def new(provider, file, sizes):
+        document = Document.read_path(provider, file)
+        changed = await document.is_changed()
+        if changed:
+            document.hide()
+            await document.build(sizes=sizes)
+            await document.upload()
+            await document.save()
+        else:
+            doc = await Document.find_one({'file': file})
+            document = Document(provider, file, doc)
+            document.ref = doc['_id']
+            await document.create_preview(sizes)
+
+        return document
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_documents
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = 'https://static.shlisselburg.org/art/uploads/{file}'
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.title = None
+        self.file_name = None
+        self.file_size = None
+        self.preview = None
+
+        self.type = 'document'  # 'award'
+        self.hidden = False
+
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+        self._set_origin()
+
+        filename = os.path.basename(self.file)
+        self.file_name = filename
+        self.file_size = self.provider.size(self.file)
+
+        if self.has_param('title'):
+            self.title = self.get_param('title')
+
+        if self.has_param('hidden'):
+            self.hidden = self.get_param('hidden')
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        await self.create_preview(sizes)
+
+        pdf_title = await get_pdf_title(self.provider, self.file)
+        if pdf_title:
+            self.title = pdf_title
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+
+            doc = await Document.find_one(query)
+            self.ref = doc['_id']
+
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def upload(self):
+        filepath = self.provider.get_local(self.file)
+        name = os.path.basename(self.url)
+
+        await request.s3_put(f'uploads/{name}', filepath)
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'file': self.file,
+            'hash': self.hash,
+            'url': self.url,
+            'type': self.type,
+            'title': self.title,
+            'origin': self.origin,
+            'hidden': self.hidden,
+            'preview': self.preview.ref,
+            'fileInfo': {
+                'name': self.file_name,
+                'size': self.file_size,
+            },
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        self.id = md5(self.file)
+
+    def __set_url(self):
+        s = os.path.splitext(self.file)
+        file = f'{self.id}{s[1]}'
+
+        self.url = self.__url_template.format(
+            file=file
+        )
+
+    def __set_hash(self):
+        doc = [getattr(self, key) for key in self.__hash_keys]
+
+        self.hash = hash_str(
+            self.__hash_salt + hash_str(doc) + self.provider.hash(self.file)
+        )
+
+    async def create_preview(self, sizes):
+        self.preview = await DocumentPreviewImage.new(self.provider, self.file, sizes)
+
+    def hide(self):
+        self.hidden = True
+        return self
+
+    def __str__(self):
+        return f'<Document file={self.file} id={self.id}>'
+
+
+async def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

modules/sync/sync/models/DocumentPreviewImage.py

@@ -0,0 +1,64 @@
+import os
+from tempfile import mkstemp
+
+from sync.models.Image import Image, default_url_factory
+from utils.image import image_magick_pdf_to_img
+
+
+class DocumentPreviewImage(Image):
+    @staticmethod
+    async def find_one(query):
+        return await Image.find_one(query)
+
+    @staticmethod
+    async def find(query):
+        return await Image.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return await Image.delete(query)
+
+    @staticmethod
+    async def new(provider, file, sizes, **kwargs):
+        img = DocumentPreviewImage(provider, file, data=None)
+        changed = await img.is_changed()
+        if changed:
+            await img.build(sizes=sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await DocumentPreviewImage.find_one({'file': file})
+            img = DocumentPreviewImage(provider, file, doc['data'])
+            img.ref = doc['_id']
+
+        return img
+
+    def __init__(self, provider, file, data):
+        super().__init__(provider, file, data, url_factory=default_url_factory)
+        self.__hash_salt = ''
+        self.temp_image_path = None
+
+    async def is_changed(self):
+        i = await DocumentPreviewImage.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def build(self, sizes):
+        self.temp_image_path = await pdf_to_jpg(self.provider, self.file)
+        await super().build(sizes)
+
+    def _get_image_file(self):
+        return self.temp_image_path
+
+    def __get_hash(self):
+        return self.provider.hash(self.file)
+
+
+async def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    await image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out

modules/sync/sync/models/Image.py

@@ -0,0 +1,253 @@
+import logging
+import os
+import tempfile
+import asyncio
+
+from typing import Optional, List
+
+import settings
+from db import collection
+from sync.data import request, list_images
+from sync.models import Model
+from utils.hash import md5, hash_str
+from utils.image import image_magick_resize
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Image')
+
+store = collection(settings.collection_images)
+
+
+def default_url_factory(file, size, ext):
+    url = settings.image_url_base + '{}-{}{}'
+    return url.format(md5(file), size, ext.lower())
+
+
+class Image(Model):
+    @staticmethod
+    async def find_one(query):
+        document = store.find_one(query)
+        return document
+
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def new(provider, file, sizes, url_factory=None):
+        url_factory = url_factory if url_factory else default_url_factory
+
+        img = Image(provider, file, data=None, url_factory=url_factory)
+        changed = await img.is_changed()
+        if changed:
+            await img.build(sizes=sizes)
+            await img.upload()
+            await img.save()
+        else:
+            doc = await Image.find_one({'file': file})
+            img = Image(provider, file, doc['data'], url_factory)
+            img.ref = doc['_id']
+
+        return img
+
+    @staticmethod
+    async def scan(provider):
+        scan_path = settings.dir_images
+        documents = list_images(provider, scan_path)
+
+        return [Image.read_path(provider, i) for i in documents]
+
+    @staticmethod
+    def read_path(provider, path):
+        return Image(
+            provider,
+            path,
+            data=None,
+            url_factory=default_url_factory
+        )
+
+    def __init__(self, provider, file, data, url_factory):
+        self.__hash_salt = settings.hash_salt_images
+
+        self.origin = None
+
+        self.data = data
+        self.url_factory = url_factory
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.id = self.__get_id()
+        self.hash = self.__get_hash()
+
+        self.origin = f"{settings.origin}:{os.path.dirname(self.file)}"
+
+    async def is_changed(self):
+        i = await Image.find_one({'file': self.file})
+        return self._is_changed_hash(i)
+
+    async def build(self, sizes):
+        logger.info(f'Building Image {self._get_abs(self.file)}')
+
+        img_in = self._get_image_file()
+        img_out = tempfile.mkdtemp()
+
+        images = await create_image(img_in, sizes, self.url_factory, img_out)
+        if not images:
+            raise Exception('Failed to create Image {}'.format(img_in))
+        data = {}
+        for i in images:
+            if i:
+                size_name = i['size']
+                data[size_name] = {
+                    **i,
+                }
+        self.data = data
+
+    async def upload(self):
+        if self.data:
+            async def fn(i):
+                file = i['file']
+                filename = os.path.basename(file)
+
+                await request.s3_put(f'images/{filename}', file)
+
+            await asyncio.wait([fn(i) for i in self.data.values()])
+
+    async def save(self):
+        c = sync_image(self.bake())
+        self.ref = c['_id']
+
+    def bake(self):
+        return {
+            'id': self.id,
+            'file': self.file,
+            'origin': self.origin,
+            'hash': self.hash,
+            'data': self.data,
+        }
+
+    def get_size(self, size):
+        if size in self.data:
+            return self.data[size]
+        return None
+
+    def _get_image_file(self):
+        return self._get_abs(self.file)
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __get_id(self):
+        return md5(self.file)
+
+    def __get_hash(self):
+        file_hash = self.provider.hash(self.file)
+        return hash_str(self.__hash_salt + file_hash)
+
+    def __str__(self):
+        return '<Image hash={} file={}>'.format(self.hash, self.file)
+
+
+def sync_image(record):
+    from db import db
+
+    q = {'file': record['file']}
+    try:
+        db().images.update_one(q, {'$set': record}, upsert=True)
+        i = db().images.find_one({'file': record['file']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
+    """
+
+    :param file: Image path to process
+    :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
+    :param url_fn: function (size_name, ext) for creating image url
+    :param output_dir: path to local folder for thumbs storing
+    :return:
+    """
+    if not os.path.exists(file):
+        return None
+
+    _, ext = os.path.splitext(file)
+    images = []
+
+    if not settings.image_processing_enabled:
+        logger.warning(
+            f'Image processing disabled {file}. Size of image will be set to original'
+        )
+
+    async def fn(size):
+        size_name, width, height = size
+
+        image_url = url_fn(file, size_name, ext)
+        image_filename = os.path.basename(image_url)
+        local_image_path = os.path.join(output_dir, image_filename)
+
+        if settings.image_processing_enabled:
+            image = await process_image(file, local_image_path, size)
+        else:
+            image = await read_image(file)
+        if not image:
+            logger.warning(f'Failed to process image {file}')
+            return None
+        width, height = image.size
+
+        images.append({
+            'file': local_image_path,
+            'url': image_url,
+            'size': size_name,
+            'width': width,
+            'height': height
+        })
+
+    await asyncio.wait([fn(s) for s in sizes])
+
+    return images
+
+
+async def process_image(input_file, output_file, size):
+    size_name, width, height = size
+
+    if size_name == 'original':
+        return await optimize(input_file, output_file, quality=90)
+    else:
+        image = await read_image(input_file)
+        if not image:
+            return None
+        image_width, image_height = image.size
+        if image_height > image_width:
+            width, height = height, width
+        return await thumbnail(input_file, output_file, (width, height), quality=90)
+
+
+async def read_image(src):
+    import aiofiles
+    from PIL import Image
+    import io
+
+    async with aiofiles.open(src, mode='rb') as f:
+        image_data = await f.read()
+        image = Image.open(io.BytesIO(image_data))
+        return image
+
+
+async def thumbnail(src, dest, size, quality):
+    await image_magick_resize(src, dest, size, quality)
+    return await read_image(dest)
+
+
+async def optimize(src, dest, quality):
+    image = await read_image(src)
+    image.save(dest, quality=quality)
+    return image

modules/sync/sync/models/Page.py

@@ -0,0 +1,204 @@
+import os
+
+import logging
+
+from lxml import etree
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from sync.data import request
+from sync.models import Model
+
+from sync import create_date, images_from_html, title_from_html, create_post
+from sync.models.Image import Image
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Page')
+store = collection(settings.collection_pages)
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Page(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_path = settings.dir_pages
+        items = [
+            Page.read(provider, path)
+            for path in provider.scan(scan_path)
+            if provider.is_dir(path)
+        ]
+        items = [i for i in items if i]
+
+        return items
+
+    @staticmethod
+    def read(provider, path):
+        """
+        :param provider:
+        :param path: path to page manifest file. For example: /Pages/<Some_Page>/<Some_Page>.md
+        :return:
+        """
+        dir_name = os.path.basename(path)
+
+        manifest_file = lambda ext: os.path.join(path, dir_name + ext)
+        manifest_path = manifest_file('.md')
+
+        if not provider.exists(manifest_path):
+            return None
+
+        params = {
+            'file': os.path.basename(manifest_path),
+            'folder': os.path.dirname(manifest_path),
+            'title': os.path.basename(manifest_path),
+        }
+
+        manifest = get_manifest(provider, manifest_path)
+        params = {
+            **params,
+            **manifest,
+        }
+
+        if 'url' not in params:
+            logger.warning('URL not specified for Page {}'.format(manifest_path))
+            return None
+
+        return Page(
+            provider,
+            manifest_path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_pages
+        self.__hash_keys = ['id', 'url']
+        self.images = []
+        self.documents = []
+        self.title = None
+        self.data = None
+        self.preview = None
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        images, documents = kazimir.extract_files(self.get_param('content'))
+        self.params['images'] = images
+        self.params['documents'] = documents
+
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+
+        markdown_post = self.get_param('content')
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.title = title_from_html(post)
+        self.data = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        images = [x.ref for x in self.images]
+        documents = [x.ref for x in self.documents]
+
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'file': self.file,
+            'data': self.data,
+            'images': images,
+            'documents': documents,
+            'title': self.title,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.get_param('id')
+        else:
+            self.id = url_encode_text(self.get_param('url'))
+
+    def __set_url(self):
+        self.url = self.params['url']
+
+    def __set_hash(self):
+        files = []
+        files += self.get_param('images')
+        files += self.get_param('documents')
+        if self.has_param('contentFile'):
+            files += [self.get_param('contentFile')]
+
+        folder = self.get_param('folder')
+        files = [os.path.join(folder, x) for x in files]
+        files = sorted([self.file] + files)
+
+        hashes = [self.provider.hash(x) for x in files]
+        self.hash = hash_str(self.__hash_salt + ''.join(hashes))
+
+    def __str__(self):
+        return '<Page file={} id={}>'.format(self.file, self.id)
+
+
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    if 'contentFile' in manifest:
+        dirname = os.path.dirname(path)
+        manifest['contentHtml'] = provider.read(os.path.join(dirname, manifest['contentFile'])).read().decode('utf-8')
+
+    return manifest

modules/sync/sync/models/Person.py

@@ -0,0 +1,130 @@
+import logging
+
+import os
+
+import settings
+from db import collection
+from sync import create_post
+from sync.data import Provider
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_collective)
+
+
+class Person(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_persons
+        documents = provider.type_filter(scan_dir, '.md')
+        documents = [Person.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Person(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_person
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.picture = None
+        self.preview = None
+        self.post = None
+        self.images = None
+        self.documents = None
+
+        super().__init__(provider, store, file, params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        image_path = self._get_relpath(self.get_param('picture'))
+
+        self.picture = await Image.new(self.provider, image_path, sizes)
+
+        markdown_post = self.get_param('content')
+        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
+        self.images = images
+        self.documents = documents
+
+        if self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = self._get_relpath(preview)
+            self.preview = await Image.new(self.provider, preview, sizes)
+
+    def bake(self):
+        return {
+            **self.params,
+            'post': self.post,
+            'images': [x.ref for x in self.images],
+            'documents': [x.ref for x in self.documents],
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'picture': self.picture.ref,
+            'preview': self.preview.ref if self.preview else None,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['name'])
+
+    def __set_hash(self):
+        files = [self.file]
+        files.append(self._get_relpath(self.get_param('picture')))
+
+        if self.has_param('preview'):
+            files.append(self._get_relpath(self.get_param('preview')))
+
+        hashes = [self.provider.hash(x) for x in files]
+        self.hash = hash_str(
+            self.__hash_salt + ''.join(hashes)
+        )
+
+    def __str__(self):
+        return f'<Person file={self.file} id={self.id}>'
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    data = provider.read(path).read().decode('utf-8')
+
+    manifest = parse_yaml_front_matter(data)
+    return {
+        **manifest,
+        'picture': manifest['picture'] if 'picture' in manifest else os.path.basename(swap_ext('.jpg')(path)),
+        'preview': manifest['preview'] if 'preview' in manifest else os.path.basename(swap_ext('.jpg')(path)),
+        'folder': os.path.dirname(path),
+        'hidden': manifest['hidden'] if 'hidden' in manifest else False,
+    }

modules/sync/sync/models/Schedule.py

@@ -0,0 +1,114 @@
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+store = collection(settings.collection_schedules)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Schedule(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_schedules
+        documents = provider.type_filter(scan_dir, '.yaml')
+        documents += provider.type_filter(scan_dir, '.toml')
+
+        documents = [Schedule.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = get_manifest(provider, path)
+
+        return Schedule(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None
+        if not doc_id:
+            period = self.__get_period()
+            doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester'])
+        self.id = doc_id
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __get_period(self):
+        value = self.params['period']
+        if isinstance(value, str):
+            return value
+
+        if isinstance(value, list):
+            return '-'.join(list(map(str, value)))
+
+    def __str__(self):
+        return '<Schedule hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def combine(fns):
+    def call(value):
+        data = value.read()
+        for f in fns:
+            try:
+                return f(data)
+            except Exception as e:
+                pass
+        return None
+
+    return call
+
+
+def toml(bytes_value):
+    s = bytes_value.decode('utf-8')
+    return toml_str(s)
+
+
+def get_manifest(provider: Provider, path: str) -> dict:
+    transform = combine([yaml, toml, constant({})])
+    manifest = get_data(provider, path, transform)
+
+    return {
+        'file': path,
+        **manifest,
+    }

modules/sync/sync/models/Settings.py

@@ -0,0 +1,64 @@
+from toml import loads as toml_str
+from yaml import load as yaml
+
+import settings
+from db import collection
+from sync.data import Provider, get_data
+from sync.models import Model
+
+from utils.fn import constant
+
+store = collection(settings.collection_settings)
+
+
+class Settings(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
+    @staticmethod
+    async def scan(provider):
+        scan_dir = settings.dir_settings
+        documents = provider.type_filter(scan_dir, '.yaml')
+
+        documents = [Settings.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        params = yaml(provider.read(path))
+
+        return Settings(
+            provider,
+            path,
+            params=params,
+        )
+
+    def __init__(self, provider, file, params=None):
+        super().__init__(provider, store, file, params=params)
+
+    def init(self):
+        pass
+        self.__set_id()
+        self.__set_hash()
+
+    def bake(self):
+        return {
+            **self.params,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+        }
+
+    def __set_id(self):
+        self.id = 'settings'
+
+    def __set_hash(self):
+        self.hash = self.provider.hash(self.file)
+
+    def __str__(self):
+        return f'<Settings file={self.file} id={self.id}>'

modules/sync/sync/models/__init__.py

@@ -0,0 +1,91 @@
+import os
+
+import settings
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Model:
+    def __init__(self, provider, store, file, params=None):
+        super().__init__()
+
+        self.provider = provider
+        self.store = store
+        self.file = file
+
+        self.origin = None
+        self.id = None
+        self.url = None
+        self.hash = None
+        self.ref = None  # Data base reference
+        self.params = params if params else {}
+
+        self.init()
+
+    def init(self):
+        pass
+
+    async def upload(self, **kwargs):
+        pass
+
+    async def build(self, **kwargs):
+        pass
+
+    def bake(self):
+        return None
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        return self._is_changed_hash(i)
+
+    def has_param(self, name):
+        if not self.params:
+            return False
+        return name in self.params
+
+    def get_param(self, name, default_value=None):
+        if not self.params:
+            return default_value
+        if not (name in self.params):
+            return default_value
+        return self.params[name]
+
+    def _set_origin(self):
+        self.origin = f'{settings.origin}:{os.path.dirname(self.file)}'
+
+    def _is_changed_hash(self, item):
+        if not item:
+            return True
+
+        if 'hash' not in item:
+            return True
+
+        return not (self.hash == item['hash'])
+
+    def _filename(self):
+        return os.path.basename(self.file)
+
+    def _get_relpath(self, path):
+        folder = self.get_param('folder', self.file)
+        return os.path.join(folder, path)
+
+    def _get_abs(self, path):
+        return self.provider.get_abs(path)
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            return self
+        except ValueError:
+            pass
+        return None

modules/sync/sync/models/product.py

@@ -0,0 +1,40 @@
+import settings
+from db import collection
+from sync.core.person import SyncPerson
+from utils.text.transform import interpolate_swift
+
+
+def resolve_value(param, store):
+    """
+
+    :param param: \(nv-andreeva)
+    :param store: function should return value by key nv-andreeva
+    :return:
+    """
+    i = interpolate_swift(param)
+    return store(i)
+
+
+def resolve_teacher(i):
+    sync = SyncPerson(
+        collection(settings.collection_collective),
+        None,
+        ''
+    )
+    return resolve_value(i, lambda teacher_id: sync.read({'id': teacher_id}))
+
+
+def create_product(doc):
+    # author, image, title, awards, teacher, date
+
+    # unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
+    # unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
+
+    product = {**doc}
+    # unpop_image(product)
+
+    # product['teacher'] = resolve_teacher(doc['teacher'])
+    # if (product['teacher']):
+        # unpop_teacher(product)
+
+    return product

modules/sync/test/sync/models/test_model.py

@@ -0,0 +1,13 @@
+import unittest
+
+from sync.models import Model
+
+
+class MyTestCase(unittest.TestCase):
+    def test_bake(self):
+        m = Model()
+        self.assertIsNone(m.bake())
+
+
+if __name__ == '__main__':
+    unittest.main()

modules/sync/test/sync/test_document.py

@@ -0,0 +1,27 @@
+import unittest
+
+import settings
+from sync.data import scan_subdirs
+from sync.data.fs import FSProvider
+
+
+class MyTestCase(unittest.TestCase):
+    def test_document(self):
+        p = FSProvider(settings.dir_documents)
+
+        documents = scan_subdirs(p, '.pdf')
+        # documents = lmap(
+        #     lambda document: create_document(p, document),
+        #     documents
+        # )
+
+        self.assertEqual(True, True)
+
+    def test_scan(self):
+        p = FSProvider(settings.dir_documents)
+        documents = scan_subdirs(p, '.pdf')
+        self.assertIsInstance(documents, list)
+
+
+if __name__ == '__main__':
+    unittest.main()

modules/sync/utils/fn.py

@@ -0,0 +1,45 @@
+import os
+from functools import reduce
+
+
+def constant(value):
+    return lambda _: value
+
+
+def combine(items):
+    if len(items) == 0:
+        return []
+
+    return reduce(
+        lambda total, i: total + i,
+        items
+    )
+
+
+def compose(*fns):
+    return lambda i: reduce(
+        lambda out, fn: fn(out),
+        fns, i
+    )
+
+
+first = lambda i: i[0] if len(i) > 0 else None
+
+
+def swap_ext(extension: str):
+    return lambda i: os.path.splitext(i)[0] + extension
+
+
+def map_cases(param, cases):
+    for predicate, fn in cases:
+        if predicate(param):
+            return fn(param)
+    return param
+
+
+def until(condition_fn, iter_fn, items, default_value=None):
+    for i in items:
+        result = iter_fn(i)
+        if condition_fn(result):
+            return result
+    return default_value

modules/sync/utils/hash.py

@@ -0,0 +1,38 @@
+import hashlib
+import json
+
+import bson.json_util
+
+from utils.fn import map_cases
+
+
+def hash_file(file):
+    with open(file, 'rb') as f:
+        data = f.read()
+        return hash_data(data)
+
+
+def hash_data(data):
+    sha = hashlib.sha256(data)
+    return sha.hexdigest()
+
+
+def hash_str(data):
+    to_json = lambda i: json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
+
+    data = map_cases(data, [
+        (lambda i: isinstance(i, dict), to_json),
+        (lambda i: isinstance(i, tuple), to_json),
+        (lambda i: isinstance(i, list), to_json),
+        (lambda i: isinstance(i, int), int),
+        (lambda i: isinstance(i, float), float),
+    ])
+    if isinstance(data, dict):
+        data = to_json(data)
+
+    return hash_data(str(data).encode())
+
+
+def md5(data: str) -> str:
+    i = hashlib.md5(data.encode('utf-8'))
+    return i.hexdigest()

modules/sync/utils/image.py

@@ -0,0 +1,35 @@
+import asyncio
+from subprocess import call
+
+
+async def image_magick_resize(input_file, output_file, size, quality=85):
+    interlace = 'Plane'
+    s = '%dx%d' % size
+
+    cmd = [
+        '/usr/local/bin/convert',
+        '"%s"' % input_file,
+
+        '-strip',
+        '-interlace', interlace,
+        '-auto-orient',
+        '-resize', s,
+        '-quality', str(quality),
+
+        '"%s"' % output_file,
+    ]
+
+    cmd = ' '.join(cmd)
+    process = await asyncio.create_subprocess_shell(cmd)
+    await process.wait()
+
+
+async def image_magick_pdf_to_img(input_file, output_file):
+    call([
+        '/usr/local/bin/gs',
+        '-q',
+        '-sDEVICE=jpeg',
+        '-dLastPage=1',
+        '-o', output_file,
+        input_file,
+    ])

modules/sync/utils/io.py

@@ -0,0 +1,47 @@
+import re
+import yaml
+
+from kazimir import markdown_to_html
+
+FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
+
+
+def read_yaml_md(data):
+    yaml_front_matter, content = split_yaml_front_matter(data)
+    return yaml_front_matter, markdown_to_html(content)
+
+
+def parse_yaml_front_matter(data: str):
+    fm, content = split_yaml_front_matter(data)
+    fm = fm if fm else {}
+    return {
+        **fm,
+        'content': content,
+    }
+
+
+def split_yaml_front_matter(text, **defaults):
+    """
+    Parse text with YAML frontmatter, return metadata and content.
+    Pass in optional metadata defaults as keyword args.
+
+    If frontmatter is not found, returns an empty metadata dictionary
+    and original text content.
+    """
+
+    # metadata starts with defaults
+    metadata = defaults.copy()
+
+    # split on the first two triple-dashes
+    try:
+        _, fm, content = FM_BOUNDARY.split(text, 2)
+    except ValueError:
+        # if we can't split, bail
+        return metadata, text
+
+    # parse yaml, now that we have frontmatter
+    fm = yaml.safe_load(fm)
+    if isinstance(fm, dict):
+        metadata.update(fm)
+
+    return metadata, content.strip()

modules/sync/utils/lxml_utils.py

@@ -0,0 +1,29 @@
+import re
+
+
+def get_following_tags(element, stop_tag_pattr):
+    stop_exp = re.compile(stop_tag_pattr)
+    following = element.xpath('./following-sibling::*')
+    tags = []
+    i = 0
+    while True:
+        try:
+            tag = following[i]
+            if stop_exp.match(tag.tag):
+                break
+            tags.append(tag)
+        except IndexError as tag:
+            break
+        i += 1
+    return tags
+
+
+def get_text_following_by_tag(element, tag, contains, strip=True):
+    e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains))
+    if len(e) > 0:
+        if strip:
+            return e[0].strip()
+        else:
+            return e[0]
+    else:
+        return None

modules/sync/utils/text/file.py

@@ -0,0 +1,29 @@
+import os
+
+import math
+
+
+def default_ext(ext: str):
+    return ext.upper()
+
+
+def get_ext(filename):
+    ext_s = ['.png', '.pdf', '.jpg', '.jpeg', '.gif', '.doc']
+    ext_d = ['ПНГ', 'ПДФ', 'ЖПГ', 'ЖПГ', 'ГИФ', 'ДОК']
+
+    _, ext = os.path.splitext(filename)
+
+    if ext in ext_s:
+        i = ext_s.index(ext)
+        return ext_d[i]
+    else:
+        default_ext(ext)
+
+
+def get_size(number_of_bytes, precision=1):
+    units = ['байт', 'КБ', 'МБ', 'ГБ', 'ТБ', 'ПБ']
+
+    number = math.floor(math.log(number_of_bytes) / math.log(1024))
+    size = (number_of_bytes / math.pow(1024, math.floor(number)))
+    size = round(size, precision)
+    return f'{size} {units[number]}'

modules/sync/utils/text/split.py

@@ -0,0 +1,18 @@
+from utils.fn import until
+
+
+def split(data, separator=' '):
+    try:
+        s = data.index(separator)
+        return data[:s], data[s:]
+    except ValueError:
+        return None, None
+
+
+def split_with(separators):
+    return lambda data: until(
+        lambda i: i[0] and i[1],
+        lambda s: split(data, s),
+        separators,
+        (None, None)
+    )

modules/sync/utils/text/transform.py

@@ -0,0 +1,94 @@
+import re
+
+from unidecode import unidecode
+
+from utils.fn import compose
+
+remove_pattr = re.compile('[«»<>#,!&"\']')
+reduce_dash_patterns = [
+    (re.compile('[-]+'), '-'),
+    (re.compile('^-'), '')
+]
+
+
+def translit(text):
+    """
+    RUSSIAN TEXT => ENGLISH TEXT
+    """
+    return unidecode(text)
+
+
+def interpolate(text, storage, key_fn=None):
+    if not key_fn:
+        key_fn = lambda i: i
+
+    key = key_fn(text)
+    return storage[key] if key in storage else text
+
+
+def interpolate_swift(i):
+    return re.search("([\d\w-]+)", i).group(1)
+
+
+def remove_punctuation(i):
+    return remove_pattr.sub('', i)
+
+
+def reduce_dashes(i):
+    value = i
+    for reg_exp, replace in reduce_dash_patterns:
+        value = reg_exp.sub(replace, value)
+    return value
+
+
+def space_to_dash(i):
+    return i.replace(' ', '-')
+
+
+def short_dash(i):
+    """
+    minus -> dash
+    tire -> dash
+    :param i:
+    :return:
+    """
+    return i.replace('–', '-').replace('—', '-')
+
+
+def dot_to_dash(i):
+    return i.replace('.', '-')
+
+
+def slash_to_dash(i):
+    return i.replace('/', '-')
+
+
+def remove_underscores(i):
+    return i.replace('_', '')
+
+
+def text_lower(i):
+    return i.lower()
+
+
+url_encode_file = compose(
+    translit,
+    space_to_dash,
+    short_dash,
+    remove_underscores,
+    remove_punctuation,
+    reduce_dashes,
+    text_lower,
+)
+
+url_encode_text = compose(
+    translit,
+    space_to_dash,
+    dot_to_dash,
+    short_dash,
+    slash_to_dash,
+    remove_underscores,
+    remove_punctuation,
+    reduce_dashes,
+    text_lower,
+)

modules/sync/utils/validation.py

@@ -0,0 +1,11 @@
+import os
+import re
+import settings
+
+__author__ = 'Roman Timashev'
+
+__check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings.description_available_chars)
+
+
+def check_description_filename(filename):
+    return __check_description_filename_exp.match(filename)

modules/sync/yandexdisk/__init__.py

@@ -0,0 +1,305 @@
+import requests
+
+
+class YDClient:
+    """
+    Implementation of https://tech.yandex.ru/disk/poligon/
+    """
+
+    base_url = 'https://cloud-api.yandex.net/v1/disk'
+
+    def __init__(self, access_token):
+        self.access_token = access_token
+
+        self.base_headers = dict(
+            Accept='application/json',
+            Authorization='OAuth %s' % self.access_token,
+        )
+
+    async def get_flat(self, limit=20):
+        """
+        https://cloud-api.yandex.net/v1/disk/resources/files?
+        [  limit=<количество файлов в списке>]
+        [& media_type=<тип запрашиваемых файлов>]
+        [& offset=<смещение относительно начала списка>]
+        [& fields=<нужные ключи ответа>]
+        [& preview_size=<размер превью>]
+        [& preview_crop=<признак обрезки превью>]
+        :return:
+        """
+
+        url = '{0}/resources/files'.format(self.base_url)
+        payload = dict(
+            limit=limit,
+            offset=0,
+        )
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        # return Directory(**json_dict)
+        return json_dict
+
+    def get_disk_metadata(self):
+        """
+        :return: disk metadata
+        """
+        url = self.base_url
+        r = requests.get(url, headers=self.base_headers)
+        self.check_code(r)
+        disk = r.json()
+        return Disk(
+            trash_size=disk['trash_size'],
+            total_space=disk['total_space'],
+            used_space=disk['used_space'],
+            system_folders=disk['system_folders']
+        )
+
+    def last_uploaded(self):
+        """
+        https://cloud-api.yandex.net/v1/disk/resources/last-uploaded?
+        [  limit=<количество файлов в списке>]
+        [& media_type=<тип запрашиваемых файлов>]
+        [& fields=<нужные ключи ответа>]
+        [& preview_size=<размер превью>]
+        [& preview_crop=<признак обрезки превью>]
+        """
+        url = '{0}/resources/last-uploaded'.format(self.base_url)
+        payload = dict(
+            limit=100,
+        )
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        # return Directory(**json_dict)
+        return json_dict
+
+    def get_content(self, path):
+        """
+        :param path: path to folder
+        :return: content of folder
+        """
+        url = '{0}/resources'.format(self.base_url)
+
+        payload = dict(
+            path=path,
+            limit=1000000
+        )
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        return Directory(**json_dict)
+
+    def create_folder(self, path_to_folder):
+        """
+        :param path_to_folder: path to folder
+        :return: created folder
+        """
+        url = '{0}/resources'.format(self.base_url)
+
+        payload = {'path': path_to_folder}
+        r = requests.put(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        return self.get_content(path_to_folder)
+
+    def remove_folder_or_file(self, path):
+        """
+        Remove folder or file
+        :param path: path to file or folder
+        """
+        url = '{0}/resources'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.delete(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def copy_folder_or_file(self, path_from, path_to):
+        """
+        Copy folder or file
+        :param path_from: from path
+        :param path_to: to path
+        """
+        url = '{0}/resources/copy'.format(self.base_url)
+
+        payload = {'path': path_to, 'from': path_from}
+        r = requests.post(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def get_download_link(self, path):
+        """
+        :param path: path to file
+        :return: download link to file
+        """
+        url = '{0}/resources/download'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        return r.json()
+
+    def get_published_elements(self):
+        """
+        :return: published elements
+        """
+        json_dict = self._get_dictionary_of_published_files()
+
+        elements = []
+        for item in json_dict['items']:
+            if item['type'] == 'dir':
+                elements.append(Directory(**item))
+            elif item['type'] == 'file':
+                elements.append(File(**item))
+        return elements
+
+    def get_public_link_to_folder_or_file(self, path):
+        """
+        :param path: path
+        :return: public link to folder or file
+        """
+        url = '{0}/resources/publish'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.put(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        files = self._get_dictionary_of_published_files()
+        for file in files['items']:
+            if str(file['path']).endswith(path):
+                return file['public_url']
+        return ''
+
+    def unpublish_folder_or_file(self, path):
+        """
+        Unpublish folder of file
+        :param path: path to file or folder
+        """
+        url = '{0}/resources/unpublish'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.put(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def get_list_of_all_files(self):
+        """
+        :return: List of all files
+        """
+        url = '{0}/resources/files'.format(self.base_url)
+
+        r = requests.get(url, headers=self.base_headers)
+        self.check_code(r)
+
+        json_dict = r.json()
+
+        files = []
+        for item in json_dict['items']:
+            f = File(**item)
+            files.append(f)
+        return files
+
+    def move_folder_or_file(self, path_from, path_to):
+        """
+        Move folder or file
+        :param path_from: path from
+        :param path_to: path to
+        """
+        url = '{0}/resources/move'.format(self.base_url)
+
+        payload = {'path': path_to, 'from': path_from}
+        r = requests.post(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def upload_file(self, path_from, path_to):
+        """
+        Upload file
+        :param path_from: path from
+        :param path_to: path to yandex disk
+        """
+        url = '{0}/resources/upload'.format(self.base_url)
+
+        payload = {'path': path_to}
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        upload_link = json_dict["href"]
+
+        with open(path_from, 'rb') as fh:
+            files = {'file': fh}
+
+            r2 = requests.put(upload_link, headers=self.base_headers, files=files)
+            self.check_code(r2)
+
+    def upload_file_from_url(self, from_url, path_to):
+        """
+        Upload file by URL
+        :param from_url: URL path from
+        :param path_to: path to yandex disk
+        """
+        url = '{0}/resources/upload'.format(self.base_url)
+
+        payload = {'path': path_to, 'url': from_url}
+        r = requests.post(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def _get_dictionary_of_published_files(self):
+        url = '{0}/resources/public'.format(self.base_url)
+
+        r = requests.get(url, headers=self.base_headers)
+        self.check_code(r)
+
+        return r.json()
+
+    def check_code(self, req):
+        if not str(req.status_code).startswith('2'):
+            raise YDException(req.status_code, req.text)
+
+
+class YDException(Exception):
+    code = None
+
+    def __init__(self, code, text):
+        super(YDException, self).__init__(text)
+        self.code = code
+
+    def __str__(self):
+        return "%d. %s" % (self.code, super(YDException, self).__str__())
+
+
+class Directory:
+    def __init__(self, **kwargs):
+        self.children = []
+
+        for key in kwargs:
+            if key is not '_embedded':
+                setattr(self, key, kwargs[key])
+
+        if "_embedded" in kwargs:
+            for item in kwargs['_embedded']['items']:
+                if item['type'] == 'dir':
+                    d = Directory(**item)
+                    self.children.append(d)
+
+                if item['type'] == 'file':
+                    f = File(**item)
+                    self.children.append(f)
+
+    def get_children(self):
+        return self.children
+
+
+class File:
+    def __init__(self, **kwargs):
+        for key in kwargs:
+            setattr(self, key, kwargs[key])
+
+
+class Disk:
+    def __init__(self, trash_size, total_space, used_space, system_folders):
+        self.trash_size = trash_size
+        self.total_space = total_space
+        self.used_space = used_space
+        self.system_folders = system_folders