sync moved to modules

modules/sync/kazimir/CSVToken.py

@@ -0,0 +1,46 @@
+import csv
+import io
+
+from kazimir.Token import FileToken, is_file
+
+extensions = ['.csv']
+
+
+class CSVToken(FileToken):
+    @staticmethod
+    def test(data: str):
+        return is_file(data, extensions)
+
+    def __init__(self, data) -> None:
+        super().__init__(data=data)
+        self.name = 'csv'
+        self.joinable = False
+        self.build = None
+
+    async def compile(self):
+        data = await self.build(self.data)
+
+        f = io.StringIO(data)
+        reader = csv.reader(f, delimiter=',', quotechar='|')
+
+        headers = next(reader, None)
+
+        row = lambda xs: '\n'.join([
+            f'<td>{x}</td>' for x in xs
+        ])
+
+        head = '\n'.join([
+            f'<th>{x}</th>' for x in headers
+        ])
+        body = '\n'.join([
+            f'<tr>{row(x)}</tr>' for x in reader
+        ])
+
+        return f'''
+            <table class="MarkerTable">
+                <thead><tr>
+                    {head}
+                </tr></thead>
+                <tbody>{body}</tbody>
+            </table>
+        '''.strip()

modules/sync/kazimir/DocumentToken.py

@@ -0,0 +1,42 @@
+from kazimir.Token import parse_file, is_document, Token
+
+
+class DocumentToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_document(data)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='file', data=data)
+        self.joinable = False
+        self.build = None
+
+    async def compile(self):
+        data = parse_file(self.data)
+        document = await self.build(data)
+
+        return create_document(document)
+
+
+def create_document(document):
+    return f'''
+    <div>
+        <div class="document-row">
+            <a href="{document['url']}" class="invisible">
+                <div class="document-row__image">
+                    <img src="{document['image_url']}" alt="{document['title']}">
+                </div>
+            </a>
+
+            <div class="document-row__file">
+                <a href="{document['url']}">{document['title']}</a>
+            </div>
+
+            <div class="document-row__file-info">
+                <a href="{document['file_url']}" target="_blank">
+                    {document['file_format']} ({document['file_size']})
+                </a>
+            </div>
+        </div>
+    </div>
+    '''.strip()

modules/sync/kazimir/DocumentUrlToken.py

@@ -0,0 +1,37 @@
+from kazimir.Token import UrlToken, parse_link
+
+
+class DocumentUrlToken(UrlToken):
+    @staticmethod
+    def test(data: str):
+        link = parse_link(data)
+        if not link:
+            return False
+        url = link['url']
+        host = url.netloc
+        path = url.path
+        return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org')
+
+    def __init__(self, data) -> None:
+        super().__init__(data)
+        self.build = None
+
+    async def compile(self):
+        from kazimir.DocumentToken import create_document
+
+        link = parse_link(self.data)
+        url = link['url']
+        doc_id = url.path.replace('/document/', '')
+        doc = await self.build(doc_id)
+
+        if not doc:
+            raise Exception(f'Document {doc_id} not found')
+
+        # return f'''DOCUMENT: {# doc['title']}'''
+        return create_document(doc)
+
+    def merge(self, token):
+        return self
+
+    def __repr__(self) -> str:
+        return f'[Token {self.name} {self.data}]'

modules/sync/kazimir/InstagramToken.py

@@ -0,0 +1,72 @@
+from urllib.parse import urlparse
+
+import re
+import requests
+from markdown.util import etree
+
+from kazimir.Token import UrlToken, is_url
+
+pattr = re.compile(r'https?://(www\.)?instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$')
+
+
+class InstagramToken(UrlToken):
+    @staticmethod
+    def test(data: str):
+        if not is_url(data):
+            return False
+
+        url = urlparse(data)
+        return url.netloc in ['www.instagram.com', 'instagram.com']
+
+    async def compile(self):
+        match = pattr.match(self.data)
+        if match:
+            m = match.group('media')
+            return await embed_instagram(m)
+
+
+async def embed_instagram(media):
+    """
+    <blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
+        <div style="padding:8px;">
+            <div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
+                <div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
+                </div>
+            </div>
+
+            <p style=" margin:8px 0 0 0; padding:0 4px;">
+                <a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
+                    Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
+                </a>
+            </p>
+
+            <p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
+                A post shared by Шлиссельбургская ДХШ (@hudozka) on
+
+                <time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
+                    May 24, 2017 at 7:05am PDT
+                </time>
+            </p>
+        </div>
+    </blockquote>
+    """
+
+    url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
+    req = requests.get(url)
+    if req.status_code == 200:
+        html = req.json()['html']
+        html = fix_html(html)
+        i = etree.fromstring(html)
+
+        obj = etree.Element('div')
+        obj.set('class', "kazimir__embed")
+        obj.append(i)
+        return etree.tostring(obj).decode()
+    else:
+        raise Exception(
+            'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
+        )
+
+
+def fix_html(html: str):
+    return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')

modules/sync/kazimir/Marker.py

@@ -0,0 +1,118 @@
+import lxml.html
+from markdown import markdown
+
+from kazimir.Token import SplitToken
+
+
+class Marker:
+    def __init__(self) -> None:
+        super().__init__()
+        self.tokens = []
+        self.tree_middlewares = []
+
+    def add_token_factory(self, token):
+        self.tokens.append(token)
+
+    def add_tree_middleware(self, callback):
+        self.tree_middlewares.append(callback)
+
+    async def create_tree(self, text: str):
+        tokens = text.split('\n')
+        tokens = self.markup_tokens(tokens)
+        tokens = join_tokens(tokens)
+        tokens = merge_tokens(tokens)
+
+        compiled = await compile_tokens(tokens)
+        tree = lxml.html.fromstring(compiled)
+        return await self.process_tree(tree)
+
+    async def process_tree(self, tree):
+        for fn in self.tree_middlewares:
+            tree = await fn(tree)
+        return tree
+
+    def markup_tokens(self, tokens: list):
+        marked_tokens = []
+
+        for data in tokens:
+            t = self.create_token(data)
+            if t:
+                marked_tokens.append(t)
+            else:
+                raise Exception(f'Token not recognized for data {data}')
+        return marked_tokens
+
+    def create_token(self, data):
+        for f in self.tokens:
+            if f.token.test(data):
+                return f.create(data=data)
+        return None
+
+
+def join_tokens(tokens):
+    is_split = lambda x: isinstance(x, SplitToken)
+
+    buffer_tag = None
+    buffer = []
+    ts = []
+    for token in tokens:
+        tag = token.name
+        changed = buffer_tag is not None and tag != buffer_tag
+
+        flush = is_split(token)
+        if changed:
+            flush = True
+
+        if not token.joinable:
+            flush = True
+
+        if flush:
+            if buffer:
+                ts.append(buffer)
+            if is_split(token):
+                buffer = []
+                buffer_tag = None
+            else:
+                buffer = [token]
+                buffer_tag = tag
+        else:
+            buffer_tag = tag
+            buffer.append(token)
+
+    if len(buffer):
+        ts.append(buffer)
+    return ts
+
+
+def merge_tokens(tokens: list):
+    tokens_out = []
+    for ts in tokens:
+        token = ts[0]
+        for x in ts[1:]:
+            token = token.merge(x)
+        tokens_out.append(token)
+    return tokens_out
+
+
+async def compile_tokens(tokens: list):
+    result = []
+    for token in tokens:
+        data = await token.compile()
+        result.append(data)
+    return '\n'.join(result)
+
+
+def extract_files(text: str) -> [str]:
+    html = markdown(text)
+    post_html = lxml.html.fromstring(html)
+
+    images = post_html.cssselect('img')
+    images = [img.get('src') for img in images]
+
+    return images, []
+
+
+def html_from_tree(tree):
+    html = lxml.html.tostring(tree, encoding='unicode')
+
+    return html

modules/sync/kazimir/MarkerRun.py

@@ -0,0 +1,113 @@
+import asyncio
+
+from kazimir.DocumentUrlToken import DocumentUrlToken
+from kazimir.DocumentToken import DocumentToken
+from kazimir.CSVToken import CSVToken
+from kazimir.InstagramToken import InstagramToken
+from kazimir.Marker import Marker, html_from_tree
+from kazimir.YoutubeToken import YoutubeToken
+from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \
+    BuildTokenFactory
+from sync import Document
+
+if __name__ == '__main__':
+    sample_text = '''
+Hello!
+
+https://www.instagram1.com/p/BUelE04l1kU
+
+https://www.youtube.com/watch?v=qnguOqWpraI
+https://www.youtube.com/watch?v=qnguOqWpraI
+![](1.jpg)
+![](2.jpg)
+
+LOL NOOB
+![](12.jpg)
+![](22.jpg)
+WTF OMG
+
+![](/document/documents-arkhiv-ustav-2013-pdf)
+
+![File](file.pdf)
+hello.pdf
+file.doc
+sheet.xlsx
+
+print.docx
+image.jpg 1
+image.png 2
+
+image.jpeg Hi
+
+Bye!
+Fye!
+
+![](1.jpg)
+
+["Ya"](http://ya.ru)
+
+More text
+
+file.csv
+    '''
+
+
+    async def build_document(data):
+        text = data['caption'] if data['caption'] else data['file']
+        # document = await self.build()
+        document = {
+            'url': data['file'],
+            'image_url': data['file'],
+            'file_url': data['file'],
+            'title': text,
+            'file_size': '0 KB',
+            'file_format': 'PDF',
+        }
+        return document
+
+    async def read_csv(data):
+        return '\n'.join([
+            'N,Name',
+            f'0,{data}',
+            '1,Pop',
+            '2,Top',
+        ])
+
+    async def read_document(data):
+        doc = await Document.find_one({'id': data})
+        return {
+            'url': doc['url'],
+            'image_url': doc['url'],
+            'file_url': doc['url'],
+            'title': doc['title'],
+            'file_size': '0 KB',
+            'file_format': 'PDF',
+        }
+
+
+    async def run():
+        from kazimir.fix_links_quotes import fix_links_quotes
+
+        m = Marker()
+        m.add_token_factory(TokenFactory(SplitToken))
+        m.add_token_factory(TokenFactory(YoutubeToken))
+        m.add_token_factory(TokenFactory(InstagramToken))
+        m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document))
+        m.add_token_factory(TokenFactory(UrlToken))
+        # m.add_token_factory(TokenFactory(ImageToken))
+        m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
+        m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv))
+        m.add_token_factory(TokenFactory(FileToken))
+        m.add_token_factory(TokenFactory(TextToken))
+        m.add_tree_middleware(fix_links_quotes)
+        sample_tree = await m.create_tree(sample_text)
+
+        # for x in sample_tree:
+        #     print(x)
+
+        print(html_from_tree(sample_tree))
+
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run())
+    loop.close()

modules/sync/kazimir/Token.py

@@ -0,0 +1,244 @@
+import os
+import re
+from markdown import markdown
+from urllib.parse import urlparse
+
+image_extensions = ['.jpg', '.jpeg', '.png']
+document_extensions = ['.doc', '.docx', '.pdf', '.xls', '.xlsx']
+
+
+class TokenFactory:
+    def __init__(self, token) -> None:
+        super().__init__()
+        self.token = token
+
+    def create(self, data):
+        return self.token(data=data)
+
+
+class BuildTokenFactory(TokenFactory):
+    def __init__(self, token, build) -> None:
+        super().__init__(token)
+        self.build = build
+
+    def create(self, data):
+        x = self.token(data=data)
+        x.build = self.build
+        return x
+
+
+class Token:
+    @staticmethod
+    def test(data: str):
+        return True
+
+    def __init__(self, name, data) -> None:
+        super().__init__()
+        self.name = name
+        self.data = data
+        self.joinable = True
+
+    async def compile(self):
+        return str(self.data)
+
+    def merge(self, token):
+        return self
+
+    def __repr__(self) -> str:
+        return f'[Token {self.name} {self.data}]'
+
+
+class SplitToken(Token):
+    @staticmethod
+    def test(data: str):
+        data = data.strip()
+        return data == ''
+
+    def __init__(self, data) -> None:
+        super().__init__(name='split', data=None)
+
+
+class TextToken(Token):
+    @staticmethod
+    def test(data: str):
+        return True
+
+    def __init__(self, data) -> None:
+        super().__init__(name='text', data=data)
+
+    def merge(self, token: Token):
+        data = self.data
+        data += '\n'
+        data += token.data
+        return TextToken(data.strip())
+
+    async def compile(self):
+        return markdown_to_html(self.data)
+
+
+class FileToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_file(data, document_extensions + image_extensions)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='file', data=data)
+        self.joinable = False
+
+    async def compile(self):
+        data = parse_file(self.data)
+        url = data['file']
+        text = data['caption'] if data['caption'] else url
+        return f'<a href="{url}">{text}</a>'
+
+
+class ImageToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_image(data)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='image', data=data)
+        self.joinable = True
+        self.build = None
+
+    def merge(self, token: Token):
+        merged_token = ImageCollectionToken([self.data, token.data])
+        merged_token.build = self.build
+        return merged_token
+
+    async def compile(self):
+        img = self.parse_data()
+        img = await self.build(img)
+        alt = img['alt']
+        caption = markdown_to_html(img['caption'])
+        src = img['src']
+
+        tpl = f'''
+            <div class="kazimir__image">
+                <figure>
+                    <img alt="{alt}" src="{src}">
+                    <figcaption>{caption}</figcaption>
+                </figure>
+            </div>
+        '''
+        return tpl.format(img).strip()
+
+    def parse_data(self):
+        return parse_file(self.data)
+
+
+class ImageCollectionToken(ImageToken):
+    def __init__(self, data) -> None:
+        super().__init__(data)
+        self.name = 'image_collection'
+        self.build = None
+
+    def merge(self, token: Token):
+        if isinstance(token.data, list):
+            data = token.data
+        else:
+            data = [token.data]
+        merged_token = ImageCollectionToken([*self.data, *data])
+        merged_token.build = self.build
+        return merged_token
+
+    async def compile(self):
+        data = self.parse_data()
+        images = []
+        for x in data:
+            img = await self.build(x)
+            images.append(img)
+
+        images = [x['src'] for x in images]
+        images = [f'<img src="{x}">' for x in images]
+        images = '\n'.join(images)
+
+        return f'''
+            <div class="kazimir__image-collection">
+                <div class="fotorama"
+                     data-width="100%"
+                     data-ratio="800/600"
+                >
+                    {images}
+                </div>
+            </div>
+        '''
+
+    def parse_data(self):
+        return [parse_file(x) for x in self.data]
+
+
+class UrlToken(Token):
+    @staticmethod
+    def test(data: str):
+        return is_url(data)
+
+    def __init__(self, data) -> None:
+        super().__init__(name='url', data=data)
+        self.joinable = False
+
+
+def markdown_to_html(text):
+    return markdown(text)
+
+
+def is_url(sample: str):
+    url = urlparse(sample)
+    return url.scheme and url.netloc
+
+
+def is_image(sample: str):
+    file = parse_file(sample)['file']
+    return is_file(file, image_extensions)
+
+
+def is_document(sample: str):
+    file = parse_file(sample)['file']
+    return is_file(file, document_extensions)
+
+
+def is_file(sample: str, extensions):
+    name, ext = os.path.splitext(sample)
+    ext = ext.lower()
+    return ext in extensions
+
+
+def parse_file(sample: str):
+    """
+        ![Caption](file.jpg) -> {file, caption}
+        file.jpg Caption -> {file, caption}
+
+        :param sample:
+        :return:
+        """
+    p = re.match(r'!\[(.*)\]\((.*)\)', sample)
+    if p:
+        caption = p.group(1)
+        file = p.group(2)
+    else:
+        sample = sample.split(' ')
+        file = sample[0]
+        caption = ' '.join(sample[1:])
+    return {
+        'file': file,
+        'caption': caption,
+    }
+
+
+def parse_link(sample: str):
+    """
+    ![Alt](href) -> {href, alt}
+
+    :param sample:
+    :return:
+    """
+    p = re.match(r'!\[(.*)\]\((.*)\)', sample)
+    if not p:
+        return None
+
+    url = p.group(2)
+    return {
+        'url': urlparse(url),
+        'alt': p.group(1),
+    }

modules/sync/kazimir/YoutubeToken.py

@@ -0,0 +1,32 @@
+from urllib.parse import urlparse, parse_qs
+from kazimir.Token import UrlToken, is_url
+
+
+class YoutubeToken(UrlToken):
+    @staticmethod
+    def test(data: str):
+        if not is_url(data):
+            return False
+
+        url = urlparse(data)
+        return url.netloc in ['www.youtube.com', 'youtube.com']
+
+    async def compile(self):
+        """
+        <div class="kazimir__video">
+            <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
+        </div>
+        """
+
+        url = urlparse(self.data)
+        query = parse_qs(url.query)
+
+        if 'v' not in query:
+            return ''
+
+        video_id = query['v'][0]
+
+        url = f'//www.youtube.com/embed/{video_id}'
+        return f'''
+            <div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>
+        '''.strip()

modules/sync/kazimir/__init__.py

@@ -0,0 +1,320 @@
+import lxml.html
+import re
+from markdown import markdown
+
+from kazimir.figure import FigureExtension
+from kazimir.instagram import InstagramExtension
+from sync.data import request
+
+quote_pairs = [
+    ('"', '"'),
+    ('«', '»'),
+]
+
+
+def markdown_to_html(text):
+    return markdown(text, extensions=[
+        FigureExtension(),
+        InstagramExtension(),
+    ])
+
+
+def extract_files(text: str) -> [str]:
+    html = markdown(text)
+    post_html = lxml.html.fromstring(html)
+
+    images = post_html.cssselect('img')
+    images = [img.get('src') for img in images]
+
+    return images, []
+
+
+def create_tree(text: str) -> str:
+    tokens = text.split('\n')
+    tokens = markup_tokens(tokens)
+    tokens = join_tokens(tokens)
+    tokens = merge_tokens(tokens)
+
+    kz = compile_tokens(tokens)
+    tree = lxml.html.fromstring(kz)
+    return clean_tree(tree)
+
+
+def clean_tree(tree):
+    return fix_links_quotes(tree)
+
+
+def fix_links_quotes(tree):
+    safe = lambda x: x if x else ''
+
+    for link in tree.iterlinks():
+        a = link[0]
+        if a.tag != 'a':
+            continue
+
+        quote = get_quote_pair(a.text)
+        if not quote:
+            continue
+
+        # New text of A without quotes
+        a.text = a.text[1:-1]
+
+        # Move first quote out of A to the prefix
+        a.getparent().text = safe(a.getparent().text) + quote[0]
+
+        # Move last quote out of A to the suffix
+        a.tail = quote[1] + safe(a.tail)
+
+    return tree
+
+
+def get_quote_pair(text: str):
+    """
+    sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
+        ? (s, t)
+        : None
+
+    :param text:
+    :return:
+    """
+    if len(text) < 2:
+        return None
+
+    first = text[0]
+    last = text[-1]
+
+    for pair_first, pair_last in quote_pairs:
+        if pair_first == first and pair_last == last:
+            return first, last
+
+    return None
+
+
+def html_from_tree(tree):
+    html = lxml.html.tostring(tree, encoding='unicode')
+
+    return html
+
+
+async def typo(text):
+    from kazimir import artlebedev
+    return await artlebedev.typograf(text)
+    # url = 'http://www.typograf.ru/webservice/'
+    # params = dict(text=text)
+    # return await request.post(url, params, {'chr': 'utf-8'})
+
+
+def join_tokens(tokens):
+    buffer_tag = None
+    buffer = []
+    ts = []
+    for token in tokens:
+        tag = token['tag']
+        changed = buffer_tag is not None and tag != buffer_tag
+
+        flush = is_tag_split(tag)
+        if changed:
+            flush = True
+
+        if not is_tag_joinable(tag):
+            flush = True
+
+        if flush:
+            if buffer:
+                ts.append(buffer)
+            if is_tag_split(tag):
+                buffer = []
+                buffer_tag = None
+            else:
+                buffer = [token]
+                buffer_tag = tag
+        else:
+            buffer_tag = tag
+            buffer.append(token)
+
+    if len(buffer):
+        ts.append(buffer)
+    return ts
+
+
+def merge_tokens(tokens: list):
+    tokens_out = []
+    for ts in tokens:
+        tag = ts[0]['tag']
+        fn = None
+        if tag == 'image':
+            fn = merge_image
+        elif tag == 'text':
+            fn = merge_text
+        else:
+            fn = merge_any
+        tokens_out.append(fn(ts))
+    return tokens_out
+
+
+def compile_tokens(tokens: list):
+    compiled = []
+    for ts in tokens:
+        tag = ts['tag']
+        fn = None
+        if tag == 'text':
+            fn = compile_text
+        elif tag == 'image_collection':
+            fn = compile_image_collection
+        elif tag == 'image':
+            fn = compile_image
+        elif tag == 'url':
+            fn = compile_url
+        else:
+            fn = compile_any
+        compiled.append(fn(ts))
+    return '\n'.join(compiled)
+
+
+def markup_tokens(tokens: list):
+    marked_tokens = []
+    for token in tokens:
+        tag = 'text'
+        if is_url(token):
+            tag = 'url'
+        elif is_image(token):
+            tag = 'image'
+        elif not token:
+            tag = 'split'
+            token = None
+
+        marked_tokens.append({
+            'tag': tag,
+            'data': token,
+        })
+    return marked_tokens
+
+
+def is_tag_split(tag):
+    return tag == 'split'
+
+
+def is_tag_joinable(tag: str):
+    return tag not in ['url']
+
+
+def is_url(sample: str):
+    from urllib.parse import urlparse
+    url = urlparse(sample)
+
+    return url.scheme and url.netloc
+
+
+def is_image(sample: str):
+    return sample[:2] == '![' and sample[-1:] == ')'
+
+
+def merge_any(tokens):
+    return tokens[0]
+
+
+def merge_text(tokens):
+    data = ''
+
+    for t in tokens:
+        data += t['data']
+        data += '\n'
+
+    return {
+        'tag': 'text',
+        'data': data.strip(),
+    }
+
+
+def merge_image(tokens):
+    if len(tokens) == 1:
+        return tokens[0]
+    else:
+        data = [t['data'] for t in tokens]
+        return {
+            'tag': 'image_collection',
+            'data': data,
+        }
+
+
+def compile_text(token):
+    return markdown_to_html(token['data'])
+
+
+def compile_image_collection(token):
+    file = lambda i: i[4:][:-1]
+    imgs = ['<img src="{}">'.format(file(i)) for i in token['data']]
+    return '''
+        <div class="kazimir__image-collection">
+            <div class="fotorama"
+                 data-width="100%"
+                 data-ratio="800/600"
+            >
+                {}
+            </div>
+        </div>
+    '''.format('\n'.join(imgs))
+
+
+def compile_image(token):
+    img = markdown_to_html(token['data'])
+    tpl = '''<div class="kazimir__image">{}</div>'''
+    return tpl.format(img)
+
+
+def compile_url(token):
+    return compile_youtube(token)
+
+
+def compile_youtube(token):
+    """
+    <div class="kazimir__video">
+     <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
+   </div>
+    """
+
+    youtube = r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P<youtubeargs>[A-Za-z0-9_&=-]+)\S*'
+    youtube_pattr = re.compile(youtube)
+    match = youtube_pattr.match(token['data'])
+    if match:
+        video_id = match.group(2)
+        url = f'//www.youtube.com/embed/{video_id}'
+        tpl = '''<div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>'''
+
+        return tpl.format(url=url)
+    else:
+        return ''
+
+
+def compile_any(token):
+    return markdown_to_html(token['data'])
+
+
+if __name__ == '__main__':
+    sample_text = '''
+Hello!
+
+https://www.instagram.com/p/BUelE04l1kU
+ku!
+
+https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka
+https://www.youtube.com/watch?v=qnguOqWpraI
+![](1.jpg)
+![](2.jpg)
+![](3.jpg)
+
+LOL NOOB
+![](12.jpg)
+![](22.jpg)
+WTF OMG
+
+Bye!
+Fye!
+
+![](1.jpg)
+
+More text
+    '''
+
+    sample_tree = create_tree(sample_text)
+    print(html_from_tree(sample_tree))

modules/sync/kazimir/artlebedev.py

@@ -0,0 +1,45 @@
+from sync.data import request
+from lxml import etree
+
+
+async def typograf(text, options={}):
+    text = text.replace('&', '&')
+    text = text.replace('<', '&lt;')
+    text = text.replace('>', '&gt;')
+
+    body = '''
+    <?xml version="1.0" encoding="UTF-8"?>
+    <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
+    <soap:Body>
+     <ProcessText xmlns="http://typograf.artlebedev.ru/webservices/">
+      <text>{text}</text>
+      <entityType>1</entityType>
+      <useBr>0</useBr>
+      <useP>1</useP>
+      <maxNobr>3</maxNobr>
+      </ProcessText>
+     </soap:Body>
+    </soap:Envelope>
+    '''.strip().format(text=text)
+
+    url = 'http://typograf.artlebedev.ru/webservices/typograf.asmx'
+    resp = await request.post(url, body, {}, {
+        'Content-Type': 'text/xml',
+        'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText',
+    })
+    resp_text = resp.encode()
+
+    try:
+        # create xml tree from response
+        response_tree = etree.fromstring(resp_text)
+
+        # look for ProcessTextResult
+        typograf_response = response_tree.getchildren()[0].getchildren()[0].getchildren()[0].text
+    except Exception as response_tree:
+        print(response_tree)
+        return text
+
+    typograf_response = typograf_response.replace('&amp;', '&')
+    typograf_response = typograf_response.replace('&lt;', '<')
+    typograf_response = typograf_response.replace('&gt;', '>')
+    return typograf_response

modules/sync/kazimir/figure.py

@@ -0,0 +1,44 @@
+from markdown import Extension
+from markdown.inlinepatterns import IMAGE_LINK_RE, IMAGE_REFERENCE_RE
+from markdown.blockprocessors import BlockProcessor
+from markdown.util import etree
+import re
+
+FIGURES = [
+    u'^\s*' + IMAGE_LINK_RE + u'\s*$',
+    u'^\s*' + IMAGE_REFERENCE_RE + u'\s*$',
+]
+
+TAG_FIGURE = 'figure'
+TAG_FIGCAPTION = 'figcaption'
+NAME = 'FigureAltCaption'
+
+
+class FigureProcessor(BlockProcessor):
+    FIGURES_RE = re.compile('|'.join(f for f in FIGURES))
+
+    def test(self, parent, block):
+        is_image = bool(self.FIGURES_RE.search(block))
+        is_single_list = (len(block.splitlines()) == 1)
+        is_in_figure = parent.tag == TAG_FIGURE
+
+        return is_image and is_single_list and not is_in_figure
+
+    def run(self, parent, blocks):
+        raw_block = blocks.pop(0)
+        caption_text = self.FIGURES_RE.search(raw_block).group(1)
+
+        figure = etree.SubElement(parent, TAG_FIGURE)
+        figure.text = raw_block
+
+        figcaption_elem = etree.SubElement(figure, TAG_FIGCAPTION)
+        figcaption_elem.text = caption_text
+
+
+class FigureExtension(Extension):
+    def extendMarkdown(self, md, md_globals):
+        md.parser.blockprocessors.add(
+            NAME,
+            FigureProcessor(md.parser),
+            '<ulist'
+        )

modules/sync/kazimir/fix_links_quotes.py

@@ -0,0 +1,50 @@
+quote_pairs = [
+    ('"', '"'),
+    ('«', '»'),
+]
+
+
+async def fix_links_quotes(tree):
+    safe = lambda x: x if x else ''
+
+    for link in tree.iterlinks():
+        a = link[0]
+        if a.tag != 'a':
+            continue
+
+        quote = get_quote_pair(a.text)
+        if not quote:
+            continue
+
+        # New text of A without quotes
+        a.text = a.text[1:-1]
+
+        # Move first quote out of A to the prefix
+        a.getparent().text = safe(a.getparent().text) + quote[0]
+
+        # Move last quote out of A to the suffix
+        a.tail = quote[1] + safe(a.tail)
+
+    return tree
+
+
+def get_quote_pair(text: str):
+    """
+    sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
+        ? (s, t)
+        : None
+
+    :param text:
+    :return:
+    """
+    if len(text) < 2:
+        return None
+
+    first = text[0]
+    last = text[-1]
+
+    for pair_first, pair_last in quote_pairs:
+        if pair_first == first and pair_last == last:
+            return first, last
+
+    return None

modules/sync/kazimir/instagram.py

@@ -0,0 +1,68 @@
+import requests
+from markdown import Extension
+from markdown.inlinepatterns import Pattern
+from markdown.util import etree
+
+
+class InstagramExtension(Extension):
+    def add_inline(self, md, name, klass, re):
+        pattern = klass(re)
+        pattern.md = md
+        pattern.ext = self
+        md.inlinePatterns.add(name, pattern, "<reference")
+
+    def extendMarkdown(self, md, md_globals):
+        match_url = r'https?://www\.instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$'
+        self.add_inline(md, 'instagram', Instagram, match_url)
+
+
+class Instagram(Pattern):
+    def handleMatch(self, m):
+        return embed_instagram(m.group('media'))
+
+
+def embed_instagram(media):
+    """
+    <blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
+        <div style="padding:8px;">
+            <div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
+                <div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
+                </div>
+            </div>
+
+            <p style=" margin:8px 0 0 0; padding:0 4px;">
+                <a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
+                    Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
+                </a>
+            </p>
+
+            <p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
+                A post shared by Шлиссельбургская ДХШ (@hudozka) on
+
+                <time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
+                    May 24, 2017 at 7:05am PDT
+                </time>
+            </p>
+        </div>
+    </blockquote>
+    """
+
+    url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
+    req = requests.get(url)
+    if req.status_code == 200:
+        html = req.json()['html']
+        html = fix_html(html)
+        i = etree.fromstring(html)
+
+        obj = etree.Element('div')
+        obj.set('class', "kazimir__embed")
+        obj.append(i)
+        return obj
+    else:
+        raise Exception(
+            'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
+        )
+
+
+def fix_html(html: str):
+    return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')