[Sync] Article post creation updated. Kazimir updated: video is not part of markdown

utils/Sync/kazimir/__init__.py

@@ -1,54 +1,45 @@
 import lxml.html
-import requests
 from markdown import markdown
 
-from kazimir.video import VideoExtension
 from kazimir.figure import FigureExtension
 from kazimir.instagram import InstagramExtension
+from sync.data import request
 
 
 def markdown_to_html(text):
     return markdown(text, extensions=[
         FigureExtension(),
-        VideoExtension(),
         InstagramExtension(),
     ])
 
 
-def kazimir_to_html(text):
-    html = make(text)
-    return typo(html)
+def get_images_src(text: str) -> [str]:
+    html = markdown(text)
+    post_html = lxml.html.fromstring(html)
+
+    images = post_html.cssselect('img')
+    return [img.get('src') for img in images]
 
 
-def html_from_tree(tree):
-    html = lxml.html.tostring(tree, encoding='unicode')
-    html = typo(html)
-
-    return html
-
-
-def typo(html):
-    url = 'http://api.tmshv.com/typograph/v1'
-    res = requests.post(
-        url=url,
-        headers={
-            'Content-Type': 'text/plain; charset=utf-8',
-        },
-        data=html.encode('utf-8')
-    )
-    if res.status_code == 200:
-        return res.text
-    return html
-
-
-def make(text):
+def kazimir_to_html(text: str) -> str:
     tokens = text.split('\n')
     tokens = markup_tokens(tokens)
     tokens = join_tokens(tokens)
     tokens = merge_tokens(tokens)
 
     return compile_tokens(tokens)
-    # return tokens
+
+
+def html_from_tree(tree):
+    html = lxml.html.tostring(tree, encoding='unicode')
+
+    return html
+
+
+async def typo(text):
+    url = 'http://www.typograf.ru/webservice/'
+    params = dict(text=text)
+    return await request.post(url, params, {'chr': 'utf-8'})
 
 
 def join_tokens(tokens):
@@ -110,6 +101,8 @@ def compile_tokens(tokens: list):
             fn = compile_image_collection
         elif tag == 'image':
             fn = compile_image
+        elif tag == 'url':
+            fn = compile_url
         else:
             fn = compile_any
         compiled.append(fn(ts))
@@ -203,11 +196,25 @@ def compile_image_collection(token):
 
 def compile_image(token):
     img = markdown_to_html(token['data'])
-    return '''
-        <div class="kazimir__image">
-            {}
-        </div>
-    '''.format(img)
+    tpl = '''<div class="kazimir__image">{}</div>'''
+    return tpl.format(img)
+
+
+def compile_url(token):
+    return compile_youtube(token)
+
+
+def compile_youtube(token):
+    """
+    <div class="kazimir-video">
+     <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
+   </div>
+    """
+
+    url = token['data']
+    tpl = '''<div class="kazimir-video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>'''
+
+    return tpl.format(url=url)
 
 
 def compile_any(token):
@@ -246,4 +253,3 @@ More text
             print(i)
     else:
         print(ts)
-

utils/Sync/kazimir/video.py

@@ -1,38 +0,0 @@
-from markdown import Extension
-from markdown.inlinepatterns import Pattern
-from markdown.util import etree
-
-
-class VideoExtension(Extension):
-    def add_inline(self, md, name, klass, re):
-        pattern = klass(re)
-        pattern.md = md
-        pattern.ext = self
-        md.inlinePatterns.add(name, pattern, "<reference")
-
-    def extendMarkdown(self, md, md_globals):
-        youtube = r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P<youtubeargs>[A-Za-z0-9_&=-]+)\S*'
-        self.add_inline(md, 'youtube', Youtube, youtube)
-
-
-class Youtube(Pattern):
-    def handleMatch(self, m):
-        url = 'https://www.youtube.com/embed/%s' % m.group('youtubeargs')
-        return flex_video(url)
-
-
-def flex_video(url):
-    """
-    <div class="kazimir-video">
-     <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
-   </div>
-    """
-    iframe = etree.Element('iframe')
-    iframe.set('src', url)
-    iframe.set('frameborder', '0')
-    iframe.set('allowfullscreen', '')
-
-    obj = etree.Element('div')
-    obj.set('class', "kazimir__video")
-    obj.append(iframe)
-    return obj

utils/Sync/settings.py

@@ -129,6 +129,8 @@ collection_collective = 'collective'
 collection_albums = 'albums'
 collection_pages = 'pages'
 
+hash_salt_articles = '0'
+
 provider_name = value('provider')['name']  # env('SYNC_PROVIDER', 'fs')
 provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
 

utils/Sync/sync/data/request.py

@@ -34,6 +34,12 @@ async def get(url):
         return await res.text()
 
 
+async def post(url, data, params={}):
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, data=data, params=params) as resp:
+            return await resp.text()
+
+
 async def put(url, data):
     async with aiohttp.ClientSession as session:
         res = session.put(url, data=data)

utils/Sync/sync/models/Article.py

@@ -49,16 +49,10 @@ class Article(Model):
             'title': doc_title,
         }
 
-        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
-        images = list_images(provider, path)
-        images = [os.path.relpath(i, path) for i in images]
-        document['images'] = images
-        document['post'] = create_post_from_image_list(images)
-
         # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
         manifest_file = md_from_folder(provider, path)
         if manifest_file:
-            manifest = get_manifest(provider, manifest_file)
+            manifest = read_manifest(provider, manifest_file)
         else:
             manifest = {}
 
@@ -76,6 +70,7 @@ class Article(Model):
         # self.__url_template: str = settings.document_url_template
         # self.__url_preview_template: str = settings.document_url_preview_template
 
+        self.hash_salt = settings.hash_salt_articles
         self.origin = settings.origin
         self.version = '2'
         self.post = None
@@ -83,6 +78,13 @@ class Article(Model):
         super().__init__(provider, store, file, params=params)
 
     def init(self):
+        if self.has_param('content'):
+            images = kazimir.get_images_src(self.get_param('content'))
+        else:
+            images = list_images(self.provider, self.file)
+            images = [os.path.relpath(i, self.file) for i in images]
+        self.params['images'] = images
+
         self.__set_id()
         self.__set_hash()
 
@@ -96,12 +98,14 @@ class Article(Model):
 
     async def build(self, **kwargs):
         sizes = kwargs['sizes']
-        post, images = await create_post(self.provider, self.id, self.params, sizes)
-        self.post = post
-        self.images = images
 
-        post, images = await create_post(self.provider, self.id, self.params, sizes)
-        self.post = kazimir.html_from_tree(post)
+        if self.has_param('content'):
+            markdown_post = self.get_param('content')
+        else:
+            markdown_post = create_post_from_image_list(self.get_param('images'))
+
+        post, images = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        self.post = post
         self.images = images
 
         if 'preview' in self.params:
@@ -140,7 +144,7 @@ class Article(Model):
         images = [self.provider.hash(i) for i in images]
 
         self.hash = hash_str(
-            combine([hash_str(self.params)] + images)
+            combine([self.hash_salt] + [hash_str(self.params)] + images)
         )
 
     def __str__(self):
@@ -151,7 +155,7 @@ def md_from_folder(provider, i):
     return first(provider.type_filter(i, '.md'))
 
 
-def get_manifest(provider, path):
+def read_manifest(provider, path):
     data = provider.read(path).read().decode('utf-8')
     manifest = parse_yaml_front_matter(data)
 
@@ -164,38 +168,30 @@ def get_manifest(provider, path):
     if 'id' in manifest:
         manifest['id'] = str(manifest['id'])
 
-    html = kazimir_to_html(manifest['content'])
     return {
         **manifest,
-        'post': html,
-        'images': images_from_html(html)  # Add images for hashing
     }
 
 
-async def create_post(provider, doc_id, document, sizes):
-    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
-
-    post_html = lxml.html.fromstring(document['post'])
+async def create_post(provider: Provider, folder: str, md: str, sizes):
+    post = kazimir.kazimir_to_html(md)
+    html = lxml.html.fromstring(post)
     images = []
-    for img in post_html.cssselect('img'):
+    for img in html.cssselect('img'):
         src = img.get('src')
-        relative_image_path = os.path.join(document['folder'], src)
+        relative_image_path = os.path.join(folder, src)
 
-        img_path = provider.get_local(relative_image_path)
-        if os.path.exists(img_path):
-            img_id = url_encode_text(os.path.splitext(src)[0])
-            url_fn = lambda file, size, ext: image_url_base.format(
-                id=doc_id,
-                img=img_id,
-                size=size,
-                ext=ext.lower()
-            )
-
-            image = await Image.new(provider, img_path, sizes, url_fn)
+        image_path = provider.get_local(relative_image_path)
+        if os.path.exists(image_path):
+            image = await Image.new(provider, image_path, sizes)
             if image:
                 url = image.get_size('big')['url']
                 images.append(image)
                 img.set('src', url)
             else:
-                logger.error('Fail to get Image', document['folder'], src)
-    return post_html, images
+                logger.error('Fail to get Image', folder, src)
+
+    post = kazimir.html_from_tree(html)
+    post = await kazimir.typo(post)
+
+    return post, images