[Sync] Kazimir update. Typograph for Articles, Unicode HTML

utils/Sync/kazimir/__init__.py

@@ -1,7 +1,35 @@
+import lxml.html
+import requests
 from markdown import markdown
 
 from kazimir.video import VideoExtension
 
 
-def kazimir_to_html(text):
+def markdown_to_html(text):
     return markdown(text, extensions=[VideoExtension()])
+
+
+def kazimir_to_html(text):
+    html = markdown_to_html(text)
+    return typo(html)
+
+
+def html_from_tree(tree):
+    html = lxml.html.tostring(tree, encoding='unicode')
+    html = typo(html)
+
+    return html
+
+
+def typo(html):
+    url = 'http://api.tmshv.com/typograph/v1'
+    res = requests.post(
+        url=url,
+        headers={
+            'Content-Type': 'text/plain; charset=utf-8',
+        },
+        data=html.encode('utf-8')
+    )
+    if res.status_code == 200:
+        return res.text
+    return html

utils/Sync/sync/core/post.py

@@ -3,6 +3,7 @@ from tempfile import mkstemp
 
 import lxml.html
 
+import kazimir
 from sync.core import Sync
 from utils.fn import last_good, lmap, combine
 from utils.hash import hash_str, hash_file
@@ -76,7 +77,7 @@ class SyncPost(Sync):
                 else:
                     print('fail: ', document['folder'], src)
 
-        document['post'] = lxml.html.tostring(post_html).decode('utf-8')
+        document['post'] = kazimir.html_from_tree(post_html)
         document['images'] = images
         document['origin'] = self.origin
         return document

utils/Sync/sync/post.py

@@ -1,12 +1,13 @@
 import os
 
 import settings
+from kazimir import markdown_to_html
 from sync.data import list_images
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
     synced_images_ids, untouched
 from sync.core.post import SyncPost
 from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
-from utils.io import read_yaml_md
+from utils.io import read_yaml_md, parse_yaml_front_matter
 
 
 def create_document_from_folder(sync, path, file_time_formats):
@@ -49,22 +50,22 @@ def create_document_from_folder(sync, path, file_time_formats):
 
 def get_manifest(provider, path):
     data = provider.read(path).read().decode('utf-8')
-    y, m = read_yaml_md(data)
-    y = y if y else {}
+    manifest = parse_yaml_front_matter(data)
 
-    if 'date' in y:
-        y['date'] = create_date(y['date'], settings.date_formats)
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
 
-    if 'until' in y:
-        y['until'] = create_date(y['until'], settings.date_formats)
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
 
-    if 'id' in y:
-        y['id'] = str(y['id'])
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
 
+    html = markdown_to_html(manifest['content'])
     return {
-        **y,
-        'post': m,
-        'images': images_from_html(m)
+        **manifest,
+        'post': html,
+        'images': images_from_html(html)  # Add images for hashing
     }
 
 
@@ -127,7 +128,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
     documents = get_folder_documents(sync, file_time_formats)
     documents += get_file_documents(sync, file_time_formats)
 
-    # # CREATE SCOPE OF CURRENT SESSION
+    # CREATE SCOPE OF CURRENT SESSION
     scope_documents_ids = lmapfn(documents)(
         lambda i: i['id']
     )

utils/Sync/utils/io.py

@@ -1,17 +1,26 @@
 import re
 import yaml
 
-from kazimir import kazimir_to_html
+from kazimir import markdown_to_html
 
 FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
 
 
 def read_yaml_md(data):
-    yaml_front_matter, content = parse_yaml_front_matter(data)
-    return yaml_front_matter, kazimir_to_html(content)
+    yaml_front_matter, content = split_yaml_front_matter(data)
+    return yaml_front_matter, markdown_to_html(content)
 
 
-def parse_yaml_front_matter(text, **defaults):
+def parse_yaml_front_matter(data: str):
+    fm, content = split_yaml_front_matter(data)
+    fm = fm if fm else {}
+    return {
+        **fm,
+        'content': content,
+    }
+
+
+def split_yaml_front_matter(text, **defaults):
     """
     Parse text with YAML frontmatter, return metadata and content.
     Pass in optional metadata defaults as keyword args.