sync simplification

- drop token compilation to html
- drop artlebedev typograph
- drop other small things

modules/sync/sync/__init__.py

@@ -10,7 +10,7 @@ from kazimir.DocumentToken import DocumentToken
 from kazimir.YoutubeToken import YoutubeToken
 from kazimir.InstagramToken import InstagramToken
 from kazimir.Marker import Marker
-from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken
+from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, HtmlToken
 from sync.data import Provider
 from sync.models import Model
 from sync.models.Document import Document
@@ -66,8 +66,6 @@ async def untouched(items: [Model]) -> [Model]:
 
 
 async def create_post(provider: Provider, folder: str, md: str, sizes):
-    from kazimir.fix_links_quotes import fix_links_quotes
-
     images = []
     documents = []
 
@@ -128,14 +126,10 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
     m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
     m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file))
     # m.add_token_factory(TokenFactory(FileToken))
+    m.add_token_factory(TokenFactory(HtmlToken))
     m.add_token_factory(TokenFactory(TextToken))
-    m.add_tree_middleware(fix_links_quotes)
 
     tokens = m.parse(md)
-    tree = await m.create_tree(tokens)
-
-    post = kazimir.html_from_tree(tree)
-    post = await kazimir.typo(post)
 
     encoded_tokens = []
     for t in tokens:
@@ -145,11 +139,4 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
             'data': data,
         })
 
-    return post, images, documents, encoded_tokens
-
-
-def title_from_html(md):
-    html = lxml.html.fromstring(md)
-    titles = html.cssselect('h1')
-    if len(titles):
-        return titles[0].text
+    return images, documents, encoded_tokens

modules/sync/sync/models/Page.py

@@ -2,21 +2,36 @@ import os
 import logging
 from lxml import etree
 import lxml.html
-import kazimir
 import settings
 from db import collection
 from sync.data import request
 from sync.models import Model
-from sync import create_date, title_from_html, create_post
+from sync import create_date, create_post
 from sync.models.Image import Image
 from utils.hash import hash_str
 from utils.front_matter import parse_yaml_front_matter
 from utils.text.transform import url_encode_text
+from markdown import markdown
+
 
 logger = logging.getLogger(settings.name + '.Page')
 store = collection(settings.collection_pages)
 
 
+def extract_files(html) -> [str]:
+    images = html.cssselect('img')
+    images = [img.get('src') for img in images]
+
+    return images, []
+
+
+def title_from_html(html):
+    titles = html.cssselect('h1')
+    if len(titles):
+        return titles[0].text
+    return ''
+
+
 def find(store, item_id: str):
     q = {'id': item_id}
     try:
@@ -103,7 +118,6 @@ class Page(Model):
         self.images = []
         self.documents = []
         self.title = None
-        self.data = None
         self.preview = None
         self.tokens = None
 
@@ -114,7 +128,12 @@ class Page(Model):
         super().__init__(provider, store, file, params=params)
 
     def init(self):
-        images, documents = kazimir.extract_files(self.get_param('content'))
+        content = self.get_param('content')
+        html = markdown(content)
+        html_tree = lxml.html.fromstring(html)
+        images, documents = extract_files(html_tree)
+        self.title = title_from_html(html_tree)
+
         self.params['images'] = images
         self.params['documents'] = documents
 
@@ -136,9 +155,7 @@ class Page(Model):
         sizes = kwargs['sizes']
 
         markdown_post = self.get_param('content')
-        post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
-        self.title = title_from_html(post)
-        self.data = post
+        images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
         self.images = images
         self.documents = documents
         self.tokens = tokens
@@ -158,7 +175,6 @@ class Page(Model):
             'hash': self.hash,
             'url': self.url,
             'file': self.file,
-            'data': self.data,
             'images': images,
             'documents': documents,
             'title': self.title,