include tokens in result data

modules/sync/kazimir/Marker.py

@@ -16,12 +16,15 @@ class Marker:
     def add_tree_middleware(self, callback):
         self.tree_middlewares.append(callback)
 
-    async def create_tree(self, text: str):
+    def parse(self, text: str):
         tokens = text.split('\n')
         tokens = self.markup_tokens(tokens)
         tokens = join_tokens(tokens)
         tokens = merge_tokens(tokens)
 
+        return tokens
+
+    async def create_tree(self, tokens: list):
         compiled = await compile_tokens(tokens)
         tree = lxml.html.fromstring(compiled)
         return await self.process_tree(tree)

modules/sync/sync/__init__.py

@@ -153,12 +153,22 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
     # m.add_token_factory(TokenFactory(FileToken))
     m.add_token_factory(TokenFactory(TextToken))
     m.add_tree_middleware(fix_links_quotes)
-    tree = await m.create_tree(md)
+
+    tokens = m.parse(md)
+    tree = await m.create_tree(tokens)
 
     post = kazimir.html_from_tree(tree)
     post = await kazimir.typo(post)
 
-    return post, images, documents
+    encoded_tokens = []
+    for t in tokens:
+        data = await t.get_data()
+        encoded_tokens.append({
+            'token': t.name,
+            'data': data,
+        })
+
+    return post, images, documents, encoded_tokens
 
 
 def images_from_html(md):

modules/sync/sync/models/Article.py

@@ -71,6 +71,7 @@ class Article(Model):
         self.preview = None
         self.images = None
         self.documents = None
+        self.tokens = None
         super().__init__(provider, store, file, params=params)
 
     def init(self):
@@ -103,10 +104,11 @@ class Article(Model):
         else:
             markdown_post = create_post_from_image_list(self.get_param('images'))
 
-        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
         self.post = post
         self.images = images
         self.documents = documents
+        self.tokens = tokens
 
         if self.has_param('preview'):
             preview = self.get_param('preview')
@@ -124,6 +126,7 @@ class Article(Model):
             'file': self.file,
             'origin': self.origin,
             'version': self.version,
+            'tokens': self.tokens,
             'preview': self.preview.ref if self.preview else None,
         }
 

modules/sync/sync/models/Page.py

@@ -96,6 +96,7 @@ class Page(Model):
         self.title = None
         self.data = None
         self.preview = None
+        self.tokens = None
 
         self.__id_template: str = '{category}-{file}'
         self.__url_template: str = settings.document_url_template
@@ -126,11 +127,12 @@ class Page(Model):
         sizes = kwargs['sizes']
 
         markdown_post = self.get_param('content')
-        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
         self.title = title_from_html(post)
         self.data = post
         self.images = images
         self.documents = documents
+        self.tokens = tokens
 
         if self.has_param('preview'):
             preview = self.get_param('preview')
@@ -151,6 +153,7 @@ class Page(Model):
             'images': images,
             'documents': documents,
             'title': self.title,
+            'tokens': self.tokens,
             'preview': self.preview.ref if self.preview else None,
         }