diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py index cad715f6..3b81c9bb 100644 --- a/modules/sync/kazimir/Marker.py +++ b/modules/sync/kazimir/Marker.py @@ -16,12 +16,15 @@ class Marker: def add_tree_middleware(self, callback): self.tree_middlewares.append(callback) - async def create_tree(self, text: str): + def parse(self, text: str): tokens = text.split('\n') tokens = self.markup_tokens(tokens) tokens = join_tokens(tokens) tokens = merge_tokens(tokens) + return tokens + + async def create_tree(self, tokens: list): compiled = await compile_tokens(tokens) tree = lxml.html.fromstring(compiled) return await self.process_tree(tree) diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py index 77aeaea7..209d6afe 100644 --- a/modules/sync/sync/__init__.py +++ b/modules/sync/sync/__init__.py @@ -153,12 +153,22 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): # m.add_token_factory(TokenFactory(FileToken)) m.add_token_factory(TokenFactory(TextToken)) m.add_tree_middleware(fix_links_quotes) - tree = await m.create_tree(md) + + tokens = m.parse(md) + tree = await m.create_tree(tokens) post = kazimir.html_from_tree(tree) post = await kazimir.typo(post) - return post, images, documents + encoded_tokens = [] + for t in tokens: + data = await t.get_data() + encoded_tokens.append({ + 'token': t.name, + 'data': data, + }) + + return post, images, documents, encoded_tokens def images_from_html(md): diff --git a/modules/sync/sync/models/Article.py b/modules/sync/sync/models/Article.py index 68b364df..40d003ce 100644 --- a/modules/sync/sync/models/Article.py +++ b/modules/sync/sync/models/Article.py @@ -71,6 +71,7 @@ class Article(Model): self.preview = None self.images = None self.documents = None + self.tokens = None super().__init__(provider, store, file, params=params) def init(self): @@ -103,10 +104,11 @@ class Article(Model): else: markdown_post = create_post_from_image_list(self.get_param('images')) - post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) + post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) self.post = post self.images = images self.documents = documents + self.tokens = tokens if self.has_param('preview'): preview = self.get_param('preview') @@ -124,6 +126,7 @@ class Article(Model): 'file': self.file, 'origin': self.origin, 'version': self.version, + 'tokens': self.tokens, 'preview': self.preview.ref if self.preview else None, } diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py index afd6b500..e72923d4 100644 --- a/modules/sync/sync/models/Page.py +++ b/modules/sync/sync/models/Page.py @@ -96,6 +96,7 @@ class Page(Model): self.title = None self.data = None self.preview = None + self.tokens = None self.__id_template: str = '{category}-{file}' self.__url_template: str = settings.document_url_template @@ -126,11 +127,12 @@ class Page(Model): sizes = kwargs['sizes'] markdown_post = self.get_param('content') - post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) + post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) self.title = title_from_html(post) self.data = post self.images = images self.documents = documents + self.tokens = tokens if self.has_param('preview'): preview = self.get_param('preview') @@ -151,6 +153,7 @@ class Page(Model): 'images': images, 'documents': documents, 'title': self.title, + 'tokens': self.tokens, 'preview': self.preview.ref if self.preview else None, }