sync moved to modules
This commit is contained in:
1 parent
1dda6ce8cc
commit
13b761a81e
50 files changed
No files matched your search
@@ -0,0 +1,118 @@
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
|
||||
from kazimir.Token import SplitToken
|
||||
|
||||
|
||||
class Marker:
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tokens = []
|
||||
self.tree_middlewares = []
|
||||
|
||||
def add_token_factory(self, token):
|
||||
self.tokens.append(token)
|
||||
|
||||
def add_tree_middleware(self, callback):
|
||||
self.tree_middlewares.append(callback)
|
||||
|
||||
async def create_tree(self, text: str):
|
||||
tokens = text.split('\n')
|
||||
tokens = self.markup_tokens(tokens)
|
||||
tokens = join_tokens(tokens)
|
||||
tokens = merge_tokens(tokens)
|
||||
|
||||
compiled = await compile_tokens(tokens)
|
||||
tree = lxml.html.fromstring(compiled)
|
||||
return await self.process_tree(tree)
|
||||
|
||||
async def process_tree(self, tree):
|
||||
for fn in self.tree_middlewares:
|
||||
tree = await fn(tree)
|
||||
return tree
|
||||
|
||||
def markup_tokens(self, tokens: list):
|
||||
marked_tokens = []
|
||||
|
||||
for data in tokens:
|
||||
t = self.create_token(data)
|
||||
if t:
|
||||
marked_tokens.append(t)
|
||||
else:
|
||||
raise Exception(f'Token not recognized for data {data}')
|
||||
return marked_tokens
|
||||
|
||||
def create_token(self, data):
|
||||
for f in self.tokens:
|
||||
if f.token.test(data):
|
||||
return f.create(data=data)
|
||||
return None
|
||||
|
||||
|
||||
def join_tokens(tokens):
|
||||
is_split = lambda x: isinstance(x, SplitToken)
|
||||
|
||||
buffer_tag = None
|
||||
buffer = []
|
||||
ts = []
|
||||
for token in tokens:
|
||||
tag = token.name
|
||||
changed = buffer_tag is not None and tag != buffer_tag
|
||||
|
||||
flush = is_split(token)
|
||||
if changed:
|
||||
flush = True
|
||||
|
||||
if not token.joinable:
|
||||
flush = True
|
||||
|
||||
if flush:
|
||||
if buffer:
|
||||
ts.append(buffer)
|
||||
if is_split(token):
|
||||
buffer = []
|
||||
buffer_tag = None
|
||||
else:
|
||||
buffer = [token]
|
||||
buffer_tag = tag
|
||||
else:
|
||||
buffer_tag = tag
|
||||
buffer.append(token)
|
||||
|
||||
if len(buffer):
|
||||
ts.append(buffer)
|
||||
return ts
|
||||
|
||||
|
||||
def merge_tokens(tokens: list):
|
||||
tokens_out = []
|
||||
for ts in tokens:
|
||||
token = ts[0]
|
||||
for x in ts[1:]:
|
||||
token = token.merge(x)
|
||||
tokens_out.append(token)
|
||||
return tokens_out
|
||||
|
||||
|
||||
async def compile_tokens(tokens: list):
|
||||
result = []
|
||||
for token in tokens:
|
||||
data = await token.compile()
|
||||
result.append(data)
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def extract_files(text: str) -> [str]:
|
||||
html = markdown(text)
|
||||
post_html = lxml.html.fromstring(html)
|
||||
|
||||
images = post_html.cssselect('img')
|
||||
images = [img.get('src') for img in images]
|
||||
|
||||
return images, []
|
||||
|
||||
|
||||
def html_from_tree(tree):
|
||||
html = lxml.html.tostring(tree, encoding='unicode')
|
||||
|
||||
return html
|
||||
Reference in new issue
Block a user