[Sync] Added Marker (refactored Kazimir)
This commit is contained in:
Roman Timashev committed
1 parent
f9f7203b1e
commit
46968ef0cf
5 files changed
+604
No files matched your search
@@ -0,0 +1,72 @@
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import re
|
||||
import requests
|
||||
from markdown.util import etree
|
||||
|
||||
from kazimir.Token import UrlToken, is_url
|
||||
|
||||
pattr = re.compile(r'https?://www\.instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$')
|
||||
|
||||
|
||||
class InstagramToken(UrlToken):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
if not is_url(data):
|
||||
return False
|
||||
|
||||
url = urlparse(data)
|
||||
return url.netloc in ['www.instagram.com', 'instagram.com']
|
||||
|
||||
async def compile(self):
|
||||
match = pattr.match(self.data)
|
||||
if match:
|
||||
m = match.group('media')
|
||||
return await embed_instagram(m)
|
||||
|
||||
|
||||
async def embed_instagram(media):
|
||||
"""
|
||||
<blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
|
||||
<div style="padding:8px;">
|
||||
<div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
|
||||
<div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p style=" margin:8px 0 0 0; padding:0 4px;">
|
||||
<a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
|
||||
Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
|
||||
</a>
|
||||
</p>
|
||||
|
||||
<p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
|
||||
A post shared by Шлиссельбургская ДХШ (@hudozka) on
|
||||
|
||||
<time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
|
||||
May 24, 2017 at 7:05am PDT
|
||||
</time>
|
||||
</p>
|
||||
</div>
|
||||
</blockquote>
|
||||
"""
|
||||
|
||||
url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
|
||||
req = requests.get(url)
|
||||
if req.status_code == 200:
|
||||
html = req.json()['html']
|
||||
html = fix_html(html)
|
||||
i = etree.fromstring(html)
|
||||
|
||||
obj = etree.Element('div')
|
||||
obj.set('class', "kazimir__embed")
|
||||
obj.append(i)
|
||||
return etree.tostring(obj).decode()
|
||||
else:
|
||||
raise Exception(
|
||||
'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
|
||||
)
|
||||
|
||||
|
||||
def fix_html(html: str):
|
||||
return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')
|
||||
@@ -0,0 +1,199 @@
|
||||
import asyncio
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
from kazimir.InstagramToken import InstagramToken
|
||||
from kazimir.YoutubeToken import YoutubeToken
|
||||
from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, DocumentToken, TokenFactory, \
|
||||
DocumentTokenFactory
|
||||
|
||||
|
||||
class Marker:
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tokens = []
|
||||
self.tree_middlewares = []
|
||||
|
||||
def add_token_factory(self, token):
|
||||
self.tokens.append(token)
|
||||
|
||||
def add_tree_middleware(self, callback):
|
||||
self.tree_middlewares.append(callback)
|
||||
|
||||
async def create_tree(self, text: str):
|
||||
tokens = text.split('\n')
|
||||
tokens = self.markup_tokens(tokens)
|
||||
tokens = join_tokens(tokens)
|
||||
tokens = merge_tokens(tokens)
|
||||
|
||||
compiled = await compile_tokens(tokens)
|
||||
tree = lxml.html.fromstring(compiled)
|
||||
return await self.process_tree(tree)
|
||||
|
||||
async def process_tree(self, tree):
|
||||
for fn in self.tree_middlewares:
|
||||
tree = await fn(tree)
|
||||
return tree
|
||||
|
||||
def markup_tokens(self, tokens: list):
|
||||
marked_tokens = []
|
||||
|
||||
for data in tokens:
|
||||
t = self.create_token(data)
|
||||
if t:
|
||||
marked_tokens.append(t)
|
||||
else:
|
||||
raise Exception(f'Token not recognized for data {data}')
|
||||
return marked_tokens
|
||||
|
||||
def create_token(self, data):
|
||||
for f in self.tokens:
|
||||
if f.token.test(data):
|
||||
return f.create(data=data)
|
||||
return None
|
||||
|
||||
|
||||
def join_tokens(tokens):
|
||||
is_split = lambda x: isinstance(x, SplitToken)
|
||||
|
||||
buffer_tag = None
|
||||
buffer = []
|
||||
ts = []
|
||||
for token in tokens:
|
||||
tag = token.name
|
||||
changed = buffer_tag is not None and tag != buffer_tag
|
||||
|
||||
flush = is_split(token)
|
||||
if changed:
|
||||
flush = True
|
||||
|
||||
if not token.joinable:
|
||||
flush = True
|
||||
|
||||
if flush:
|
||||
if buffer:
|
||||
ts.append(buffer)
|
||||
if is_split(token):
|
||||
buffer = []
|
||||
buffer_tag = None
|
||||
else:
|
||||
buffer = [token]
|
||||
buffer_tag = tag
|
||||
else:
|
||||
buffer_tag = tag
|
||||
buffer.append(token)
|
||||
|
||||
if len(buffer):
|
||||
ts.append(buffer)
|
||||
return ts
|
||||
|
||||
|
||||
def merge_tokens(tokens: list):
|
||||
tokens_out = []
|
||||
for ts in tokens:
|
||||
token = ts[0]
|
||||
for x in ts[1:]:
|
||||
token = token.merge(x)
|
||||
tokens_out.append(token)
|
||||
return tokens_out
|
||||
|
||||
|
||||
async def compile_tokens(tokens: list):
|
||||
result = []
|
||||
for token in tokens:
|
||||
data = await token.compile()
|
||||
result.append(data)
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def extract_files(text: str) -> [str]:
|
||||
html = markdown(text)
|
||||
post_html = lxml.html.fromstring(html)
|
||||
|
||||
images = post_html.cssselect('img')
|
||||
images = [img.get('src') for img in images]
|
||||
|
||||
return images, []
|
||||
|
||||
|
||||
def html_from_tree(tree):
|
||||
html = lxml.html.tostring(tree, encoding='unicode')
|
||||
|
||||
return html
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sample_text = '''
|
||||
Hello!
|
||||
|
||||
https://www.instagram1.com/p/BUelE04l1kU
|
||||
|
||||
https://www.youtube.com/watch?v=qnguOqWpraI
|
||||
https://www.youtube.com/watch?v=qnguOqWpraI
|
||||
|
||||
|
||||
|
||||
LOL NOOB
|
||||
|
||||
|
||||
WTF OMG
|
||||
|
||||
|
||||
hello.pdf
|
||||
file.doc
|
||||
sheet.xlsx
|
||||
|
||||
print.docx
|
||||
image.jpg 1
|
||||
image.png 2
|
||||
|
||||
image.jpeg Hi
|
||||
|
||||
Bye!
|
||||
Fye!
|
||||
|
||||
|
||||
|
||||
["Ya"](http://ya.ru)
|
||||
|
||||
More text
|
||||
'''
|
||||
|
||||
|
||||
async def build_document(data):
|
||||
text = data['caption'] if data['caption'] else data['file']
|
||||
# document = await self.build()
|
||||
document = {
|
||||
'url': data['file'],
|
||||
'image_url': data['file'],
|
||||
'file_url': data['file'],
|
||||
'title': text,
|
||||
'file_size': '0 KB',
|
||||
'file_format': 'PDF',
|
||||
}
|
||||
return document
|
||||
|
||||
|
||||
async def run():
|
||||
from kazimir.fix_links_quotes import fix_links_quotes
|
||||
|
||||
m = Marker()
|
||||
m.add_token_factory(TokenFactory(SplitToken))
|
||||
m.add_token_factory(TokenFactory(YoutubeToken))
|
||||
m.add_token_factory(TokenFactory(InstagramToken))
|
||||
m.add_token_factory(TokenFactory(UrlToken))
|
||||
m.add_token_factory(TokenFactory(ImageToken))
|
||||
m.add_token_factory(DocumentTokenFactory(build=build_document))
|
||||
m.add_token_factory(TokenFactory(FileToken))
|
||||
m.add_token_factory(TokenFactory(TextToken))
|
||||
m.add_tree_middleware(fix_links_quotes)
|
||||
sample_tree = await m.create_tree(sample_text)
|
||||
|
||||
# for x in sample_tree:
|
||||
# print(x)
|
||||
|
||||
print(html_from_tree(sample_tree))
|
||||
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(run())
|
||||
loop.close()
|
||||
@@ -0,0 +1,251 @@
|
||||
import os
|
||||
import re
|
||||
from markdown import markdown
|
||||
from urllib.parse import urlparse
|
||||
|
||||
image_extensions = ['.jpg', '.jpeg', '.png']
|
||||
document_extensions = ['.doc', '.docx', '.pdf', '.xls', '.xlsx']
|
||||
|
||||
|
||||
class TokenFactory:
|
||||
def __init__(self, token) -> None:
|
||||
super().__init__()
|
||||
self.token = token
|
||||
|
||||
def create(self, data):
|
||||
return self.token(data=data)
|
||||
|
||||
|
||||
class DocumentTokenFactory(TokenFactory):
|
||||
def __init__(self, build) -> None:
|
||||
super().__init__(DocumentToken)
|
||||
self.build = build
|
||||
|
||||
def create(self, data):
|
||||
x = self.token(data=data)
|
||||
x.build = self.build
|
||||
return x
|
||||
|
||||
|
||||
class Token:
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return True
|
||||
|
||||
def __init__(self, name, data) -> None:
|
||||
super().__init__()
|
||||
self.name = name
|
||||
self.data = data
|
||||
self.joinable = True
|
||||
|
||||
async def compile(self):
|
||||
return str(self.data)
|
||||
|
||||
def merge(self, token):
|
||||
return self
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'[Token {self.name} {self.data}]'
|
||||
|
||||
|
||||
class SplitToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
data = data.strip()
|
||||
return data == ''
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='split', data=None)
|
||||
|
||||
|
||||
class TextToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return True
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='text', data=data)
|
||||
|
||||
def merge(self, token: Token):
|
||||
data = self.data
|
||||
data += '\n'
|
||||
data += token.data
|
||||
return TextToken(data.strip())
|
||||
|
||||
async def compile(self):
|
||||
return markdown_to_html(self.data)
|
||||
|
||||
|
||||
class FileToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return is_file(data, document_extensions + image_extensions)
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='file', data=data)
|
||||
self.joinable = False
|
||||
|
||||
async def compile(self):
|
||||
data = parse_file(self.data)
|
||||
url = data['file']
|
||||
text = data['caption'] if data['caption'] else url
|
||||
return f'<a href="{url}">{text}</a>'
|
||||
|
||||
|
||||
class DocumentToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return is_document(data)
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='file', data=data)
|
||||
self.joinable = False
|
||||
self.build = None
|
||||
|
||||
async def compile(self):
|
||||
data = parse_file(self.data)
|
||||
document = await self.build(data)
|
||||
|
||||
return f'''
|
||||
<div>
|
||||
<div class="document-row">
|
||||
<a href="{document['url']}" class="invisible">
|
||||
<div class="document-row__image">
|
||||
<img src="{document['image_url']}" alt="{document['title']}">
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<div class="document-row__file">
|
||||
<a href="{document['file_url']}">{document['title']}</a>
|
||||
</div>
|
||||
|
||||
<div class="document-row__file-info">
|
||||
<a href="{document['file_url']}" target="_blank">
|
||||
{document['file_format']} ({document['file_size']})
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
'''.strip()
|
||||
|
||||
|
||||
class ImageToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return is_image(data)
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='image', data=data)
|
||||
self.joinable = True
|
||||
|
||||
def merge(self, token: Token):
|
||||
return ImageCollectionToken([self.data, token.data])
|
||||
|
||||
async def compile(self):
|
||||
img = self.parse_data()
|
||||
caption = img['caption']
|
||||
file = img['file']
|
||||
|
||||
tpl = f'''
|
||||
<div class="kazimir__image">
|
||||
<figure>
|
||||
<img alt="{caption}" src="{file}">
|
||||
<figcaption>{caption}</figcaption>
|
||||
</figure>
|
||||
</div>
|
||||
'''
|
||||
return tpl.format(img).strip()
|
||||
|
||||
def parse_data(self):
|
||||
return parse_file(self.data)
|
||||
|
||||
|
||||
class ImageCollectionToken(ImageToken):
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(data)
|
||||
self.name = 'image_collection'
|
||||
|
||||
def merge(self, token: Token):
|
||||
if isinstance(token.data, list):
|
||||
data = token.data
|
||||
else:
|
||||
data = [token.data]
|
||||
return ImageCollectionToken([*self.data, *data])
|
||||
|
||||
async def compile(self):
|
||||
data = self.parse_data()
|
||||
images = [f'<img src="{x}">' for x in data]
|
||||
images = '\n'.join(images)
|
||||
|
||||
return f'''
|
||||
<div class="kazimir__image-collection">
|
||||
<div class="fotorama"
|
||||
data-width="100%"
|
||||
data-ratio="800/600"
|
||||
>
|
||||
{images}
|
||||
</div>
|
||||
</div>
|
||||
'''
|
||||
|
||||
def parse_data(self):
|
||||
images = [parse_file(x) for x in self.data]
|
||||
return [x['file'] for x in images]
|
||||
|
||||
|
||||
class UrlToken(Token):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
return is_url(data)
|
||||
|
||||
def __init__(self, data) -> None:
|
||||
super().__init__(name='url', data=data)
|
||||
self.joinable = False
|
||||
|
||||
|
||||
def markdown_to_html(text):
|
||||
return markdown(text)
|
||||
|
||||
|
||||
def is_url(sample: str):
|
||||
url = urlparse(sample)
|
||||
|
||||
return url.scheme and url.netloc
|
||||
|
||||
|
||||
def is_image(sample: str):
|
||||
file = parse_file(sample)['file']
|
||||
return is_file(file, image_extensions)
|
||||
|
||||
|
||||
def is_document(sample: str):
|
||||
file = parse_file(sample)['file']
|
||||
return is_file(file, document_extensions)
|
||||
|
||||
|
||||
def is_file(sample: str, extensions):
|
||||
name, ext = os.path.splitext(sample)
|
||||
ext = ext.lower()
|
||||
return ext in extensions
|
||||
|
||||
|
||||
def parse_file(sample: str):
|
||||
"""
|
||||
 -> {file, caption}
|
||||
file.jpg Caption -> {file, caption}
|
||||
|
||||
:param sample:
|
||||
:return:
|
||||
"""
|
||||
p = re.match(r'!\[(.*)\]\((.*)\)', sample)
|
||||
if p:
|
||||
caption = p.group(1)
|
||||
file = p.group(2)
|
||||
else:
|
||||
sample = sample.split(' ')
|
||||
file = sample[0]
|
||||
caption = ' '.join(sample[1:])
|
||||
return {
|
||||
'file': file,
|
||||
'caption': caption,
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
from kazimir.Token import UrlToken, is_url
|
||||
|
||||
pattr = re.compile(r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P<youtubeargs>[A-Za-z0-9_&=-]+)\S*')
|
||||
|
||||
|
||||
class YoutubeToken(UrlToken):
|
||||
@staticmethod
|
||||
def test(data: str):
|
||||
if not is_url(data):
|
||||
return False
|
||||
|
||||
url = urlparse(data)
|
||||
return url.netloc in ['www.youtube.com', 'youtube.com']
|
||||
|
||||
async def compile(self):
|
||||
"""
|
||||
<div class="kazimir__video">
|
||||
<iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
|
||||
</div>
|
||||
"""
|
||||
|
||||
match = pattr.match(self.data)
|
||||
if match:
|
||||
video_id = match.group(2)
|
||||
url = f'//www.youtube.com/embed/{video_id}'
|
||||
return f'''
|
||||
<div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>
|
||||
'''.strip()
|
||||
else:
|
||||
return ''
|
||||
@@ -0,0 +1,50 @@
|
||||
quote_pairs = [
|
||||
('"', '"'),
|
||||
('«', '»'),
|
||||
]
|
||||
|
||||
|
||||
async def fix_links_quotes(tree):
|
||||
safe = lambda x: x if x else ''
|
||||
|
||||
for link in tree.iterlinks():
|
||||
a = link[0]
|
||||
if a.tag != 'a':
|
||||
continue
|
||||
|
||||
quote = get_quote_pair(a.text)
|
||||
if not quote:
|
||||
continue
|
||||
|
||||
# New text of A without quotes
|
||||
a.text = a.text[1:-1]
|
||||
|
||||
# Move first quote out of A to the prefix
|
||||
a.getparent().text = safe(a.getparent().text) + quote[0]
|
||||
|
||||
# Move last quote out of A to the suffix
|
||||
a.tail = quote[1] + safe(a.tail)
|
||||
|
||||
return tree
|
||||
|
||||
|
||||
def get_quote_pair(text: str):
|
||||
"""
|
||||
sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
|
||||
? (s, t)
|
||||
: None
|
||||
|
||||
:param text:
|
||||
:return:
|
||||
"""
|
||||
if len(text) < 2:
|
||||
return None
|
||||
|
||||
first = text[0]
|
||||
last = text[-1]
|
||||
|
||||
for pair_first, pair_last in quote_pairs:
|
||||
if pair_first == first and pair_last == last:
|
||||
return first, last
|
||||
|
||||
return None
|
||||
Reference in new issue
Block a user