sync simplification

- drop token compilation to html
- drop artlebedev typograph
- drop other small things
This commit is contained in:
tmshv committed 2020-03-21 20:02:59 +03:00
1 parent 5f51a6e00d
commit c1a3af3a3e
13 files changed
+73 -443

No files matched your search

-42
View File
@@ -1,5 +1,4 @@
from kazimir.Token import parse_file, is_document, Token
import math
class DocumentToken(Token):
@@ -16,44 +15,3 @@ class DocumentToken(Token):
data = parse_file(self.data)
document = await self.build(data)
return document
async def compile(self):
document = await self.get_data()
return create_document(document)
def create_document(document):
file_size = get_size(document['file_size'], 1)
file_format = 'ПДФ'
return f'''
<div>
<div class="document-row">
<a href="{document['url']}" class="invisible">
<div class="document-row__image">
<img src="{document['image_url']}" alt="{document['title']}">
</div>
</a>
<div class="document-row__file">
<a href="{document['url']}">{document['title']}</a>
</div>
<div class="document-row__file-info">
<a href="{document['file_url']}" target="_blank">
{file_format} ({file_size})
</a>
</div>
</div>
</div>
'''.strip()
def get_size(number_of_bytes, precision=1):
units = ['байт', 'КБ', 'МБ', 'ГБ', 'ТБ', 'ПБ']
number = math.floor(math.log(number_of_bytes) / math.log(1024))
size = (number_of_bytes / math.pow(1024, math.floor(number)))
size = round(size, precision)
return f'{size} {units[number]}'
+25 -12
View File
@@ -18,11 +18,21 @@ class InstagramToken(UrlToken):
url = urlparse(data)
return url.netloc in ['www.instagram.com', 'instagram.com']
async def compile(self):
match = pattr.match(self.data)
if match:
m = match.group('media')
return await embed_instagram(m)
def __init__(self, data: str) -> None:
super().__init__(data=data)
self.name = 'instagram'
async def get_data(self):
m = pattr.match(self.data)
embed = None
if m:
m = m.group('media')
embed = await embed_instagram(m)
return {
'url': self.data,
'embed': embed,
}
async def embed_instagram(media):
@@ -51,20 +61,23 @@ async def embed_instagram(media):
</blockquote>
"""
url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
url = f'https://api.instagram.com/oembed?url=http://instagr.am/p/{media}&hidecaption=true&omitscript=true'
req = requests.get(url)
if req.status_code == 200:
html = req.json()['html']
html = fix_html(html)
i = etree.fromstring(html)
obj = etree.Element('div')
obj.set('class', "kazimir__embed")
obj.append(i)
return etree.tostring(obj).decode()
return html
# i = etree.fromstring(html)
# obj = etree.Element('div')
# obj.set('class', "kazimir__embed")
# obj.append(i)
# return etree.tostring(obj).decode()
else:
raise Exception(
'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
'Failed to compile Instagram {media}: Status Code: {status}'.format(
media=media, status=req.status_code)
)
-29
View File
@@ -8,14 +8,10 @@ class Marker:
def __init__(self) -> None:
super().__init__()
self.tokens = []
self.tree_middlewares = []
def add_token_factory(self, token):
self.tokens.append(token)
def add_tree_middleware(self, callback):
self.tree_middlewares.append(callback)
def parse(self, text: str):
tokens = text.split('\n')
tokens = self.markup_tokens(tokens)
@@ -24,16 +20,6 @@ class Marker:
return tokens
async def create_tree(self, tokens: list):
compiled = await compile_tokens(tokens)
tree = lxml.html.fromstring(compiled)
return await self.process_tree(tree)
async def process_tree(self, tree):
for fn in self.tree_middlewares:
tree = await fn(tree)
return tree
def markup_tokens(self, tokens: list):
marked_tokens = []
@@ -95,18 +81,3 @@ def merge_tokens(tokens: list):
token = token.merge(x)
tokens_out.append(token)
return tokens_out
async def compile_tokens(tokens: list):
result = []
for token in tokens:
data = await token.compile()
result.append(data)
return '\n'.join(result)
+13 -76
View File
@@ -41,10 +41,6 @@ class Token:
async def get_data(self):
return self.data
async def compile(self) -> str:
data = await self.get_data()
return str(data)
def merge(self, token):
return self
@@ -76,9 +72,19 @@ class TextToken(Token):
data += token.data
return TextToken(data.strip())
async def compile(self):
data = await self.get_data()
return markdown_to_html(data)
class HtmlToken(Token):
@staticmethod
def test(data: str):
"""
Result of compiling markdown to html is the same as source data meaning that the source is html
"""
html = markdown(data)
return html == data
def __init__(self, data: str) -> None:
super().__init__(name='html', data=data)
self.joinable = False
class FileToken(Token):
@@ -93,12 +99,6 @@ class FileToken(Token):
async def get_data(self):
return parse_file(self.data)
async def compile(self):
data = await self.get_data()
url = data['file']
text = data['caption'] if data['caption'] else url
return f'<a href="{url}">{text}</a>'
class ImageToken(Token):
@staticmethod
@@ -110,74 +110,11 @@ class ImageToken(Token):
self.joinable = False
self.build = None
def merge(self, token: Token):
merged_token = ImageCollectionToken([self.data, token.data])
merged_token.build = self.build
return merged_token
async def get_data(self):
data = parse_file(self.data)
data = await self.build(data)
return data
async def compile(self):
s = 1500 # ('big', 1500, 667)
img = await self.get_data()
alt = img['alt']
caption = markdown_to_html(img['caption'])
src = img['src']
src = f'https://images.weserv.nl/?url={src}&w={s}&h={s}'
tpl = f'''
<div class="kazimir__image">
<figure>
<img alt="{alt}" src="{src}">
<figcaption>{caption}</figcaption>
</figure>
</div>
'''
return tpl.format(img).strip()
class ImageCollectionToken(ImageToken):
def __init__(self, data) -> None:
super().__init__(data)
self.name = 'image_collection'
self.build = None
def merge(self, token: Token):
if isinstance(token.data, list):
data = token.data
else:
data = [token.data]
merged_token = ImageCollectionToken([*self.data, *data])
merged_token.build = self.build
return merged_token
async def get_data(self):
items = [parse_file(x) for x in self.data]
images = []
for x in items:
img = await self.build(x)
images.append(img)
return images
async def compile(self):
images = await self.get_data()
images = [x['src'] for x in images]
images = [f'<img src="{x}">' for x in images]
images = '\n'.join(images)
return f'''
<div class="kazimir__image-collection">
<div class="fotorama"
data-width="100%"
data-ratio="800/600"
>
{images}
</div>
</div>
'''
class UrlToken(Token):
@staticmethod
+7 -18
View File
@@ -11,22 +11,11 @@ class YoutubeToken(UrlToken):
url = urlparse(data)
return url.netloc in ['www.youtube.com', 'youtube.com']
async def compile(self):
"""
<div class="kazimir__video">
<iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
</div>
"""
def __init__(self, data: str) -> None:
super().__init__(data=data)
self.name = 'youtube'
url = urlparse(self.data)
query = parse_qs(url.query)
if 'v' not in query:
return ''
video_id = query['v'][0]
url = f'//www.youtube.com/embed/{video_id}'
return f'''
<div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>
'''.strip()
async def get_data(self):
return {
'url': self.data
}
-34
View File
@@ -1,34 +0,0 @@
import lxml.html
import re
from markdown import markdown
from kazimir.figure import FigureExtension
from kazimir.instagram import InstagramExtension
from sync.data import request
def extract_files(text: str) -> [str]:
html = markdown(text)
post_html = lxml.html.fromstring(html)
images = post_html.cssselect('img')
images = [img.get('src') for img in images]
return images, []
def html_from_tree(tree):
html = lxml.html.tostring(tree, encoding='unicode')
return html
async def typo(text):
from kazimir import artlebedev
result = await artlebedev.typograf(text)
if result == 'Размер текста ограничен 32 КБ':
return text
return result
# url = 'http://www.typograf.ru/webservice/'
# params = dict(text=text)
# return await request.post(url, params, {'chr': 'utf-8'})
-45
View File
@@ -1,45 +0,0 @@
from sync.data import request
from lxml import etree
async def typograf(text, options={}):
text = text.replace('&', '&amp;')
text = text.replace('<', '&lt;')
text = text.replace('>', '&gt;')
body = '''
<?xml version="1.0" encoding="UTF-8"?>
<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
<soap:Body>
<ProcessText xmlns="http://typograf.artlebedev.ru/webservices/">
<text>{text}</text>
<entityType>1</entityType>
<useBr>0</useBr>
<useP>1</useP>
<maxNobr>3</maxNobr>
</ProcessText>
</soap:Body>
</soap:Envelope>
'''.strip().format(text=text)
url = 'http://typograf.artlebedev.ru/webservices/typograf.asmx'
resp = await request.post(url, body, {}, {
'Content-Type': 'text/xml',
'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText',
})
resp_text = resp.encode()
try:
# create xml tree from response
response_tree = etree.fromstring(resp_text)
# look for ProcessTextResult
typograf_response = response_tree.getchildren()[0].getchildren()[0].getchildren()[0].text
except Exception as response_tree:
print(response_tree)
return text
typograf_response = typograf_response.replace('&amp;', '&')
typograf_response = typograf_response.replace('&lt;', '<')
typograf_response = typograf_response.replace('&gt;', '>')
return typograf_response
-44
View File
@@ -1,44 +0,0 @@
from markdown import Extension
from markdown.inlinepatterns import IMAGE_LINK_RE, IMAGE_REFERENCE_RE
from markdown.blockprocessors import BlockProcessor
from markdown.util import etree
import re
FIGURES = [
u'^\s*' + IMAGE_LINK_RE + u'\s*$',
u'^\s*' + IMAGE_REFERENCE_RE + u'\s*$',
]
TAG_FIGURE = 'figure'
TAG_FIGCAPTION = 'figcaption'
NAME = 'FigureAltCaption'
class FigureProcessor(BlockProcessor):
FIGURES_RE = re.compile('|'.join(f for f in FIGURES))
def test(self, parent, block):
is_image = bool(self.FIGURES_RE.search(block))
is_single_list = (len(block.splitlines()) == 1)
is_in_figure = parent.tag == TAG_FIGURE
return is_image and is_single_list and not is_in_figure
def run(self, parent, blocks):
raw_block = blocks.pop(0)
caption_text = self.FIGURES_RE.search(raw_block).group(1)
figure = etree.SubElement(parent, TAG_FIGURE)
figure.text = raw_block
figcaption_elem = etree.SubElement(figure, TAG_FIGCAPTION)
figcaption_elem.text = caption_text
class FigureExtension(Extension):
def extendMarkdown(self, md, md_globals):
md.parser.blockprocessors.add(
NAME,
FigureProcessor(md.parser),
'<ulist'
)
-50
View File
@@ -1,50 +0,0 @@
quote_pairs = [
('"', '"'),
('«', '»'),
]
async def fix_links_quotes(tree):
safe = lambda x: x if x else ''
for link in tree.iterlinks():
a = link[0]
if a.tag != 'a':
continue
quote = get_quote_pair(a.text)
if not quote:
continue
# New text of A without quotes
a.text = a.text[1:-1]
# Move first quote out of A to the prefix
a.getparent().text = safe(a.getparent().text) + quote[0]
# Move last quote out of A to the suffix
a.tail = quote[1] + safe(a.tail)
return tree
def get_quote_pair(text: str):
"""
sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
? (s, t)
: None
:param text:
:return:
"""
if len(text) < 2:
return None
first = text[0]
last = text[-1]
for pair_first, pair_last in quote_pairs:
if pair_first == first and pair_last == last:
return first, last
return None
-68
View File
@@ -1,68 +0,0 @@
import requests
from markdown import Extension
from markdown.inlinepatterns import Pattern
from markdown.util import etree
class InstagramExtension(Extension):
def add_inline(self, md, name, klass, re):
pattern = klass(re)
pattern.md = md
pattern.ext = self
md.inlinePatterns.add(name, pattern, "<reference")
def extendMarkdown(self, md, md_globals):
match_url = r'https?://www\.instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$'
self.add_inline(md, 'instagram', Instagram, match_url)
class Instagram(Pattern):
def handleMatch(self, m):
return embed_instagram(m.group('media'))
def embed_instagram(media):
"""
<blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
<div style="padding:8px;">
<div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
<div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
</div>
</div>
<p style=" margin:8px 0 0 0; padding:0 4px;">
<a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
</a>
</p>
<p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
A post shared by Шлиссельбургская ДХШ (@hudozka) on
<time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
May 24, 2017 at 7:05am PDT
</time>
</p>
</div>
</blockquote>
"""
url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
req = requests.get(url)
if req.status_code == 200:
html = req.json()['html']
html = fix_html(html)
i = etree.fromstring(html)
obj = etree.Element('div')
obj.set('class', "kazimir__embed")
obj.append(i)
return obj
else:
raise Exception(
'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
)
def fix_html(html: str):
return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')