tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

sync simplification

Browse Source 
- drop token compilation to html
- drop artlebedev typograph
- drop other small things
This commit is contained in:
tmshv committed 2020-03-21 20:02:59 +03:00
1 parent 5f51a6e00d
commit c1a3af3a3e
13 files changed
+73 -443
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

modules/sync/kazimir/DocumentToken.py
-42
View File
@@ -1,5 +1,4 @@
from kazimir.Token import parse_file, is_document, Token
import math
class DocumentToken(Token):
@@ -16,44 +15,3 @@ class DocumentToken(Token):
data = parse_file(self.data)
document = await self.build(data)
return document
async def compile(self):
document = await self.get_data()
return create_document(document)
def create_document(document):
file_size = get_size(document['file_size'], 1)
file_format = 'ПДФ'
return f'''
<div>
<div class="document-row">
<a href="{document['url']}" class="invisible">
<div class="document-row__image">
<img src="{document['image_url']}" alt="{document['title']}">
</div>
</a>
<div class="document-row__file">
<a href="{document['url']}">{document['title']}</a>
</div>
<div class="document-row__file-info">
<a href="{document['file_url']}" target="_blank">
{file_format} ({file_size})
</a>
</div>
</div>
</div>
'''.strip()
def get_size(number_of_bytes, precision=1):
units = ['байт', 'КБ', 'МБ', 'ГБ', 'ТБ', 'ПБ']
number = math.floor(math.log(number_of_bytes) / math.log(1024))
size = (number_of_bytes / math.pow(1024, math.floor(number)))
size = round(size, precision)
return f'{size} {units[number]}'
modules/sync/kazimir/InstagramToken.py
+25 -12
View File
@@ -18,11 +18,21 @@ class InstagramToken(UrlToken):
url = urlparse(data)
return url.netloc in ['www.instagram.com', 'instagram.com']
async def compile(self):
match = pattr.match(self.data)
if match:
m = match.group('media')
return await embed_instagram(m)
def __init__(self, data: str) -> None:
super().__init__(data=data)
self.name = 'instagram'
async def get_data(self):
m = pattr.match(self.data)
embed = None
if m:
m = m.group('media')
embed = await embed_instagram(m)
return {
'url': self.data,
'embed': embed,
}
async def embed_instagram(media):
@@ -51,20 +61,23 @@ async def embed_instagram(media):
</blockquote>
"""
url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
url = f'https://api.instagram.com/oembed?url=http://instagr.am/p/{media}&hidecaption=true&omitscript=true'
req = requests.get(url)
if req.status_code == 200:
html = req.json()['html']
html = fix_html(html)
i = etree.fromstring(html)
obj = etree.Element('div')
obj.set('class', "kazimir__embed")
obj.append(i)
return etree.tostring(obj).decode()
return html
# i = etree.fromstring(html)
# obj = etree.Element('div')
# obj.set('class', "kazimir__embed")
# obj.append(i)
# return etree.tostring(obj).decode()
else:
raise Exception(
'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
'Failed to compile Instagram {media}: Status Code: {status}'.format(
media=media, status=req.status_code)
)
modules/sync/kazimir/Marker.py
-29
View File
@@ -8,14 +8,10 @@ class Marker:
def __init__(self) -> None:
super().__init__()
self.tokens = []
self.tree_middlewares = []
def add_token_factory(self, token):
self.tokens.append(token)
def add_tree_middleware(self, callback):
self.tree_middlewares.append(callback)
def parse(self, text: str):
tokens = text.split('\n')
tokens = self.markup_tokens(tokens)
@@ -24,16 +20,6 @@ class Marker:
return tokens
async def create_tree(self, tokens: list):
compiled = await compile_tokens(tokens)
tree = lxml.html.fromstring(compiled)
return await self.process_tree(tree)
async def process_tree(self, tree):
for fn in self.tree_middlewares:
tree = await fn(tree)
return tree
def markup_tokens(self, tokens: list):
marked_tokens = []
@@ -95,18 +81,3 @@ def merge_tokens(tokens: list):
token = token.merge(x)
tokens_out.append(token)
return tokens_out
async def compile_tokens(tokens: list):
result = []
for token in tokens:
data = await token.compile()
result.append(data)
return '\n'.join(result)
modules/sync/kazimir/Token.py
+13 -76
View File
@@ -41,10 +41,6 @@ class Token:
async def get_data(self):
return self.data
async def compile(self) -> str:
data = await self.get_data()
return str(data)
def merge(self, token):
return self
@@ -76,9 +72,19 @@ class TextToken(Token):
data += token.data
return TextToken(data.strip())
async def compile(self):
data = await self.get_data()
return markdown_to_html(data)
class HtmlToken(Token):
@staticmethod
def test(data: str):
"""
Result of compiling markdown to html is the same as source data meaning that the source is html
"""
html = markdown(data)
return html == data
def __init__(self, data: str) -> None:
super().__init__(name='html', data=data)
self.joinable = False
class FileToken(Token):
@@ -93,12 +99,6 @@ class FileToken(Token):
async def get_data(self):
return parse_file(self.data)
async def compile(self):
data = await self.get_data()
url = data['file']
text = data['caption'] if data['caption'] else url
return f'<a href="{url}">{text}</a>'
class ImageToken(Token):
@staticmethod
@@ -110,74 +110,11 @@ class ImageToken(Token):
self.joinable = False
self.build = None
def merge(self, token: Token):
merged_token = ImageCollectionToken([self.data, token.data])
merged_token.build = self.build
return merged_token
async def get_data(self):
data = parse_file(self.data)
data = await self.build(data)
return data
async def compile(self):
s = 1500 # ('big', 1500, 667)
img = await self.get_data()
alt = img['alt']
caption = markdown_to_html(img['caption'])
src = img['src']
src = f'https://images.weserv.nl/?url={src}&w={s}&h={s}'
tpl = f'''
<div class="kazimir__image">
<figure>
<img alt="{alt}" src="{src}">
<figcaption>{caption}</figcaption>
</figure>
</div>
'''
return tpl.format(img).strip()
class ImageCollectionToken(ImageToken):
def __init__(self, data) -> None:
super().__init__(data)
self.name = 'image_collection'
self.build = None
def merge(self, token: Token):
if isinstance(token.data, list):
data = token.data
else:
data = [token.data]
merged_token = ImageCollectionToken([*self.data, *data])
merged_token.build = self.build
return merged_token
async def get_data(self):
items = [parse_file(x) for x in self.data]
images = []
for x in items:
img = await self.build(x)
images.append(img)
return images
async def compile(self):
images = await self.get_data()
images = [x['src'] for x in images]
images = [f'<img src="{x}">' for x in images]
images = '\n'.join(images)
return f'''
<div class="kazimir__image-collection">
<div class="fotorama"
data-width="100%"
data-ratio="800/600"
>
{images}
</div>
</div>
'''
class UrlToken(Token):
@staticmethod
modules/sync/kazimir/YoutubeToken.py
+7 -18
View File
@@ -11,22 +11,11 @@ class YoutubeToken(UrlToken):
url = urlparse(data)
return url.netloc in ['www.youtube.com', 'youtube.com']
async def compile(self):
"""
<div class="kazimir__video">
<iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
</div>
"""
def __init__(self, data: str) -> None:
super().__init__(data=data)
self.name = 'youtube'
url = urlparse(self.data)
query = parse_qs(url.query)
if 'v' not in query:
return ''
video_id = query['v'][0]
url = f'//www.youtube.com/embed/{video_id}'
return f'''
<div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>
'''.strip()
async def get_data(self):
return {
'url': self.data
}
modules/sync/kazimir/__init__.py
-34
View File
@@ -1,34 +0,0 @@
import lxml.html
import re
from markdown import markdown
from kazimir.figure import FigureExtension
from kazimir.instagram import InstagramExtension
from sync.data import request
def extract_files(text: str) -> [str]:
html = markdown(text)
post_html = lxml.html.fromstring(html)
images = post_html.cssselect('img')
images = [img.get('src') for img in images]
return images, []
def html_from_tree(tree):
html = lxml.html.tostring(tree, encoding='unicode')
return html
async def typo(text):
from kazimir import artlebedev
result = await artlebedev.typograf(text)
if result == 'Размер текста ограничен 32 КБ':
return text
return result
# url = 'http://www.typograf.ru/webservice/'
# params = dict(text=text)
# return await request.post(url, params, {'chr': 'utf-8'})
modules/sync/kazimir/artlebedev.py
-45
View File
@@ -1,45 +0,0 @@
from sync.data import request
from lxml import etree
async def typograf(text, options={}):
text = text.replace('&', '&amp;')
text = text.replace('<', '&lt;')
text = text.replace('>', '&gt;')
body = '''
<?xml version="1.0" encoding="UTF-8"?>
<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
<soap:Body>
<ProcessText xmlns="http://typograf.artlebedev.ru/webservices/">
<text>{text}</text>
<entityType>1</entityType>
<useBr>0</useBr>
<useP>1</useP>
<maxNobr>3</maxNobr>
</ProcessText>
</soap:Body>
</soap:Envelope>
'''.strip().format(text=text)
url = 'http://typograf.artlebedev.ru/webservices/typograf.asmx'
resp = await request.post(url, body, {}, {
'Content-Type': 'text/xml',
'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText',
})
resp_text = resp.encode()
try:
# create xml tree from response
response_tree = etree.fromstring(resp_text)
# look for ProcessTextResult
typograf_response = response_tree.getchildren()[0].getchildren()[0].getchildren()[0].text
except Exception as response_tree:
print(response_tree)
return text
typograf_response = typograf_response.replace('&amp;', '&')
typograf_response = typograf_response.replace('&lt;', '<')
typograf_response = typograf_response.replace('&gt;', '>')
return typograf_response
modules/sync/kazimir/figure.py
-44
View File
@@ -1,44 +0,0 @@
from markdown import Extension
from markdown.inlinepatterns import IMAGE_LINK_RE, IMAGE_REFERENCE_RE
from markdown.blockprocessors import BlockProcessor
from markdown.util import etree
import re
FIGURES = [
u'^\s*' + IMAGE_LINK_RE + u'\s*$',
u'^\s*' + IMAGE_REFERENCE_RE + u'\s*$',
]
TAG_FIGURE = 'figure'
TAG_FIGCAPTION = 'figcaption'
NAME = 'FigureAltCaption'
class FigureProcessor(BlockProcessor):
FIGURES_RE = re.compile('|'.join(f for f in FIGURES))
def test(self, parent, block):
is_image = bool(self.FIGURES_RE.search(block))
is_single_list = (len(block.splitlines()) == 1)
is_in_figure = parent.tag == TAG_FIGURE
return is_image and is_single_list and not is_in_figure
def run(self, parent, blocks):
raw_block = blocks.pop(0)
caption_text = self.FIGURES_RE.search(raw_block).group(1)
figure = etree.SubElement(parent, TAG_FIGURE)
figure.text = raw_block
figcaption_elem = etree.SubElement(figure, TAG_FIGCAPTION)
figcaption_elem.text = caption_text
class FigureExtension(Extension):
def extendMarkdown(self, md, md_globals):
md.parser.blockprocessors.add(
NAME,
FigureProcessor(md.parser),
'<ulist'
)
modules/sync/kazimir/fix_links_quotes.py
-50
View File
@@ -1,50 +0,0 @@
quote_pairs = [
('"', '"'),
('«', '»'),
]
async def fix_links_quotes(tree):
safe = lambda x: x if x else ''
for link in tree.iterlinks():
a = link[0]
if a.tag != 'a':
continue
quote = get_quote_pair(a.text)
if not quote:
continue
# New text of A without quotes
a.text = a.text[1:-1]
# Move first quote out of A to the prefix
a.getparent().text = safe(a.getparent().text) + quote[0]
# Move last quote out of A to the suffix
a.tail = quote[1] + safe(a.tail)
return tree
def get_quote_pair(text: str):
"""
sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
? (s, t)
: None
:param text:
:return:
"""
if len(text) < 2:
return None
first = text[0]
last = text[-1]
for pair_first, pair_last in quote_pairs:
if pair_first == first and pair_last == last:
return first, last
return None
modules/sync/kazimir/instagram.py
-68
View File
@@ -1,68 +0,0 @@
import requests
from markdown import Extension
from markdown.inlinepatterns import Pattern
from markdown.util import etree
class InstagramExtension(Extension):
def add_inline(self, md, name, klass, re):
pattern = klass(re)
pattern.md = md
pattern.ext = self
md.inlinePatterns.add(name, pattern, "<reference")
def extendMarkdown(self, md, md_globals):
match_url = r'https?://www\.instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$'
self.add_inline(md, 'instagram', Instagram, match_url)
class Instagram(Pattern):
def handleMatch(self, m):
return embed_instagram(m.group('media'))
def embed_instagram(media):
"""
<blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
<div style="padding:8px;">
<div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
<div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
</div>
</div>
<p style=" margin:8px 0 0 0; padding:0 4px;">
<a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
</a>
</p>
<p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
A post shared by Шлиссельбургская ДХШ (@hudozka) on
<time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
May 24, 2017 at 7:05am PDT
</time>
</p>
</div>
</blockquote>
"""
url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media)
req = requests.get(url)
if req.status_code == 200:
html = req.json()['html']
html = fix_html(html)
i = etree.fromstring(html)
obj = etree.Element('div')
obj.set('class', "kazimir__embed")
obj.append(i)
return obj
else:
raise Exception(
'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code)
)
def fix_html(html: str):
return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')
modules/sync/settings.py
+1 -1
View File
@@ -65,7 +65,7 @@ collection_images = 'images'
collection_documents = 'documents'
collection_pages = 'pages'
hash_salt_pages = '4'
hash_salt_pages = '6'
hash_salt_documents = '4'
hash_salt_images = '5'
modules/sync/sync/__init__.py
+3 -16
View File
@@ -10,7 +10,7 @@ from kazimir.DocumentToken import DocumentToken
from kazimir.YoutubeToken import YoutubeToken
from kazimir.InstagramToken import InstagramToken
from kazimir.Marker import Marker
from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken
from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, HtmlToken
from sync.data import Provider
from sync.models import Model
from sync.models.Document import Document
@@ -66,8 +66,6 @@ async def untouched(items: [Model]) -> [Model]:
async def create_post(provider: Provider, folder: str, md: str, sizes):
from kazimir.fix_links_quotes import fix_links_quotes
images = []
documents = []
@@ -128,14 +126,10 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file))
# m.add_token_factory(TokenFactory(FileToken))
m.add_token_factory(TokenFactory(HtmlToken))
m.add_token_factory(TokenFactory(TextToken))
m.add_tree_middleware(fix_links_quotes)
tokens = m.parse(md)
tree = await m.create_tree(tokens)
post = kazimir.html_from_tree(tree)
post = await kazimir.typo(post)
encoded_tokens = []
for t in tokens:
@@ -145,11 +139,4 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
'data': data,
})
return post, images, documents, encoded_tokens
def title_from_html(md):
html = lxml.html.fromstring(md)
titles = html.cssselect('h1')
if len(titles):
return titles[0].text
return images, documents, encoded_tokens
modules/sync/sync/models/Page.py
+24 -8
View File
@@ -2,21 +2,36 @@ import os
import logging
from lxml import etree
import lxml.html
import kazimir
import settings
from db import collection
from sync.data import request
from sync.models import Model
from sync import create_date, title_from_html, create_post
from sync import create_date, create_post
from sync.models.Image import Image
from utils.hash import hash_str
from utils.front_matter import parse_yaml_front_matter
from utils.text.transform import url_encode_text
from markdown import markdown
logger = logging.getLogger(settings.name + '.Page')
store = collection(settings.collection_pages)
def extract_files(html) -> [str]:
images = html.cssselect('img')
images = [img.get('src') for img in images]
return images, []
def title_from_html(html):
titles = html.cssselect('h1')
if len(titles):
return titles[0].text
return ''
def find(store, item_id: str):
q = {'id': item_id}
try:
@@ -103,7 +118,6 @@ class Page(Model):
self.images = []
self.documents = []
self.title = None
self.data = None
self.preview = None
self.tokens = None
@@ -114,7 +128,12 @@ class Page(Model):
super().__init__(provider, store, file, params=params)
def init(self):
images, documents = kazimir.extract_files(self.get_param('content'))
content = self.get_param('content')
html = markdown(content)
html_tree = lxml.html.fromstring(html)
images, documents = extract_files(html_tree)
self.title = title_from_html(html_tree)
self.params['images'] = images
self.params['documents'] = documents
@@ -136,9 +155,7 @@ class Page(Model):
sizes = kwargs['sizes']
markdown_post = self.get_param('content')
post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
self.title = title_from_html(post)
self.data = post
images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
self.images = images
self.documents = documents
self.tokens = tokens
@@ -158,7 +175,6 @@ class Page(Model):
'hash': self.hash,
'url': self.url,
'file': self.file,
'data': self.data,
'images': images,
'documents': documents,
'title': self.title,