diff --git a/utils/Sync/kazimir/__init__.py b/utils/Sync/kazimir/__init__.py
index 81d330b3..9bdd7af5 100644
--- a/utils/Sync/kazimir/__init__.py
+++ b/utils/Sync/kazimir/__init__.py
@@ -1,54 +1,45 @@
import lxml.html
-import requests
from markdown import markdown
-from kazimir.video import VideoExtension
from kazimir.figure import FigureExtension
from kazimir.instagram import InstagramExtension
+from sync.data import request
def markdown_to_html(text):
return markdown(text, extensions=[
FigureExtension(),
- VideoExtension(),
InstagramExtension(),
])
-def kazimir_to_html(text):
- html = make(text)
- return typo(html)
+def get_images_src(text: str) -> [str]:
+ html = markdown(text)
+ post_html = lxml.html.fromstring(html)
+
+ images = post_html.cssselect('img')
+ return [img.get('src') for img in images]
-def html_from_tree(tree):
- html = lxml.html.tostring(tree, encoding='unicode')
- html = typo(html)
-
- return html
-
-
-def typo(html):
- url = 'http://api.tmshv.com/typograph/v1'
- res = requests.post(
- url=url,
- headers={
- 'Content-Type': 'text/plain; charset=utf-8',
- },
- data=html.encode('utf-8')
- )
- if res.status_code == 200:
- return res.text
- return html
-
-
-def make(text):
+def kazimir_to_html(text: str) -> str:
tokens = text.split('\n')
tokens = markup_tokens(tokens)
tokens = join_tokens(tokens)
tokens = merge_tokens(tokens)
return compile_tokens(tokens)
- # return tokens
+
+
+def html_from_tree(tree):
+ html = lxml.html.tostring(tree, encoding='unicode')
+
+ return html
+
+
+async def typo(text):
+ url = 'http://www.typograf.ru/webservice/'
+ params = dict(text=text)
+ return await request.post(url, params, {'chr': 'utf-8'})
def join_tokens(tokens):
@@ -110,6 +101,8 @@ def compile_tokens(tokens: list):
fn = compile_image_collection
elif tag == 'image':
fn = compile_image
+ elif tag == 'url':
+ fn = compile_url
else:
fn = compile_any
compiled.append(fn(ts))
@@ -203,11 +196,25 @@ def compile_image_collection(token):
def compile_image(token):
img = markdown_to_html(token['data'])
- return '''
-
- {}
-
- '''.format(img)
+ tpl = '''{}
'''
+ return tpl.format(img)
+
+
+def compile_url(token):
+ return compile_youtube(token)
+
+
+def compile_youtube(token):
+ """
+
+
+
+ """
+
+ url = token['data']
+ tpl = ''''''
+
+ return tpl.format(url=url)
def compile_any(token):
@@ -246,4 +253,3 @@ More text
print(i)
else:
print(ts)
-
diff --git a/utils/Sync/kazimir/video.py b/utils/Sync/kazimir/video.py
deleted file mode 100644
index 8cc2df47..00000000
--- a/utils/Sync/kazimir/video.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from markdown import Extension
-from markdown.inlinepatterns import Pattern
-from markdown.util import etree
-
-
-class VideoExtension(Extension):
- def add_inline(self, md, name, klass, re):
- pattern = klass(re)
- pattern.md = md
- pattern.ext = self
- md.inlinePatterns.add(name, pattern, "[A-Za-z0-9_&=-]+)\S*'
- self.add_inline(md, 'youtube', Youtube, youtube)
-
-
-class Youtube(Pattern):
- def handleMatch(self, m):
- url = 'https://www.youtube.com/embed/%s' % m.group('youtubeargs')
- return flex_video(url)
-
-
-def flex_video(url):
- """
-
-
-
- """
- iframe = etree.Element('iframe')
- iframe.set('src', url)
- iframe.set('frameborder', '0')
- iframe.set('allowfullscreen', '')
-
- obj = etree.Element('div')
- obj.set('class', "kazimir__video")
- obj.append(iframe)
- return obj
diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py
index 9a15b1c2..fdbc283b 100644
--- a/utils/Sync/settings.py
+++ b/utils/Sync/settings.py
@@ -129,6 +129,8 @@ collection_collective = 'collective'
collection_albums = 'albums'
collection_pages = 'pages'
+hash_salt_articles = '0'
+
provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs')
provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs')
diff --git a/utils/Sync/sync/data/request.py b/utils/Sync/sync/data/request.py
index 3b5e4a3a..34c9e538 100644
--- a/utils/Sync/sync/data/request.py
+++ b/utils/Sync/sync/data/request.py
@@ -34,6 +34,12 @@ async def get(url):
return await res.text()
+async def post(url, data, params={}):
+ async with aiohttp.ClientSession() as session:
+ async with session.post(url, data=data, params=params) as resp:
+ return await resp.text()
+
+
async def put(url, data):
async with aiohttp.ClientSession as session:
res = session.put(url, data=data)
diff --git a/utils/Sync/sync/models/Article.py b/utils/Sync/sync/models/Article.py
index 9280ffb1..d279bc6d 100644
--- a/utils/Sync/sync/models/Article.py
+++ b/utils/Sync/sync/models/Article.py
@@ -49,16 +49,10 @@ class Article(Model):
'title': doc_title,
}
- # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
- images = list_images(provider, path)
- images = [os.path.relpath(i, path) for i in images]
- document['images'] = images
- document['post'] = create_post_from_image_list(images)
-
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
manifest_file = md_from_folder(provider, path)
if manifest_file:
- manifest = get_manifest(provider, manifest_file)
+ manifest = read_manifest(provider, manifest_file)
else:
manifest = {}
@@ -76,6 +70,7 @@ class Article(Model):
# self.__url_template: str = settings.document_url_template
# self.__url_preview_template: str = settings.document_url_preview_template
+ self.hash_salt = settings.hash_salt_articles
self.origin = settings.origin
self.version = '2'
self.post = None
@@ -83,6 +78,13 @@ class Article(Model):
super().__init__(provider, store, file, params=params)
def init(self):
+ if self.has_param('content'):
+ images = kazimir.get_images_src(self.get_param('content'))
+ else:
+ images = list_images(self.provider, self.file)
+ images = [os.path.relpath(i, self.file) for i in images]
+ self.params['images'] = images
+
self.__set_id()
self.__set_hash()
@@ -96,12 +98,14 @@ class Article(Model):
async def build(self, **kwargs):
sizes = kwargs['sizes']
- post, images = await create_post(self.provider, self.id, self.params, sizes)
- self.post = post
- self.images = images
- post, images = await create_post(self.provider, self.id, self.params, sizes)
- self.post = kazimir.html_from_tree(post)
+ if self.has_param('content'):
+ markdown_post = self.get_param('content')
+ else:
+ markdown_post = create_post_from_image_list(self.get_param('images'))
+
+ post, images = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+ self.post = post
self.images = images
if 'preview' in self.params:
@@ -140,7 +144,7 @@ class Article(Model):
images = [self.provider.hash(i) for i in images]
self.hash = hash_str(
- combine([hash_str(self.params)] + images)
+ combine([self.hash_salt] + [hash_str(self.params)] + images)
)
def __str__(self):
@@ -151,7 +155,7 @@ def md_from_folder(provider, i):
return first(provider.type_filter(i, '.md'))
-def get_manifest(provider, path):
+def read_manifest(provider, path):
data = provider.read(path).read().decode('utf-8')
manifest = parse_yaml_front_matter(data)
@@ -164,38 +168,30 @@ def get_manifest(provider, path):
if 'id' in manifest:
manifest['id'] = str(manifest['id'])
- html = kazimir_to_html(manifest['content'])
return {
**manifest,
- 'post': html,
- 'images': images_from_html(html) # Add images for hashing
}
-async def create_post(provider, doc_id, document, sizes):
- image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
-
- post_html = lxml.html.fromstring(document['post'])
+async def create_post(provider: Provider, folder: str, md: str, sizes):
+ post = kazimir.kazimir_to_html(md)
+ html = lxml.html.fromstring(post)
images = []
- for img in post_html.cssselect('img'):
+ for img in html.cssselect('img'):
src = img.get('src')
- relative_image_path = os.path.join(document['folder'], src)
+ relative_image_path = os.path.join(folder, src)
- img_path = provider.get_local(relative_image_path)
- if os.path.exists(img_path):
- img_id = url_encode_text(os.path.splitext(src)[0])
- url_fn = lambda file, size, ext: image_url_base.format(
- id=doc_id,
- img=img_id,
- size=size,
- ext=ext.lower()
- )
-
- image = await Image.new(provider, img_path, sizes, url_fn)
+ image_path = provider.get_local(relative_image_path)
+ if os.path.exists(image_path):
+ image = await Image.new(provider, image_path, sizes)
if image:
url = image.get_size('big')['url']
images.append(image)
img.set('src', url)
else:
- logger.error('Fail to get Image', document['folder'], src)
- return post_html, images
+ logger.error('Fail to get Image', folder, src)
+
+ post = kazimir.html_from_tree(html)
+ post = await kazimir.typo(post)
+
+ return post, images