From bb393becc44e4e9c564977c47e0aa077240e86a7 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Thu, 19 Mar 2020 22:34:28 +0300 Subject: [PATCH 1/7] render page content based on tokens; ignore rendered content field --- package-lock.json | 35 ++++++++++++++++ package.json | 2 + pages/[...slug].tsx | 67 +++++++++++++++++++++++++++++-- src/components/Markdown/index.tsx | 19 +++++++++ src/components/Page/index.tsx | 8 +--- src/lib/tokens.ts | 36 +++++++++++++++++ src/style/article.css | 4 -- src/types.ts | 32 +++++++++++++++ 8 files changed, 188 insertions(+), 15 deletions(-) create mode 100644 src/components/Markdown/index.tsx create mode 100644 src/lib/tokens.ts diff --git a/package-lock.json b/package-lock.json index cb5cdfb7..2880e81e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4451,6 +4451,14 @@ } } }, + "linkify-it": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz", + "integrity": "sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==", + "requires": { + "uc.micro": "^1.0.1" + } + }, "loader-runner": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", @@ -4667,6 +4675,18 @@ "object-visit": "^1.0.0" } }, + "markdown-it": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-10.0.0.tgz", + "integrity": "sha512-YWOP1j7UbDNz+TumYP1kpwnP0aEa711cJjrAQrzd0UXlbJfc5aAq0F/PZHjiioqDC1NKgvIMX+o+9Bk7yuM2dg==", + "requires": { + "argparse": "^1.0.7", + "entities": "~2.0.0", + "linkify-it": "^2.0.0", + "mdurl": "^1.0.1", + "uc.micro": "^1.0.5" + } + }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -4684,6 +4704,11 @@ } } }, + "mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=" + }, "memory-fs": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", @@ -8077,6 +8102,16 @@ "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", "dev": true }, + "typograf": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/typograf/-/typograf-6.11.0.tgz", + "integrity": "sha512-7PcX6gyocBoNoCnLNmgqzd7ohp7SrW44U6YIqk91+7x4ivVlzpRltN4JK7mtBov9CVPsfX6W8yXTnoUxImSzwQ==" + }, + "uc.micro": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", + "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==" + }, "underscore": { "version": "1.8.3", "resolved": "http://storage.mds.yandex.net/get-npm/35308/underscore-1.8.3.tgz", diff --git a/package.json b/package.json index 89e7e3dc..d26bd04f 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,12 @@ "classnames": "^2.2.6", "date-fns": "^2.11.0", "lodash": "^4.17.15", + "markdown-it": "^10.0.0", "next": "^9.3.1", "react": "^16.13.0", "react-dom": "^16.13.0", "sitemap": "^1.5.0", + "typograf": "^6.11.0", "use-media": "^1.4.0" }, "devDependencies": { diff --git a/pages/[...slug].tsx b/pages/[...slug].tsx index b7a4031b..f47235f2 100644 --- a/pages/[...slug].tsx +++ b/pages/[...slug].tsx @@ -2,11 +2,13 @@ import Head from 'next/head' import { App } from 'src/components/App' import { tail } from 'lodash' import { Page } from 'src/components/Page' +import { Markdown } from 'src/components/Markdown' import { Meta } from 'src/components/Meta' import { MetaBuilder } from 'src/lib/meta' import { createApiUrl, requestGet, IResponseItems } from 'src/next-lib' import { NextPage } from 'next' -import { IBreadcumbsPart, IMeta, IPage, ITag } from 'src/types' +import { IBreadcumbsPart, IMeta, IPage, ITag, FileTokenData, Token } from 'src/types' +import { joinTokens } from 'src/lib/tokens' function array(value: T | T[]) { return Array.isArray(value) @@ -14,13 +16,33 @@ function array(value: T | T[]) { : [value] } +const File: React.SFC = props => ( +
+ +
+ {props['title']} +
+ + + + + +
+) + type Props = { title: string tags: ITag[] date: string - content: string breadcrumb: IBreadcumbsPart[] meta?: IMeta + tokens?: Token[] } const Index: NextPage = props => ( @@ -42,7 +64,43 @@ const Index: NextPage = props => ( tags={props.tags} date={props.date ? new Date(props.date) : null} > - {props.content} +
+ {joinTokens(props.tokens ?? []).map((x, i) => { + switch (x.token) { + case 'text': + return ( + + ) + + case 'image': + return ( +
+
+ {x.data.alt} +
{x.data.caption}
+
+
+ ) + + case 'file': + return ( + + ) + + default: + return ( + 
+                                    {JSON.stringify(x)}
+
+ ) + } + })} +
) @@ -66,10 +124,11 @@ export const getStaticProps = async (ctx: any) => { .setTitle(page.title) .setDescription(description) .build() + const tokens = page.tokens return { props: { - content: page.data, + tokens, title: page.title, tags: page.tags, date: page.date, diff --git a/src/components/Markdown/index.tsx b/src/components/Markdown/index.tsx new file mode 100644 index 00000000..2ede54d2 --- /dev/null +++ b/src/components/Markdown/index.tsx @@ -0,0 +1,19 @@ +import { Html } from 'src/components/Html' +import MarkdownIt from 'markdown-it' +import Typograf from 'typograf' +import { memo } from 'react' + +const tp = new Typograf({ locale: ['ru'] }) +const md = new MarkdownIt({ + html: true, +}) + +export type MarkdownProps = { + data: string +} + +export const Markdown: React.SFC = memo(props => ( + +)) diff --git a/src/components/Page/index.tsx b/src/components/Page/index.tsx index efb74fb5..64b83694 100644 --- a/src/components/Page/index.tsx +++ b/src/components/Page/index.tsx @@ -7,19 +7,13 @@ import { ITag } from 'src/types' export type PageProps = { style?: React.CSSProperties - children: string date: Date tags: ITag[] } export const Page: React.FC = props => (
- - + {props.children}
diff --git a/src/lib/tokens.ts b/src/lib/tokens.ts new file mode 100644 index 00000000..fedd2cd8 --- /dev/null +++ b/src/lib/tokens.ts @@ -0,0 +1,36 @@ +import { TextToken, Token } from 'src/types' + +export function joinTextTokens(tokens: TextToken[]): TextToken { + return { + token: 'text', + data: tokens + .map(x => x.data) + .join('\n\n'), + } +} + +export function joinTokens(tokens: Token[]): Token[] { + let buffer: TextToken[] = [] + const result: Token[] = [] + + for (const token of tokens) { + if (token.token === 'text') { + buffer.push(token) + continue + } + + const compiledToken = joinTextTokens(buffer) + buffer = [] + + result.push(compiledToken) + result.push(token) + } + + if (buffer.length) { + result.push( + joinTextTokens(buffer) + ) + } + + return result +} diff --git a/src/style/article.css b/src/style/article.css index c53f0270..d5b23251 100644 --- a/src/style/article.css +++ b/src/style/article.css @@ -27,10 +27,6 @@ line-height: var(--article-line-height); } -.article p:last-child { - margin-bottom: 0; -} - .article blockquote { padding: var(--size-m); font-size: var(--font-size-accent); diff --git a/src/types.ts b/src/types.ts index e2b41594..bab0b2df 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,3 +103,35 @@ export type ImageDefinition = { height: number src: string } + +export type TextToken = { + token: 'text' + data: string +} + +export type ImageToken = { + token: 'image' + data: { + src: string + alt: string + caption: string + } +} + +export type FileTokenData = { + url: string + slug: string + image_url: string + file_url: string + title: string + file_size: number + file_format: 'application/pdf' +} + +export type FileToken = { + token: 'file' + data: FileTokenData +} + +export type Token = TextToken | ImageToken | FileToken + From b78d78710c354a675aba9c280e9e9b066d4ea9e0 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Thu, 19 Mar 2020 23:12:22 +0300 Subject: [PATCH 2/7] Markdown component refactoring: move core to lib/text --- src/components/Markdown/index.tsx | 10 ++-------- src/lib/text.ts | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 src/lib/text.ts diff --git a/src/components/Markdown/index.tsx b/src/components/Markdown/index.tsx index 2ede54d2..575e4b77 100644 --- a/src/components/Markdown/index.tsx +++ b/src/components/Markdown/index.tsx @@ -1,12 +1,6 @@ import { Html } from 'src/components/Html' -import MarkdownIt from 'markdown-it' -import Typograf from 'typograf' import { memo } from 'react' - -const tp = new Typograf({ locale: ['ru'] }) -const md = new MarkdownIt({ - html: true, -}) +import { typograf, markdownToHtml } from 'src/lib/text' export type MarkdownProps = { data: string @@ -14,6 +8,6 @@ export type MarkdownProps = { export const Markdown: React.SFC = memo(props => ( )) diff --git a/src/lib/text.ts b/src/lib/text.ts new file mode 100644 index 00000000..ebcf5956 --- /dev/null +++ b/src/lib/text.ts @@ -0,0 +1,19 @@ +import MarkdownIt from 'markdown-it' +import Typograf from 'typograf' + +const tp = new Typograf({ locale: ['ru'] }) +const md = new MarkdownIt({ + html: true, +}) + +export type MarkdownProps = { + data: string +} + +export function markdownToHtml(data: string): string { + return md.render(data) +} + +export function typograf(data: string): string { + return tp.execute(data) +} From 5f51a6e00d47be3c2c5f4f55d0e0fab22c4c6d2b Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 21 Mar 2020 18:31:54 +0300 Subject: [PATCH 3/7] style.css moved to style folder --- pages/_app.tsx | 2 +- src/{ => style}/style.css | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{ => style}/style.css (100%) diff --git a/pages/_app.tsx b/pages/_app.tsx index 02c1c587..df3a3342 100644 --- a/pages/_app.tsx +++ b/pages/_app.tsx @@ -1,4 +1,4 @@ -import 'src/style.css' +import 'src/style/style.css' import 'src/style/document-row.css' import 'src/style/content.css' import 'src/style/article.css' diff --git a/src/style.css b/src/style/style.css similarity index 100% rename from src/style.css rename to src/style/style.css From c1a3af3a3e5bf321ea716f8de3adafc63809d7b4 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 21 Mar 2020 20:02:51 +0300 Subject: [PATCH 4/7] sync simplification - drop token compilation to html - drop artlebedev typograph - drop other small things --- modules/sync/kazimir/DocumentToken.py | 42 ----------- modules/sync/kazimir/InstagramToken.py | 37 ++++++---- modules/sync/kazimir/Marker.py | 29 -------- modules/sync/kazimir/Token.py | 89 ++++-------------------- modules/sync/kazimir/YoutubeToken.py | 25 ++----- modules/sync/kazimir/__init__.py | 34 --------- modules/sync/kazimir/artlebedev.py | 45 ------------ modules/sync/kazimir/figure.py | 44 ------------ modules/sync/kazimir/fix_links_quotes.py | 50 ------------- modules/sync/kazimir/instagram.py | 68 ------------------ modules/sync/settings.py | 2 +- modules/sync/sync/__init__.py | 19 +---- modules/sync/sync/models/Page.py | 32 ++++++--- 13 files changed, 73 insertions(+), 443 deletions(-) delete mode 100644 modules/sync/kazimir/__init__.py delete mode 100644 modules/sync/kazimir/artlebedev.py delete mode 100644 modules/sync/kazimir/figure.py delete mode 100644 modules/sync/kazimir/fix_links_quotes.py delete mode 100644 modules/sync/kazimir/instagram.py diff --git a/modules/sync/kazimir/DocumentToken.py b/modules/sync/kazimir/DocumentToken.py index e24b5072..c9c97c25 100644 --- a/modules/sync/kazimir/DocumentToken.py +++ b/modules/sync/kazimir/DocumentToken.py @@ -1,5 +1,4 @@ from kazimir.Token import parse_file, is_document, Token -import math class DocumentToken(Token): @@ -16,44 +15,3 @@ class DocumentToken(Token): data = parse_file(self.data) document = await self.build(data) return document - - async def compile(self): - document = await self.get_data() - - return create_document(document) - - -def create_document(document): - file_size = get_size(document['file_size'], 1) - file_format = 'ПДФ' - - return f''' - - '''.strip() - - -def get_size(number_of_bytes, precision=1): - units = ['байт', 'КБ', 'МБ', 'ГБ', 'ТБ', 'ПБ'] - number = math.floor(math.log(number_of_bytes) / math.log(1024)) - size = (number_of_bytes / math.pow(1024, math.floor(number))) - size = round(size, precision) - - return f'{size} {units[number]}' diff --git a/modules/sync/kazimir/InstagramToken.py b/modules/sync/kazimir/InstagramToken.py index ad7757ea..e3d3fc78 100644 --- a/modules/sync/kazimir/InstagramToken.py +++ b/modules/sync/kazimir/InstagramToken.py @@ -18,11 +18,21 @@ class InstagramToken(UrlToken): url = urlparse(data) return url.netloc in ['www.instagram.com', 'instagram.com'] - async def compile(self): - match = pattr.match(self.data) - if match: - m = match.group('media') - return await embed_instagram(m) + def __init__(self, data: str) -> None: + super().__init__(data=data) + self.name = 'instagram' + + async def get_data(self): + m = pattr.match(self.data) + embed = None + if m: + m = m.group('media') + embed = await embed_instagram(m) + + return { + 'url': self.data, + 'embed': embed, + } async def embed_instagram(media): @@ -51,20 +61,23 @@ async def embed_instagram(media): """ - url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media) + url = f'https://api.instagram.com/oembed?url=http://instagr.am/p/{media}&hidecaption=true&omitscript=true' req = requests.get(url) if req.status_code == 200: html = req.json()['html'] html = fix_html(html) - i = etree.fromstring(html) - obj = etree.Element('div') - obj.set('class', "kazimir__embed") - obj.append(i) - return etree.tostring(obj).decode() + return html + # i = etree.fromstring(html) + + # obj = etree.Element('div') + # obj.set('class', "kazimir__embed") + # obj.append(i) + # return etree.tostring(obj).decode() else: raise Exception( - 'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code) + 'Failed to compile Instagram {media}: Status Code: {status}'.format( + media=media, status=req.status_code) ) diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py index 3b81c9bb..ee013743 100644 --- a/modules/sync/kazimir/Marker.py +++ b/modules/sync/kazimir/Marker.py @@ -8,14 +8,10 @@ class Marker: def __init__(self) -> None: super().__init__() self.tokens = [] - self.tree_middlewares = [] def add_token_factory(self, token): self.tokens.append(token) - def add_tree_middleware(self, callback): - self.tree_middlewares.append(callback) - def parse(self, text: str): tokens = text.split('\n') tokens = self.markup_tokens(tokens) @@ -24,16 +20,6 @@ class Marker: return tokens - async def create_tree(self, tokens: list): - compiled = await compile_tokens(tokens) - tree = lxml.html.fromstring(compiled) - return await self.process_tree(tree) - - async def process_tree(self, tree): - for fn in self.tree_middlewares: - tree = await fn(tree) - return tree - def markup_tokens(self, tokens: list): marked_tokens = [] @@ -95,18 +81,3 @@ def merge_tokens(tokens: list): token = token.merge(x) tokens_out.append(token) return tokens_out - - -async def compile_tokens(tokens: list): - result = [] - for token in tokens: - data = await token.compile() - result.append(data) - return '\n'.join(result) - - - - - - - diff --git a/modules/sync/kazimir/Token.py b/modules/sync/kazimir/Token.py index 4de5ae35..0ac9a9c6 100644 --- a/modules/sync/kazimir/Token.py +++ b/modules/sync/kazimir/Token.py @@ -41,10 +41,6 @@ class Token: async def get_data(self): return self.data - async def compile(self) -> str: - data = await self.get_data() - return str(data) - def merge(self, token): return self @@ -76,9 +72,19 @@ class TextToken(Token): data += token.data return TextToken(data.strip()) - async def compile(self): - data = await self.get_data() - return markdown_to_html(data) + +class HtmlToken(Token): + @staticmethod + def test(data: str): + """ + Result of compiling markdown to html is the same as source data meaning that the source is html + """ + html = markdown(data) + return html == data + + def __init__(self, data: str) -> None: + super().__init__(name='html', data=data) + self.joinable = False class FileToken(Token): @@ -93,12 +99,6 @@ class FileToken(Token): async def get_data(self): return parse_file(self.data) - async def compile(self): - data = await self.get_data() - url = data['file'] - text = data['caption'] if data['caption'] else url - return f'{text}' - class ImageToken(Token): @staticmethod @@ -110,74 +110,11 @@ class ImageToken(Token): self.joinable = False self.build = None - def merge(self, token: Token): - merged_token = ImageCollectionToken([self.data, token.data]) - merged_token.build = self.build - return merged_token - async def get_data(self): data = parse_file(self.data) data = await self.build(data) return data - async def compile(self): - s = 1500 # ('big', 1500, 667) - img = await self.get_data() - alt = img['alt'] - caption = markdown_to_html(img['caption']) - src = img['src'] - src = f'https://images.weserv.nl/?url={src}&w={s}&h={s}' - - tpl = f''' -
-
- {alt} -
{caption}
-
-
- ''' - return tpl.format(img).strip() - - -class ImageCollectionToken(ImageToken): - def __init__(self, data) -> None: - super().__init__(data) - self.name = 'image_collection' - self.build = None - - def merge(self, token: Token): - if isinstance(token.data, list): - data = token.data - else: - data = [token.data] - merged_token = ImageCollectionToken([*self.data, *data]) - merged_token.build = self.build - return merged_token - - async def get_data(self): - items = [parse_file(x) for x in self.data] - images = [] - for x in items: - img = await self.build(x) - images.append(img) - return images - - async def compile(self): - images = await self.get_data() - images = [x['src'] for x in images] - images = [f'' for x in images] - images = '\n'.join(images) - - return f''' -
-
- {images} -
-
- ''' class UrlToken(Token): @staticmethod diff --git a/modules/sync/kazimir/YoutubeToken.py b/modules/sync/kazimir/YoutubeToken.py index c6133d52..2157d957 100644 --- a/modules/sync/kazimir/YoutubeToken.py +++ b/modules/sync/kazimir/YoutubeToken.py @@ -11,22 +11,11 @@ class YoutubeToken(UrlToken): url = urlparse(data) return url.netloc in ['www.youtube.com', 'youtube.com'] - async def compile(self): - """ -
- -
- """ + def __init__(self, data: str) -> None: + super().__init__(data=data) + self.name = 'youtube' - url = urlparse(self.data) - query = parse_qs(url.query) - - if 'v' not in query: - return '' - - video_id = query['v'][0] - - url = f'//www.youtube.com/embed/{video_id}' - return f''' -
- '''.strip() + async def get_data(self): + return { + 'url': self.data + } diff --git a/modules/sync/kazimir/__init__.py b/modules/sync/kazimir/__init__.py deleted file mode 100644 index fa0281ed..00000000 --- a/modules/sync/kazimir/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -import lxml.html -import re -from markdown import markdown - -from kazimir.figure import FigureExtension -from kazimir.instagram import InstagramExtension -from sync.data import request - -def extract_files(text: str) -> [str]: - html = markdown(text) - post_html = lxml.html.fromstring(html) - - images = post_html.cssselect('img') - images = [img.get('src') for img in images] - - return images, [] - - -def html_from_tree(tree): - html = lxml.html.tostring(tree, encoding='unicode') - - return html - - -async def typo(text): - from kazimir import artlebedev - result = await artlebedev.typograf(text) - if result == 'Размер текста ограничен 32 КБ': - return text - - return result - # url = 'http://www.typograf.ru/webservice/' - # params = dict(text=text) - # return await request.post(url, params, {'chr': 'utf-8'}) diff --git a/modules/sync/kazimir/artlebedev.py b/modules/sync/kazimir/artlebedev.py deleted file mode 100644 index c1c42289..00000000 --- a/modules/sync/kazimir/artlebedev.py +++ /dev/null @@ -1,45 +0,0 @@ -from sync.data import request -from lxml import etree - - -async def typograf(text, options={}): - text = text.replace('&', '&') - text = text.replace('<', '<') - text = text.replace('>', '>') - - body = ''' - - - - - {text} - 1 - 0 - 1 - 3 - - - - '''.strip().format(text=text) - - url = 'http://typograf.artlebedev.ru/webservices/typograf.asmx' - resp = await request.post(url, body, {}, { - 'Content-Type': 'text/xml', - 'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText', - }) - resp_text = resp.encode() - - try: - # create xml tree from response - response_tree = etree.fromstring(resp_text) - - # look for ProcessTextResult - typograf_response = response_tree.getchildren()[0].getchildren()[0].getchildren()[0].text - except Exception as response_tree: - print(response_tree) - return text - - typograf_response = typograf_response.replace('&', '&') - typograf_response = typograf_response.replace('<', '<') - typograf_response = typograf_response.replace('>', '>') - return typograf_response diff --git a/modules/sync/kazimir/figure.py b/modules/sync/kazimir/figure.py deleted file mode 100644 index 1d0a9668..00000000 --- a/modules/sync/kazimir/figure.py +++ /dev/null @@ -1,44 +0,0 @@ -from markdown import Extension -from markdown.inlinepatterns import IMAGE_LINK_RE, IMAGE_REFERENCE_RE -from markdown.blockprocessors import BlockProcessor -from markdown.util import etree -import re - -FIGURES = [ - u'^\s*' + IMAGE_LINK_RE + u'\s*$', - u'^\s*' + IMAGE_REFERENCE_RE + u'\s*$', -] - -TAG_FIGURE = 'figure' -TAG_FIGCAPTION = 'figcaption' -NAME = 'FigureAltCaption' - - -class FigureProcessor(BlockProcessor): - FIGURES_RE = re.compile('|'.join(f for f in FIGURES)) - - def test(self, parent, block): - is_image = bool(self.FIGURES_RE.search(block)) - is_single_list = (len(block.splitlines()) == 1) - is_in_figure = parent.tag == TAG_FIGURE - - return is_image and is_single_list and not is_in_figure - - def run(self, parent, blocks): - raw_block = blocks.pop(0) - caption_text = self.FIGURES_RE.search(raw_block).group(1) - - figure = etree.SubElement(parent, TAG_FIGURE) - figure.text = raw_block - - figcaption_elem = etree.SubElement(figure, TAG_FIGCAPTION) - figcaption_elem.text = caption_text - - -class FigureExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.parser.blockprocessors.add( - NAME, - FigureProcessor(md.parser), - ' [s]ample tex[t] -> (s, t) -> match pair from quote_pair - ? (s, t) - : None - - :param text: - :return: - """ - if len(text) < 2: - return None - - first = text[0] - last = text[-1] - - for pair_first, pair_last in quote_pairs: - if pair_first == first and pair_last == last: - return first, last - - return None diff --git a/modules/sync/kazimir/instagram.py b/modules/sync/kazimir/instagram.py deleted file mode 100644 index ee5f8c0c..00000000 --- a/modules/sync/kazimir/instagram.py +++ /dev/null @@ -1,68 +0,0 @@ -import requests -from markdown import Extension -from markdown.inlinepatterns import Pattern -from markdown.util import etree - - -class InstagramExtension(Extension): - def add_inline(self, md, name, klass, re): - pattern = klass(re) - pattern.md = md - pattern.ext = self - md.inlinePatterns.add(name, pattern, "[\w\d]+)/?(.*)$' - self.add_inline(md, 'instagram', Instagram, match_url) - - -class Instagram(Pattern): - def handleMatch(self, m): - return embed_instagram(m.group('media')) - - -def embed_instagram(media): - """ -
- -
- """ - - url = 'https://api.instagram.com/oembed?url=http://instagr.am/p/{}&hidecaption=true&omitscript=true'.format(media) - req = requests.get(url) - if req.status_code == 200: - html = req.json()['html'] - html = fix_html(html) - i = etree.fromstring(html) - - obj = etree.Element('div') - obj.set('class', "kazimir__embed") - obj.append(i) - return obj - else: - raise Exception( - 'Failed to compile Instagram {media}: Status Code: {status}'.format(media=media, status=req.status_code) - ) - - -def fix_html(html: str): - return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""') diff --git a/modules/sync/settings.py b/modules/sync/settings.py index 808b1c08..13d79699 100644 --- a/modules/sync/settings.py +++ b/modules/sync/settings.py @@ -65,7 +65,7 @@ collection_images = 'images' collection_documents = 'documents' collection_pages = 'pages' -hash_salt_pages = '4' +hash_salt_pages = '6' hash_salt_documents = '4' hash_salt_images = '5' diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py index 5ccfef90..4ad4e024 100644 --- a/modules/sync/sync/__init__.py +++ b/modules/sync/sync/__init__.py @@ -10,7 +10,7 @@ from kazimir.DocumentToken import DocumentToken from kazimir.YoutubeToken import YoutubeToken from kazimir.InstagramToken import InstagramToken from kazimir.Marker import Marker -from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken +from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, HtmlToken from sync.data import Provider from sync.models import Model from sync.models.Document import Document @@ -66,8 +66,6 @@ async def untouched(items: [Model]) -> [Model]: async def create_post(provider: Provider, folder: str, md: str, sizes): - from kazimir.fix_links_quotes import fix_links_quotes - images = [] documents = [] @@ -128,14 +126,10 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document)) m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file)) # m.add_token_factory(TokenFactory(FileToken)) + m.add_token_factory(TokenFactory(HtmlToken)) m.add_token_factory(TokenFactory(TextToken)) - m.add_tree_middleware(fix_links_quotes) tokens = m.parse(md) - tree = await m.create_tree(tokens) - - post = kazimir.html_from_tree(tree) - post = await kazimir.typo(post) encoded_tokens = [] for t in tokens: @@ -145,11 +139,4 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): 'data': data, }) - return post, images, documents, encoded_tokens - - -def title_from_html(md): - html = lxml.html.fromstring(md) - titles = html.cssselect('h1') - if len(titles): - return titles[0].text + return images, documents, encoded_tokens diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py index aa2b22e6..aec6a836 100644 --- a/modules/sync/sync/models/Page.py +++ b/modules/sync/sync/models/Page.py @@ -2,21 +2,36 @@ import os import logging from lxml import etree import lxml.html -import kazimir import settings from db import collection from sync.data import request from sync.models import Model -from sync import create_date, title_from_html, create_post +from sync import create_date, create_post from sync.models.Image import Image from utils.hash import hash_str from utils.front_matter import parse_yaml_front_matter from utils.text.transform import url_encode_text +from markdown import markdown + logger = logging.getLogger(settings.name + '.Page') store = collection(settings.collection_pages) +def extract_files(html) -> [str]: + images = html.cssselect('img') + images = [img.get('src') for img in images] + + return images, [] + + +def title_from_html(html): + titles = html.cssselect('h1') + if len(titles): + return titles[0].text + return '' + + def find(store, item_id: str): q = {'id': item_id} try: @@ -103,7 +118,6 @@ class Page(Model): self.images = [] self.documents = [] self.title = None - self.data = None self.preview = None self.tokens = None @@ -114,7 +128,12 @@ class Page(Model): super().__init__(provider, store, file, params=params) def init(self): - images, documents = kazimir.extract_files(self.get_param('content')) + content = self.get_param('content') + html = markdown(content) + html_tree = lxml.html.fromstring(html) + images, documents = extract_files(html_tree) + self.title = title_from_html(html_tree) + self.params['images'] = images self.params['documents'] = documents @@ -136,9 +155,7 @@ class Page(Model): sizes = kwargs['sizes'] markdown_post = self.get_param('content') - post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) - self.title = title_from_html(post) - self.data = post + images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) self.images = images self.documents = documents self.tokens = tokens @@ -158,7 +175,6 @@ class Page(Model): 'hash': self.hash, 'url': self.url, 'file': self.file, - 'data': self.data, 'images': images, 'documents': documents, 'title': self.title, From a13b6429793a2fa33af40d7a14005e365d6dea53 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 21 Mar 2020 20:03:31 +0300 Subject: [PATCH 5/7] pages Page to Index --- pages/_error.tsx | 4 ++-- pages/collective.tsx | 4 ++-- pages/document/[slug].tsx | 4 ++-- pages/index.tsx | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pages/_error.tsx b/pages/_error.tsx index 11f31acc..16f029a2 100644 --- a/pages/_error.tsx +++ b/pages/_error.tsx @@ -5,7 +5,7 @@ import { Copyright } from 'src/components/Copyright' import { Block } from 'src/components/Block' import { Spacer } from 'src/components/Spacer' -const Page: NextPage = () => ( +const Index: NextPage = () => ( ( ) -export default Page +export default Index diff --git a/pages/collective.tsx b/pages/collective.tsx index f111d753..1596e7ef 100644 --- a/pages/collective.tsx +++ b/pages/collective.tsx @@ -43,7 +43,7 @@ type Props = { data: CardData[] } -const Page: NextPage = props => { +const Index: NextPage = props => { if (!props.data) { console.log('kek error') return @@ -131,4 +131,4 @@ export const getStaticProps = async (ctx: NextPageContext) => { } } -export default Page +export default Index diff --git a/pages/document/[slug].tsx b/pages/document/[slug].tsx index 667a4beb..bd1780a8 100644 --- a/pages/document/[slug].tsx +++ b/pages/document/[slug].tsx @@ -13,7 +13,7 @@ type Props = { meta: IMeta } -const Page: NextPage = props => { +const Index: NextPage = props => { if (!props.data) { console.log('kek error', props) @@ -71,4 +71,4 @@ export const getStaticPaths = async () => { } } -export default Page +export default Index diff --git a/pages/index.tsx b/pages/index.tsx index 0df49ce2..e47a29ea 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -8,13 +8,13 @@ import { getPagesCardsByTags } from 'src/rest' import { NextPage } from 'next' import { IMeta, PageCardData, PageCardDto } from 'src/types' -interface IProps { +type Props = { title: string meta: IMeta items: PageCardDto[] } -const Index: NextPage = props => { +const Index: NextPage = props => { const items = props.items.map(item => ({ url: item.url, title: item.title, From be836c42be3d6fb96792bb56c36f6698d7cdf0b4 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 21 Mar 2020 20:03:56 +0300 Subject: [PATCH 6/7] youtube, intagram, html tokens support --- pages/[...slug].tsx | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pages/[...slug].tsx b/pages/[...slug].tsx index f47235f2..c0585948 100644 --- a/pages/[...slug].tsx +++ b/pages/[...slug].tsx @@ -9,6 +9,7 @@ import { createApiUrl, requestGet, IResponseItems } from 'src/next-lib' import { NextPage } from 'next' import { IBreadcumbsPart, IMeta, IPage, ITag, FileTokenData, Token } from 'src/types' import { joinTokens } from 'src/lib/tokens' +import { Html } from 'src/components/Html' function array(value: T | T[]) { return Array.isArray(value) @@ -16,6 +17,26 @@ function array(value: T | T[]) { : [value] } +const Youtube: React.SFC<{ url: string }> = props => { + const url = new URL(props.url) + const videoId = url.searchParams.get('v') + if (!videoId) { + return null + } + + const src = `//www.youtube.com/embed/${videoId}` + + return ( +
+