Merge pull request #37 from tmshv/fix/drop-documents-page

Drop documents page

modules/sync/kazimir/DocumentToken.py

@@ -10,10 +10,14 @@ class DocumentToken(Token):
         super().__init__(name='file', data=data)
         self.joinable = False
         self.build = None
-
-    async def compile(self):
+    
+    async def get_data(self):
         data = parse_file(self.data)
         document = await self.build(data)
+        return document
+
+    async def compile(self):
+        document = await self.get_data()
 
         return create_document(document)
 

modules/sync/kazimir/DocumentUrlToken.py

@@ -1,37 +0,0 @@
-from kazimir.Token import UrlToken, parse_link
-
-
-class DocumentUrlToken(UrlToken):
-    @staticmethod
-    def test(data: str):
-        link = parse_link(data)
-        if not link:
-            return False
-        url = link['url']
-        host = url.netloc
-        path = url.path
-        return path.startswith('/document/') and (host == '' or host == 'art.shlisselburg.org')
-
-    def __init__(self, data) -> None:
-        super().__init__(data)
-        self.build = None
-
-    async def compile(self):
-        from kazimir.DocumentToken import create_document
-
-        link = parse_link(self.data)
-        url = link['url']
-        doc_id = url.path.replace('/document/', '')
-        doc = await self.build(doc_id)
-
-        if not doc:
-            raise Exception(f'Document {doc_id} not found')
-
-        # return f'''DOCUMENT: {# doc['title']}'''
-        return create_document(doc)
-
-    def merge(self, token):
-        return self
-
-    def __repr__(self) -> str:
-        return f'[Token {self.name} {self.data}]'

modules/sync/kazimir/Marker.py

@@ -16,12 +16,15 @@ class Marker:
     def add_tree_middleware(self, callback):
         self.tree_middlewares.append(callback)
 
-    async def create_tree(self, text: str):
+    def parse(self, text: str):
         tokens = text.split('\n')
         tokens = self.markup_tokens(tokens)
         tokens = join_tokens(tokens)
         tokens = merge_tokens(tokens)
 
+        return tokens
+
+    async def create_tree(self, tokens: list):
         compiled = await compile_tokens(tokens)
         tree = lxml.html.fromstring(compiled)
         return await self.process_tree(tree)
@@ -102,17 +105,8 @@ async def compile_tokens(tokens: list):
     return '\n'.join(result)
 
 
-def extract_files(text: str) -> [str]:
-    html = markdown(text)
-    post_html = lxml.html.fromstring(html)
-
-    images = post_html.cssselect('img')
-    images = [img.get('src') for img in images]
-
-    return images, []
 
 
-def html_from_tree(tree):
-    html = lxml.html.tostring(tree, encoding='unicode')
 
-    return html
+
+

modules/sync/kazimir/MarkerRun.py

@@ -1,113 +0,0 @@
-import asyncio
-
-from kazimir.DocumentUrlToken import DocumentUrlToken
-from kazimir.DocumentToken import DocumentToken
-from kazimir.CSVToken import CSVToken
-from kazimir.InstagramToken import InstagramToken
-from kazimir.Marker import Marker, html_from_tree
-from kazimir.YoutubeToken import YoutubeToken
-from kazimir.Token import SplitToken, ImageToken, UrlToken, FileToken, TextToken, TokenFactory, \
-    BuildTokenFactory
-from sync import Document
-
-if __name__ == '__main__':
-    sample_text = '''
-Hello!
-
-https://www.instagram1.com/p/BUelE04l1kU
-
-https://www.youtube.com/watch?v=qnguOqWpraI
-https://www.youtube.com/watch?v=qnguOqWpraI
-![](1.jpg)
-![](2.jpg)
-
-LOL NOOB
-![](12.jpg)
-![](22.jpg)
-WTF OMG
-
-![](/document/documents-arkhiv-ustav-2013-pdf)
-
-![File](file.pdf)
-hello.pdf
-file.doc
-sheet.xlsx
-
-print.docx
-image.jpg 1
-image.png 2
-
-image.jpeg Hi
-
-Bye!
-Fye!
-
-![](1.jpg)
-
-["Ya"](http://ya.ru)
-
-More text
-
-file.csv
-    '''
-
-
-    async def build_document(data):
-        text = data['caption'] if data['caption'] else data['file']
-        # document = await self.build()
-        document = {
-            'url': data['file'],
-            'image_url': data['file'],
-            'file_url': data['file'],
-            'title': text,
-            'file_size': '0 KB',
-            'file_format': 'PDF',
-        }
-        return document
-
-    async def read_csv(data):
-        return '\n'.join([
-            'N,Name',
-            f'0,{data}',
-            '1,Pop',
-            '2,Top',
-        ])
-
-    async def read_document(data):
-        doc = await Document.find_one({'id': data})
-        return {
-            'url': doc['url'],
-            'image_url': doc['url'],
-            'file_url': doc['url'],
-            'title': doc['title'],
-            'file_size': '0 KB',
-            'file_format': 'PDF',
-        }
-
-
-    async def run():
-        from kazimir.fix_links_quotes import fix_links_quotes
-
-        m = Marker()
-        m.add_token_factory(TokenFactory(SplitToken))
-        m.add_token_factory(TokenFactory(YoutubeToken))
-        m.add_token_factory(TokenFactory(InstagramToken))
-        m.add_token_factory(BuildTokenFactory(DocumentUrlToken, build=read_document))
-        m.add_token_factory(TokenFactory(UrlToken))
-        # m.add_token_factory(TokenFactory(ImageToken))
-        m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
-        m.add_token_factory(BuildTokenFactory(CSVToken, build=read_csv))
-        m.add_token_factory(TokenFactory(FileToken))
-        m.add_token_factory(TokenFactory(TextToken))
-        m.add_tree_middleware(fix_links_quotes)
-        sample_tree = await m.create_tree(sample_text)
-
-        # for x in sample_tree:
-        #     print(x)
-
-        print(html_from_tree(sample_tree))
-
-
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(run())
-    loop.close()

modules/sync/kazimir/Token.py

@@ -38,8 +38,12 @@ class Token:
         self.data = data
         self.joinable = True
 
-    async def compile(self):
-        return str(self.data)
+    async def get_data(self):
+        return self.data
+
+    async def compile(self) -> str:
+        data = await self.get_data()
+        return str(data)
 
     def merge(self, token):
         return self
@@ -73,7 +77,8 @@ class TextToken(Token):
         return TextToken(data.strip())
 
     async def compile(self):
-        return markdown_to_html(self.data)
+        data = await self.get_data()
+        return markdown_to_html(data)
 
 
 class FileToken(Token):
@@ -85,8 +90,11 @@ class FileToken(Token):
         super().__init__(name='file', data=data)
         self.joinable = False
 
+    async def get_data(self):
+        return parse_file(self.data)
+
     async def compile(self):
-        data = parse_file(self.data)
+        data = await self.get_data()
         url = data['file']
         text = data['caption'] if data['caption'] else url
         return f'<a href="{url}">{text}</a>'
@@ -107,9 +115,13 @@ class ImageToken(Token):
         merged_token.build = self.build
         return merged_token
 
+    async def get_data(self):
+        data = parse_file(self.data)
+        data = await self.build(data)
+        return data
+
     async def compile(self):
-        img = self.parse_data()
-        img = await self.build(img)
+        img = await self.get_data()
         alt = img['alt']
         caption = markdown_to_html(img['caption'])
         src = img['src']
@@ -124,9 +136,6 @@ class ImageToken(Token):
         '''
         return tpl.format(img).strip()
 
-    def parse_data(self):
-        return parse_file(self.data)
-
 
 class ImageCollectionToken(ImageToken):
     def __init__(self, data) -> None:
@@ -143,13 +152,16 @@ class ImageCollectionToken(ImageToken):
         merged_token.build = self.build
         return merged_token
 
-    async def compile(self):
-        data = self.parse_data()
+    async def get_data(self):
+        items = [parse_file(x) for x in self.data]
         images = []
-        for x in data:
+        for x in items:
             img = await self.build(x)
             images.append(img)
+        return images
 
+    async def compile(self):
+        images = await self.get_data()
         images = [x['src'] for x in images]
         images = [f'<img src="{x}">' for x in images]
         images = '\n'.join(images)
@@ -165,10 +177,6 @@ class ImageCollectionToken(ImageToken):
             </div>
         '''
 
-    def parse_data(self):
-        return [parse_file(x) for x in self.data]
-
-
 class UrlToken(Token):
     @staticmethod
     def test(data: str):

modules/sync/kazimir/__init__.py

@@ -6,19 +6,6 @@ from kazimir.figure import FigureExtension
 from kazimir.instagram import InstagramExtension
 from sync.data import request
 
-quote_pairs = [
-    ('"', '"'),
-    ('«', '»'),
-]
-
-
-def markdown_to_html(text):
-    return markdown(text, extensions=[
-        FigureExtension(),
-        InstagramExtension(),
-    ])
-
-
 def extract_files(text: str) -> [str]:
     html = markdown(text)
     post_html = lxml.html.fromstring(html)
@@ -29,67 +16,6 @@ def extract_files(text: str) -> [str]:
     return images, []
 
 
-def create_tree(text: str) -> str:
-    tokens = text.split('\n')
-    tokens = markup_tokens(tokens)
-    tokens = join_tokens(tokens)
-    tokens = merge_tokens(tokens)
-
-    kz = compile_tokens(tokens)
-    tree = lxml.html.fromstring(kz)
-    return clean_tree(tree)
-
-
-def clean_tree(tree):
-    return fix_links_quotes(tree)
-
-
-def fix_links_quotes(tree):
-    safe = lambda x: x if x else ''
-
-    for link in tree.iterlinks():
-        a = link[0]
-        if a.tag != 'a':
-            continue
-
-        quote = get_quote_pair(a.text)
-        if not quote:
-            continue
-
-        # New text of A without quotes
-        a.text = a.text[1:-1]
-
-        # Move first quote out of A to the prefix
-        a.getparent().text = safe(a.getparent().text) + quote[0]
-
-        # Move last quote out of A to the suffix
-        a.tail = quote[1] + safe(a.tail)
-
-    return tree
-
-
-def get_quote_pair(text: str):
-    """
-    sample text -> [s]ample tex[t] -> (s, t) -> match pair from quote_pair
-        ? (s, t)
-        : None
-
-    :param text:
-    :return:
-    """
-    if len(text) < 2:
-        return None
-
-    first = text[0]
-    last = text[-1]
-
-    for pair_first, pair_last in quote_pairs:
-        if pair_first == first and pair_last == last:
-            return first, last
-
-    return None
-
-
 def html_from_tree(tree):
     html = lxml.html.tostring(tree, encoding='unicode')
 
@@ -98,223 +24,11 @@ def html_from_tree(tree):
 
 async def typo(text):
     from kazimir import artlebedev
-    return await artlebedev.typograf(text)
+    result = await artlebedev.typograf(text)
+    if result == 'Размер текста ограничен 32 КБ':
+        return text
+
+    return result
     # url = 'http://www.typograf.ru/webservice/'
     # params = dict(text=text)
     # return await request.post(url, params, {'chr': 'utf-8'})
-
-
-def join_tokens(tokens):
-    buffer_tag = None
-    buffer = []
-    ts = []
-    for token in tokens:
-        tag = token['tag']
-        changed = buffer_tag is not None and tag != buffer_tag
-
-        flush = is_tag_split(tag)
-        if changed:
-            flush = True
-
-        if not is_tag_joinable(tag):
-            flush = True
-
-        if flush:
-            if buffer:
-                ts.append(buffer)
-            if is_tag_split(tag):
-                buffer = []
-                buffer_tag = None
-            else:
-                buffer = [token]
-                buffer_tag = tag
-        else:
-            buffer_tag = tag
-            buffer.append(token)
-
-    if len(buffer):
-        ts.append(buffer)
-    return ts
-
-
-def merge_tokens(tokens: list):
-    tokens_out = []
-    for ts in tokens:
-        tag = ts[0]['tag']
-        fn = None
-        if tag == 'image':
-            fn = merge_image
-        elif tag == 'text':
-            fn = merge_text
-        else:
-            fn = merge_any
-        tokens_out.append(fn(ts))
-    return tokens_out
-
-
-def compile_tokens(tokens: list):
-    compiled = []
-    for ts in tokens:
-        tag = ts['tag']
-        fn = None
-        if tag == 'text':
-            fn = compile_text
-        elif tag == 'image_collection':
-            fn = compile_image_collection
-        elif tag == 'image':
-            fn = compile_image
-        elif tag == 'url':
-            fn = compile_url
-        else:
-            fn = compile_any
-        compiled.append(fn(ts))
-    return '\n'.join(compiled)
-
-
-def markup_tokens(tokens: list):
-    marked_tokens = []
-    for token in tokens:
-        tag = 'text'
-        if is_url(token):
-            tag = 'url'
-        elif is_image(token):
-            tag = 'image'
-        elif not token:
-            tag = 'split'
-            token = None
-
-        marked_tokens.append({
-            'tag': tag,
-            'data': token,
-        })
-    return marked_tokens
-
-
-def is_tag_split(tag):
-    return tag == 'split'
-
-
-def is_tag_joinable(tag: str):
-    return tag not in ['url']
-
-
-def is_url(sample: str):
-    from urllib.parse import urlparse
-    url = urlparse(sample)
-
-    return url.scheme and url.netloc
-
-
-def is_image(sample: str):
-    return sample[:2] == '![' and sample[-1:] == ')'
-
-
-def merge_any(tokens):
-    return tokens[0]
-
-
-def merge_text(tokens):
-    data = ''
-
-    for t in tokens:
-        data += t['data']
-        data += '\n'
-
-    return {
-        'tag': 'text',
-        'data': data.strip(),
-    }
-
-
-def merge_image(tokens):
-    if len(tokens) == 1:
-        return tokens[0]
-    else:
-        data = [t['data'] for t in tokens]
-        return {
-            'tag': 'image_collection',
-            'data': data,
-        }
-
-
-def compile_text(token):
-    return markdown_to_html(token['data'])
-
-
-def compile_image_collection(token):
-    file = lambda i: i[4:][:-1]
-    imgs = ['<img src="{}">'.format(file(i)) for i in token['data']]
-    return '''
-        <div class="kazimir__image-collection">
-            <div class="fotorama"
-                 data-width="100%"
-                 data-ratio="800/600"
-            >
-                {}
-            </div>
-        </div>
-    '''.format('\n'.join(imgs))
-
-
-def compile_image(token):
-    img = markdown_to_html(token['data'])
-    tpl = '''<div class="kazimir__image">{}</div>'''
-    return tpl.format(img)
-
-
-def compile_url(token):
-    return compile_youtube(token)
-
-
-def compile_youtube(token):
-    """
-    <div class="kazimir__video">
-     <iframe src="http://www.youtube.com/embed/9otNWTHOJi8" frameborder="0" allowfullscreen></iframe>
-   </div>
-    """
-
-    youtube = r'([^(]|^)https?://www\.youtube\.com/watch\?\S*v=(?P<youtubeargs>[A-Za-z0-9_&=-]+)\S*'
-    youtube_pattr = re.compile(youtube)
-    match = youtube_pattr.match(token['data'])
-    if match:
-        video_id = match.group(2)
-        url = f'//www.youtube.com/embed/{video_id}'
-        tpl = '''<div class="kazimir__video"><iframe src="{url}" frameborder="0" allowfullscreen></iframe></div>'''
-
-        return tpl.format(url=url)
-    else:
-        return ''
-
-
-def compile_any(token):
-    return markdown_to_html(token['data'])
-
-
-if __name__ == '__main__':
-    sample_text = '''
-Hello!
-
-https://www.instagram.com/p/BUelE04l1kU
-ku!
-
-https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka
-https://www.youtube.com/watch?v=qnguOqWpraI
-![](1.jpg)
-![](2.jpg)
-![](3.jpg)
-
-LOL NOOB
-![](12.jpg)
-![](22.jpg)
-WTF OMG
-
-Bye!
-Fye!
-
-![](1.jpg)
-
-More text
-    '''
-
-    sample_tree = create_tree(sample_text)
-    print(html_from_tree(sample_tree))

modules/sync/settings.py

@@ -91,7 +91,6 @@ provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
 
 f = abs_fn(provider_root)
 
-dir_documents = './Documents'
 dir_articles = './Articles'
 dir_pages = './Pages'
 

modules/sync/sync.py

@@ -1,6 +1,5 @@
 import settings
 from sync.core import Sync
-from sync.core.SyncDocument import SyncDocument
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
 
@@ -41,12 +40,6 @@ async def run_sync(sync: Sync, **kwargs):
 async def run(run_interval=0):
     io = lambda root: get_provider(settings.provider_name, root)
 
-    sync_documents = SyncDocument(
-        provider=io(settings.provider_root),
-        sizes=settings.image_sizes,
-    )
-    sync_documents.strict_origin = True
-
     sync_pages = Sync(
         provider=io(settings.provider_root),
         model=Page,
@@ -62,7 +55,6 @@ async def run(run_interval=0):
 
     while True:
         await asyncio.wait([
-            run_sync(sync_documents),
             run_sync(sync_pages),
             run_sync(sync_articles),
         ])

modules/sync/sync/__init__.py

@@ -3,8 +3,6 @@ import re
 from datetime import datetime
 
 import lxml.html
-from markdown import markdown
-
 import kazimir
 import settings
 from kazimir.CSVToken import CSVToken
@@ -155,12 +153,22 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
     # m.add_token_factory(TokenFactory(FileToken))
     m.add_token_factory(TokenFactory(TextToken))
     m.add_tree_middleware(fix_links_quotes)
-    tree = await m.create_tree(md)
+
+    tokens = m.parse(md)
+    tree = await m.create_tree(tokens)
 
     post = kazimir.html_from_tree(tree)
     post = await kazimir.typo(post)
 
-    return post, images, documents
+    encoded_tokens = []
+    for t in tokens:
+        data = await t.get_data()
+        encoded_tokens.append({
+            'token': t.name,
+            'data': data,
+        })
+
+    return post, images, documents, encoded_tokens
 
 
 def images_from_html(md):
@@ -181,11 +189,3 @@ def title_from_html(md):
     titles = html.cssselect('h1')
     if len(titles):
         return titles[0].text
-
-
-create_post_from_image_list = lambda images: markdown(
-    '\n'.join(map(
-        lambda i: '![]({img})'.format(img=i),
-        images
-    ))
-)

modules/sync/sync/core/SyncDocument.py

@@ -1,14 +0,0 @@
-from sync.core import Sync
-from sync.data import Provider
-from sync.models.Document import Document
-
-
-class SyncDocument(Sync):
-    def __init__(self, provider: Provider, sizes):
-        super().__init__(provider, Document)
-        self.sizes = sizes
-
-    def _get_build_args(self):
-        return {
-            'sizes': self.sizes,
-        }

modules/sync/sync/models/Article.py

@@ -5,8 +5,8 @@ import os
 import kazimir
 import settings
 from db import collection
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, \
-    create_post
+from markdown import markdown
+from sync import create_date_and_title_from_folder_name, create_date, create_post
 from sync.data import list_images
 from sync.models import Model
 from sync.models.Image import Image
@@ -71,6 +71,7 @@ class Article(Model):
         self.preview = None
         self.images = None
         self.documents = None
+        self.tokens = None
         super().__init__(provider, store, file, params=params)
 
     def init(self):
@@ -103,10 +104,11 @@ class Article(Model):
         else:
             markdown_post = create_post_from_image_list(self.get_param('images'))
 
-        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
         self.post = post
         self.images = images
         self.documents = documents
+        self.tokens = tokens
 
         if self.has_param('preview'):
             preview = self.get_param('preview')
@@ -124,6 +126,7 @@ class Article(Model):
             'file': self.file,
             'origin': self.origin,
             'version': self.version,
+            'tokens': self.tokens,
             'preview': self.preview.ref if self.preview else None,
         }
 
@@ -172,3 +175,12 @@ def read_manifest(provider, path):
     return {
         **manifest,
     }
+
+
+def create_post_from_image_list(images):
+    return markdown(
+        '\n'.join(map(
+            lambda i: '![]({img})'.format(img=i),
+            images
+        ))
+    )

modules/sync/sync/models/Document.py

@@ -3,7 +3,7 @@ from tempfile import mkstemp
 
 import settings
 from db import collection
-from sync.data import scan_subdirs, request
+from sync.data import request
 from sync.models import Model
 from sync.models.DocumentPreviewImage import DocumentPreviewImage
 from utils.hash import hash_str, md5
@@ -25,12 +25,6 @@ class Document(Model):
     async def delete(query):
         return store.find_one_and_delete(query)
 
-    @staticmethod
-    async def scan(provider):
-        scan_dir = settings.dir_documents
-        paths = scan_subdirs(provider, '.pdf', scan_dir)
-        return [Document.read_path(provider, path) for path in paths]
-
     @staticmethod
     def read_path(provider, path):
         name = os.path.basename(path)
@@ -181,14 +175,21 @@ async def get_pdf_title(provider, file):
     from PyPDF2.generic import TextStringObject
     from PyPDF2.generic import IndirectObject
 
+    def t(value):
+        value = str(value)
+        value = value.replace('\u0000', '')
+        if value == 'None':
+            return None
+        return value
+
     pdf = PdfFileReader(provider.read(file))
     info = pdf.getDocumentInfo()
 
     if info:
         if type(info.title) == TextStringObject:
-            return str(info.title)
+            return t(info.title)
 
         if type(info.title_raw) == IndirectObject:
             o = pdf.getObject(info.title_raw)
-            return str(o)
+            return t(info.title)
     return None

modules/sync/sync/models/Image.py

@@ -7,7 +7,7 @@ from typing import Optional, List
 
 import settings
 from db import collection
-from sync.data import request, list_images
+from sync.data import request 
 from sync.models import Model
 from utils.hash import md5, hash_str
 from utils.image import image_magick_resize
@@ -54,13 +54,6 @@ class Image(Model):
 
         return img
 
-    @staticmethod
-    async def scan(provider):
-        scan_path = settings.dir_images
-        documents = list_images(provider, scan_path)
-
-        return [Image.read_path(provider, i) for i in documents]
-
     @staticmethod
     def read_path(provider, path):
         return Image(

modules/sync/sync/models/Page.py

@@ -14,7 +14,7 @@ from sync.models import Model
 from sync import create_date, images_from_html, title_from_html, create_post
 from sync.models.Image import Image
 from utils.hash import hash_str
-from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.io import parse_yaml_front_matter
 from utils.text.transform import url_encode_text
 
 logger = logging.getLogger(settings.name + '.Page')
@@ -96,6 +96,7 @@ class Page(Model):
         self.title = None
         self.data = None
         self.preview = None
+        self.tokens = None
 
         self.__id_template: str = '{category}-{file}'
         self.__url_template: str = settings.document_url_template
@@ -126,11 +127,12 @@ class Page(Model):
         sizes = kwargs['sizes']
 
         markdown_post = self.get_param('content')
-        post, images, documents = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
+        post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
         self.title = title_from_html(post)
         self.data = post
         self.images = images
         self.documents = documents
+        self.tokens = tokens
 
         if self.has_param('preview'):
             preview = self.get_param('preview')
@@ -151,6 +153,7 @@ class Page(Model):
             'images': images,
             'documents': documents,
             'title': self.title,
+            'tokens': self.tokens,
             'preview': self.preview.ref if self.preview else None,
         }
 

modules/sync/test/sync/test_document.py

@@ -1,27 +0,0 @@
-import unittest
-
-import settings
-from sync.data import scan_subdirs
-from sync.data.fs import FSProvider
-
-
-class MyTestCase(unittest.TestCase):
-    def test_document(self):
-        p = FSProvider(settings.dir_documents)
-
-        documents = scan_subdirs(p, '.pdf')
-        # documents = lmap(
-        #     lambda document: create_document(p, document),
-        #     documents
-        # )
-
-        self.assertEqual(True, True)
-
-    def test_scan(self):
-        p = FSProvider(settings.dir_documents)
-        documents = scan_subdirs(p, '.pdf')
-        self.assertIsInstance(documents, list)
-
-
-if __name__ == '__main__':
-    unittest.main()

modules/sync/utils/io.py

@@ -1,14 +1,10 @@
 import re
 import yaml
 
-from kazimir import markdown_to_html
 
 FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
 
 
-def read_yaml_md(data):
-    yaml_front_matter, content = split_yaml_front_matter(data)
-    return yaml_front_matter, markdown_to_html(content)
 
 
 def parse_yaml_front_matter(data: str):

pages/documents.tsx

@@ -1,138 +0,0 @@
-import Head from 'next/head'
-import { App } from 'src/components/App'
-import { Meta } from 'src/components/Meta'
-import { meta } from 'src/lib/meta'
-import { IDocument } from 'src/types'
-import { DocumentList } from 'src/components/DocumentList'
-import { unique } from 'src/utils/common'
-import { createApiUrl, requestGet, IResponseItems } from 'src/next-lib'
-import { NextPage } from 'next'
-
-interface IDocumentCategory {
-    name: string
-    items: IDocument[]
-}
-
-function getSorted(documents: IDocument[]): IDocumentCategory[] {
-    const compose = (...fns) => value => fns
-        .map(fn => fn)
-        .reverse()
-        .reduce((acc, fn) => fn(acc), value)
-
-    const pass = fn => value => {
-        fn(value)
-        return value
-    }
-
-    const subtract = (a, b) => a - b
-
-    const uniqueCategories = unique<IDocument, string>(i => i.category)
-
-    const priority = {
-        'Основные документы': [
-            'https://static.shlisselburg.org/art/uploads/ustav-2015.pdf',
-            'https://static.shlisselburg.org/art/uploads/obrazovatelnaia-litsenziia-2015.pdf',
-            'https://static.shlisselburg.org/art/uploads/svidetelstvo-na-pomeshchenie.pdf',
-            'https://static.shlisselburg.org/art/uploads/egriul.pdf',
-            'https://static.shlisselburg.org/art/uploads/inn.pdf',
-            'https://static.shlisselburg.org/art/uploads/reshenie-o-sozdanii-shkoly.pdf',
-            'https://static.shlisselburg.org/art/uploads/kopiia-resheniia-o-naznachenii-rukovoditelia.pdf',
-            'https://static.shlisselburg.org/art/uploads/postanovlenie-o-sozdanii-munitsipalnogo-obrazovatelnogo-uchrezhdeniia-kultury-detskaia-khudozhestvennaia-shkola.pdf',
-        ],
-        'Учебные программы': [
-            'https://static.shlisselburg.org/art/uploads/uchebnaia-programma-zhivopis.pdf',
-        ]
-    }
-
-    const CATEGORY_PRIORITY = [
-        'Основные документы',
-        'Планы ФХД',
-        'Самообследование деятельности',
-        'Инструкции',
-        'Образование',
-        'Учебные программы',
-        'Платные образовательные услуги',
-        'Дополнительные общеразвивающие программы',
-        'Документы для поступления',
-    ]
-
-    function sortBy<T>(fn: (value: T) => number): (a: T, b: T) => number {
-        return (a, b) => [a, b].map(fn).reduce(subtract)
-    }
-
-    function getPriorityFn<T>(m: number, list: T[]): (value: T) => number {
-        return (value: T) => {
-            const has = list.includes(value)
-            return has ? list.indexOf(value) : m
-        }
-    }
-
-    const safeSelect = (d, store, key) => key in store
-        ? store[key]
-        : d
-
-    const categoryPriority = getPriorityFn(10000, CATEGORY_PRIORITY)
-
-    const documentPriority = ({ category, fileUrl }) => compose(
-        getPriorityFn.bind(null, 10000),
-        safeSelect.bind(null, [], priority)
-    )(category)(fileUrl)
-
-    const documentsOf = category => documents.filter(i => i.category === category)
-    const sorted = documents => documents.sort(sortBy(documentPriority))
-    const collectDocumentsByCategory = compose(sorted, documentsOf)
-
-    return uniqueCategories(documents)
-        .sort(sortBy(categoryPriority))
-        .reduce((acc, name) => [...acc, {
-            name,
-            items: collectDocumentsByCategory(name)
-        }], [])
-}
-
-interface IProps {
-    collections: IDocumentCategory[]
-    title: string
-    meta: any
-}
-
-const Page: NextPage<IProps> = props => (
-    <App
-        showAuthor={true}
-    >
-        <Head>
-            <title>{props.title}</title>
-            <Meta meta={props.meta} />
-        </Head>
-
-        <h1>{props.title}</h1>
-
-        {props.collections.map((collection, index) => (
-            <DocumentList
-                key={index}
-                name={collection.name}
-                documents={collection.items}
-            />
-        ))}
-    </App>
-)
-
-export const unstable_getStaticProps = async (ctx) => {
-    const res = await requestGet<IResponseItems<IDocument>>(createApiUrl('/api/files'), null)
-    const files = res.items
-    const title = 'Документы'
-    const collections = getSorted(files)
-
-    return {
-        props: {
-            collections,
-            title,
-            meta: meta({
-                title,
-                description: 'Документы Шлиссельбургской ДХШ',
-            })
-        }
-    }
-}
-
-export default Page

scripts/sitemap.js

@@ -12,7 +12,6 @@ async function getMenuUrls(frequency = 'daily') {
     const menu = [
         '/',
         '/schedule',
-        '/documents',
         '/collective',
     ]
     return menu

src/components/DocumentList/index.tsx

@@ -1,22 +0,0 @@
-import * as React from 'react'
-import { IDocument } from '../../types'
-import { DocumentListItem } from '../DocumentListItem'
-
-export interface IDocumentListProps {
-    documents: IDocument[]
-    name: string
-}
-
-export const DocumentList: React.FC<IDocumentListProps> = props => (
-    <div className="documents-container">
-        <div className="documents-container__head">
-            <h2>{props.name}</h2>
-        </div>
-
-        <div className="documents-container__body">
-            {props.documents.map((d, i) => (
-                <DocumentListItem key={i} {...d} />
-            ))}
-        </div>
-    </div>
-)

src/types.ts

@@ -20,7 +20,7 @@ export interface IPage {
 }
 
 export interface IDocument {
-    category: string
+    // category: string
     fileName: string
     fileSize: number
     fileUrl: string

src/utils/common.ts

@@ -36,26 +36,6 @@ export function choise(scope, selector) {
     return scope.reduce(selector, null);
 }
 
-export function unique<T, U>(fn: (value: T) => U): (list: T[]) => U[] {
-    return list => Array.from(
-        new Set(list.map(fn))
-    )
-}
-
-export function branch(fn) {
-    let keys = unique(fn);
-
-    return list => keys(list)
-        .reduce((dict: any, key: any) => {
-            dict[key] = list.filter(
-                i => fn(i) === key
-            );
-
-            return dict;
-        }, {})
-
-}
-
 export function assign(dict, key, value) {
     dict[key] = value;
     return dict;