sync util update

utils/Sync/sync/__init__.py

@@ -89,6 +89,9 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
 
 
 def images_from_html(md):
+    if not md:
+        return []
+
     post_html = lxml.html.fromstring(md)
 
     images = []

utils/Sync/sync/album.py

@@ -1,6 +1,8 @@
 import os
+import re
 from glob import glob
 
+import frontmatter
 import lxml.html
 from lxml import etree
 
@@ -13,7 +15,7 @@ from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, firs
     key_mapper, lprint
 from utils.hash import hash_file, hash_str
 from utils.image import create_image
-from utils.io import read_yaml_md
+from utils.io import read_yaml_md, read_file, parse_yaml_front_matter
 from utils.text.transform import url_encode_text
 
 
@@ -326,7 +328,25 @@ def main(dir_documents, sync):
 if __name__ == '__main__':
     image_creator = ImageCreator(cache=None)
     alb = SyncAlbum(image_creator=image_creator, images_dir=settings.image_output)
-    alb.skip_image_processing = True
+    # alb.skip_image_processing = True
 
-    albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
+    # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
-    main(dir_documents=albums_dir, sync=alb)
+    # dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Gallery'
+    dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/Gallery'
+    main(dir_documents=dir_docs, sync=alb)
+
+    # dir_docs += '/2013 Валяние Зверушки'
+    # dir_docs += '/Валяние Зверушки.md'
+
+    # dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/News/2015 Выставка 70-летия победы.md'
+
+    # print(frontmatter.load(dir_docs).content)
+    # d = read_file(dir_docs)
+    # d = d.replace('\u0D0A', '\n')
+    # print(frontmatter.loads(d).metadata)
+    # print(parse_yaml_front_matter(d))
+    #
+    # print(len(re.split('^---', d, 2, re.MULTILINE)))
+    # print(re.split('^---', d, 2)[1])
+    #
+    # print(read_yaml_md(dir_docs))

utils/Sync/sync/document.py

@@ -5,8 +5,8 @@ from tempfile import mkstemp
 
 import settings
 from db import db
-from sync import Sync, untouched, synced_image_id
+from sync import Sync, synced_image_id, untouched
-from utils.fn import combine, lmap, lprint_json, lprint, key_mapper
+from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn
 from utils.hash import hash_file, hash_str
 from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
@@ -15,10 +15,9 @@ from utils.text.transform import url_encode_text, url_encode_file
 
 
 class SyncDocument(Sync):
-    def __init__(self, collection_name, document_type, sizes):
+    def __init__(self, collection_name, sizes):
         super().__init__()
         self.collection = db()[collection_name]
-        self.document_type = document_type
         self.sizes = sizes
         self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
         self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
@@ -28,7 +27,7 @@ class SyncDocument(Sync):
         file = document['file']
         filename = os.path.basename(document['file'])
 
-        document['type'] = self.document_type
+        document['type'] = self.document_type(document)
         document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
         document['file'] = {
             'name': filename,
@@ -37,10 +36,14 @@ class SyncDocument(Sync):
 
         return document
 
+    def document_type(self, document):
+        if document['category'] == 'Награды':
+            return 'award'
+        return 'document'
+
     def create_id(self, document):
         bn = os.path.basename(document['file'])
-        document['id'] = url_encode_text('{type}-{category}-{file}'.format(
+        document['id'] = url_encode_text('{category}-{file}'.format(
-            type=self.document_type,
             category=document['category'],
             file=bn
         ))
@@ -57,11 +60,12 @@ class SyncDocument(Sync):
         )
         return document
 
-    def create_preview(self, pdf, sizes, preview_dir):
+    def create_preview(self, file, sizes, preview_dir):
-        temp_preview_path = pdf_to_jpg(pdf)
+        temp_preview_path = pdf_to_jpg(file)
 
-        id = url_encode_text(pdf)
+        file = os.path.basename(file)
-        url = lambda size, ext: self.url_base_preview.format(id=id, size=size, ext=ext)
+        file = url_encode_text(file)
+        url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
 
         img = create_image(temp_preview_path, sizes, url, preview_dir)
         os.remove(temp_preview_path)
@@ -153,12 +157,49 @@ def documents_from_subdirs():
     return documents
 
 
+def get_pdf_title(file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(open(file, 'rb'))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None
+
+
+def until_none(ls):
+    better = None
+    for i in ls:
+        if (i is not None) and i != '':
+            better = i
+    return better
+
+
 def main(dir_documents, dir_static_uploads, sync):
     os.chdir(dir_documents)
 
     # documents = documents_from_yaml()
     documents = documents_from_subdirs()
 
+    # CHOICE BETTER TITLE
+    documents = lmapfn(documents)(
+        lambda i: {
+            **i,
+            'title': until_none([i['title'], get_pdf_title(i['file'])])
+        }
+    )
+
+    # lprint_json(documents)
+    # return
+
     # CREATE DOCUMENT IDENTITY
     documents = lmap(sync.create_id, documents)
 
@@ -242,7 +283,6 @@ if __name__ == '__main__':
         dir_static_uploads,
         SyncDocument(
             'documents',
-            'document',
             sizes=settings.image_sizes
         )
     )

utils/Sync/sync/post.py

@@ -7,7 +7,7 @@ import lxml.html
 import settings
 from db import db
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \
-    images_from_html, list_images, synced_images_ids
+    images_from_html, list_images, synced_images_ids, untouched
 from sync.image import sync_image
 from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn
 from utils.hash import hash_file, hash_str
@@ -78,6 +78,14 @@ class SyncPost(Sync):
         document['images'] = images
         return document
 
+    def create_remove_query(self, query):
+        if self.type:
+            query = {
+                **query,
+                'type': self.type
+            }
+        return query
+
 
 def get_manifest(path):
     y, m = read_yaml_md(path)
@@ -223,15 +231,7 @@ def main(dir_documents, sync, file_time_formats):
     )
 
     # SKIP UNTOUCHED DOCUMENTS
-    documents = lmap(
+    documents = untouched(documents, sync)
-        lambda i: i[0],
-        filter(
-            lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
-            lmap(
-                lambda document: (document, sync.read(document)),
-                documents
-            )
-        ))
 
     # MAP EVENT MANIFEST -> EVENT_OBJECT
     documents = lmap(
@@ -254,7 +254,8 @@ def main(dir_documents, sync, file_time_formats):
         documents
     )
 
-    documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
+    remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
+    documents_to_remove = sync.query(remove_query)
     documents_to_remove = lmap(
         sync.delete,
         map(
@@ -280,14 +281,16 @@ def main(dir_documents, sync, file_time_formats):
         lambda i: i['id'],
         documents
     ))
-    lprint_json(documents)
+    # lprint_json(documents)
     print('[SYNC DOCUMENTS DONE]')
 
 
 if __name__ == '__main__':
     image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
+    # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
+    dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Events'
     main(
-        '/Users/tmshv/Dropbox/Dev/Hud school/Events',
+        dir_docs,
         SyncPost(
             'events',
             None,
@@ -299,8 +302,10 @@ if __name__ == '__main__':
     )
 
     image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
+    # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/News'
+    dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/News'
     main(
-        '/Users/tmshv/Dropbox/Dev/Hud school/News',
+        dir_docs,
         SyncPost(
             'timeline',
             'post',

utils/Sync/utils/io.py

@@ -1,8 +1,13 @@
+import re
+
+import frontmatter
 import yaml
 from markdown import markdown
 
 from utils.text.split import split, split_with
 
+# FM_BOUNDARY = re.compile(r'^-{3,}$', re.MULTILINE)
+FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
 
 def read_file(file):
     with open(file, 'rb') as f:
@@ -23,19 +28,11 @@ def read_md(file):
 
 
 def read_yaml_md(file):
-    data = read_file(file)
+    yaml, content = parse_yaml_front_matter(read_file(file))
-    yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data)
+    return yaml, markdown(content)
 
-    if not yaml_data and not md_data:
+    # data = frontmatter.load(file)
-        return (read_yaml(file), None)
+    # return data.metadata, markdown(data.content)
-
-    try:
-        y = yaml.load(yaml_data)
-    except:
-        y = None
-
-    m = markdown(md_data) if md_data else None
-    return y, m
 
 
 def read_md_yaml(file):
@@ -52,3 +49,30 @@ def read_md_yaml(file):
 
     m = markdown(md_data) if md_data else None
     return y, m
+
+
+def parse_yaml_front_matter(text, **defaults):
+    """
+    Parse text with YAML frontmatter, return metadata and content.
+    Pass in optional metadata defaults as keyword args.
+
+    If frontmatter is not found, returns an empty metadata dictionary
+    and original text content.
+    """
+
+    # metadata starts with defaults
+    metadata = defaults.copy()
+
+    # split on the first two triple-dashes
+    try:
+        _, fm, content = FM_BOUNDARY.split(text, 2)
+    except ValueError:
+        # if we can't split, bail
+        return metadata, text
+
+    # parse yaml, now that we have frontmatter
+    fm = yaml.safe_load(fm)
+    if isinstance(fm, dict):
+        metadata.update(fm)
+
+    return metadata, content.strip()