diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index 89d01182..a2528dd6 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -89,6 +89,9 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None): def images_from_html(md): + if not md: + return [] + post_html = lxml.html.fromstring(md) images = [] diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py index 2a70b2b6..b4e0d55b 100644 --- a/utils/Sync/sync/album.py +++ b/utils/Sync/sync/album.py @@ -1,6 +1,8 @@ import os +import re from glob import glob +import frontmatter import lxml.html from lxml import etree @@ -13,7 +15,7 @@ from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, firs key_mapper, lprint from utils.hash import hash_file, hash_str from utils.image import create_image -from utils.io import read_yaml_md +from utils.io import read_yaml_md, read_file, parse_yaml_front_matter from utils.text.transform import url_encode_text @@ -326,7 +328,25 @@ def main(dir_documents, sync): if __name__ == '__main__': image_creator = ImageCreator(cache=None) alb = SyncAlbum(image_creator=image_creator, images_dir=settings.image_output) - alb.skip_image_processing = True + # alb.skip_image_processing = True - albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' - main(dir_documents=albums_dir, sync=alb) + # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' + # dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Gallery' + dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/Gallery' + main(dir_documents=dir_docs, sync=alb) + + # dir_docs += '/2013 Валяние Зверушки' + # dir_docs += '/Валяние Зверушки.md' + + # dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/News/2015 Выставка 70-летия победы.md' + + # print(frontmatter.load(dir_docs).content) + # d = read_file(dir_docs) + # d = d.replace('\u0D0A', '\n') + # print(frontmatter.loads(d).metadata) + # print(parse_yaml_front_matter(d)) + # + # print(len(re.split('^---', d, 2, re.MULTILINE))) + # print(re.split('^---', d, 2)[1]) + # + # print(read_yaml_md(dir_docs)) diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 2b760f80..3b23fe53 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -5,8 +5,8 @@ from tempfile import mkstemp import settings from db import db -from sync import Sync, untouched, synced_image_id -from utils.fn import combine, lmap, lprint_json, lprint, key_mapper +from sync import Sync, synced_image_id, untouched +from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn from utils.hash import hash_file, hash_str from utils.image import create_image from utils.image.resize import image_magick_pdf_to_img @@ -15,10 +15,9 @@ from utils.text.transform import url_encode_text, url_encode_file class SyncDocument(Sync): - def __init__(self, collection_name, document_type, sizes): + def __init__(self, collection_name, sizes): super().__init__() self.collection = db()[collection_name] - self.document_type = document_type self.sizes = sizes self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' @@ -28,7 +27,7 @@ class SyncDocument(Sync): file = document['file'] filename = os.path.basename(document['file']) - document['type'] = self.document_type + document['type'] = self.document_type(document) document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) document['file'] = { 'name': filename, @@ -37,10 +36,14 @@ class SyncDocument(Sync): return document + def document_type(self, document): + if document['category'] == 'Награды': + return 'award' + return 'document' + def create_id(self, document): bn = os.path.basename(document['file']) - document['id'] = url_encode_text('{type}-{category}-{file}'.format( - type=self.document_type, + document['id'] = url_encode_text('{category}-{file}'.format( category=document['category'], file=bn )) @@ -57,11 +60,12 @@ class SyncDocument(Sync): ) return document - def create_preview(self, pdf, sizes, preview_dir): - temp_preview_path = pdf_to_jpg(pdf) + def create_preview(self, file, sizes, preview_dir): + temp_preview_path = pdf_to_jpg(file) - id = url_encode_text(pdf) - url = lambda size, ext: self.url_base_preview.format(id=id, size=size, ext=ext) + file = os.path.basename(file) + file = url_encode_text(file) + url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext) img = create_image(temp_preview_path, sizes, url, preview_dir) os.remove(temp_preview_path) @@ -153,12 +157,49 @@ def documents_from_subdirs(): return documents +def get_pdf_title(file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(open(file, 'rb')) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None + + +def until_none(ls): + better = None + for i in ls: + if (i is not None) and i != '': + better = i + return better + + def main(dir_documents, dir_static_uploads, sync): os.chdir(dir_documents) # documents = documents_from_yaml() documents = documents_from_subdirs() + # CHOICE BETTER TITLE + documents = lmapfn(documents)( + lambda i: { + **i, + 'title': until_none([i['title'], get_pdf_title(i['file'])]) + } + ) + + # lprint_json(documents) + # return + # CREATE DOCUMENT IDENTITY documents = lmap(sync.create_id, documents) @@ -242,7 +283,6 @@ if __name__ == '__main__': dir_static_uploads, SyncDocument( 'documents', - 'document', sizes=settings.image_sizes ) ) diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 104c0e50..1998bce9 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -7,7 +7,7 @@ import lxml.html import settings from db import db from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \ - images_from_html, list_images, synced_images_ids + images_from_html, list_images, synced_images_ids, untouched from sync.image import sync_image from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn from utils.hash import hash_file, hash_str @@ -78,6 +78,14 @@ class SyncPost(Sync): document['images'] = images return document + def create_remove_query(self, query): + if self.type: + query = { + **query, + 'type': self.type + } + return query + def get_manifest(path): y, m = read_yaml_md(path) @@ -223,15 +231,7 @@ def main(dir_documents, sync, file_time_formats): ) # SKIP UNTOUCHED DOCUMENTS - documents = lmap( - lambda i: i[0], - filter( - lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), - lmap( - lambda document: (document, sync.read(document)), - documents - ) - )) + documents = untouched(documents, sync) # MAP EVENT MANIFEST -> EVENT_OBJECT documents = lmap( @@ -254,7 +254,8 @@ def main(dir_documents, sync, file_time_formats): documents ) - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) + remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = sync.query(remove_query) documents_to_remove = lmap( sync.delete, map( @@ -280,14 +281,16 @@ def main(dir_documents, sync, file_time_formats): lambda i: i['id'], documents )) - lprint_json(documents) + # lprint_json(documents) print('[SYNC DOCUMENTS DONE]') if __name__ == '__main__': image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' + # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Events' + dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Events' main( - '/Users/tmshv/Dropbox/Dev/Hud school/Events', + dir_docs, SyncPost( 'events', None, @@ -299,8 +302,10 @@ if __name__ == '__main__': ) image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' + # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/News' + dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/News' main( - '/Users/tmshv/Dropbox/Dev/Hud school/News', + dir_docs, SyncPost( 'timeline', 'post', diff --git a/utils/Sync/utils/io.py b/utils/Sync/utils/io.py index a0316729..31073301 100644 --- a/utils/Sync/utils/io.py +++ b/utils/Sync/utils/io.py @@ -1,8 +1,13 @@ +import re + +import frontmatter import yaml from markdown import markdown from utils.text.split import split, split_with +# FM_BOUNDARY = re.compile(r'^-{3,}$', re.MULTILINE) +FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE) def read_file(file): with open(file, 'rb') as f: @@ -23,19 +28,11 @@ def read_md(file): def read_yaml_md(file): - data = read_file(file) - yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data) + yaml, content = parse_yaml_front_matter(read_file(file)) + return yaml, markdown(content) - if not yaml_data and not md_data: - return (read_yaml(file), None) - - try: - y = yaml.load(yaml_data) - except: - y = None - - m = markdown(md_data) if md_data else None - return y, m + # data = frontmatter.load(file) + # return data.metadata, markdown(data.content) def read_md_yaml(file): @@ -52,3 +49,30 @@ def read_md_yaml(file): m = markdown(md_data) if md_data else None return y, m + + +def parse_yaml_front_matter(text, **defaults): + """ + Parse text with YAML frontmatter, return metadata and content. + Pass in optional metadata defaults as keyword args. + + If frontmatter is not found, returns an empty metadata dictionary + and original text content. + """ + + # metadata starts with defaults + metadata = defaults.copy() + + # split on the first two triple-dashes + try: + _, fm, content = FM_BOUNDARY.split(text, 2) + except ValueError: + # if we can't split, bail + return metadata, text + + # parse yaml, now that we have frontmatter + fm = yaml.safe_load(fm) + if isinstance(fm, dict): + metadata.update(fm) + + return metadata, content.strip() \ No newline at end of file