From 10dd3d6316a260bad61b600d274a65a8f63b990e Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Wed, 14 Sep 2016 02:11:37 +0300 Subject: [PATCH] [sync] post updated --- utils/Sync/settings.py | 1 + utils/Sync/sync.py | 14 ++- utils/Sync/sync/__init__.py | 8 -- utils/Sync/sync/core/post.py | 87 ++++++++++++++++ utils/Sync/sync/data/__init__.py | 8 ++ utils/Sync/sync/post.py | 167 ++++++------------------------- utils/Sync/utils/io.py | 6 +- 7 files changed, 141 insertions(+), 150 deletions(-) create mode 100644 utils/Sync/sync/core/post.py diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 417c2837..ae1d1468 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -96,6 +96,7 @@ date_formats_reverse = [ collection_documents = 'documents' collection_schedules = 'schedules' +collection_events = 'posts' sync_provider_type = env('SYNC_PROVIDER', 'fs') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index e6b23f13..815d784f 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -2,8 +2,7 @@ import settings from db import db from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider -from sync.document import sync_documents -from sync.schedule import sync_schedules +from sync.post import sync_posts from utils.fn import lprint_json @@ -19,17 +18,24 @@ def main(provider, fn, collection_name): if provider: collection = db()[collection_name] + print('SYNC: %s' % (collection_name.upper())) u, d = fn(provider, collection, update=settings.do_update, delete=settings.do_update) + print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d)))) lprint_json(d) print('UPDATE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_update else str(len(u)))) lprint_json(u) + print('') + if __name__ == '__main__': # client = get_provider(settings.sync_provider_type, settings.dir_documents) # main(client, sync_documents, settings.collection_documents) - client = get_provider(settings.sync_provider_type, settings.dir_schedules) - main(client, sync_schedules, settings.collection_schedules) + # client = get_provider(settings.sync_provider_type, settings.dir_schedules) + # main(client, sync_schedules, settings.collection_schedules) + + client = get_provider(settings.sync_provider_type, settings.dir_events) + main(client, sync_posts, settings.collection_events) diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index 8e241298..d2a37c33 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -10,14 +10,6 @@ from sync.image import sync_image from utils.fn import lmap, combine -def list_images(dir): - image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG'] - return combine(map( - lambda type: glob(os.path.join(dir, type)), - image_types - )) - - def create_date(date_str, date_formats=None): """ 2016 diff --git a/utils/Sync/sync/core/post.py b/utils/Sync/sync/core/post.py new file mode 100644 index 00000000..4de3776c --- /dev/null +++ b/utils/Sync/sync/core/post.py @@ -0,0 +1,87 @@ +import os +from tempfile import mkstemp + +import lxml.html + +from sync.core import Sync +from utils.fn import last_good, lmap, combine +from utils.hash import hash_str, hash_file +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + + +class SyncPost(Sync): + def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes): + super().__init__() + self.collection = collection + self.provider = provider + self.sizes = sizes + self.type = document_type + self.image_url_base = image_url_base + self.dir_local_images = dir_local_images + + def create_id(self, document): + document['id'] = document['id'] if 'id' in document else url_encode_text(document['title']) + return document + + def create_hash(self, document): + hash_images = lmap( + self.provider.hash, + lmap( + lambda i: os.path.join(document['folder'], i), + document['images'] + ) + ) + + document['hash'] = hash_str( + combine([hash_str(document)] + hash_images) + ) + + return document + + def create(self, document): + if 'date' not in document or not document['date']: + return None + + if 'title' not in document or not document['title']: + return None + + if self.type: + document['type'] = self.type + + post_html = lxml.html.fromstring(document['post']) + images = [] + for img in post_html.cssselect('img'): + src = img.get('src') + img_path = os.path.abspath( + os.path.join(document['folder'], src) + ) + + if os.path.exists(img_path): + img_id = url_encode_text(os.path.splitext(src)[0]) + url_fn = lambda size, ext: self.image_url_base.format( + id=document['id'], + img=img_id, + size=size, + ext=ext.lower() + ) + + image = create_image(img_path, self.sizes, url_fn, self.dir_local_images) + if image: + images.append(image) + img.set('src', image['data']['big']['url']) + else: + print('fail: ', document['folder'], src) + + document['post'] = lxml.html.tostring(post_html).decode('utf-8') + document['images'] = images + return document + + def create_remove_query(self, query): + if self.type: + query = { + **query, + 'type': self.type + } + return query diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index 4bec6475..9815e3e8 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -69,3 +69,11 @@ def scan_subdirs(provider, ext, path='.'): def get_data(provider, path, transform): data = provider.read(path) return transform(data) + + +def list_images(provider, path): + image_types = ['.jpg', '.JPG', '.png', '.PNG'] + return combine(lmap( + lambda ext: provider.type_filter(path, ext), + image_types + )) diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 5aff9095..3d1fbf7e 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -5,90 +5,22 @@ from glob import glob import lxml.html import settings +from sync.data import list_images from db import db -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \ - images_from_html, list_images, synced_images_ids, untouched +from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ + synced_images_ids, untouched +from sync.core.post import SyncPost from sync.image import sync_image -from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn +from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn, constant from utils.hash import hash_file, hash_str from utils.image import create_image from utils.io import read_yaml_md from utils.text.transform import url_encode_text -class SyncPost(Sync): - def __init__(self, collection_name, document_type, image_url_base, dir_local_images, sizes): - super().__init__() - self.collection = db()[collection_name] - self.sizes = sizes - self.type = document_type - self.image_url_base = image_url_base - self.dir_local_images = dir_local_images - - def create_id(self, document): - document['id'] = document['id'] if 'id' in document else url_encode_text(document['title']) - return document - - def create_hash(self, document): - hash_images = lmap( - hash_file, - lmap( - lambda i: os.path.join(document['folder'], i), - document['images'] - ) - ) - - document['hash'] = hash_str( - combine([hash_str(document)] + hash_images) - ) - - return document - - def create(self, document): - if 'date' not in document or not document['date']: - return None - - if 'title' not in document or not document['title']: - return None - - if self.type: - document['type'] = self.type - - post_html = lxml.html.fromstring(document['post']) - images = [] - for img in post_html.cssselect('img'): - src = img.get('src') - img_path = os.path.abspath( - os.path.join(document['folder'], src) - ) - - if os.path.exists(img_path): - img_id = url_encode_text(os.path.splitext(src)[0]) - url_fn = lambda size, ext: self.image_url_base.format(id=document['id'], img=img_id, size=size, - ext=ext.lower()) - - image = create_image(img_path, self.sizes, url_fn, self.dir_local_images) - if image: - images.append(image) - img.set('src', image['data']['big']['url']) - else: - print('fail: ', document['folder'], src) - - document['post'] = lxml.html.tostring(post_html).decode('utf-8') - document['images'] = images - return document - - def create_remove_query(self, query): - if self.type: - query = { - **query, - 'type': self.type - } - return query - - -def get_manifest(path): - y, m = read_yaml_md(path) +def get_manifest(provider, path): + data = provider.read(path).read().decode('utf-8') + y, m = read_yaml_md(data) y = y if y else {} if 'date' in y: @@ -107,25 +39,23 @@ def get_manifest(path): } -def md_from_folder(i): - return first(glob(i + '/*.md')) +def md_from_folder(provider, i): + return first(provider.type_filter(i, '.md')) def get_folder_documents(sync, file_time_formats): # GET EVENT FOLDERS documents = lmap( - lambda i: i.path, + lambda i: i, filter( - lambda i: i.is_dir(), - os.scandir('.') + lambda i: sync.provider.is_dir(i), + sync.provider.scan('.') ) ) # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE documents = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name( - os.path.basename(i), file_time_formats - ), + lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats), documents ) documents = lmap( @@ -143,7 +73,7 @@ def get_folder_documents(sync, file_time_formats): **i, 'images': lmap( lambda path: os.path.relpath(path, i['folder']), - list_images(i['folder']) + list_images(sync.provider, i['folder']) ) }, documents @@ -163,10 +93,10 @@ def get_folder_documents(sync, file_time_formats): **map_cases( i, [( - lambda i: md_from_folder(i['folder']), - lambda i: get_manifest(md_from_folder(i['folder'])), + lambda doc: md_from_folder(sync.provider, doc['folder']), + lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, doc['folder'])), )], - lambda i: {} + constant({}) ) }, documents @@ -183,11 +113,9 @@ def get_folder_documents(sync, file_time_formats): # GET EVENT FILES def get_file_documents(sync, file_time_formats): - documents = glob('*.md') + documents = sync.provider.type_filter('.', '.md') documents = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name( - os.path.basename(i), file_time_formats - ), + lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats), documents ) @@ -204,7 +132,7 @@ def get_file_documents(sync, file_time_formats): documents = lmap( lambda i: { **i, - **get_manifest(i['file']) + **get_manifest(sync.provider, i['file']) }, documents ) @@ -218,14 +146,11 @@ def get_file_documents(sync, file_time_formats): return documents -def main(dir_documents, sync, file_time_formats, update_documents=True, delete_documents=True): - # INIT - os.chdir(dir_documents) - +def main(sync, file_time_formats, update_documents=True, delete_documents=True): documents = get_folder_documents(sync, file_time_formats) documents += get_file_documents(sync, file_time_formats) - # CREATE SCOPE OF CURRENT SESSION + # # CREATE SCOPE OF CURRENT SESSION scope_documents_ids = lmapfn(documents)( lambda i: i['id'] ) @@ -268,50 +193,22 @@ def main(dir_documents, sync, file_time_formats, update_documents=True, delete_d else: documents_to_delete = [] - # SCOPE - print('SCOPE:') - lprint(scope_documents_ids) - - # DELETE - print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete)))) - lprint_json(lmap( - lambda i: i['id'], - documents_to_delete - )) - - # DELETE - print('UPDATE DOCUMENTS:') - lprint_json(lmap( - lambda i: i['id'], - documents - )) - # lprint_json(documents) - print('[SYNC DOCUMENTS DONE]') + return documents, documents_to_delete -if __name__ == '__main__': +def sync_posts(provider, collection, update=True, delete=True): image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' - main( - settings.dir_events, + + return main( SyncPost( - 'events', + collection, + provider, None, image_url_base, settings.dir_static_images, settings.event_image_sizes ), - file_time_formats=['%Y.%m.%d'] - ) - - image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' - main( - settings.dir_news, - SyncPost( - 'timeline', - 'post', - image_url_base, - settings.dir_static_images, - settings.event_image_sizes - ), - file_time_formats=settings.date_formats_reverse + file_time_formats=['%Y.%m.%d'], + update_documents=update, + delete_documents=delete ) diff --git a/utils/Sync/utils/io.py b/utils/Sync/utils/io.py index 31073301..6288f874 100644 --- a/utils/Sync/utils/io.py +++ b/utils/Sync/utils/io.py @@ -27,9 +27,9 @@ def read_md(file): return markdown(data) -def read_yaml_md(file): - yaml, content = parse_yaml_front_matter(read_file(file)) - return yaml, markdown(content) +def read_yaml_md(data): + yaml_front_matter, content = parse_yaml_front_matter(data) + return yaml_front_matter, markdown(content) # data = frontmatter.load(file) # return data.metadata, markdown(data.content)