From 927674e53e005f9674e22854729f10afd4cb393f Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Fri, 5 Aug 2016 02:04:06 +0300 Subject: [PATCH] sync update: + data provider; refactoring; documents update --- utils/Sync/settings.py | 11 +- utils/Sync/sync/__init__.py | 45 ----- utils/Sync/sync/core/__init__.py | 46 +++++ utils/Sync/sync/core/document.py | 78 ++++++++ utils/Sync/sync/data/__init__.py | 32 ++++ utils/Sync/sync/data/fs.py | 30 ++++ utils/Sync/sync/document.py | 293 ++++++++++++------------------- utils/Sync/sync/teacher.py | 2 +- utils/Sync/utils/fn.py | 3 + 9 files changed, 311 insertions(+), 229 deletions(-) create mode 100644 utils/Sync/sync/core/__init__.py create mode 100644 utils/Sync/sync/core/document.py create mode 100644 utils/Sync/sync/data/__init__.py create mode 100644 utils/Sync/sync/data/fs.py diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index d4390da0..3f52b56b 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -17,8 +17,13 @@ DIGITS = '0123456789' description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-' translit_table = ( - ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'], - ['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', '_'] + ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', + 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', + 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'], + ['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', + 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', + '_'] ) teachers_names = { @@ -89,6 +94,8 @@ date_formats_reverse = [ '%Y.%m.%d' ] +collection_documents = 'documents' + dir_documents = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Documents' dir_events = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Events' dir_news = '/Users/tmshv/Yandex.Disk/Hudozka/Site/News' diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index a2528dd6..8e241298 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -107,48 +107,3 @@ create_post_from_image_list = lambda images: markdown( images )) ) - - -class Sync: - def __init__(self): - super().__init__() - - self.collection = None - - def read(self, document): - if isinstance(document, list): - return lmap(self.read, document) - - q = {'id': document['id']} - try: - return self.collection.find_one(q) - except ValueError: - pass - return None - - def update(self, document): - if isinstance(document, list): - return lmap(self.update, document) - - q = {'id': document['id']} - try: - self.collection.update_one(q, {'$set': document}, upsert=True) - return self.read(document) - except ValueError: - pass - return None - - def compile(self, document): - return document - - def query(self, q): - return self.collection.find(q) - - def delete(self, q): - return self.collection.find_one_and_delete(q) - - def create_id(self, document): - return None - - def create_hash(self, document): - return None diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py new file mode 100644 index 00000000..4628ec59 --- /dev/null +++ b/utils/Sync/sync/core/__init__.py @@ -0,0 +1,46 @@ +from utils.fn import lmap + + +class Sync: + def __init__(self): + super().__init__() + + self.collection = None + + def read(self, document): + if isinstance(document, list): + return lmap(self.read, document) + + q = {'id': document['id']} + try: + return self.collection.find_one(q) + except ValueError: + pass + return None + + def update(self, document): + if isinstance(document, list): + return lmap(self.update, document) + + q = {'id': document['id']} + try: + self.collection.update_one(q, {'$set': document}, upsert=True) + return self.read(document) + except ValueError: + pass + return None + + def compile(self, document): + return document + + def query(self, q): + return self.collection.find(q) + + def delete(self, q): + return self.collection.find_one_and_delete(q) + + def create_id(self, document): + return None + + def create_hash(self, document): + return None diff --git a/utils/Sync/sync/core/document.py b/utils/Sync/sync/core/document.py new file mode 100644 index 00000000..09bc6e96 --- /dev/null +++ b/utils/Sync/sync/core/document.py @@ -0,0 +1,78 @@ +import os +from tempfile import mkstemp + +from sync.core import Sync +from utils.hash import hash_file, hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + + +class SyncDocument(Sync): + def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document): + super().__init__() + self.collection = collection + self.provider = provider + self.sizes = sizes + self.dir_static_previews = dir_static_previews + self.url_base_preview = url_base_preview + self.url_base_document = url_base_document + + def create(self, document): + file = document['file'] + filename = os.path.basename(file) + + document['type'] = self.document_type(document) + document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) + document['file'] = { + 'name': filename, + 'size': self.provider.size(file) + } + + return document + + def document_type(self, document): + if document['category'] == 'Награды': + return 'award' + return 'document' + + def create_id(self, document): + bn = os.path.basename(document['file']) + document['id'] = url_encode_text('{category}-{file}'.format( + category=document['category'], + file=bn + )) + return document + + def create_url(self, document): + filename = os.path.basename(document['file']) + document['url'] = self.url_base_document.format(file=url_encode_file(filename)) + return document + + def create_hash(self, document): + document['hash'] = hash_str( + hash_str(document) + self.provider.hash(document['file']) + ) + return document + + def create_preview(self, file, sizes, preview_dir): + temp_preview_path = pdf_to_jpg(self.provider, file) + + file = os.path.basename(file) + file = url_encode_text(file) + url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext) + + img = create_image(temp_preview_path, sizes, url, preview_dir) + os.remove(temp_preview_path) + return img + + +def pdf_to_jpg(provider, pdf): + # cwd = os.getcwd() + # abspdf = os.path.join(cwd, pdf) + abspdf = provider.get_abs(pdf) + _, temp = mkstemp('.jpg') + image_magick_pdf_to_img(abspdf, temp) + return temp + + diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py new file mode 100644 index 00000000..cfeea574 --- /dev/null +++ b/utils/Sync/sync/data/__init__.py @@ -0,0 +1,32 @@ +import os + + +class Provider: + def __init__(self, root): + super().__init__() + self.root = root + + def open(self, path, mode): + return '' + + def scan(self, path): + return [] + + def glob(self, pattr): + return [] + + def copy(self, path, out): + pass + + def hash(self, path): + return None + + def size(self, path): + return 0 + + def get_abs(self, path): + abs = os.path.join(self.root, path) + return os.path.normpath(abs) + + def get_rel(self, path): + return os.path.relpath(path, self.root) \ No newline at end of file diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py new file mode 100644 index 00000000..64099c20 --- /dev/null +++ b/utils/Sync/sync/data/fs.py @@ -0,0 +1,30 @@ +import os +from glob import glob + +from shutil import copyfile + +from sync.data import Provider +from utils.hash import hash_file + + +class FSProvider(Provider): + def open(self, path, mode): + return open(self.get_abs(path), mode) + + def scan(self, path): + return os.scandir(self.get_abs(path)) + + def glob(self, pattr): + return glob(self.get_abs(pattr)) + + def hash(self, path): + return hash_file(self.get_abs(path)) + + def copy(self, path, out): + file = self.get_abs(path) + return copyfile(file, out) + + def size(self, path): + file = self.get_abs(path) + return os.path.getsize(file) + diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 2e41616b..1dde3c63 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -1,83 +1,128 @@ import os from glob import glob -from shutil import copyfile -from tempfile import mkstemp import settings from db import db -from sync import Sync, synced_image_id, untouched -from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn -from utils.hash import hash_file, hash_str -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img +from sync import synced_image_id, untouched +from sync.core.document import SyncDocument +from sync.data.fs import FSProvider +from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap from utils.io import read_yaml -from utils.text.transform import url_encode_text, url_encode_file -class SyncDocument(Sync): - def __init__(self, collection_name, sizes, dir_static_previews, url_base_preview, url_base_document): - super().__init__() - self.collection = db()[collection_name] - self.sizes = sizes - self.dir_static_previews = dir_static_previews - self.url_base_preview = url_base_preview - self.url_base_document = url_base_document +def sync_documents(update=True, delete=True): + provider = FSProvider(settings.dir_documents) - def create(self, document): - file = document['file'] - filename = os.path.basename(document['file']) + static_path = static_path_fn(settings.dir_static_uploads) - document['type'] = self.document_type(document) - document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) - document['file'] = { - 'name': filename, - 'size': os.path.getsize((file)) - } + u, d = main( + SyncDocument( + db()[settings.collection_documents], + provider, + sizes=settings.image_sizes, + dir_static_previews=settings.dir_static_images, + url_base_preview=settings.url_base_preview, + url_base_document=settings.url_base_document, + ), + static_path, + update_documents=update, + delete_documents=delete + ) - return document + print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d)))) + lprint_json(d) - def document_type(self, document): - if document['category'] == 'Награды': - return 'award' - return 'document' + print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u)))) + lprint_json(u) - def create_id(self, document): - bn = os.path.basename(document['file']) - document['id'] = url_encode_text('{category}-{file}'.format( - category=document['category'], - file=bn - )) - return document - def create_url(self, document): - filename = os.path.basename(document['file']) - document['url'] = self.url_base_document.format(file=url_encode_file(filename)) - return document +def main(sync, static_path, update_documents, delete_documents): + documents = get_documents(sync) + documents_id = lmap( + lambda i: i['id'], + documents + ) - def create_hash(self, document): - document['hash'] = hash_str( - hash_str(document) + hash_file(document['file']) + # CHECKING + file_names = set(lmap( + lambda i: i['file'], + documents + )) + if len(file_names) != len(documents): + raise Exception('File names should be unique') + + # SKIP UNTOUCHED DOCUMENTS + documents = untouched(documents, sync) + + # COPY FILE -> STATIC_DIR/URL_FILENAME + for i in documents: + sync.provider.copy( + i['file'], + static_path(i) ) - return document - def create_preview(self, file, sizes, preview_dir): - temp_preview_path = pdf_to_jpg(file) + # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT + documents = lmap( + sync.create, + documents + ) - file = os.path.basename(file) - file = url_encode_text(file) - url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext) + if update_documents: + # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('preview', lambda i: synced_image_id(i)), + documents + ) - img = create_image(temp_preview_path, sizes, url, preview_dir) - os.remove(temp_preview_path) - return img + # SYNC DOCUMENT_OBJECT WITH DB + documents = lmap( + sync.update, + documents + ) + + # DELETING + documents_delete = sync.query({'id': {'$nin': documents_id}}) + if delete_documents: + documents_delete = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_delete + ) + ) + + return documents, documents_delete -def pdf_to_jpg(pdf): - cwd = os.getcwd() - abspdf = os.path.join(cwd, pdf) - _, temp = mkstemp('.jpg') - image_magick_pdf_to_img(abspdf, temp) - return temp +def get_documents(sync): + # documents = documents_from_yaml() + documents = documents_from_subdirs(sync.provider) + + # SETUP TITLE BASED ON FILE NAME + documents = lmapfn(documents)( + lambda i: { + **i, + 'title': os.path.splitext(os.path.basename(i['file']))[0] + } + ) + + # CHOICE BETTER TITLE + documents = lmapfn(documents)( + lambda i: { + **i, + 'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])]) + } + ) + + documents = lmap(sync.create_id, documents) + documents = lmap(sync.create_hash, documents) + documents = lmap(sync.create_url, documents) + + return documents + + +def static_path_fn(dir): + return lambda document: os.path.join(dir, os.path.basename(document['url'])) def categorize_files_list(files, title): @@ -131,23 +176,22 @@ def documents_from_yaml(dirpath): return documents -def documents_from_subdirs(): +def documents_from_subdirs(provider): documents = lmap( lambda i: i.path, filter( lambda i: i.is_dir(), - os.scandir('.') + provider.scan('.') ) ) documents = lmap( lambda folder: lmap( lambda path: { - 'file': path, - 'title': os.path.splitext(os.path.basename(os.path.relpath(path, folder)))[0], + 'file': provider.get_rel(path), 'category': os.path.basename(folder) }, - glob(folder + '/*.pdf') + provider.glob(folder + '/*.pdf') ), documents ) @@ -156,12 +200,12 @@ def documents_from_subdirs(): return documents -def get_pdf_title(file): +def get_pdf_title(provider, file): from PyPDF2 import PdfFileReader from PyPDF2.generic import TextStringObject from PyPDF2.generic import IndirectObject - pdf = PdfFileReader(open(file, 'rb')) + pdf = PdfFileReader(provider.open(file, 'rb')) info = pdf.getDocumentInfo() if info: @@ -182,122 +226,9 @@ def until_none(ls): return better -def main(dir_documents, dir_static_uploads, sync, update_documents=True, delete_documents=True): - """ - - :param dir_documents: - :param dir_static_uploads: - :param sync: - :param update_documents: - :param delete_documents: - :return: - """ - - os.chdir(dir_documents) - - # documents = documents_from_yaml() - documents = documents_from_subdirs() - - # CHOICE BETTER TITLE - documents = lmapfn(documents)( - lambda i: { - **i, - 'title': until_none([i['title'], get_pdf_title(i['file'])]) - } - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) - - # CREATE HASH OF DOCUMENT FILE - documents = lmap(sync.create_hash, documents) - - # CREATE SCOPE - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - file_names = set(lmap( - lambda i: i['file'], - documents - )) - if len(file_names) != len(documents): - raise Exception('File names should be unique') - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # URL - documents = lmap(sync.create_url, documents) - - # COPY FILE -> STATIC_DIR/URL_FILENAME - out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url'])) - lmap( - lambda doc: copyfile( - doc['file'], - out_path(doc) - ), - documents - ) - - # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT - documents = lmap( - sync.create, - documents - ) - - if update_documents: - # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('preview', lambda i: synced_image_id(i)), - documents - ) - - # SYNC DOCUMENT_OBJECT WITH DB - documents = lmap( - sync.update, - documents - ) - - if delete_documents: - documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}}) - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - else: - documents_to_delete = [] - - print('SCOPE:', len(scope_documents_ids)) - lprint(scope_documents_ids) - - print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete)))) - lprint_json(documents_to_delete) - - print('UPDATE DOCUMENTS:', len(documents)) - lprint_json(documents) - - print('[SYNC DOCUMENTS DONE]') - - if __name__ == '__main__': - main( - settings.dir_documents, - settings.dir_static_uploads, - SyncDocument( - 'documents', - sizes=settings.image_sizes, - dir_static_previews=settings.dir_static_images, - url_base_preview=settings.url_base_preview, - url_base_document=settings.url_base_document, - ), - update_documents=True, - delete_documents=True - ) + sync_documents() + # sync_documents(delete=False, update=False) # DOCUMENT_OBJECT SAMPLE # { diff --git a/utils/Sync/sync/teacher.py b/utils/Sync/sync/teacher.py index 63e1f571..10cb2fe4 100644 --- a/utils/Sync/sync/teacher.py +++ b/utils/Sync/sync/teacher.py @@ -94,7 +94,7 @@ def main(dir_teachers, sync): filter( lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), lmap( - lambda document: (document, sync.read(document)), + lambda document: (document, sync.open(document)), documents ) )) diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index 4b7e751d..b03bbfa9 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -7,6 +7,9 @@ import bson.json_util def combine(items): + if len(items) == 0: + return [] + return reduce( lambda total, i: total + i, items