From 24b28e6e4dbe8168d91b9f46d724ac50bb718479 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Thu, 4 Aug 2016 23:35:39 +0300 Subject: [PATCH] utils update: moving some parameters to settings --- utils/Sync/settings.py | 24 +++++--- utils/Sync/sync/document.py | 90 +++++++++++++++++------------- utils/Sync/sync/image.py | 6 ++ utils/Sync/sync/post.py | 44 +++++++-------- utils/Sync/utils/text/transform.py | 11 +++- 5 files changed, 106 insertions(+), 69 deletions(-) diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index e7793a94..d4390da0 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -1,3 +1,5 @@ +import os + __author__ = 'Roman Timashev' root_dir_gallery = './Gallery' @@ -5,7 +7,7 @@ root_dir_schedule = './Schedules' default_description_version = 1 -database_uri = 'mongodb://localhost/hudozka' +database_uri = os.environ['MONGO_URI'] CYR_LOWER = 'йцукенгшщзхъфывапролджэячсмитьбю' CYR_UPPER = 'ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮ' @@ -14,7 +16,6 @@ EN_UPPER = 'QWERTYUIOPASDFGHJKLZXCVBNM' DIGITS = '0123456789' description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-' - translit_table = ( ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'], ['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', '_'] @@ -27,15 +28,11 @@ teachers_names = { 'В.А.Саржин': 'va-sarzhin', 'Р.К.Тимашев': 'rk-timashev', 'А.С.Тимашева': 'as-timasheva', - # '': '', } image_processing = True image_save_dir = './static_images' -image_base_url = 'https://static.shburg.org/art/images/' -image_output = '/Users/tmshv/Desktop/Hudozka Static/images' - album_html_img_class = 'hudozka-product' image_ext = '.jpg' @@ -90,4 +87,17 @@ date_formats_reverse = [ '%Y', '%Y.%m', '%Y.%m.%d' -] \ No newline at end of file +] + +dir_documents = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Documents' +dir_events = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Events' +dir_news = '/Users/tmshv/Yandex.Disk/Hudozka/Site/News' + +dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' +dir_static_images = '/Users/tmshv/Desktop/Hudozka Static/images' +url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' +url_base_document = 'https://static.shburg.org/art/uploads/{file}' + +image_base_url = 'https://static.shburg.org/art/images/' +image_name_format = '{type}-{id}-{img}-{size}{ext}' +image_output = '/Users/tmshv/Desktop/Hudozka Static/images' diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 3b23fe53..2e41616b 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -15,13 +15,13 @@ from utils.text.transform import url_encode_text, url_encode_file class SyncDocument(Sync): - def __init__(self, collection_name, sizes): + def __init__(self, collection_name, sizes, dir_static_previews, url_base_preview, url_base_document): super().__init__() self.collection = db()[collection_name] self.sizes = sizes - self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' - self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' - self.url_base_document = 'https://static.shburg.org/art/uploads/{file}' + self.dir_static_previews = dir_static_previews + self.url_base_preview = url_base_preview + self.url_base_document = url_base_document def create(self, document): file = document['file'] @@ -115,10 +115,9 @@ def unwrap_manifest(param): return param -def documents_from_yaml(): +def documents_from_yaml(dirpath): # GET DOCUMENT YAML_MANIFEST FILES - - documents = glob(dir_documents + '/*.yaml') + documents = glob(dirpath + '/*.yaml') # READ YAML_MANIFEST FILES documents = lmap( @@ -183,7 +182,17 @@ def until_none(ls): return better -def main(dir_documents, dir_static_uploads, sync): +def main(dir_documents, dir_static_uploads, sync, update_documents=True, delete_documents=True): + """ + + :param dir_documents: + :param dir_static_uploads: + :param sync: + :param update_documents: + :param delete_documents: + :return: + """ + os.chdir(dir_documents) # documents = documents_from_yaml() @@ -197,9 +206,6 @@ def main(dir_documents, dir_static_uploads, sync): } ) - # lprint_json(documents) - # return - # CREATE DOCUMENT IDENTITY documents = lmap(sync.create_id, documents) @@ -241,32 +247,36 @@ def main(dir_documents, dir_static_uploads, sync): documents ) - # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('preview', lambda i: synced_image_id(i)), - documents - ) - - # SYNC DOCUMENT_OBJECT WITH DB - documents = lmap( - sync.update, - documents - ) - - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove + if update_documents: + # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('preview', lambda i: synced_image_id(i)), + documents ) - ) + + # SYNC DOCUMENT_OBJECT WITH DB + documents = lmap( + sync.update, + documents + ) + + if delete_documents: + documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}}) + documents_to_delete = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_delete + ) + ) + else: + documents_to_delete = [] print('SCOPE:', len(scope_documents_ids)) lprint(scope_documents_ids) - print('DELETE DOCUMENTS:', len(documents_to_remove)) - lprint_json(documents_to_remove) + print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete)))) + lprint_json(documents_to_delete) print('UPDATE DOCUMENTS:', len(documents)) lprint_json(documents) @@ -275,16 +285,18 @@ def main(dir_documents, dir_static_uploads, sync): if __name__ == '__main__': - dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents' - dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' - main( - dir_documents, - dir_static_uploads, + settings.dir_documents, + settings.dir_static_uploads, SyncDocument( 'documents', - sizes=settings.image_sizes - ) + sizes=settings.image_sizes, + dir_static_previews=settings.dir_static_images, + url_base_preview=settings.url_base_preview, + url_base_document=settings.url_base_document, + ), + update_documents=True, + delete_documents=True ) # DOCUMENT_OBJECT SAMPLE diff --git a/utils/Sync/sync/image.py b/utils/Sync/sync/image.py index 81f210a7..3f3a9967 100644 --- a/utils/Sync/sync/image.py +++ b/utils/Sync/sync/image.py @@ -1,4 +1,5 @@ from db import db +from settings import image_name_format, image_base_url from utils.fn import lmap @@ -58,3 +59,8 @@ def sync_image(record): pass return None + + +def image_create_url(file): + url = image_base_url + image_name_format + return url.format(file) diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 1998bce9..25cb75d9 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -218,7 +218,7 @@ def get_file_documents(sync, file_time_formats): return documents -def main(dir_documents, sync, file_time_formats): +def main(dir_documents, sync, file_time_formats, update_documents=True, delete_documents=True): # INIT os.chdir(dir_documents) @@ -249,30 +249,34 @@ def main(dir_documents, sync, file_time_formats): ) # SYNC EVENT_OBJECT WITH DB - documents = lmap( - sync.update, - documents - ) - - remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}}) - documents_to_remove = sync.query(remove_query) - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove + if update_documents: + documents = lmap( + sync.update, + documents ) - ) + + if delete_documents: + remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}}) + documents_to_delete = sync.query(remove_query) + documents_to_delete = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_delete + ) + ) + else: + documents_to_delete = [] # SCOPE print('SCOPE:') lprint(scope_documents_ids) # DELETE - print('DELETE DOCUMENTS:') + print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete)))) lprint_json(lmap( lambda i: i['id'], - documents_to_remove + documents_to_delete )) # DELETE @@ -287,10 +291,8 @@ def main(dir_documents, sync, file_time_formats): if __name__ == '__main__': image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' - # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Events' - dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Events' main( - dir_docs, + settings.dir_events, SyncPost( 'events', None, @@ -302,10 +304,8 @@ if __name__ == '__main__': ) image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' - # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/News' - dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/News' main( - dir_docs, + settings.dir_news, SyncPost( 'timeline', 'post', diff --git a/utils/Sync/utils/text/transform.py b/utils/Sync/utils/text/transform.py index 871a9610..088a3b8e 100644 --- a/utils/Sync/utils/text/transform.py +++ b/utils/Sync/utils/text/transform.py @@ -76,7 +76,16 @@ def text_lower(i): return i.lower() -url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower]) +url_encode_file = iterate_over_fns([ + translit, + space_to_dash, + short_dash, + remove_underscores, + remove_punctuation, + reduce_dashes, + text_lower +]) + url_encode_text = iterate_over_fns([ translit, space_to_dash,