diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py deleted file mode 100644 index 426cd4fb..00000000 --- a/utils/Sync/sync/document.py +++ /dev/null @@ -1,206 +0,0 @@ -import os - -import requests - -import settings -from sync import synced_image_id, untouched -from sync.core.SyncDocument import SyncDocument -from sync.data import scan_subdirs -from sync.models.Document import Document -from utils.fn import combine, lmap, key_mapper - - -def sync_documents(provider, collection, update=True, delete=True): - """ - DOCUMENT_OBJECT SAMPLE - { - "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0", - "_id": { - "$oid": "56d08411ace9573958e5e497" - }, - "type": "document", - "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf", - "category": "Для поступающих", - "file": { - "size": 53088, - "name": "Договор о пожертвовании.pdf" - }, - "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf", - "preview": { - "$oid": "56d08007ace9573958e5e48f" - }, - "title": "Договор о пожертвовании" - } - - :param provider: - :param update: - :param delete: - :return: - """ - - static_path = static_path_fn(settings.dir_static_uploads) - - return main( - SyncDocument( - collection, - provider, - sizes=settings.image_sizes, - dir_static_previews=settings.dir_static_images, - url_base_preview=settings.url_base_preview, - url_base_document=settings.document_url_template, - ), - static_path, - update_documents=update, - delete_documents=delete - ) - - -def upload(provider, document): - f = document['file'] - p = provider.get_local(f) - n = os.path.basename(document['url']) - - files = {'file': open(p, 'rb')} - # values = {'DB': 'photcat', 'OUT': 'csv', 'SHORT': 'short'} - - r = requests.put( - url='https://static.shlisselburg.org/upload/art/uploads/{}'.format(n), - files=files, - auth=('uploader', '1207690962638465') - ) - if r.status_code == 200: - print('Document {} uploaded'.format(document['url'])) - else: - print('Failed to upload document {}'.format(document['url'])) - print('https://static.shlisselburg.org/upload/art/uploads/{}'.format(n)) - print(r.request.headers) - print(r.headers) - print(r.text) - print() - - -def main(sync, static_path, update_documents, delete_documents): - documents = scan_subdirs(sync.provider, '.pdf') - documents = lmap( - lambda document: read_document(sync, document), - documents - ) - documents_id = lmap( - lambda document: document['id'], - documents - ) - - # CHECKING - file_names = set(lmap( - lambda document: document['file'], - documents - )) - if len(file_names) != len(documents): - raise Exception('File names should be unique') - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # COPY FILE -> STATIC_DIR/URL_FILENAME - for i in documents: - sync.provider.copy( - i['file'], - static_path(i) - ) - - for i in documents: - upload(sync.provider, i) - - # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT - documents = lmap( - sync.create, - documents - ) - - if update_documents: - # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('preview', lambda img: synced_image_id(img)), - documents - ) - - # SYNC DOCUMENT_OBJECT WITH DB - documents = lmap( - sync.update, - documents - ) - - # DELETING - documents_delete = sync.query({'id': {'$nin': documents_id}}) - if delete_documents: - documents_delete = lmap( - sync.delete, - map( - lambda document: {'_id': document['_id']}, - documents_delete - ) - ) - - return documents, documents_delete - - -def read_document(sync, path): - name = os.path.basename(path) - - document = { - 'file': path, - 'category': os.path.dirname(path), - 'title': os.path.splitext(name)[0] - } - - document = sync.create_id(document) - document = sync.create_url(document) - document = sync.create_hash(document) - - return document - - -def create_document(provider, m): - # dir_static_previews = settings.dir_static_images, - # url_base_preview = settings.url_base_preview, - # url_base_document = settings.url_base_document - return Document(provider, m, {}) - - -def static_path_fn(dir_path): - return lambda document: os.path.join(dir_path, os.path.basename(document['url'])) - - -def categorize_files_list(files, title): - """ - Adds specified category to each file in files list - :param files: - :param title: - :return: - """ - return map( - lambda profile: { - **profile, - 'category': title - }, - files - ) - - -def unwrap_manifest(param): - if isinstance(param, list): - return combine(map( - unwrap_manifest, - param - )) - - if 'files' in param: - return lmap( - unwrap_manifest, - categorize_files_list(param['files'], param['title']) - ) - - if 'file' not in param: - return None - - return param diff --git a/utils/Sync/test/sync/test_document.py b/utils/Sync/test/sync/test_document.py index b824c1f0..9c13dc39 100644 --- a/utils/Sync/test/sync/test_document.py +++ b/utils/Sync/test/sync/test_document.py @@ -3,7 +3,6 @@ import unittest import settings from sync.data import scan_subdirs from sync.data.fs import FSProvider -from sync.document import create_document from sync.models.Document import Document from utils.fn import lmap