diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 3f52b56b..ca9122fd 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -2,6 +2,11 @@ import os __author__ = 'Roman Timashev' + +def abs_fn(root): + return lambda path: os.path.join(root, path) + + root_dir_gallery = './Gallery' root_dir_schedule = './Schedules' @@ -96,9 +101,17 @@ date_formats_reverse = [ collection_documents = 'documents' -dir_documents = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Documents' -dir_events = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Events' -dir_news = '/Users/tmshv/Yandex.Disk/Hudozka/Site/News' +sync_provider_type = os.environ['SYNC_PROVIDER'] + +_providers_roots = { + 'fs': abs_fn('/Users/tmshv/Yandex.Disk'), + 'yd': abs_fn('/'), +} +f = _providers_roots[sync_provider_type] + +dir_documents = f('Hudozka/Site/Documents') +dir_events = f('Hudozka/Site/Events') +dir_news = f('Hudozka/Site/News') dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' dir_static_images = '/Users/tmshv/Desktop/Hudozka Static/images' @@ -108,3 +121,5 @@ url_base_document = 'https://static.shburg.org/art/uploads/{file}' image_base_url = 'https://static.shburg.org/art/images/' image_name_format = '{type}-{id}-{img}-{size}{ext}' image_output = '/Users/tmshv/Desktop/Hudozka Static/images' + +yandex_disk_access_token = os.environ['YANDEX_DISK_ACCESS_TOKEN'] diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py new file mode 100644 index 00000000..d3b42b2b --- /dev/null +++ b/utils/Sync/sync.py @@ -0,0 +1,47 @@ +import settings +from sync.core.document import pdf_to_jpg +from sync.data.fs import FSProvider +from sync.data.yandexdisk import YDProvider +from sync.document import sync_documents +from utils.fn import lprint_json +from utils.image.resize import image_magick_pdf_to_img +from yandexdisk import YDClient + + +def get_provider(provider_type, root): + if provider_type == 'fs': + return FSProvider(root) + elif provider_type == 'yd': + return YDProvider(root, settings.yandex_disk_access_token) + return None + + +if __name__ == '__main__': + provider = get_provider(settings.sync_provider_type, settings.dir_documents) + + update = False + delete = False + + if provider: + u, d = sync_documents(provider, update=update, delete=delete) + print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d)))) + lprint_json(d) + + print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u)))) + lprint_json(u) + + + # print(api.scan('.')) + # print(list(filter( + # lambda i: api.is_dir(i), + # api.scan('.') + # ))) + + # print(api.hash('./Архив/Устав 2013.pdf')) + # print(api.size('./Архив/Устав 2013.pdf')) + + # print(api.glob('./Архив/*.pdf')) + # print(api.type_filter('./Архив', '.pdf')) + + # path = api.type_filter('./Архив', '.pdf')[0] + # print(api.copy(path, '/Users/tmshv/Desktop/fafo')) diff --git a/utils/Sync/sync/core/document.py b/utils/Sync/sync/core/document.py index 09bc6e96..d4f77eab 100644 --- a/utils/Sync/sync/core/document.py +++ b/utils/Sync/sync/core/document.py @@ -22,7 +22,7 @@ class SyncDocument(Sync): file = document['file'] filename = os.path.basename(file) - document['type'] = self.document_type(document) + document['type'] = document_type(document) document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) document['file'] = { 'name': filename, @@ -31,11 +31,6 @@ class SyncDocument(Sync): return document - def document_type(self, document): - if document['category'] == 'Награды': - return 'award' - return 'document' - def create_id(self, document): bn = os.path.basename(document['file']) document['id'] = url_encode_text('{category}-{file}'.format( @@ -56,23 +51,29 @@ class SyncDocument(Sync): return document def create_preview(self, file, sizes, preview_dir): - temp_preview_path = pdf_to_jpg(self.provider, file) + def url(size, ext): + return self.url_base_preview.format(id=file, size=size, ext=ext) + temp_preview_path = pdf_to_jpg(self.provider, file) file = os.path.basename(file) file = url_encode_text(file) - url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext) - img = create_image(temp_preview_path, sizes, url, preview_dir) os.remove(temp_preview_path) return img +def document_type(document): + if document['category'] == 'Награды': + return 'award' + return 'document' + + def pdf_to_jpg(provider, pdf): - # cwd = os.getcwd() - # abspdf = os.path.join(cwd, pdf) - abspdf = provider.get_abs(pdf) - _, temp = mkstemp('.jpg') - image_magick_pdf_to_img(abspdf, temp) - return temp + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + os.remove(temp_in) + return temp_out diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index cfeea574..bf251dac 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -1,4 +1,5 @@ import os +from io import BytesIO class Provider: @@ -6,8 +7,8 @@ class Provider: super().__init__() self.root = root - def open(self, path, mode): - return '' + def read(self, path): + return BytesIO(bytes()) def scan(self, path): return [] @@ -15,6 +16,9 @@ class Provider: def glob(self, pattr): return [] + def type_filter(self, path, ext): + return [] + def copy(self, path, out): pass @@ -29,4 +33,8 @@ class Provider: return os.path.normpath(abs) def get_rel(self, path): - return os.path.relpath(path, self.root) \ No newline at end of file + rel = os.path.relpath(path, self.root) + return rel + + def is_dir(self, path): + return False \ No newline at end of file diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py index 64099c20..ccf0cc3a 100644 --- a/utils/Sync/sync/data/fs.py +++ b/utils/Sync/sync/data/fs.py @@ -4,18 +4,31 @@ from glob import glob from shutil import copyfile from sync.data import Provider +from utils.fn import lmap from utils.hash import hash_file class FSProvider(Provider): - def open(self, path, mode): - return open(self.get_abs(path), mode) + def open(self, path): + return open(self.get_abs(path), 'rb') def scan(self, path): - return os.scandir(self.get_abs(path)) + return lmap( + self.get_rel, + map( + lambda i: i.path, + os.scandir(self.get_abs(path)) + ) + ) def glob(self, pattr): - return glob(self.get_abs(pattr)) + return lmap(self.get_rel, glob(self.get_abs(pattr))) + + def type_filter(self, path, ext): + return list(filter( + lambda fn: os.path.splitext(fn)[1] == ext, + self.scan(path) + )) def hash(self, path): return hash_file(self.get_abs(path)) @@ -28,3 +41,6 @@ class FSProvider(Provider): file = self.get_abs(path) return os.path.getsize(file) + def is_dir(self, path): + return os.path.isdir(self.get_abs(path)) + diff --git a/utils/Sync/sync/data/yandexdisk.py b/utils/Sync/sync/data/yandexdisk.py new file mode 100644 index 00000000..ed42ddd1 --- /dev/null +++ b/utils/Sync/sync/data/yandexdisk.py @@ -0,0 +1,64 @@ +import os + +import re + +import io +import requests + +from sync.data import Provider +from utils.fn import lmap +from yandexdisk import YDClient + + +class YDProvider(Provider): + def __init__(self, root, access_token): + super().__init__(root) + self.api = YDClient(access_token) + + def read(self, path): + remote = self.api.get_download_link(self.get_abs(path)) + url = remote['href'] + req = requests.get(url) + data = req.content + return io.BytesIO(data) + + def scan(self, path): + res = self.api.get_content(self.get_abs(path)) + files = [] + for item in res['_embedded']['items']: + files.append(item['path']) + + files = map( + lambda fp: re.sub('disk:', '', fp), + files + ) + return lmap(self.get_rel, files) + + def glob(self, pattr): + return [] + + def type_filter(self, path, ext): + return list(filter( + lambda fn: os.path.splitext(fn)[1] == ext, + self.scan(path) + )) + + def hash(self, path): + res = self.api.get_content(self.get_abs(path)) + return res['sha256'] + + def copy(self, path, out): + data = self.read(path) + f = open(out, 'wb') + f.write(data.read()) + f.close() + return out + + def size(self, path): + res = self.api.get_content(self.get_abs(path)) + return res['size'] + + def is_dir(self, path): + res = self.api.get_content(self.get_abs(path)) + return res['type'] == 'dir' + diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 1dde3c63..eb2e1a4d 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -5,17 +5,41 @@ import settings from db import db from sync import synced_image_id, untouched from sync.core.document import SyncDocument -from sync.data.fs import FSProvider -from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap +from utils.fn import combine, lmap, key_mapper, lmapfn from utils.io import read_yaml -def sync_documents(update=True, delete=True): - provider = FSProvider(settings.dir_documents) +def sync_documents(provider, update=True, delete=True): + """ + DOCUMENT_OBJECT SAMPLE + { + "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0", + "_id": { + "$oid": "56d08411ace9573958e5e497" + }, + "type": "document", + "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf", + "category": "Для поступающих", + "file": { + "size": 53088, + "name": "Договор о пожертвовании.pdf" + }, + "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf", + "preview": { + "$oid": "56d08007ace9573958e5e48f" + }, + "title": "Договор о пожертвовании" + } + + :param provider: + :param update: + :param delete: + :return: + """ static_path = static_path_fn(settings.dir_static_uploads) - u, d = main( + return main( SyncDocument( db()[settings.collection_documents], provider, @@ -29,23 +53,17 @@ def sync_documents(update=True, delete=True): delete_documents=delete ) - print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d)))) - lprint_json(d) - - print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u)))) - lprint_json(u) - def main(sync, static_path, update_documents, delete_documents): documents = get_documents(sync) documents_id = lmap( - lambda i: i['id'], + lambda document: document['id'], documents ) # CHECKING file_names = set(lmap( - lambda i: i['file'], + lambda document: document['file'], documents )) if len(file_names) != len(documents): @@ -70,7 +88,7 @@ def main(sync, static_path, update_documents, delete_documents): if update_documents: # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB documents = lmap( - key_mapper('preview', lambda i: synced_image_id(i)), + key_mapper('preview', lambda img: synced_image_id(img)), documents ) @@ -86,7 +104,7 @@ def main(sync, static_path, update_documents, delete_documents): documents_delete = lmap( sync.delete, map( - lambda i: {'_id': i['_id']}, + lambda document: {'_id': document['_id']}, documents_delete ) ) @@ -107,12 +125,12 @@ def get_documents(sync): ) # CHOICE BETTER TITLE - documents = lmapfn(documents)( - lambda i: { - **i, - 'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])]) - } - ) + # documents = lmapfn(documents)( + # lambda i: { + # **i, + # 'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])]) + # } + # ) documents = lmap(sync.create_id, documents) documents = lmap(sync.create_hash, documents) @@ -121,8 +139,8 @@ def get_documents(sync): return documents -def static_path_fn(dir): - return lambda document: os.path.join(dir, os.path.basename(document['url'])) +def static_path_fn(dir_path): + return lambda document: os.path.join(dir_path, os.path.basename(document['url'])) def categorize_files_list(files, title): @@ -177,21 +195,20 @@ def documents_from_yaml(dirpath): def documents_from_subdirs(provider): - documents = lmap( - lambda i: i.path, - filter( - lambda i: i.is_dir(), - provider.scan('.') - ) - ) + documents = provider.scan('.') + documents = list(filter( + lambda i: provider.is_dir(i), + documents + )) documents = lmap( lambda folder: lmap( lambda path: { - 'file': provider.get_rel(path), + # 'file': provider.get_rel(path), + 'file': path, 'category': os.path.basename(folder) }, - provider.glob(folder + '/*.pdf') + provider.type_filter(folder, '.pdf') ), documents ) @@ -205,7 +222,7 @@ def get_pdf_title(provider, file): from PyPDF2.generic import TextStringObject from PyPDF2.generic import IndirectObject - pdf = PdfFileReader(provider.open(file, 'rb')) + pdf = PdfFileReader(provider.read(file)) info = pdf.getDocumentInfo() if info: @@ -224,28 +241,3 @@ def until_none(ls): if (i is not None) and i != '': better = i return better - - -if __name__ == '__main__': - sync_documents() - # sync_documents(delete=False, update=False) - - # DOCUMENT_OBJECT SAMPLE - # { - # "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0", - # "_id": { - # "$oid": "56d08411ace9573958e5e497" - # }, - # "type": "document", - # "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf", - # "category": "Для поступающих", - # "file": { - # "size": 53088, - # "name": "Договор о пожертвовании.pdf" - # }, - # "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf", - # "preview": { - # "$oid": "56d08007ace9573958e5e48f" - # }, - # "title": "Договор о пожертвовании" - # } diff --git a/utils/Sync/yandexdisk/__init__.py b/utils/Sync/yandexdisk/__init__.py new file mode 100644 index 00000000..6781604a --- /dev/null +++ b/utils/Sync/yandexdisk/__init__.py @@ -0,0 +1,230 @@ +import requests + + +class YDClient: + ''' + Implementation of https://tech.yandex.ru/disk/poligon/ + ''' + + base_url = 'https://cloud-api.yandex.net:443/v1/disk' + + def __init__(self, access_token): + self.access_token = access_token + + self.base_headers = dict( + Accept='application/json', + Authorization='OAuth %s' % self.access_token, + ) + + def get_disk_metadata(self): + """ + :return: disk metadata + """ + url = self.base_url + r = requests.get(url, headers=self.base_headers) + self.check_code(r) + + disk = r.json() + return dict( + trash_size=disk['trash_size'], + total_space=disk['total_space'], + used_space=disk['used_space'], + system_folders=disk['system_folders'] + ) + + def get_content(self, path): + """ + :param path: path to folder + :return: content of folder + """ + url = '{0}/resources'.format(self.base_url) + + payload = {'path': path} + r = requests.get(url, headers=self.base_headers, params=payload) + self.check_code(r) + + json_dict = r.json() + return json_dict + # return Directory(**json_dict) + + def create_folder(self, path_to_folder): + """ + :param path_to_folder: path to folder + :return: created folder + """ + url = '{0}/resources'.format(self.base_url) + + payload = {'path': path_to_folder} + r = requests.put(url, headers=self.base_headers, params=payload) + self.check_code(r) + + return self.get_content(path_to_folder) + + def remove_folder_or_file(self, path): + """ + Remove folder or file + :param path: path to file or folder + """ + url = '{0}/resources'.format(self.base_url) + + payload = {'path': path} + r = requests.delete(url, headers=self.base_headers, params=payload) + self.check_code(r) + + def copy_folder_or_file(self, path_from, path_to): + """ + Copy folder or file + :param path_from: from path + :param path_to: to path + """ + url = '{0}/resources/copy'.format(self.base_url) + + payload = {'path': path_to, 'from': path_from} + r = requests.post(url, headers=self.base_headers, params=payload) + self.check_code(r) + + def get_download_link(self, path): + """ + :param path: path to file + :return: download link to file + """ + url = '{0}/resources/download'.format(self.base_url) + + payload = {'path': path} + r = requests.get(url, headers=self.base_headers, params=payload) + self.check_code(r) + + return r.json() + + def get_published_elements(self): + """ + :return: published elements + """ + json_dict = self._get_dictionary_of_published_files() + + elements = [] + + # for item in json_dict["items"]: + # if item["type"] == "dir": + # elements.append(Directory(**item)) + # elif item["type"] == "file": + # elements.append(File(**item)) + + return elements + + def get_public_link_to_folder_or_file(self, path): + """ + :param path: path + :return: public link to folder or file + """ + url = self.base_url + "/resources/publish" + + payload = {'path': path} + r = requests.put(url, headers=self.base_headers, params=payload) + self.check_code(r) + + files = self._get_dictionary_of_published_files() + + for file in files["items"]: + if str(file["path"]).endswith(path): + return file["public_url"] + + return "" + + def unpublish_folder_or_file(self, path): + """ + Unpublish folder of file + :param path: path to file or folder + """ + url = self.base_url + "/resources/unpublish" + + payload = {'path': path} + r = requests.put(url, headers=self.base_headers, params=payload) + self.check_code(r) + + def get_list_of_all_files(self): + """ + :return: List of all files + """ + url = self.base_url + "/resources/files" + + r = requests.get(url, headers=self.base_headers) + self.check_code(r) + + json_dict = r.json() + + files = [] + + # for item in json_dict["items"]: + # f = File(**item) + # files.append(f) + + return files + + def move_folder_or_file(self, path_from, path_to): + """ + Move folder or file + :param path_from: path from + :param path_to: path to + """ + url = self.base_url + "/resources/move" + + payload = {'path': path_to, 'from': path_from} + r = requests.post(url, headers=self.base_headers, params=payload) + self.check_code(r) + + def upload_file(self, path_from, path_to): + """ + Upload file + :param path_from: path from + :param path_to: path to yandex disk + """ + url = self.base_url + "/resources/upload" + + payload = {'path': path_to} + r = requests.get(url, headers=self.base_headers, params=payload) + self.check_code(r) + + json_dict = r.json() + upload_link = json_dict["href"] + + with open(path_from, 'rb') as fh: + files = {'file': fh} + + r2 = requests.put(upload_link, headers=self.base_headers, files=files) + self.check_code(r2) + + def upload_file_from_url(self, from_url, path_to): + """ + Upload file by URL + :param from_url: URL path from + :param path_to: path to yandex disk + """ + url = self.base_url + "/resources/upload" + + payload = {'path': path_to, 'url': from_url} + r = requests.post(url, headers=self.base_headers, params=payload) + self.check_code(r) + + def _get_dictionary_of_published_files(self): + url = self.base_url + "/resources/public" + + r = requests.get(url, headers=self.base_headers) + self.check_code(r) + + return r.json() + + def check_code(self, req): + if not str(req.status_code).startswith("2"): + raise YDException(req.status_code, req.text) + + +class YDException(Exception): + code = None + + def __init__(self, code, text): + super(YDException, self).__init__(text) + self.code = code + + def __str__(self): + return "%d. %s" % (self.code, super(YDException, self).__str__()) \ No newline at end of file