sync update: moving to ./sync.py as entry point

utils/Sync/settings.py

@@ -2,6 +2,11 @@ import os
 
 __author__ = 'Roman Timashev'
 
+
+def abs_fn(root):
+    return lambda path: os.path.join(root, path)
+
+
 root_dir_gallery = './Gallery'
 root_dir_schedule = './Schedules'
 
@@ -96,9 +101,17 @@ date_formats_reverse = [
 
 collection_documents = 'documents'
 
-dir_documents = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Documents'
-dir_events = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Events'
-dir_news = '/Users/tmshv/Yandex.Disk/Hudozka/Site/News'
+sync_provider_type = os.environ['SYNC_PROVIDER']
+
+_providers_roots = {
+    'fs': abs_fn('/Users/tmshv/Yandex.Disk'),
+    'yd': abs_fn('/'),
+}
+f = _providers_roots[sync_provider_type]
+
+dir_documents = f('Hudozka/Site/Documents')
+dir_events = f('Hudozka/Site/Events')
+dir_news = f('Hudozka/Site/News')
 
 dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
 dir_static_images = '/Users/tmshv/Desktop/Hudozka Static/images'
@@ -108,3 +121,5 @@ url_base_document = 'https://static.shburg.org/art/uploads/{file}'
 image_base_url = 'https://static.shburg.org/art/images/'
 image_name_format = '{type}-{id}-{img}-{size}{ext}'
 image_output = '/Users/tmshv/Desktop/Hudozka Static/images'
+
+yandex_disk_access_token = os.environ['YANDEX_DISK_ACCESS_TOKEN']

utils/Sync/sync.py

@@ -0,0 +1,47 @@
+import settings
+from sync.core.document import pdf_to_jpg
+from sync.data.fs import FSProvider
+from sync.data.yandexdisk import YDProvider
+from sync.document import sync_documents
+from utils.fn import lprint_json
+from utils.image.resize import image_magick_pdf_to_img
+from yandexdisk import YDClient
+
+
+def get_provider(provider_type, root):
+    if provider_type == 'fs':
+        return FSProvider(root)
+    elif provider_type == 'yd':
+        return YDProvider(root, settings.yandex_disk_access_token)
+    return None
+
+
+if __name__ == '__main__':
+    provider = get_provider(settings.sync_provider_type, settings.dir_documents)
+
+    update = False
+    delete = False
+
+    if provider:
+        u, d = sync_documents(provider, update=update, delete=delete)
+        print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d))))
+        lprint_json(d)
+
+        print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
+        lprint_json(u)
+
+
+        # print(api.scan('.'))
+        # print(list(filter(
+        #     lambda i: api.is_dir(i),
+        #     api.scan('.')
+        # )))
+
+        # print(api.hash('./Архив/Устав 2013.pdf'))
+        # print(api.size('./Архив/Устав 2013.pdf'))
+
+        # print(api.glob('./Архив/*.pdf'))
+        # print(api.type_filter('./Архив', '.pdf'))
+
+        # path = api.type_filter('./Архив', '.pdf')[0]
+        # print(api.copy(path, '/Users/tmshv/Desktop/fafo'))

utils/Sync/sync/core/document.py

@@ -22,7 +22,7 @@ class SyncDocument(Sync):
         file = document['file']
         filename = os.path.basename(file)
 
-        document['type'] = self.document_type(document)
+        document['type'] = document_type(document)
         document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
         document['file'] = {
             'name': filename,
@@ -31,11 +31,6 @@ class SyncDocument(Sync):
 
         return document
 
-    def document_type(self, document):
-        if document['category'] == 'Награды':
-            return 'award'
-        return 'document'
-
     def create_id(self, document):
         bn = os.path.basename(document['file'])
         document['id'] = url_encode_text('{category}-{file}'.format(
@@ -56,23 +51,29 @@ class SyncDocument(Sync):
         return document
 
     def create_preview(self, file, sizes, preview_dir):
-        temp_preview_path = pdf_to_jpg(self.provider, file)
+        def url(size, ext):
+            return self.url_base_preview.format(id=file, size=size, ext=ext)
 
+        temp_preview_path = pdf_to_jpg(self.provider, file)
         file = os.path.basename(file)
         file = url_encode_text(file)
-        url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
-
         img = create_image(temp_preview_path, sizes, url, preview_dir)
         os.remove(temp_preview_path)
         return img
 
 
+def document_type(document):
+    if document['category'] == 'Награды':
+        return 'award'
+    return 'document'
+
+
 def pdf_to_jpg(provider, pdf):
-    # cwd = os.getcwd()
-    # abspdf = os.path.join(cwd, pdf)
-    abspdf = provider.get_abs(pdf)
-    _, temp = mkstemp('.jpg')
-    image_magick_pdf_to_img(abspdf, temp)
-    return temp
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
 
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
 
+    os.remove(temp_in)
+    return temp_out

utils/Sync/sync/data/__init__.py

@@ -1,4 +1,5 @@
 import os
+from io import BytesIO
 
 
 class Provider:
@@ -6,8 +7,8 @@ class Provider:
         super().__init__()
         self.root = root
 
-    def open(self, path, mode):
-        return ''
+    def read(self, path):
+        return BytesIO(bytes())
 
     def scan(self, path):
         return []
@@ -15,6 +16,9 @@ class Provider:
     def glob(self, pattr):
         return []
 
+    def type_filter(self, path, ext):
+        return []
+
     def copy(self, path, out):
         pass
 
@@ -29,4 +33,8 @@ class Provider:
         return os.path.normpath(abs)
 
     def get_rel(self, path):
-        return os.path.relpath(path, self.root)
+        rel = os.path.relpath(path, self.root)
+        return rel
+
+    def is_dir(self, path):
+        return False

utils/Sync/sync/data/fs.py

@@ -4,18 +4,31 @@ from glob import glob
 from shutil import copyfile
 
 from sync.data import Provider
+from utils.fn import lmap
 from utils.hash import hash_file
 
 
 class FSProvider(Provider):
-    def open(self, path, mode):
-        return open(self.get_abs(path), mode)
+    def open(self, path):
+        return open(self.get_abs(path), 'rb')
 
     def scan(self, path):
-        return os.scandir(self.get_abs(path))
+        return lmap(
+            self.get_rel,
+            map(
+                lambda i: i.path,
+                os.scandir(self.get_abs(path))
+            )
+        )
 
     def glob(self, pattr):
-        return glob(self.get_abs(pattr))
+        return lmap(self.get_rel, glob(self.get_abs(pattr)))
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
 
     def hash(self, path):
         return hash_file(self.get_abs(path))
@@ -28,3 +41,6 @@ class FSProvider(Provider):
         file = self.get_abs(path)
         return os.path.getsize(file)
 
+    def is_dir(self, path):
+        return os.path.isdir(self.get_abs(path))
+

utils/Sync/sync/data/yandexdisk.py

@@ -0,0 +1,64 @@
+import os
+
+import re
+
+import io
+import requests
+
+from sync.data import Provider
+from utils.fn import lmap
+from yandexdisk import YDClient
+
+
+class YDProvider(Provider):
+    def __init__(self, root, access_token):
+        super().__init__(root)
+        self.api = YDClient(access_token)
+
+    def read(self, path):
+        remote = self.api.get_download_link(self.get_abs(path))
+        url = remote['href']
+        req = requests.get(url)
+        data = req.content
+        return io.BytesIO(data)
+
+    def scan(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        files = []
+        for item in res['_embedded']['items']:
+            files.append(item['path'])
+
+        files = map(
+            lambda fp: re.sub('disk:', '', fp),
+            files
+        )
+        return lmap(self.get_rel, files)
+
+    def glob(self, pattr):
+        return []
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
+
+    def hash(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['sha256']
+
+    def copy(self, path, out):
+        data = self.read(path)
+        f = open(out, 'wb')
+        f.write(data.read())
+        f.close()
+        return out
+
+    def size(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['size']
+
+    def is_dir(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['type'] == 'dir'
+

utils/Sync/sync/document.py

@@ -5,17 +5,41 @@ import settings
 from db import db
 from sync import synced_image_id, untouched
 from sync.core.document import SyncDocument
-from sync.data.fs import FSProvider
-from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap
+from utils.fn import combine, lmap, key_mapper, lmapfn
 from utils.io import read_yaml
 
 
-def sync_documents(update=True, delete=True):
-    provider = FSProvider(settings.dir_documents)
+def sync_documents(provider, update=True, delete=True):
+    """
+    DOCUMENT_OBJECT SAMPLE
+    {
+        "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
+        "_id": {
+            "$oid": "56d08411ace9573958e5e497"
+        },
+        "type": "document",
+        "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
+        "category": "Для поступающих",
+        "file": {
+            "size": 53088,
+            "name": "Договор о пожертвовании.pdf"
+        },
+        "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
+        "preview": {
+            "$oid": "56d08007ace9573958e5e48f"
+        },
+        "title": "Договор о пожертвовании"
+    }
+
+    :param provider:
+    :param update:
+    :param delete:
+    :return:
+    """
 
     static_path = static_path_fn(settings.dir_static_uploads)
 
-    u, d = main(
+    return main(
         SyncDocument(
             db()[settings.collection_documents],
             provider,
@@ -29,23 +53,17 @@ def sync_documents(update=True, delete=True):
         delete_documents=delete
     )
 
-    print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d))))
-    lprint_json(d)
-
-    print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
-    lprint_json(u)
-
 
 def main(sync, static_path, update_documents, delete_documents):
     documents = get_documents(sync)
     documents_id = lmap(
-        lambda i: i['id'],
+        lambda document: document['id'],
         documents
     )
 
     # CHECKING
     file_names = set(lmap(
-        lambda i: i['file'],
+        lambda document: document['file'],
         documents
     ))
     if len(file_names) != len(documents):
@@ -70,7 +88,7 @@ def main(sync, static_path, update_documents, delete_documents):
     if update_documents:
         # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
         documents = lmap(
-            key_mapper('preview', lambda i: synced_image_id(i)),
+            key_mapper('preview', lambda img: synced_image_id(img)),
             documents
         )
 
@@ -86,7 +104,7 @@ def main(sync, static_path, update_documents, delete_documents):
         documents_delete = lmap(
             sync.delete,
             map(
-                lambda i: {'_id': i['_id']},
+                lambda document: {'_id': document['_id']},
                 documents_delete
             )
         )
@@ -107,12 +125,12 @@ def get_documents(sync):
     )
 
     # CHOICE BETTER TITLE
-    documents = lmapfn(documents)(
-        lambda i: {
-            **i,
-            'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
-        }
-    )
+    # documents = lmapfn(documents)(
+    #     lambda i: {
+    #         **i,
+    #         'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
+    #     }
+    # )
 
     documents = lmap(sync.create_id, documents)
     documents = lmap(sync.create_hash, documents)
@@ -121,8 +139,8 @@ def get_documents(sync):
     return documents
 
 
-def static_path_fn(dir):
-    return lambda document: os.path.join(dir, os.path.basename(document['url']))
+def static_path_fn(dir_path):
+    return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
 
 
 def categorize_files_list(files, title):
@@ -177,21 +195,20 @@ def documents_from_yaml(dirpath):
 
 
 def documents_from_subdirs(provider):
-    documents = lmap(
-        lambda i: i.path,
-        filter(
-            lambda i: i.is_dir(),
-            provider.scan('.')
-        )
-    )
+    documents = provider.scan('.')
+    documents = list(filter(
+        lambda i: provider.is_dir(i),
+        documents
+    ))
 
     documents = lmap(
         lambda folder: lmap(
             lambda path: {
-                'file': provider.get_rel(path),
+                # 'file': provider.get_rel(path),
+                'file': path,
                 'category': os.path.basename(folder)
             },
-            provider.glob(folder + '/*.pdf')
+            provider.type_filter(folder, '.pdf')
         ),
         documents
     )
@@ -205,7 +222,7 @@ def get_pdf_title(provider, file):
     from PyPDF2.generic import TextStringObject
     from PyPDF2.generic import IndirectObject
 
-    pdf = PdfFileReader(provider.open(file, 'rb'))
+    pdf = PdfFileReader(provider.read(file))
     info = pdf.getDocumentInfo()
 
     if info:
@@ -224,28 +241,3 @@ def until_none(ls):
         if (i is not None) and i != '':
             better = i
     return better
-
-
-if __name__ == '__main__':
-    sync_documents()
-    # sync_documents(delete=False, update=False)
-
-    # DOCUMENT_OBJECT SAMPLE
-    # {
-    #     "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
-    #     "_id": {
-    #         "$oid": "56d08411ace9573958e5e497"
-    #     },
-    #     "type": "document",
-    #     "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
-    #     "category": "Для поступающих",
-    #     "file": {
-    #         "size": 53088,
-    #         "name": "Договор о пожертвовании.pdf"
-    #     },
-    #     "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
-    #     "preview": {
-    #         "$oid": "56d08007ace9573958e5e48f"
-    #     },
-    #     "title": "Договор о пожертвовании"
-    # }

utils/Sync/yandexdisk/__init__.py

@@ -0,0 +1,230 @@
+import requests
+
+
+class YDClient:
+    '''
+    Implementation of https://tech.yandex.ru/disk/poligon/
+    '''
+
+    base_url = 'https://cloud-api.yandex.net:443/v1/disk'
+
+    def __init__(self, access_token):
+        self.access_token = access_token
+
+        self.base_headers = dict(
+            Accept='application/json',
+            Authorization='OAuth %s' % self.access_token,
+        )
+
+    def get_disk_metadata(self):
+        """
+        :return: disk metadata
+        """
+        url = self.base_url
+        r = requests.get(url, headers=self.base_headers)
+        self.check_code(r)
+
+        disk = r.json()
+        return dict(
+            trash_size=disk['trash_size'],
+            total_space=disk['total_space'],
+            used_space=disk['used_space'],
+            system_folders=disk['system_folders']
+        )
+
+    def get_content(self, path):
+        """
+        :param path: path to folder
+        :return: content of folder
+        """
+        url = '{0}/resources'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        return json_dict
+        # return Directory(**json_dict)
+
+    def create_folder(self, path_to_folder):
+        """
+        :param path_to_folder: path to folder
+        :return: created folder
+        """
+        url = '{0}/resources'.format(self.base_url)
+
+        payload = {'path': path_to_folder}
+        r = requests.put(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        return self.get_content(path_to_folder)
+
+    def remove_folder_or_file(self, path):
+        """
+        Remove folder or file
+        :param path: path to file or folder
+        """
+        url = '{0}/resources'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.delete(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def copy_folder_or_file(self, path_from, path_to):
+        """
+        Copy folder or file
+        :param path_from: from path
+        :param path_to: to path
+        """
+        url = '{0}/resources/copy'.format(self.base_url)
+
+        payload = {'path': path_to, 'from': path_from}
+        r = requests.post(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def get_download_link(self, path):
+        """
+        :param path: path to file
+        :return: download link to file
+        """
+        url = '{0}/resources/download'.format(self.base_url)
+
+        payload = {'path': path}
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        return r.json()
+
+    def get_published_elements(self):
+        """
+        :return: published elements
+        """
+        json_dict = self._get_dictionary_of_published_files()
+
+        elements = []
+
+        # for item in json_dict["items"]:
+        #     if item["type"] == "dir":
+        #         elements.append(Directory(**item))
+        #     elif item["type"] == "file":
+        #         elements.append(File(**item))
+
+        return elements
+
+    def get_public_link_to_folder_or_file(self, path):
+        """
+        :param path: path
+        :return: public link to folder or file
+        """
+        url = self.base_url + "/resources/publish"
+
+        payload = {'path': path}
+        r = requests.put(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        files = self._get_dictionary_of_published_files()
+
+        for file in files["items"]:
+            if str(file["path"]).endswith(path):
+                return file["public_url"]
+
+        return ""
+
+    def unpublish_folder_or_file(self, path):
+        """
+        Unpublish folder of file
+        :param path: path to file or folder
+        """
+        url = self.base_url + "/resources/unpublish"
+
+        payload = {'path': path}
+        r = requests.put(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def get_list_of_all_files(self):
+        """
+        :return: List of all files
+        """
+        url = self.base_url + "/resources/files"
+
+        r = requests.get(url, headers=self.base_headers)
+        self.check_code(r)
+
+        json_dict = r.json()
+
+        files = []
+
+        # for item in json_dict["items"]:
+        #     f = File(**item)
+        #     files.append(f)
+
+        return files
+
+    def move_folder_or_file(self, path_from, path_to):
+        """
+        Move folder or file
+        :param path_from: path from
+        :param path_to: path to
+        """
+        url = self.base_url + "/resources/move"
+
+        payload = {'path': path_to, 'from': path_from}
+        r = requests.post(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def upload_file(self, path_from, path_to):
+        """
+        Upload file
+        :param path_from: path from
+        :param path_to: path to yandex disk
+        """
+        url = self.base_url + "/resources/upload"
+
+        payload = {'path': path_to}
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        upload_link = json_dict["href"]
+
+        with open(path_from, 'rb') as fh:
+            files = {'file': fh}
+
+            r2 = requests.put(upload_link, headers=self.base_headers, files=files)
+            self.check_code(r2)
+
+    def upload_file_from_url(self, from_url, path_to):
+        """
+        Upload file by URL
+        :param from_url: URL path from
+        :param path_to: path to yandex disk
+        """
+        url = self.base_url + "/resources/upload"
+
+        payload = {'path': path_to, 'url': from_url}
+        r = requests.post(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+    def _get_dictionary_of_published_files(self):
+        url = self.base_url + "/resources/public"
+
+        r = requests.get(url, headers=self.base_headers)
+        self.check_code(r)
+
+        return r.json()
+
+    def check_code(self, req):
+        if not str(req.status_code).startswith("2"):
+            raise YDException(req.status_code, req.text)
+
+
+class YDException(Exception):
+    code = None
+
+    def __init__(self, code, text):
+        super(YDException, self).__init__(text)
+        self.code = code
+
+    def __str__(self):
+        return "%d. %s" % (self.code, super(YDException, self).__str__())