sync update: moving to ./sync.py as entry point

utils/Sync/sync/core/document.py

@@ -22,7 +22,7 @@ class SyncDocument(Sync):
         file = document['file']
         filename = os.path.basename(file)
 
-        document['type'] = self.document_type(document)
+        document['type'] = document_type(document)
         document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
         document['file'] = {
             'name': filename,
@@ -31,11 +31,6 @@ class SyncDocument(Sync):
 
         return document
 
-    def document_type(self, document):
-        if document['category'] == 'Награды':
-            return 'award'
-        return 'document'
-
     def create_id(self, document):
         bn = os.path.basename(document['file'])
         document['id'] = url_encode_text('{category}-{file}'.format(
@@ -56,23 +51,29 @@ class SyncDocument(Sync):
         return document
 
     def create_preview(self, file, sizes, preview_dir):
-        temp_preview_path = pdf_to_jpg(self.provider, file)
+        def url(size, ext):
+            return self.url_base_preview.format(id=file, size=size, ext=ext)
 
+        temp_preview_path = pdf_to_jpg(self.provider, file)
         file = os.path.basename(file)
         file = url_encode_text(file)
-        url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
-
         img = create_image(temp_preview_path, sizes, url, preview_dir)
         os.remove(temp_preview_path)
         return img
 
 
+def document_type(document):
+    if document['category'] == 'Награды':
+        return 'award'
+    return 'document'
+
+
 def pdf_to_jpg(provider, pdf):
-    # cwd = os.getcwd()
-    # abspdf = os.path.join(cwd, pdf)
-    abspdf = provider.get_abs(pdf)
-    _, temp = mkstemp('.jpg')
-    image_magick_pdf_to_img(abspdf, temp)
-    return temp
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
 
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
 
+    os.remove(temp_in)
+    return temp_out

utils/Sync/sync/data/__init__.py

@@ -1,4 +1,5 @@
 import os
+from io import BytesIO
 
 
 class Provider:
@@ -6,8 +7,8 @@ class Provider:
         super().__init__()
         self.root = root
 
-    def open(self, path, mode):
-        return ''
+    def read(self, path):
+        return BytesIO(bytes())
 
     def scan(self, path):
         return []
@@ -15,6 +16,9 @@ class Provider:
     def glob(self, pattr):
         return []
 
+    def type_filter(self, path, ext):
+        return []
+
     def copy(self, path, out):
         pass
 
@@ -29,4 +33,8 @@ class Provider:
         return os.path.normpath(abs)
 
     def get_rel(self, path):
-        return os.path.relpath(path, self.root)
+        rel = os.path.relpath(path, self.root)
+        return rel
+
+    def is_dir(self, path):
+        return False

utils/Sync/sync/data/fs.py

@@ -4,18 +4,31 @@ from glob import glob
 from shutil import copyfile
 
 from sync.data import Provider
+from utils.fn import lmap
 from utils.hash import hash_file
 
 
 class FSProvider(Provider):
-    def open(self, path, mode):
-        return open(self.get_abs(path), mode)
+    def open(self, path):
+        return open(self.get_abs(path), 'rb')
 
     def scan(self, path):
-        return os.scandir(self.get_abs(path))
+        return lmap(
+            self.get_rel,
+            map(
+                lambda i: i.path,
+                os.scandir(self.get_abs(path))
+            )
+        )
 
     def glob(self, pattr):
-        return glob(self.get_abs(pattr))
+        return lmap(self.get_rel, glob(self.get_abs(pattr)))
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
 
     def hash(self, path):
         return hash_file(self.get_abs(path))
@@ -28,3 +41,6 @@ class FSProvider(Provider):
         file = self.get_abs(path)
         return os.path.getsize(file)
 
+    def is_dir(self, path):
+        return os.path.isdir(self.get_abs(path))
+

utils/Sync/sync/data/yandexdisk.py

@@ -0,0 +1,64 @@
+import os
+
+import re
+
+import io
+import requests
+
+from sync.data import Provider
+from utils.fn import lmap
+from yandexdisk import YDClient
+
+
+class YDProvider(Provider):
+    def __init__(self, root, access_token):
+        super().__init__(root)
+        self.api = YDClient(access_token)
+
+    def read(self, path):
+        remote = self.api.get_download_link(self.get_abs(path))
+        url = remote['href']
+        req = requests.get(url)
+        data = req.content
+        return io.BytesIO(data)
+
+    def scan(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        files = []
+        for item in res['_embedded']['items']:
+            files.append(item['path'])
+
+        files = map(
+            lambda fp: re.sub('disk:', '', fp),
+            files
+        )
+        return lmap(self.get_rel, files)
+
+    def glob(self, pattr):
+        return []
+
+    def type_filter(self, path, ext):
+        return list(filter(
+            lambda fn: os.path.splitext(fn)[1] == ext,
+            self.scan(path)
+        ))
+
+    def hash(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['sha256']
+
+    def copy(self, path, out):
+        data = self.read(path)
+        f = open(out, 'wb')
+        f.write(data.read())
+        f.close()
+        return out
+
+    def size(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['size']
+
+    def is_dir(self, path):
+        res = self.api.get_content(self.get_abs(path))
+        return res['type'] == 'dir'
+

utils/Sync/sync/document.py

@@ -5,17 +5,41 @@ import settings
 from db import db
 from sync import synced_image_id, untouched
 from sync.core.document import SyncDocument
-from sync.data.fs import FSProvider
-from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap
+from utils.fn import combine, lmap, key_mapper, lmapfn
 from utils.io import read_yaml
 
 
-def sync_documents(update=True, delete=True):
-    provider = FSProvider(settings.dir_documents)
+def sync_documents(provider, update=True, delete=True):
+    """
+    DOCUMENT_OBJECT SAMPLE
+    {
+        "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
+        "_id": {
+            "$oid": "56d08411ace9573958e5e497"
+        },
+        "type": "document",
+        "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
+        "category": "Для поступающих",
+        "file": {
+            "size": 53088,
+            "name": "Договор о пожертвовании.pdf"
+        },
+        "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
+        "preview": {
+            "$oid": "56d08007ace9573958e5e48f"
+        },
+        "title": "Договор о пожертвовании"
+    }
+
+    :param provider:
+    :param update:
+    :param delete:
+    :return:
+    """
 
     static_path = static_path_fn(settings.dir_static_uploads)
 
-    u, d = main(
+    return main(
         SyncDocument(
             db()[settings.collection_documents],
             provider,
@@ -29,23 +53,17 @@ def sync_documents(update=True, delete=True):
         delete_documents=delete
     )
 
-    print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d))))
-    lprint_json(d)
-
-    print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
-    lprint_json(u)
-
 
 def main(sync, static_path, update_documents, delete_documents):
     documents = get_documents(sync)
     documents_id = lmap(
-        lambda i: i['id'],
+        lambda document: document['id'],
         documents
     )
 
     # CHECKING
     file_names = set(lmap(
-        lambda i: i['file'],
+        lambda document: document['file'],
         documents
     ))
     if len(file_names) != len(documents):
@@ -70,7 +88,7 @@ def main(sync, static_path, update_documents, delete_documents):
     if update_documents:
         # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
         documents = lmap(
-            key_mapper('preview', lambda i: synced_image_id(i)),
+            key_mapper('preview', lambda img: synced_image_id(img)),
             documents
         )
 
@@ -86,7 +104,7 @@ def main(sync, static_path, update_documents, delete_documents):
         documents_delete = lmap(
             sync.delete,
             map(
-                lambda i: {'_id': i['_id']},
+                lambda document: {'_id': document['_id']},
                 documents_delete
             )
         )
@@ -107,12 +125,12 @@ def get_documents(sync):
     )
 
     # CHOICE BETTER TITLE
-    documents = lmapfn(documents)(
-        lambda i: {
-            **i,
-            'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
-        }
-    )
+    # documents = lmapfn(documents)(
+    #     lambda i: {
+    #         **i,
+    #         'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
+    #     }
+    # )
 
     documents = lmap(sync.create_id, documents)
     documents = lmap(sync.create_hash, documents)
@@ -121,8 +139,8 @@ def get_documents(sync):
     return documents
 
 
-def static_path_fn(dir):
-    return lambda document: os.path.join(dir, os.path.basename(document['url']))
+def static_path_fn(dir_path):
+    return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
 
 
 def categorize_files_list(files, title):
@@ -177,21 +195,20 @@ def documents_from_yaml(dirpath):
 
 
 def documents_from_subdirs(provider):
-    documents = lmap(
-        lambda i: i.path,
-        filter(
-            lambda i: i.is_dir(),
-            provider.scan('.')
-        )
-    )
+    documents = provider.scan('.')
+    documents = list(filter(
+        lambda i: provider.is_dir(i),
+        documents
+    ))
 
     documents = lmap(
         lambda folder: lmap(
             lambda path: {
-                'file': provider.get_rel(path),
+                # 'file': provider.get_rel(path),
+                'file': path,
                 'category': os.path.basename(folder)
             },
-            provider.glob(folder + '/*.pdf')
+            provider.type_filter(folder, '.pdf')
         ),
         documents
     )
@@ -205,7 +222,7 @@ def get_pdf_title(provider, file):
     from PyPDF2.generic import TextStringObject
     from PyPDF2.generic import IndirectObject
 
-    pdf = PdfFileReader(provider.open(file, 'rb'))
+    pdf = PdfFileReader(provider.read(file))
     info = pdf.getDocumentInfo()
 
     if info:
@@ -224,28 +241,3 @@ def until_none(ls):
         if (i is not None) and i != '':
             better = i
     return better
-
-
-if __name__ == '__main__':
-    sync_documents()
-    # sync_documents(delete=False, update=False)
-
-    # DOCUMENT_OBJECT SAMPLE
-    # {
-    #     "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
-    #     "_id": {
-    #         "$oid": "56d08411ace9573958e5e497"
-    #     },
-    #     "type": "document",
-    #     "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
-    #     "category": "Для поступающих",
-    #     "file": {
-    #         "size": 53088,
-    #         "name": "Договор о пожертвовании.pdf"
-    #     },
-    #     "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
-    #     "preview": {
-    #         "$oid": "56d08007ace9573958e5e48f"
-    #     },
-    #     "title": "Договор о пожертвовании"
-    # }