sync: fixed & updated

utils/Sync/sync.py

@@ -1,10 +1,11 @@
 import settings
-from sync.core.document import pdf_to_jpg
+from sync.core.document import pdf_to_jpg, SyncDocument
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
-from sync.document import sync_documents
+from sync.document import sync_documents, read_document, get_documents
 from utils.fn import lprint_json
 from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text
 from yandexdisk import YDClient
 
 
@@ -21,6 +22,8 @@ if __name__ == '__main__':
 
     update = False
     delete = False
+    # update = True
+    # delete = True
 
     if provider:
         u, d = sync_documents(provider, update=update, delete=delete)
@@ -30,6 +33,16 @@ if __name__ == '__main__':
         print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
         lprint_json(u)
 
+        # s = SyncDocument(
+        #     {},
+        #     provider,
+        #     sizes=settings.image_sizes,
+        #     dir_static_previews=settings.dir_static_images,
+        #     url_base_preview=settings.url_base_preview,
+        #     url_base_document=settings.url_base_document,
+        # )
+        # # print(read_document(s, 'Награды/2016 В мире фантазий — Диплом.pdf'))
+        # lprint_json(get_documents(s))
 
         # print(api.scan('.'))
         # print(list(filter(
@@ -41,7 +54,7 @@ if __name__ == '__main__':
         # print(api.size('./Архив/Устав 2013.pdf'))
 
         # print(api.glob('./Архив/*.pdf'))
-        # print(api.type_filter('./Архив', '.pdf'))
+        # lprint_json(provider.type_filter('Награды', '.pdf'))
 
         # path = api.type_filter('./Архив', '.pdf')[0]
         # print(api.copy(path, '/Users/tmshv/Desktop/fafo'))

utils/Sync/sync/core/document.py

@@ -2,6 +2,7 @@ import os
 from tempfile import mkstemp
 
 from sync.core import Sync
+from utils.fn import last_good, lmap
 from utils.hash import hash_file, hash_str
 from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
@@ -24,6 +25,10 @@ class SyncDocument(Sync):
 
         document['type'] = document_type(document)
         document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
+        document['title'] = last_good([
+            document['title'],
+            get_pdf_title(self.provider, document['file'])
+        ])
         document['file'] = {
             'name': filename,
             'size': self.provider.size(file)
@@ -45,8 +50,13 @@ class SyncDocument(Sync):
         return document
 
     def create_hash(self, document):
+        hash_keys = ['id', 'url']
+        hash_doc = lmap(
+            lambda key: document[key],
+            hash_keys
+        )
         document['hash'] = hash_str(
-            hash_str(document) + self.provider.hash(document['file'])
+            hash_str(hash_doc) + self.provider.hash(document['file'])
         )
         return document
 
@@ -77,3 +87,21 @@ def pdf_to_jpg(provider, pdf):
 
     os.remove(temp_in)
     return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/data/fs.py

@@ -9,7 +9,7 @@ from utils.hash import hash_file
 
 
 class FSProvider(Provider):
-    def open(self, path):
+    def read(self, path):
         return open(self.get_abs(path), 'rb')
 
     def scan(self, path):

utils/Sync/sync/document.py

@@ -116,29 +116,29 @@ def get_documents(sync):
     # documents = documents_from_yaml()
     documents = documents_from_subdirs(sync.provider)
 
-    # SETUP TITLE BASED ON FILE NAME
-    documents = lmapfn(documents)(
-        lambda i: {
-            **i,
-            'title': os.path.splitext(os.path.basename(i['file']))[0]
-        }
+    documents = lmap(
+        lambda document: read_document(sync, document),
+        documents
     )
-
-    # CHOICE BETTER TITLE
-    # documents = lmapfn(documents)(
-    #     lambda i: {
-    #         **i,
-    #         'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
-    #     }
-    # )
-
-    documents = lmap(sync.create_id, documents)
-    documents = lmap(sync.create_hash, documents)
-    documents = lmap(sync.create_url, documents)
-
     return documents
 
 
+def read_document(sync, path):
+    name = os.path.basename(path)
+
+    document = {
+        'file': path,
+        'category': os.path.dirname(path),
+        'title': os.path.splitext(name)[0]
+    }
+
+    document = sync.create_id(document)
+    document = sync.create_url(document)
+    document = sync.create_hash(document)
+
+    return document
+
+
 def static_path_fn(dir_path):
     return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
 
@@ -202,42 +202,9 @@ def documents_from_subdirs(provider):
     ))
 
     documents = lmap(
-        lambda folder: lmap(
-            lambda path: {
-                # 'file': provider.get_rel(path),
-                'file': path,
-                'category': os.path.basename(folder)
-            },
-            provider.type_filter(folder, '.pdf')
-        ),
+        lambda folder: provider.type_filter(folder, '.pdf'),
         documents
     )
 
     documents = combine(documents)
     return documents
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None
-
-
-def until_none(ls):
-    better = None
-    for i in ls:
-        if (i is not None) and i != '':
-            better = i
-    return better

utils/Sync/utils/fn.py

@@ -34,6 +34,7 @@ iterate_iter_over_fns = lambda fns: lambda i: lmap(
 def lmap(fn, ls):
     return list(map(fn, ls))
 
+
 def lmapfn(ls):
     return lambda fn: list(map(fn, ls))
 
@@ -89,3 +90,11 @@ dir_globber = lambda mask_list: lambda dir: combine(map(
     lambda mask: glob(os.path.join(dir, mask)),
     mask_list
 ))
+
+
+def last_good(ls):
+    better = None
+    for i in ls:
+        if (i is not None) and i != '':
+            better = i
+    return better