sync update: + data provider; refactoring; documents update

utils/Sync/settings.py

@@ -17,8 +17,13 @@ DIGITS = '0123456789'
 description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-'
 
 translit_table = (
-    ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж',  'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч',  'ш',   'щ',  'ъ', 'ы', 'ь', 'э', 'ю',  'я',  'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж',  'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч',   'Ш', 'Щ',   'Ъ', 'Ы', 'Ь', 'Э', 'Ю',   'Я', '̆'],
-    ['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', '_']
+    ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х',
+     'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л',
+     'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'],
+    ['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h',
+     'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J',
+     'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA',
+     '_']
 )
 
 teachers_names = {
@@ -89,6 +94,8 @@ date_formats_reverse = [
     '%Y.%m.%d'
 ]
 
+collection_documents = 'documents'
+
 dir_documents = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Documents'
 dir_events = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Events'
 dir_news = '/Users/tmshv/Yandex.Disk/Hudozka/Site/News'

utils/Sync/sync/__init__.py

@@ -107,48 +107,3 @@ create_post_from_image_list = lambda images: markdown(
         images
     ))
 )
-
-
-class Sync:
-    def __init__(self):
-        super().__init__()
-
-        self.collection = None
-
-    def read(self, document):
-        if isinstance(document, list):
-            return lmap(self.read, document)
-
-        q = {'id': document['id']}
-        try:
-            return self.collection.find_one(q)
-        except ValueError:
-            pass
-        return None
-
-    def update(self, document):
-        if isinstance(document, list):
-            return lmap(self.update, document)
-
-        q = {'id': document['id']}
-        try:
-            self.collection.update_one(q, {'$set': document}, upsert=True)
-            return self.read(document)
-        except ValueError:
-            pass
-        return None
-
-    def compile(self, document):
-        return document
-
-    def query(self, q):
-        return self.collection.find(q)
-
-    def delete(self, q):
-        return self.collection.find_one_and_delete(q)
-
-    def create_id(self, document):
-        return None
-
-    def create_hash(self, document):
-        return None

utils/Sync/sync/core/__init__.py

@@ -0,0 +1,46 @@
+from utils.fn import lmap
+
+
+class Sync:
+    def __init__(self):
+        super().__init__()
+
+        self.collection = None
+
+    def read(self, document):
+        if isinstance(document, list):
+            return lmap(self.read, document)
+
+        q = {'id': document['id']}
+        try:
+            return self.collection.find_one(q)
+        except ValueError:
+            pass
+        return None
+
+    def update(self, document):
+        if isinstance(document, list):
+            return lmap(self.update, document)
+
+        q = {'id': document['id']}
+        try:
+            self.collection.update_one(q, {'$set': document}, upsert=True)
+            return self.read(document)
+        except ValueError:
+            pass
+        return None
+
+    def compile(self, document):
+        return document
+
+    def query(self, q):
+        return self.collection.find(q)
+
+    def delete(self, q):
+        return self.collection.find_one_and_delete(q)
+
+    def create_id(self, document):
+        return None
+
+    def create_hash(self, document):
+        return None

utils/Sync/sync/core/document.py

@@ -0,0 +1,78 @@
+import os
+from tempfile import mkstemp
+
+from sync.core import Sync
+from utils.hash import hash_file, hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+class SyncDocument(Sync):
+    def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+        self.sizes = sizes
+        self.dir_static_previews = dir_static_previews
+        self.url_base_preview = url_base_preview
+        self.url_base_document = url_base_document
+
+    def create(self, document):
+        file = document['file']
+        filename = os.path.basename(file)
+
+        document['type'] = self.document_type(document)
+        document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
+        document['file'] = {
+            'name': filename,
+            'size': self.provider.size(file)
+        }
+
+        return document
+
+    def document_type(self, document):
+        if document['category'] == 'Награды':
+            return 'award'
+        return 'document'
+
+    def create_id(self, document):
+        bn = os.path.basename(document['file'])
+        document['id'] = url_encode_text('{category}-{file}'.format(
+            category=document['category'],
+            file=bn
+        ))
+        return document
+
+    def create_url(self, document):
+        filename = os.path.basename(document['file'])
+        document['url'] = self.url_base_document.format(file=url_encode_file(filename))
+        return document
+
+    def create_hash(self, document):
+        document['hash'] = hash_str(
+            hash_str(document) + self.provider.hash(document['file'])
+        )
+        return document
+
+    def create_preview(self, file, sizes, preview_dir):
+        temp_preview_path = pdf_to_jpg(self.provider, file)
+
+        file = os.path.basename(file)
+        file = url_encode_text(file)
+        url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
+
+        img = create_image(temp_preview_path, sizes, url, preview_dir)
+        os.remove(temp_preview_path)
+        return img
+
+
+def pdf_to_jpg(provider, pdf):
+    # cwd = os.getcwd()
+    # abspdf = os.path.join(cwd, pdf)
+    abspdf = provider.get_abs(pdf)
+    _, temp = mkstemp('.jpg')
+    image_magick_pdf_to_img(abspdf, temp)
+    return temp
+
+

utils/Sync/sync/data/__init__.py

@@ -0,0 +1,32 @@
+import os
+
+
+class Provider:
+    def __init__(self, root):
+        super().__init__()
+        self.root = root
+
+    def open(self, path, mode):
+        return ''
+
+    def scan(self, path):
+        return []
+
+    def glob(self, pattr):
+        return []
+
+    def copy(self, path, out):
+        pass
+
+    def hash(self, path):
+        return None
+
+    def size(self, path):
+        return 0
+
+    def get_abs(self, path):
+        abs = os.path.join(self.root, path)
+        return os.path.normpath(abs)
+
+    def get_rel(self, path):
+        return os.path.relpath(path, self.root)

utils/Sync/sync/data/fs.py

@@ -0,0 +1,30 @@
+import os
+from glob import glob
+
+from shutil import copyfile
+
+from sync.data import Provider
+from utils.hash import hash_file
+
+
+class FSProvider(Provider):
+    def open(self, path, mode):
+        return open(self.get_abs(path), mode)
+
+    def scan(self, path):
+        return os.scandir(self.get_abs(path))
+
+    def glob(self, pattr):
+        return glob(self.get_abs(pattr))
+
+    def hash(self, path):
+        return hash_file(self.get_abs(path))
+
+    def copy(self, path, out):
+        file = self.get_abs(path)
+        return copyfile(file, out)
+
+    def size(self, path):
+        file = self.get_abs(path)
+        return os.path.getsize(file)
+

utils/Sync/sync/document.py

@@ -1,83 +1,128 @@
 import os
 from glob import glob
-from shutil import copyfile
-from tempfile import mkstemp
 
 import settings
 from db import db
-from sync import Sync, synced_image_id, untouched
-from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn
-from utils.hash import hash_file, hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
+from sync import synced_image_id, untouched
+from sync.core.document import SyncDocument
+from sync.data.fs import FSProvider
+from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap
 from utils.io import read_yaml
-from utils.text.transform import url_encode_text, url_encode_file
 
 
-class SyncDocument(Sync):
-    def __init__(self, collection_name, sizes, dir_static_previews, url_base_preview, url_base_document):
-        super().__init__()
-        self.collection = db()[collection_name]
-        self.sizes = sizes
-        self.dir_static_previews = dir_static_previews
-        self.url_base_preview = url_base_preview
-        self.url_base_document = url_base_document
+def sync_documents(update=True, delete=True):
+    provider = FSProvider(settings.dir_documents)
 
-    def create(self, document):
-        file = document['file']
-        filename = os.path.basename(document['file'])
+    static_path = static_path_fn(settings.dir_static_uploads)
 
-        document['type'] = self.document_type(document)
-        document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
-        document['file'] = {
-            'name': filename,
-            'size': os.path.getsize((file))
-        }
+    u, d = main(
+        SyncDocument(
+            db()[settings.collection_documents],
+            provider,
+            sizes=settings.image_sizes,
+            dir_static_previews=settings.dir_static_images,
+            url_base_preview=settings.url_base_preview,
+            url_base_document=settings.url_base_document,
+        ),
+        static_path,
+        update_documents=update,
+        delete_documents=delete
+    )
 
-        return document
+    print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d))))
+    lprint_json(d)
 
-    def document_type(self, document):
-        if document['category'] == 'Награды':
-            return 'award'
-        return 'document'
+    print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
+    lprint_json(u)
 
-    def create_id(self, document):
-        bn = os.path.basename(document['file'])
-        document['id'] = url_encode_text('{category}-{file}'.format(
-            category=document['category'],
-            file=bn
-        ))
-        return document
 
-    def create_url(self, document):
-        filename = os.path.basename(document['file'])
-        document['url'] = self.url_base_document.format(file=url_encode_file(filename))
-        return document
+def main(sync, static_path, update_documents, delete_documents):
+    documents = get_documents(sync)
+    documents_id = lmap(
+        lambda i: i['id'],
+        documents
+    )
 
-    def create_hash(self, document):
-        document['hash'] = hash_str(
-            hash_str(document) + hash_file(document['file'])
+    # CHECKING
+    file_names = set(lmap(
+        lambda i: i['file'],
+        documents
+    ))
+    if len(file_names) != len(documents):
+        raise Exception('File names should be unique')
+
+    # SKIP UNTOUCHED DOCUMENTS
+    documents = untouched(documents, sync)
+
+    # COPY FILE -> STATIC_DIR/URL_FILENAME
+    for i in documents:
+        sync.provider.copy(
+            i['file'],
+            static_path(i)
         )
-        return document
 
-    def create_preview(self, file, sizes, preview_dir):
-        temp_preview_path = pdf_to_jpg(file)
+    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
+    documents = lmap(
+        sync.create,
+        documents
+    )
 
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
+    if update_documents:
+        # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
+        documents = lmap(
+            key_mapper('preview', lambda i: synced_image_id(i)),
+            documents
+        )
 
-        img = create_image(temp_preview_path, sizes, url, preview_dir)
-        os.remove(temp_preview_path)
-        return img
+        # SYNC DOCUMENT_OBJECT WITH DB
+        documents = lmap(
+            sync.update,
+            documents
+        )
+
+    # DELETING
+    documents_delete = sync.query({'id': {'$nin': documents_id}})
+    if delete_documents:
+        documents_delete = lmap(
+            sync.delete,
+            map(
+                lambda i: {'_id': i['_id']},
+                documents_delete
+            )
+        )
+
+    return documents, documents_delete
 
 
-def pdf_to_jpg(pdf):
-    cwd = os.getcwd()
-    abspdf = os.path.join(cwd, pdf)
-    _, temp = mkstemp('.jpg')
-    image_magick_pdf_to_img(abspdf, temp)
-    return temp
+def get_documents(sync):
+    # documents = documents_from_yaml()
+    documents = documents_from_subdirs(sync.provider)
+
+    # SETUP TITLE BASED ON FILE NAME
+    documents = lmapfn(documents)(
+        lambda i: {
+            **i,
+            'title': os.path.splitext(os.path.basename(i['file']))[0]
+        }
+    )
+
+    # CHOICE BETTER TITLE
+    documents = lmapfn(documents)(
+        lambda i: {
+            **i,
+            'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
+        }
+    )
+
+    documents = lmap(sync.create_id, documents)
+    documents = lmap(sync.create_hash, documents)
+    documents = lmap(sync.create_url, documents)
+
+    return documents
+
+
+def static_path_fn(dir):
+    return lambda document: os.path.join(dir, os.path.basename(document['url']))
 
 
 def categorize_files_list(files, title):
@@ -131,23 +176,22 @@ def documents_from_yaml(dirpath):
     return documents
 
 
-def documents_from_subdirs():
+def documents_from_subdirs(provider):
     documents = lmap(
         lambda i: i.path,
         filter(
             lambda i: i.is_dir(),
-            os.scandir('.')
+            provider.scan('.')
         )
     )
 
     documents = lmap(
         lambda folder: lmap(
             lambda path: {
-                'file': path,
-                'title': os.path.splitext(os.path.basename(os.path.relpath(path, folder)))[0],
+                'file': provider.get_rel(path),
                 'category': os.path.basename(folder)
             },
-            glob(folder + '/*.pdf')
+            provider.glob(folder + '/*.pdf')
         ),
         documents
     )
@@ -156,12 +200,12 @@ def documents_from_subdirs():
     return documents
 
 
-def get_pdf_title(file):
+def get_pdf_title(provider, file):
     from PyPDF2 import PdfFileReader
     from PyPDF2.generic import TextStringObject
     from PyPDF2.generic import IndirectObject
 
-    pdf = PdfFileReader(open(file, 'rb'))
+    pdf = PdfFileReader(provider.open(file, 'rb'))
     info = pdf.getDocumentInfo()
 
     if info:
@@ -182,122 +226,9 @@ def until_none(ls):
     return better
 
 
-def main(dir_documents, dir_static_uploads, sync, update_documents=True, delete_documents=True):
-    """
-
-    :param dir_documents:
-    :param dir_static_uploads:
-    :param sync:
-    :param update_documents:
-    :param delete_documents:
-    :return:
-    """
-
-    os.chdir(dir_documents)
-
-    # documents = documents_from_yaml()
-    documents = documents_from_subdirs()
-
-    # CHOICE BETTER TITLE
-    documents = lmapfn(documents)(
-        lambda i: {
-            **i,
-            'title': until_none([i['title'], get_pdf_title(i['file'])])
-        }
-    )
-
-    # CREATE DOCUMENT IDENTITY
-    documents = lmap(sync.create_id, documents)
-
-    # CREATE HASH OF DOCUMENT FILE
-    documents = lmap(sync.create_hash, documents)
-
-    # CREATE SCOPE
-    scope_documents_ids = lmap(
-        lambda i: i['id'],
-        documents
-    )
-
-    file_names = set(lmap(
-        lambda i: i['file'],
-        documents
-    ))
-    if len(file_names) != len(documents):
-        raise Exception('File names should be unique')
-
-    # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
-
-    # URL
-    documents = lmap(sync.create_url, documents)
-
-    # COPY FILE -> STATIC_DIR/URL_FILENAME
-    out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
-    lmap(
-        lambda doc: copyfile(
-            doc['file'],
-            out_path(doc)
-        ),
-        documents
-    )
-
-    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
-    documents = lmap(
-        sync.create,
-        documents
-    )
-
-    if update_documents:
-        # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
-        documents = lmap(
-            key_mapper('preview', lambda i: synced_image_id(i)),
-            documents
-        )
-
-        # SYNC DOCUMENT_OBJECT WITH DB
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    if delete_documents:
-        documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
-        documents_to_delete = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_delete
-            )
-        )
-    else:
-        documents_to_delete = []
-
-    print('SCOPE:', len(scope_documents_ids))
-    lprint(scope_documents_ids)
-
-    print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete))))
-    lprint_json(documents_to_delete)
-
-    print('UPDATE DOCUMENTS:', len(documents))
-    lprint_json(documents)
-
-    print('[SYNC DOCUMENTS DONE]')
-
-
 if __name__ == '__main__':
-    main(
-        settings.dir_documents,
-        settings.dir_static_uploads,
-        SyncDocument(
-            'documents',
-            sizes=settings.image_sizes,
-            dir_static_previews=settings.dir_static_images,
-            url_base_preview=settings.url_base_preview,
-            url_base_document=settings.url_base_document,
-        ),
-        update_documents=True,
-        delete_documents=True
-    )
+    sync_documents()
+    # sync_documents(delete=False, update=False)
 
     # DOCUMENT_OBJECT SAMPLE
     # {

utils/Sync/sync/teacher.py

@@ -94,7 +94,7 @@ def main(dir_teachers, sync):
         filter(
             lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
             lmap(
-                lambda document: (document, sync.read(document)),
+                lambda document: (document, sync.open(document)),
                 documents
             )
         ))

utils/Sync/utils/fn.py

@@ -7,6 +7,9 @@ import bson.json_util
 
 
 def combine(items):
+    if len(items) == 0:
+        return []
+
     return reduce(
         lambda total, i: total + i,
         items