[sync] updated

utils/Sync/sync/album.py

@@ -1,21 +1,19 @@
 import os
-import re
 from glob import glob
 
-import frontmatter
 import lxml.html
 from lxml import etree
 
 import settings
 from db import db
-from models.product import create_product
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \
     untouched, synced_images_ids
+from sync.models import create_product
 from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \
     key_mapper, lprint
 from utils.hash import hash_file, hash_str
 from utils.image import create_image
-from utils.io import read_yaml_md, read_file, parse_yaml_front_matter
+from utils.io import read_yaml_md
 from utils.text.transform import url_encode_text
 
 

utils/Sync/sync/compile.py

@@ -0,0 +1,36 @@
+import os
+from tempfile import mkstemp
+
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text
+
+
+class Preview:
+    def __init__(self, provider, url_base, output, sizes):
+        super().__init__()
+        self.provider = provider
+        self.url_base = url_base
+        self.output = output
+        self.sizes = sizes
+
+    def create(self, file):
+        def url(size, ext):
+            return self.url_base.format(id=file, size=size, ext=ext)
+
+        temp_preview_path = self.pdf_to_jpg(file)
+        file = os.path.basename(file)
+        file = url_encode_text(file)
+        img = create_image(temp_preview_path, self.sizes, url, self.output)
+        os.remove(temp_preview_path)
+        return img
+
+    def pdf_to_jpg(self, pdf):
+        _, temp_in = mkstemp(suffix='.pdf')
+        _, temp_out = mkstemp(suffix='.jpg')
+
+        abspdf = self.provider.copy(pdf, temp_in)
+        image_magick_pdf_to_img(abspdf, temp_out)
+
+        os.remove(temp_in)
+        return temp_out

utils/Sync/sync/core/__init__.py

@@ -1,7 +1,12 @@
+from sync.models import Model
 from utils.fn import lmap
 
 
 class Sync:
+    @staticmethod
+    def compile(document):
+        return document.bake()
+
     def __init__(self):
         super().__init__()
 
@@ -22,6 +27,9 @@ class Sync:
         if isinstance(document, list):
             return lmap(self.update, document)
 
+        if isinstance(document, Model):
+            return self.update(self.compile(document))
+
         q = {'id': document['id']}
         try:
             self.collection.update_one(q, {'$set': document}, upsert=True)
@@ -30,9 +38,6 @@ class Sync:
             pass
         return None
 
-    def compile(self, document):
-        return document
-
     def query(self, q):
         return self.collection.find(q)
 

utils/Sync/sync/core/document.py

@@ -3,7 +3,7 @@ from tempfile import mkstemp
 
 from sync.core import Sync
 from utils.fn import last_good, lmap
-from utils.hash import hash_file, hash_str
+from utils.hash import hash_str
 from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
 from utils.text.transform import url_encode_text, url_encode_file

utils/Sync/sync/data/__init__.py

@@ -1,6 +1,8 @@
 import os
 from io import BytesIO
 
+from utils.fn import combine, lmap
+
 
 class Provider:
     def __init__(self, root):
@@ -29,12 +31,36 @@ class Provider:
         return 0
 
     def get_abs(self, path):
-        abs = os.path.join(self.root, path)
-        return os.path.normpath(abs)
+        a = os.path.join(self.root, path)
+        return os.path.normpath(a)
 
     def get_rel(self, path):
         rel = os.path.relpath(path, self.root)
         return rel
 
     def is_dir(self, path):
-        return False
+        return False
+
+
+def scan_subdirs(provider, ext, path='.'):
+    """
+    Creates flat list of files with specific ext
+    It iterates only one level of sub directories
+    :param provider: instance of Provider
+    :param ext: file extension like .jpg of .pdf for file filtering
+    :param path: optional path prefix
+    :return: list of files relative to provider.root
+    """
+    items = provider.scan(path)
+    items = list(filter(
+        lambda i: provider.is_dir(i),
+        items
+    ))
+
+    items = lmap(
+        lambda folder: provider.type_filter(folder, ext),
+        items
+    )
+
+    items = combine(items)
+    return items

utils/Sync/sync/document.py

@@ -5,11 +5,13 @@ import settings
 from db import db
 from sync import synced_image_id, untouched
 from sync.core.document import SyncDocument
+from sync.data import scan_subdirs
+from sync.models.document import Document
 from utils.fn import combine, lmap, key_mapper, lmapfn
 from utils.io import read_yaml
 
 
-def sync_documents(provider, update=True, delete=True):
+def sync_documents(provider, collection, update=True, delete=True):
     """
     DOCUMENT_OBJECT SAMPLE
     {
@@ -41,7 +43,7 @@ def sync_documents(provider, update=True, delete=True):
 
     return main(
         SyncDocument(
-            db()[settings.collection_documents],
+            collection,
             provider,
             sizes=settings.image_sizes,
             dir_static_previews=settings.dir_static_images,
@@ -55,7 +57,11 @@ def sync_documents(provider, update=True, delete=True):
 
 
 def main(sync, static_path, update_documents, delete_documents):
-    documents = get_documents(sync)
+    documents = scan_subdirs(sync.provider, '.pdf')
+    documents = lmap(
+        lambda document: read_document(sync, document),
+        documents
+    )
     documents_id = lmap(
         lambda document: document['id'],
         documents
@@ -112,17 +118,6 @@ def main(sync, static_path, update_documents, delete_documents):
     return documents, documents_delete
 
 
-def get_documents(sync):
-    # documents = documents_from_yaml()
-    documents = documents_from_subdirs(sync.provider)
-
-    documents = lmap(
-        lambda document: read_document(sync, document),
-        documents
-    )
-    return documents
-
-
 def read_document(sync, path):
     name = os.path.basename(path)
 
@@ -139,6 +134,13 @@ def read_document(sync, path):
     return document
 
 
+def create_document(provider, m):
+    # dir_static_previews = settings.dir_static_images,
+    # url_base_preview = settings.url_base_preview,
+    # url_base_document = settings.url_base_document
+    return Document(provider, m, {})
+
+
 def static_path_fn(dir_path):
     return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
 
@@ -192,19 +194,3 @@ def documents_from_yaml(dirpath):
     documents = unwrap_manifest(documents)
 
     return documents
-
-
-def documents_from_subdirs(provider):
-    documents = provider.scan('.')
-    documents = list(filter(
-        lambda i: provider.is_dir(i),
-        documents
-    ))
-
-    documents = lmap(
-        lambda folder: provider.type_filter(folder, '.pdf'),
-        documents
-    )
-
-    documents = combine(documents)
-    return documents

utils/Sync/sync/models/Album.py

@@ -0,0 +1,21 @@
+__author__ = 'Roman Timashev'
+
+
+class Album:
+    def __init__(self, type, title, uri, course, course_uri, date, teacher, comment):
+        super().__init__()
+        self.products = []
+        self.type = type
+        self.title = title
+        self.uri = uri
+        self.course = course
+        self.course_uri = course_uri
+        self.date = date
+        self.teacher = teacher
+        self.comment = comment
+
+    def add_product(self, product):
+        self.products.append(product)
+
+    def __repr__(self, *args, **kwargs):
+        return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products))

utils/Sync/sync/models/__init__.py

@@ -0,0 +1,6 @@
+class Model:
+    def __init__(self):
+        super().__init__()
+
+    def bake(self):
+        return None

utils/Sync/sync/models/document.py

@@ -0,0 +1,124 @@
+import os
+from tempfile import mkstemp
+
+import settings
+from sync.models import Model
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+class Document(Model):
+    @staticmethod
+    def read(path):
+        name = os.path.basename(path)
+
+        document = {
+            'file': path,
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return document
+
+    def __init__(self, provider, path, url_base, preview_compiler):
+        super().__init__()
+
+        self.path = path
+
+        self.category = None
+        self.file = None
+        self.id = None
+        self.type = None
+        self.url = None
+        self.title = None
+        self.preview = None
+
+        for k, v in m.items():
+            setattr(self, k, v)
+
+        self.provider = provider
+
+    def bake(self):
+        file = self.file
+        filename = os.path.basename(file)
+
+        self.type = document_type(self.type)
+        self.preview = self.create_preview(file)
+        self.title = last_good([
+            self.title,
+            get_pdf_title(self.provider, self.file)
+        ])
+        self.file = {
+            'name': filename,
+            'size': self.provider.size(file)
+        }
+
+    def __set_id(self):
+        bn = os.path.basename(self.file)
+        self.id = url_encode_text('{category}-{file}'.format(
+            category=self.category,
+            file=bn
+        ))
+
+    def __set_url(self):
+        filename = os.path.basename(self.file)
+        self.url = self.url_base_document.format(file=url_encode_file(filename))
+
+    def __set_hash(self):
+        hash_keys = ['id', 'url']
+        hash_doc = lmap(
+            lambda key: getattr(self, key),
+            hash_keys
+        )
+        self.hash = hash_str(
+            hash_str(hash_doc) + self.provider.hash(self.file)
+        )
+
+    def create_preview(self, file):
+        def url(size, ext):
+            return self.url_base_preview.format(id=file, size=size, ext=ext)
+
+        temp_preview_path = pdf_to_jpg(self.provider, file)
+        file = os.path.basename(file)
+        file = url_encode_text(file)
+        img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images)
+        os.remove(temp_preview_path)
+        return img
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/models/product.py

@@ -0,0 +1,39 @@
+from sync.image import sync_image
+from sync.teacher import Teacher
+from utils.fn import key_mapper
+from utils.text.transform import interpolate_swift
+
+
+def resolve_value(param, store):
+    """
+
+    :param param: \(nv-andreeva)
+    :param store: function should return value by key nv-andreeva
+    :return:
+    """
+    i = interpolate_swift(param)
+    return store(i)
+
+
+unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
+
+
+
+def resolve_teacher(i):
+    sync = Teacher()
+    return resolve_value(i, lambda i: sync.read({'id': i}))
+
+
+def create_product(doc):
+    # author, image, title, awards, teacher, date
+
+    unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
+
+    product = {**doc}
+    unpop_image(product)
+
+    product['teacher'] = resolve_teacher(doc['teacher'])
+    if(product['teacher']):
+        unpop_teacher(product)
+
+    return product

utils/Sync/utils/hash.py

@@ -1,5 +1,4 @@
 import hashlib
-import json
 
 from utils.fn import map_cases, to_json
 

utils/Sync/yandexdisk/__init__.py

@@ -2,9 +2,9 @@ import requests
 
 
 class YDClient:
-    '''
+    """
     Implementation of https://tech.yandex.ru/disk/poligon/
-    '''
+    """
 
     base_url = 'https://cloud-api.yandex.net:443/v1/disk'
 
@@ -23,9 +23,8 @@ class YDClient:
         url = self.base_url
         r = requests.get(url, headers=self.base_headers)
         self.check_code(r)
-
         disk = r.json()
-        return dict(
+        return Disk(
             trash_size=disk['trash_size'],
             total_space=disk['total_space'],
             used_space=disk['used_space'],