diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py index eac78e6d..183d326b 100644 --- a/utils/Sync/sync/album.py +++ b/utils/Sync/sync/album.py @@ -1,21 +1,19 @@ import os -import re from glob import glob -import frontmatter import lxml.html from lxml import etree import settings from db import db -from models.product import create_product from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \ untouched, synced_images_ids +from sync.models import create_product from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \ key_mapper, lprint from utils.hash import hash_file, hash_str from utils.image import create_image -from utils.io import read_yaml_md, read_file, parse_yaml_front_matter +from utils.io import read_yaml_md from utils.text.transform import url_encode_text diff --git a/utils/Sync/sync/compile.py b/utils/Sync/sync/compile.py new file mode 100644 index 00000000..c9da54d8 --- /dev/null +++ b/utils/Sync/sync/compile.py @@ -0,0 +1,36 @@ +import os +from tempfile import mkstemp + +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text + + +class Preview: + def __init__(self, provider, url_base, output, sizes): + super().__init__() + self.provider = provider + self.url_base = url_base + self.output = output + self.sizes = sizes + + def create(self, file): + def url(size, ext): + return self.url_base.format(id=file, size=size, ext=ext) + + temp_preview_path = self.pdf_to_jpg(file) + file = os.path.basename(file) + file = url_encode_text(file) + img = create_image(temp_preview_path, self.sizes, url, self.output) + os.remove(temp_preview_path) + return img + + def pdf_to_jpg(self, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = self.provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py index 4628ec59..483d75f5 100644 --- a/utils/Sync/sync/core/__init__.py +++ b/utils/Sync/sync/core/__init__.py @@ -1,7 +1,12 @@ +from sync.models import Model from utils.fn import lmap class Sync: + @staticmethod + def compile(document): + return document.bake() + def __init__(self): super().__init__() @@ -22,6 +27,9 @@ class Sync: if isinstance(document, list): return lmap(self.update, document) + if isinstance(document, Model): + return self.update(self.compile(document)) + q = {'id': document['id']} try: self.collection.update_one(q, {'$set': document}, upsert=True) @@ -30,9 +38,6 @@ class Sync: pass return None - def compile(self, document): - return document - def query(self, q): return self.collection.find(q) diff --git a/utils/Sync/sync/core/document.py b/utils/Sync/sync/core/document.py index dab662a3..e541619d 100644 --- a/utils/Sync/sync/core/document.py +++ b/utils/Sync/sync/core/document.py @@ -3,7 +3,7 @@ from tempfile import mkstemp from sync.core import Sync from utils.fn import last_good, lmap -from utils.hash import hash_file, hash_str +from utils.hash import hash_str from utils.image import create_image from utils.image.resize import image_magick_pdf_to_img from utils.text.transform import url_encode_text, url_encode_file diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index bf251dac..398d9db1 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -1,6 +1,8 @@ import os from io import BytesIO +from utils.fn import combine, lmap + class Provider: def __init__(self, root): @@ -29,12 +31,36 @@ class Provider: return 0 def get_abs(self, path): - abs = os.path.join(self.root, path) - return os.path.normpath(abs) + a = os.path.join(self.root, path) + return os.path.normpath(a) def get_rel(self, path): rel = os.path.relpath(path, self.root) return rel def is_dir(self, path): - return False \ No newline at end of file + return False + + +def scan_subdirs(provider, ext, path='.'): + """ + Creates flat list of files with specific ext + It iterates only one level of sub directories + :param provider: instance of Provider + :param ext: file extension like .jpg of .pdf for file filtering + :param path: optional path prefix + :return: list of files relative to provider.root + """ + items = provider.scan(path) + items = list(filter( + lambda i: provider.is_dir(i), + items + )) + + items = lmap( + lambda folder: provider.type_filter(folder, ext), + items + ) + + items = combine(items) + return items diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index f79d5a58..6d85033f 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -5,11 +5,13 @@ import settings from db import db from sync import synced_image_id, untouched from sync.core.document import SyncDocument +from sync.data import scan_subdirs +from sync.models.document import Document from utils.fn import combine, lmap, key_mapper, lmapfn from utils.io import read_yaml -def sync_documents(provider, update=True, delete=True): +def sync_documents(provider, collection, update=True, delete=True): """ DOCUMENT_OBJECT SAMPLE { @@ -41,7 +43,7 @@ def sync_documents(provider, update=True, delete=True): return main( SyncDocument( - db()[settings.collection_documents], + collection, provider, sizes=settings.image_sizes, dir_static_previews=settings.dir_static_images, @@ -55,7 +57,11 @@ def sync_documents(provider, update=True, delete=True): def main(sync, static_path, update_documents, delete_documents): - documents = get_documents(sync) + documents = scan_subdirs(sync.provider, '.pdf') + documents = lmap( + lambda document: read_document(sync, document), + documents + ) documents_id = lmap( lambda document: document['id'], documents @@ -112,17 +118,6 @@ def main(sync, static_path, update_documents, delete_documents): return documents, documents_delete -def get_documents(sync): - # documents = documents_from_yaml() - documents = documents_from_subdirs(sync.provider) - - documents = lmap( - lambda document: read_document(sync, document), - documents - ) - return documents - - def read_document(sync, path): name = os.path.basename(path) @@ -139,6 +134,13 @@ def read_document(sync, path): return document +def create_document(provider, m): + # dir_static_previews = settings.dir_static_images, + # url_base_preview = settings.url_base_preview, + # url_base_document = settings.url_base_document + return Document(provider, m, {}) + + def static_path_fn(dir_path): return lambda document: os.path.join(dir_path, os.path.basename(document['url'])) @@ -192,19 +194,3 @@ def documents_from_yaml(dirpath): documents = unwrap_manifest(documents) return documents - - -def documents_from_subdirs(provider): - documents = provider.scan('.') - documents = list(filter( - lambda i: provider.is_dir(i), - documents - )) - - documents = lmap( - lambda folder: provider.type_filter(folder, '.pdf'), - documents - ) - - documents = combine(documents) - return documents diff --git a/utils/Sync/sync/models/Album.py b/utils/Sync/sync/models/Album.py new file mode 100644 index 00000000..f30f6348 --- /dev/null +++ b/utils/Sync/sync/models/Album.py @@ -0,0 +1,21 @@ +__author__ = 'Roman Timashev' + + +class Album: + def __init__(self, type, title, uri, course, course_uri, date, teacher, comment): + super().__init__() + self.products = [] + self.type = type + self.title = title + self.uri = uri + self.course = course + self.course_uri = course_uri + self.date = date + self.teacher = teacher + self.comment = comment + + def add_product(self, product): + self.products.append(product) + + def __repr__(self, *args, **kwargs): + return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products)) diff --git a/utils/Sync/sync/models/__init__.py b/utils/Sync/sync/models/__init__.py new file mode 100644 index 00000000..5c6d8dea --- /dev/null +++ b/utils/Sync/sync/models/__init__.py @@ -0,0 +1,6 @@ +class Model: + def __init__(self): + super().__init__() + + def bake(self): + return None diff --git a/utils/Sync/sync/models/document.py b/utils/Sync/sync/models/document.py new file mode 100644 index 00000000..29724b4a --- /dev/null +++ b/utils/Sync/sync/models/document.py @@ -0,0 +1,124 @@ +import os +from tempfile import mkstemp + +import settings +from sync.models import Model +from utils.fn import last_good, lmap +from utils.hash import hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + + +class Document(Model): + @staticmethod + def read(path): + name = os.path.basename(path) + + document = { + 'file': path, + 'category': os.path.dirname(path), + 'title': os.path.splitext(name)[0] + } + + return document + + def __init__(self, provider, path, url_base, preview_compiler): + super().__init__() + + self.path = path + + self.category = None + self.file = None + self.id = None + self.type = None + self.url = None + self.title = None + self.preview = None + + for k, v in m.items(): + setattr(self, k, v) + + self.provider = provider + + def bake(self): + file = self.file + filename = os.path.basename(file) + + self.type = document_type(self.type) + self.preview = self.create_preview(file) + self.title = last_good([ + self.title, + get_pdf_title(self.provider, self.file) + ]) + self.file = { + 'name': filename, + 'size': self.provider.size(file) + } + + def __set_id(self): + bn = os.path.basename(self.file) + self.id = url_encode_text('{category}-{file}'.format( + category=self.category, + file=bn + )) + + def __set_url(self): + filename = os.path.basename(self.file) + self.url = self.url_base_document.format(file=url_encode_file(filename)) + + def __set_hash(self): + hash_keys = ['id', 'url'] + hash_doc = lmap( + lambda key: getattr(self, key), + hash_keys + ) + self.hash = hash_str( + hash_str(hash_doc) + self.provider.hash(self.file) + ) + + def create_preview(self, file): + def url(size, ext): + return self.url_base_preview.format(id=file, size=size, ext=ext) + + temp_preview_path = pdf_to_jpg(self.provider, file) + file = os.path.basename(file) + file = url_encode_text(file) + img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images) + os.remove(temp_preview_path) + return img + + +def document_type(doctype): + if doctype == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None diff --git a/utils/Sync/sync/models/product.py b/utils/Sync/sync/models/product.py new file mode 100644 index 00000000..c7abf5ad --- /dev/null +++ b/utils/Sync/sync/models/product.py @@ -0,0 +1,39 @@ +from sync.image import sync_image +from sync.teacher import Teacher +from utils.fn import key_mapper +from utils.text.transform import interpolate_swift + + +def resolve_value(param, store): + """ + + :param param: \(nv-andreeva) + :param store: function should return value by key nv-andreeva + :return: + """ + i = interpolate_swift(param) + return store(i) + + +unpop_image = key_mapper('image', lambda i: sync_image(i)['_id']) + + + +def resolve_teacher(i): + sync = Teacher() + return resolve_value(i, lambda i: sync.read({'id': i})) + + +def create_product(doc): + # author, image, title, awards, teacher, date + + unpop_teacher = key_mapper('teacher', lambda i: i['_id']) + + product = {**doc} + unpop_image(product) + + product['teacher'] = resolve_teacher(doc['teacher']) + if(product['teacher']): + unpop_teacher(product) + + return product diff --git a/utils/Sync/utils/hash.py b/utils/Sync/utils/hash.py index 564e89c2..87891e57 100644 --- a/utils/Sync/utils/hash.py +++ b/utils/Sync/utils/hash.py @@ -1,5 +1,4 @@ import hashlib -import json from utils.fn import map_cases, to_json diff --git a/utils/Sync/yandexdisk/__init__.py b/utils/Sync/yandexdisk/__init__.py index fcbd5a3c..17f2bcf4 100644 --- a/utils/Sync/yandexdisk/__init__.py +++ b/utils/Sync/yandexdisk/__init__.py @@ -2,9 +2,9 @@ import requests class YDClient: - ''' + """ Implementation of https://tech.yandex.ru/disk/poligon/ - ''' + """ base_url = 'https://cloud-api.yandex.net:443/v1/disk' @@ -23,9 +23,8 @@ class YDClient: url = self.base_url r = requests.get(url, headers=self.base_headers) self.check_code(r) - disk = r.json() - return dict( + return Disk( trash_size=disk['trash_size'], total_space=disk['total_space'], used_space=disk['used_space'],