diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index d3b42b2b..d8ea0628 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,10 +1,11 @@ import settings -from sync.core.document import pdf_to_jpg +from sync.core.document import pdf_to_jpg, SyncDocument from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider -from sync.document import sync_documents +from sync.document import sync_documents, read_document, get_documents from utils.fn import lprint_json from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text from yandexdisk import YDClient @@ -21,6 +22,8 @@ if __name__ == '__main__': update = False delete = False + # update = True + # delete = True if provider: u, d = sync_documents(provider, update=update, delete=delete) @@ -30,6 +33,16 @@ if __name__ == '__main__': print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u)))) lprint_json(u) + # s = SyncDocument( + # {}, + # provider, + # sizes=settings.image_sizes, + # dir_static_previews=settings.dir_static_images, + # url_base_preview=settings.url_base_preview, + # url_base_document=settings.url_base_document, + # ) + # # print(read_document(s, 'Награды/2016 В мире фантазий — Диплом.pdf')) + # lprint_json(get_documents(s)) # print(api.scan('.')) # print(list(filter( @@ -41,7 +54,7 @@ if __name__ == '__main__': # print(api.size('./Архив/Устав 2013.pdf')) # print(api.glob('./Архив/*.pdf')) - # print(api.type_filter('./Архив', '.pdf')) + # lprint_json(provider.type_filter('Награды', '.pdf')) # path = api.type_filter('./Архив', '.pdf')[0] # print(api.copy(path, '/Users/tmshv/Desktop/fafo')) diff --git a/utils/Sync/sync/core/document.py b/utils/Sync/sync/core/document.py index d4f77eab..dab662a3 100644 --- a/utils/Sync/sync/core/document.py +++ b/utils/Sync/sync/core/document.py @@ -2,6 +2,7 @@ import os from tempfile import mkstemp from sync.core import Sync +from utils.fn import last_good, lmap from utils.hash import hash_file, hash_str from utils.image import create_image from utils.image.resize import image_magick_pdf_to_img @@ -24,6 +25,10 @@ class SyncDocument(Sync): document['type'] = document_type(document) document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) + document['title'] = last_good([ + document['title'], + get_pdf_title(self.provider, document['file']) + ]) document['file'] = { 'name': filename, 'size': self.provider.size(file) @@ -45,8 +50,13 @@ class SyncDocument(Sync): return document def create_hash(self, document): + hash_keys = ['id', 'url'] + hash_doc = lmap( + lambda key: document[key], + hash_keys + ) document['hash'] = hash_str( - hash_str(document) + self.provider.hash(document['file']) + hash_str(hash_doc) + self.provider.hash(document['file']) ) return document @@ -77,3 +87,21 @@ def pdf_to_jpg(provider, pdf): os.remove(temp_in) return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py index ccf0cc3a..3ce94064 100644 --- a/utils/Sync/sync/data/fs.py +++ b/utils/Sync/sync/data/fs.py @@ -9,7 +9,7 @@ from utils.hash import hash_file class FSProvider(Provider): - def open(self, path): + def read(self, path): return open(self.get_abs(path), 'rb') def scan(self, path): diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index eb2e1a4d..f79d5a58 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -116,29 +116,29 @@ def get_documents(sync): # documents = documents_from_yaml() documents = documents_from_subdirs(sync.provider) - # SETUP TITLE BASED ON FILE NAME - documents = lmapfn(documents)( - lambda i: { - **i, - 'title': os.path.splitext(os.path.basename(i['file']))[0] - } + documents = lmap( + lambda document: read_document(sync, document), + documents ) - - # CHOICE BETTER TITLE - # documents = lmapfn(documents)( - # lambda i: { - # **i, - # 'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])]) - # } - # ) - - documents = lmap(sync.create_id, documents) - documents = lmap(sync.create_hash, documents) - documents = lmap(sync.create_url, documents) - return documents +def read_document(sync, path): + name = os.path.basename(path) + + document = { + 'file': path, + 'category': os.path.dirname(path), + 'title': os.path.splitext(name)[0] + } + + document = sync.create_id(document) + document = sync.create_url(document) + document = sync.create_hash(document) + + return document + + def static_path_fn(dir_path): return lambda document: os.path.join(dir_path, os.path.basename(document['url'])) @@ -202,42 +202,9 @@ def documents_from_subdirs(provider): )) documents = lmap( - lambda folder: lmap( - lambda path: { - # 'file': provider.get_rel(path), - 'file': path, - 'category': os.path.basename(folder) - }, - provider.type_filter(folder, '.pdf') - ), + lambda folder: provider.type_filter(folder, '.pdf'), documents ) documents = combine(documents) return documents - - -def get_pdf_title(provider, file): - from PyPDF2 import PdfFileReader - from PyPDF2.generic import TextStringObject - from PyPDF2.generic import IndirectObject - - pdf = PdfFileReader(provider.read(file)) - info = pdf.getDocumentInfo() - - if info: - if type(info.title) == TextStringObject: - return str(info.title) - - if type(info.title_raw) == IndirectObject: - o = pdf.getObject(info.title_raw) - return str(o) - return None - - -def until_none(ls): - better = None - for i in ls: - if (i is not None) and i != '': - better = i - return better diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index b03bbfa9..00b68f4c 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -34,6 +34,7 @@ iterate_iter_over_fns = lambda fns: lambda i: lmap( def lmap(fn, ls): return list(map(fn, ls)) + def lmapfn(ls): return lambda fn: list(map(fn, ls)) @@ -89,3 +90,11 @@ dir_globber = lambda mask_list: lambda dir: combine(map( lambda mask: glob(os.path.join(dir, mask)), mask_list )) + + +def last_good(ls): + better = None + for i in ls: + if (i is not None) and i != '': + better = i + return better