sync update: + data provider; refactoring; documents update
This commit is contained in:
tmshv committed
1 parent
144e69e6c4
commit
927674e53e
9 files changed
+311
-229
No files matched your search
+112
-181
@@ -1,83 +1,128 @@
|
||||
import os
|
||||
from glob import glob
|
||||
from shutil import copyfile
|
||||
from tempfile import mkstemp
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync import Sync, synced_image_id, untouched
|
||||
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from sync import synced_image_id, untouched
|
||||
from sync.core.document import SyncDocument
|
||||
from sync.data.fs import FSProvider
|
||||
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap
|
||||
from utils.io import read_yaml
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
|
||||
class SyncDocument(Sync):
|
||||
def __init__(self, collection_name, sizes, dir_static_previews, url_base_preview, url_base_document):
|
||||
super().__init__()
|
||||
self.collection = db()[collection_name]
|
||||
self.sizes = sizes
|
||||
self.dir_static_previews = dir_static_previews
|
||||
self.url_base_preview = url_base_preview
|
||||
self.url_base_document = url_base_document
|
||||
def sync_documents(update=True, delete=True):
|
||||
provider = FSProvider(settings.dir_documents)
|
||||
|
||||
def create(self, document):
|
||||
file = document['file']
|
||||
filename = os.path.basename(document['file'])
|
||||
static_path = static_path_fn(settings.dir_static_uploads)
|
||||
|
||||
document['type'] = self.document_type(document)
|
||||
document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
|
||||
document['file'] = {
|
||||
'name': filename,
|
||||
'size': os.path.getsize((file))
|
||||
}
|
||||
u, d = main(
|
||||
SyncDocument(
|
||||
db()[settings.collection_documents],
|
||||
provider,
|
||||
sizes=settings.image_sizes,
|
||||
dir_static_previews=settings.dir_static_images,
|
||||
url_base_preview=settings.url_base_preview,
|
||||
url_base_document=settings.url_base_document,
|
||||
),
|
||||
static_path,
|
||||
update_documents=update,
|
||||
delete_documents=delete
|
||||
)
|
||||
|
||||
return document
|
||||
print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d))))
|
||||
lprint_json(d)
|
||||
|
||||
def document_type(self, document):
|
||||
if document['category'] == 'Награды':
|
||||
return 'award'
|
||||
return 'document'
|
||||
print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
|
||||
lprint_json(u)
|
||||
|
||||
def create_id(self, document):
|
||||
bn = os.path.basename(document['file'])
|
||||
document['id'] = url_encode_text('{category}-{file}'.format(
|
||||
category=document['category'],
|
||||
file=bn
|
||||
))
|
||||
return document
|
||||
|
||||
def create_url(self, document):
|
||||
filename = os.path.basename(document['file'])
|
||||
document['url'] = self.url_base_document.format(file=url_encode_file(filename))
|
||||
return document
|
||||
def main(sync, static_path, update_documents, delete_documents):
|
||||
documents = get_documents(sync)
|
||||
documents_id = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
def create_hash(self, document):
|
||||
document['hash'] = hash_str(
|
||||
hash_str(document) + hash_file(document['file'])
|
||||
# CHECKING
|
||||
file_names = set(lmap(
|
||||
lambda i: i['file'],
|
||||
documents
|
||||
))
|
||||
if len(file_names) != len(documents):
|
||||
raise Exception('File names should be unique')
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
for i in documents:
|
||||
sync.provider.copy(
|
||||
i['file'],
|
||||
static_path(i)
|
||||
)
|
||||
return document
|
||||
|
||||
def create_preview(self, file, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(file)
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
sync.create,
|
||||
documents
|
||||
)
|
||||
|
||||
file = os.path.basename(file)
|
||||
file = url_encode_text(file)
|
||||
url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
|
||||
if update_documents:
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('preview', lambda i: synced_image_id(i)),
|
||||
documents
|
||||
)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
# DELETING
|
||||
documents_delete = sync.query({'id': {'$nin': documents_id}})
|
||||
if delete_documents:
|
||||
documents_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_delete
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_delete
|
||||
|
||||
|
||||
def pdf_to_jpg(pdf):
|
||||
cwd = os.getcwd()
|
||||
abspdf = os.path.join(cwd, pdf)
|
||||
_, temp = mkstemp('.jpg')
|
||||
image_magick_pdf_to_img(abspdf, temp)
|
||||
return temp
|
||||
def get_documents(sync):
|
||||
# documents = documents_from_yaml()
|
||||
documents = documents_from_subdirs(sync.provider)
|
||||
|
||||
# SETUP TITLE BASED ON FILE NAME
|
||||
documents = lmapfn(documents)(
|
||||
lambda i: {
|
||||
**i,
|
||||
'title': os.path.splitext(os.path.basename(i['file']))[0]
|
||||
}
|
||||
)
|
||||
|
||||
# CHOICE BETTER TITLE
|
||||
documents = lmapfn(documents)(
|
||||
lambda i: {
|
||||
**i,
|
||||
'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
|
||||
}
|
||||
)
|
||||
|
||||
documents = lmap(sync.create_id, documents)
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
documents = lmap(sync.create_url, documents)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def static_path_fn(dir):
|
||||
return lambda document: os.path.join(dir, os.path.basename(document['url']))
|
||||
|
||||
|
||||
def categorize_files_list(files, title):
|
||||
@@ -131,23 +176,22 @@ def documents_from_yaml(dirpath):
|
||||
return documents
|
||||
|
||||
|
||||
def documents_from_subdirs():
|
||||
def documents_from_subdirs(provider):
|
||||
documents = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
provider.scan('.')
|
||||
)
|
||||
)
|
||||
|
||||
documents = lmap(
|
||||
lambda folder: lmap(
|
||||
lambda path: {
|
||||
'file': path,
|
||||
'title': os.path.splitext(os.path.basename(os.path.relpath(path, folder)))[0],
|
||||
'file': provider.get_rel(path),
|
||||
'category': os.path.basename(folder)
|
||||
},
|
||||
glob(folder + '/*.pdf')
|
||||
provider.glob(folder + '/*.pdf')
|
||||
),
|
||||
documents
|
||||
)
|
||||
@@ -156,12 +200,12 @@ def documents_from_subdirs():
|
||||
return documents
|
||||
|
||||
|
||||
def get_pdf_title(file):
|
||||
def get_pdf_title(provider, file):
|
||||
from PyPDF2 import PdfFileReader
|
||||
from PyPDF2.generic import TextStringObject
|
||||
from PyPDF2.generic import IndirectObject
|
||||
|
||||
pdf = PdfFileReader(open(file, 'rb'))
|
||||
pdf = PdfFileReader(provider.open(file, 'rb'))
|
||||
info = pdf.getDocumentInfo()
|
||||
|
||||
if info:
|
||||
@@ -182,122 +226,9 @@ def until_none(ls):
|
||||
return better
|
||||
|
||||
|
||||
def main(dir_documents, dir_static_uploads, sync, update_documents=True, delete_documents=True):
|
||||
"""
|
||||
|
||||
:param dir_documents:
|
||||
:param dir_static_uploads:
|
||||
:param sync:
|
||||
:param update_documents:
|
||||
:param delete_documents:
|
||||
:return:
|
||||
"""
|
||||
|
||||
os.chdir(dir_documents)
|
||||
|
||||
# documents = documents_from_yaml()
|
||||
documents = documents_from_subdirs()
|
||||
|
||||
# CHOICE BETTER TITLE
|
||||
documents = lmapfn(documents)(
|
||||
lambda i: {
|
||||
**i,
|
||||
'title': until_none([i['title'], get_pdf_title(i['file'])])
|
||||
}
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
# CREATE SCOPE
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
file_names = set(lmap(
|
||||
lambda i: i['file'],
|
||||
documents
|
||||
))
|
||||
if len(file_names) != len(documents):
|
||||
raise Exception('File names should be unique')
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# URL
|
||||
documents = lmap(sync.create_url, documents)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
|
||||
lmap(
|
||||
lambda doc: copyfile(
|
||||
doc['file'],
|
||||
out_path(doc)
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
sync.create,
|
||||
documents
|
||||
)
|
||||
|
||||
if update_documents:
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('preview', lambda i: synced_image_id(i)),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
if delete_documents:
|
||||
documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_delete
|
||||
)
|
||||
)
|
||||
else:
|
||||
documents_to_delete = []
|
||||
|
||||
print('SCOPE:', len(scope_documents_ids))
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete))))
|
||||
lprint_json(documents_to_delete)
|
||||
|
||||
print('UPDATE DOCUMENTS:', len(documents))
|
||||
lprint_json(documents)
|
||||
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(
|
||||
settings.dir_documents,
|
||||
settings.dir_static_uploads,
|
||||
SyncDocument(
|
||||
'documents',
|
||||
sizes=settings.image_sizes,
|
||||
dir_static_previews=settings.dir_static_images,
|
||||
url_base_preview=settings.url_base_preview,
|
||||
url_base_document=settings.url_base_document,
|
||||
),
|
||||
update_documents=True,
|
||||
delete_documents=True
|
||||
)
|
||||
sync_documents()
|
||||
# sync_documents(delete=False, update=False)
|
||||
|
||||
# DOCUMENT_OBJECT SAMPLE
|
||||
# {
|
||||
|
||||
Reference in new issue
Block a user