[sync] updated: config based; async loop; Documents adopted to new system
This commit is contained in:
Roman Timashev committed
1 parent
b0b518fd8c
commit
06965bebee
23 files changed
+1078
-346
No files matched your search
@@ -0,0 +1,188 @@
|
||||
import os
|
||||
from tempfile import mkstemp
|
||||
|
||||
import requests
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import scan_subdirs, request
|
||||
from sync.models import Model
|
||||
from sync.models.Image import Image
|
||||
from utils.fn import last_good, lmap
|
||||
from utils.hash import hash_str
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
|
||||
def find(store, item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Document(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
paths = scan_subdirs(provider, '.pdf')
|
||||
return [Document.read_path(provider, path) for path in paths]
|
||||
|
||||
@staticmethod
|
||||
def read_path(provider, path):
|
||||
name = os.path.basename(path)
|
||||
|
||||
document = {
|
||||
'category': os.path.dirname(path),
|
||||
'title': os.path.splitext(name)[0]
|
||||
}
|
||||
|
||||
return Document(
|
||||
provider,
|
||||
path,
|
||||
**document
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, category=None, title=None):
|
||||
store = collection(settings.collection_documents)
|
||||
|
||||
self.__hash_keys = ['id', 'url']
|
||||
|
||||
self.__id_template: str = '{category}-{file}'
|
||||
self.__url_template: str = settings.document_url_template
|
||||
self.__url_preview_template: str = settings.document_url_preview_template
|
||||
|
||||
self.category = category
|
||||
self.title = title
|
||||
self.type = None
|
||||
self.preview = None
|
||||
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
self.__set_id()
|
||||
self.__set_url()
|
||||
self.__set_hash()
|
||||
|
||||
async def is_changed(self):
|
||||
i = find(self.store, self.id)
|
||||
if not i:
|
||||
return True
|
||||
|
||||
if 'hash' not in i:
|
||||
return True
|
||||
|
||||
return not (self.hash == i['hash'])
|
||||
|
||||
async def save(self):
|
||||
document = self.bake()
|
||||
|
||||
query = {'id': document['id']}
|
||||
try:
|
||||
self.store.update_one(query, {'$set': document}, upsert=True)
|
||||
# return self.read(document)
|
||||
return self
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
async def upload(self):
|
||||
filepath = self.provider.get_local(self.file)
|
||||
name = os.path.basename(self.url)
|
||||
url = settings.document_url_upload_template.format(file=name)
|
||||
|
||||
await request.upload(url, filepath)
|
||||
|
||||
def bake(self):
|
||||
file = self.file
|
||||
filename = os.path.basename(file)
|
||||
|
||||
return {
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'url': self.url,
|
||||
'type': document_type(self.type),
|
||||
'file': {
|
||||
'name': filename,
|
||||
'size': self.provider.size(file)
|
||||
},
|
||||
'preview': self.preview,
|
||||
'title': last_good([
|
||||
self.title,
|
||||
get_pdf_title(self.provider, self.file)
|
||||
]),
|
||||
}
|
||||
|
||||
def __filename(self):
|
||||
return os.path.basename(self.file)
|
||||
|
||||
def __set_id(self):
|
||||
self.id = self.__id_template.format(
|
||||
category=url_encode_text(self.category),
|
||||
file=url_encode_text(self.__filename())
|
||||
)
|
||||
|
||||
def __set_url(self):
|
||||
self.url = self.__url_template.format(
|
||||
file=url_encode_file(self.__filename())
|
||||
)
|
||||
|
||||
def __set_hash(self):
|
||||
doc = [getattr(self, key) for key in self.__hash_keys]
|
||||
|
||||
self.hash = hash_str(
|
||||
hash_str(doc) + self.provider.hash(self.file)
|
||||
)
|
||||
|
||||
async def create_preview(self, sizes):
|
||||
def url(_, size, ext):
|
||||
file = url_encode_text(self.__filename())
|
||||
return self.__url_preview_template.format(id=file, size=size, ext=ext)
|
||||
|
||||
temp_preview_path = pdf_to_jpg(self.provider, self.file)
|
||||
preview = await Image.new(self.provider, temp_preview_path, sizes, url)
|
||||
|
||||
if preview:
|
||||
self.preview = preview.id
|
||||
|
||||
os.remove(temp_preview_path)
|
||||
|
||||
def __str__(self):
|
||||
return '<Document hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def document_type(doctype):
|
||||
if doctype == 'Награды':
|
||||
return 'award'
|
||||
return 'document'
|
||||
|
||||
|
||||
def pdf_to_jpg(provider, pdf):
|
||||
_, temp_in = mkstemp(suffix='.pdf')
|
||||
_, temp_out = mkstemp(suffix='.jpg')
|
||||
|
||||
abspdf = provider.copy(pdf, temp_in)
|
||||
image_magick_pdf_to_img(abspdf, temp_out)
|
||||
|
||||
os.remove(temp_in)
|
||||
return temp_out
|
||||
|
||||
|
||||
def get_pdf_title(provider, file):
|
||||
from PyPDF2 import PdfFileReader
|
||||
from PyPDF2.generic import TextStringObject
|
||||
from PyPDF2.generic import IndirectObject
|
||||
|
||||
pdf = PdfFileReader(provider.read(file))
|
||||
info = pdf.getDocumentInfo()
|
||||
|
||||
if info:
|
||||
if type(info.title) == TextStringObject:
|
||||
return str(info.title)
|
||||
|
||||
if type(info.title_raw) == IndirectObject:
|
||||
o = pdf.getObject(info.title_raw)
|
||||
return str(o)
|
||||
return None
|
||||
@@ -0,0 +1,207 @@
|
||||
import os
|
||||
import tempfile
|
||||
from tempfile import mkstemp
|
||||
from typing import Optional, List
|
||||
|
||||
import requests
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import scan_subdirs, request
|
||||
from sync.models import Model
|
||||
from utils.fn import last_good, lmap
|
||||
from utils.hash import hash_str
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync.data import upload
|
||||
from utils.hash import hash_file
|
||||
from utils.image.resize import optimize, thumbnail, orient
|
||||
|
||||
logger = logging.getLogger(settings.name)
|
||||
|
||||
|
||||
def get_upload_url(url):
|
||||
return os.path.join(settings.image_url_upload, os.path.basename(url))
|
||||
|
||||
|
||||
class Image(Model):
|
||||
@staticmethod
|
||||
async def new(provider, file, sizes, url_factory):
|
||||
img = Image(provider, file, url_factory)
|
||||
await img.compile(sizes)
|
||||
await img.upload()
|
||||
await img.sync()
|
||||
return img
|
||||
|
||||
def __init__(self, provider, file, url_factory):
|
||||
store = collection(settings.collection_images)
|
||||
self.url_factory = url_factory
|
||||
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
self.hash = self.__get_hash()
|
||||
|
||||
async def compile(self, sizes):
|
||||
img_out = tempfile.mkdtemp()
|
||||
self.image = create_image(self.file, sizes, self.url_factory, img_out)
|
||||
|
||||
async def upload(self):
|
||||
if self.image:
|
||||
for i in self.image:
|
||||
url = get_upload_url(i['url'])
|
||||
file = i['file']
|
||||
|
||||
logger.info('Uploading image {}'.format(file))
|
||||
|
||||
await request.upload(url, file)
|
||||
|
||||
async def sync(self):
|
||||
c = sync_image(self.bake())
|
||||
self.id = c['_id']
|
||||
|
||||
def bake(self):
|
||||
data = {}
|
||||
if self.image:
|
||||
for i in self.image:
|
||||
size_name = i['size']
|
||||
data[size_name] = {
|
||||
'url': i['url'],
|
||||
'size': size_name,
|
||||
'width': i['width'],
|
||||
'height': i['height'],
|
||||
}
|
||||
|
||||
return {
|
||||
'file': self.file,
|
||||
'hash': self.hash,
|
||||
'data': data,
|
||||
}
|
||||
|
||||
def __filename(self):
|
||||
return os.path.basename(self.file)
|
||||
|
||||
def __get_hash(self):
|
||||
return self.provider.hash(self.file)
|
||||
|
||||
def __str__(self):
|
||||
return '<Image hash={} file={}>'.format(self.hash, self.file)
|
||||
|
||||
|
||||
def sync_image(record):
|
||||
from db import db
|
||||
|
||||
q = {'hash': record['hash']}
|
||||
try:
|
||||
db().images.update_one(q, {'$set': record}, upsert=True)
|
||||
i = db().images.find_one({'hash': record['hash']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def document_type(doctype):
|
||||
if doctype == 'Награды':
|
||||
return 'award'
|
||||
return 'document'
|
||||
|
||||
|
||||
def pdf_to_jpg(provider, pdf):
|
||||
_, temp_in = mkstemp(suffix='.pdf')
|
||||
_, temp_out = mkstemp(suffix='.jpg')
|
||||
|
||||
abspdf = provider.copy(pdf, temp_in)
|
||||
image_magick_pdf_to_img(abspdf, temp_out)
|
||||
|
||||
os.remove(temp_in)
|
||||
return temp_out
|
||||
|
||||
|
||||
def get_pdf_title(provider, file):
|
||||
from PyPDF2 import PdfFileReader
|
||||
from PyPDF2.generic import TextStringObject
|
||||
from PyPDF2.generic import IndirectObject
|
||||
|
||||
pdf = PdfFileReader(provider.read(file))
|
||||
info = pdf.getDocumentInfo()
|
||||
|
||||
if info:
|
||||
if type(info.title) == TextStringObject:
|
||||
return str(info.title)
|
||||
|
||||
if type(info.title_raw) == IndirectObject:
|
||||
o = pdf.getObject(info.title_raw)
|
||||
return str(o)
|
||||
return None
|
||||
|
||||
|
||||
def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
|
||||
"""
|
||||
|
||||
:param file: Image path to process
|
||||
:param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
|
||||
:param url_fn: function (size_name, ext) for creating image url
|
||||
:param output_dir: path to local folder for thumbs storing
|
||||
:return:
|
||||
"""
|
||||
if not os.path.exists(file):
|
||||
return None
|
||||
|
||||
_, ext = os.path.splitext(file)
|
||||
|
||||
def fn(size):
|
||||
size_name, width, height = size
|
||||
|
||||
image_url = url_fn(file, size_name, ext)
|
||||
image_filename = os.path.basename(image_url)
|
||||
local_image_path = os.path.join(output_dir, image_filename)
|
||||
|
||||
if settings.image_processing_enabled:
|
||||
image = process_image(file, local_image_path, size)
|
||||
if not image:
|
||||
logger.warning('Failed to process image {}'.format(file))
|
||||
return None
|
||||
width, height = image.size
|
||||
|
||||
return {
|
||||
'file': local_image_path,
|
||||
'url': image_url,
|
||||
'size': size_name,
|
||||
'width': width,
|
||||
'height': height
|
||||
}
|
||||
|
||||
return [fn(s) for s in sizes]
|
||||
|
||||
|
||||
def process_image(input_file, output_file, size):
|
||||
size_name, width, height = size
|
||||
|
||||
if size_name == 'original':
|
||||
return optimize(input_file, output_file, quality=90)
|
||||
else:
|
||||
image = read_image(input_file)
|
||||
if not image:
|
||||
return None
|
||||
image_width, image_height = image.size
|
||||
if image_height > image_width:
|
||||
width, height = height, width
|
||||
return thumbnail(input_file, output_file, (width, height))
|
||||
|
||||
|
||||
def read_image(src):
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
i = Image.open(src)
|
||||
return i
|
||||
except Exception as e:
|
||||
print(src, e)
|
||||
return None
|
||||
@@ -1,6 +1,25 @@
|
||||
class Model:
|
||||
def __init__(self):
|
||||
def __init__(self, provider, store, file):
|
||||
super().__init__()
|
||||
|
||||
self.provider = provider
|
||||
self.store = store
|
||||
self.file = file
|
||||
|
||||
self.id = None
|
||||
self.url = None
|
||||
self.hash = None
|
||||
|
||||
self.init()
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
|
||||
def bake(self):
|
||||
return None
|
||||
|
||||
async def is_changed(self):
|
||||
return False
|
||||
|
||||
async def save(self):
|
||||
pass
|
||||
@@ -1,124 +0,0 @@
|
||||
import os
|
||||
from tempfile import mkstemp
|
||||
|
||||
import settings
|
||||
from sync.models import Model
|
||||
from utils.fn import last_good, lmap
|
||||
from utils.hash import hash_str
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
|
||||
class Document(Model):
|
||||
@staticmethod
|
||||
def read(path):
|
||||
name = os.path.basename(path)
|
||||
|
||||
document = {
|
||||
'file': path,
|
||||
'category': os.path.dirname(path),
|
||||
'title': os.path.splitext(name)[0]
|
||||
}
|
||||
|
||||
return document
|
||||
|
||||
def __init__(self, provider, path, url_base, preview_compiler):
|
||||
super().__init__()
|
||||
|
||||
self.path = path
|
||||
|
||||
self.category = None
|
||||
self.file = None
|
||||
self.id = None
|
||||
self.type = None
|
||||
self.url = None
|
||||
self.title = None
|
||||
self.preview = None
|
||||
|
||||
for k, v in m.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
self.provider = provider
|
||||
|
||||
def bake(self):
|
||||
file = self.file
|
||||
filename = os.path.basename(file)
|
||||
|
||||
self.type = document_type(self.type)
|
||||
self.preview = self.create_preview(file)
|
||||
self.title = last_good([
|
||||
self.title,
|
||||
get_pdf_title(self.provider, self.file)
|
||||
])
|
||||
self.file = {
|
||||
'name': filename,
|
||||
'size': self.provider.size(file)
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
bn = os.path.basename(self.file)
|
||||
self.id = url_encode_text('{category}-{file}'.format(
|
||||
category=self.category,
|
||||
file=bn
|
||||
))
|
||||
|
||||
def __set_url(self):
|
||||
filename = os.path.basename(self.file)
|
||||
self.url = self.url_base_document.format(file=url_encode_file(filename))
|
||||
|
||||
def __set_hash(self):
|
||||
hash_keys = ['id', 'url']
|
||||
hash_doc = lmap(
|
||||
lambda key: getattr(self, key),
|
||||
hash_keys
|
||||
)
|
||||
self.hash = hash_str(
|
||||
hash_str(hash_doc) + self.provider.hash(self.file)
|
||||
)
|
||||
|
||||
def create_preview(self, file):
|
||||
def url(size, ext):
|
||||
return self.url_base_preview.format(id=file, size=size, ext=ext)
|
||||
|
||||
temp_preview_path = pdf_to_jpg(self.provider, file)
|
||||
file = os.path.basename(file)
|
||||
file = url_encode_text(file)
|
||||
img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
|
||||
|
||||
def document_type(doctype):
|
||||
if doctype == 'Награды':
|
||||
return 'award'
|
||||
return 'document'
|
||||
|
||||
|
||||
def pdf_to_jpg(provider, pdf):
|
||||
_, temp_in = mkstemp(suffix='.pdf')
|
||||
_, temp_out = mkstemp(suffix='.jpg')
|
||||
|
||||
abspdf = provider.copy(pdf, temp_in)
|
||||
image_magick_pdf_to_img(abspdf, temp_out)
|
||||
|
||||
os.remove(temp_in)
|
||||
return temp_out
|
||||
|
||||
|
||||
def get_pdf_title(provider, file):
|
||||
from PyPDF2 import PdfFileReader
|
||||
from PyPDF2.generic import TextStringObject
|
||||
from PyPDF2.generic import IndirectObject
|
||||
|
||||
pdf = PdfFileReader(provider.read(file))
|
||||
info = pdf.getDocumentInfo()
|
||||
|
||||
if info:
|
||||
if type(info.title) == TextStringObject:
|
||||
return str(info.title)
|
||||
|
||||
if type(info.title_raw) == IndirectObject:
|
||||
o = pdf.getObject(info.title_raw)
|
||||
return str(o)
|
||||
return None
|
||||
Reference in new issue
Block a user