93 lines
2.9 KiB
Python
93 lines
2.9 KiB
Python
import os
|
|
from tempfile import mkstemp
|
|
|
|
import lxml.html
|
|
|
|
import kazimir
|
|
from sync.core import Sync
|
|
from utils.fn import last_good, lmap, combine
|
|
from utils.hash import hash_str, hash_file
|
|
from utils.image import create_image
|
|
from utils.image.resize import image_magick_pdf_to_img
|
|
from utils.text.transform import url_encode_text, url_encode_file
|
|
|
|
|
|
class SyncPost(Sync):
|
|
def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
|
|
super().__init__()
|
|
self.collection = collection
|
|
self.provider = provider
|
|
self.sizes = sizes
|
|
self.type = document_type
|
|
self.image_url_base = image_url_base
|
|
self.dir_local_images = dir_local_images
|
|
self.origin = origin
|
|
|
|
def create_id(self, document):
|
|
new_id = url_encode_text(document['title'])
|
|
document['id'] = document['id'] if 'id' in document else new_id
|
|
return document
|
|
|
|
def create_hash(self, document):
|
|
files = sorted(document['images'])
|
|
hashes = lmap(
|
|
self.provider.hash,
|
|
lmap(
|
|
lambda i: os.path.join(document['folder'], i),
|
|
files
|
|
)
|
|
)
|
|
|
|
document['hash'] = hash_str(
|
|
combine([hash_str(document)] + hashes)
|
|
)
|
|
|
|
return document
|
|
|
|
def create(self, document):
|
|
if 'date' not in document or not document['date']:
|
|
return None
|
|
|
|
if 'title' not in document or not document['title']:
|
|
return None
|
|
|
|
if self.type:
|
|
document['type'] = self.type
|
|
|
|
post_html = lxml.html.fromstring(document['post'])
|
|
images = []
|
|
for img in post_html.cssselect('img'):
|
|
src = img.get('src')
|
|
relative_image_path = os.path.join(document['folder'], src)
|
|
|
|
img_path = self.provider.get_local(relative_image_path)
|
|
if os.path.exists(img_path):
|
|
img_id = url_encode_text(os.path.splitext(src)[0])
|
|
url_fn = lambda size, ext: self.image_url_base.format(
|
|
id=document['id'],
|
|
img=img_id,
|
|
size=size,
|
|
ext=ext.lower()
|
|
)
|
|
|
|
image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
|
|
if image:
|
|
images.append(image)
|
|
img.set('src', image['data']['big']['url'])
|
|
else:
|
|
print('fail: ', document['folder'], src)
|
|
|
|
document['post'] = kazimir.html_from_tree(post_html)
|
|
document['images'] = images
|
|
document['origin'] = self.origin
|
|
return document
|
|
|
|
def create_remove_query(self, query):
|
|
if self.type:
|
|
query = {
|
|
**query,
|
|
'type': self.type,
|
|
'origin': self.origin
|
|
}
|
|
return query
|