[Sync] + new Album sync
This commit is contained in:
Roman Timashev committed
1 parent
3c8f6418b5
commit
62982a9945
7 files changed
+307
-330
No files matched your search
@@ -138,7 +138,7 @@ dir_awards = f('Awards')
|
||||
dir_schedules = f('Schedules')
|
||||
dir_articles = f('Articles')
|
||||
dir_collective = f('Collective')
|
||||
dir_gallery = f('Gallery')
|
||||
dir_albums = f('Gallery')
|
||||
dir_pages = f('Pages')
|
||||
|
||||
page_url_base = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}'
|
||||
|
||||
+6
-2
@@ -1,6 +1,7 @@
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.core import Sync
|
||||
from sync.core.SyncAlbum import SyncAlbum
|
||||
from sync.core.SyncArticle import SyncArticle
|
||||
from sync.core.SyncDocument import SyncDocument
|
||||
from sync.core.SyncPage import SyncPage
|
||||
@@ -68,16 +69,19 @@ async def run(run_interval=0):
|
||||
provider=io(settings.dir_schedules),
|
||||
collection=c(settings.collection_schedules),
|
||||
)
|
||||
albums = SyncAlbum(
|
||||
provider=io(settings.dir_albums),
|
||||
collection=c(settings.collection_albums),
|
||||
)
|
||||
|
||||
while True:
|
||||
# main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
|
||||
|
||||
await asyncio.wait([
|
||||
main(documents),
|
||||
main(pages),
|
||||
main(persons),
|
||||
main(articles),
|
||||
main(schedules),
|
||||
main(albums),
|
||||
])
|
||||
|
||||
if run_interval == 0:
|
||||
|
||||
@@ -89,12 +89,12 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
|
||||
return date, title
|
||||
|
||||
|
||||
def create_post(md, image_path_fn):
|
||||
async def create_post(md, image_path_fn):
|
||||
html = lxml.html.fromstring(md)
|
||||
|
||||
for img in html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
path = image_path_fn(src)
|
||||
path = await image_path_fn(src)
|
||||
|
||||
if path:
|
||||
img.set('src', path)
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
import os
|
||||
|
||||
import settings
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \
|
||||
synced_images_ids, synced_image_id
|
||||
from sync.core.album import SyncAlbum
|
||||
from sync.data import list_images
|
||||
from utils.fn import lmap, lmapfn, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
|
||||
|
||||
class ImageCreator:
|
||||
def __init__(self, provider, cache):
|
||||
super().__init__()
|
||||
self.provider = provider
|
||||
self.cache = cache if cache else set()
|
||||
|
||||
def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
|
||||
filehash = self.provider.hash(file)
|
||||
cached = self.cached(filehash)
|
||||
return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir,
|
||||
skip_processing=skip_processing) if not cached else cached
|
||||
|
||||
def cached(self, name):
|
||||
return self.cache[name] if name in self.cache else None
|
||||
|
||||
|
||||
def get_manifest(provider, filepath):
|
||||
data = provider.read(filepath).read().decode('utf-8')
|
||||
y, m = read_yaml_md(data)
|
||||
|
||||
data = {**y} if y else {}
|
||||
_ = lambda key, d=None: data[key] if key in data else d
|
||||
|
||||
date = create_date(str(_('date')), settings.date_formats_direct)
|
||||
if date:
|
||||
data['date'] = date
|
||||
|
||||
album_id = _('id')
|
||||
if album_id:
|
||||
data['id'] = str(album_id)
|
||||
|
||||
post = {'post': m} if m else {}
|
||||
return {**data, **post}
|
||||
|
||||
|
||||
def album_from_folder(sync, path):
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
date, title = create_date_and_title_from_folder_name(
|
||||
os.path.basename(path)
|
||||
)
|
||||
|
||||
images = lmap(
|
||||
lambda i: os.path.relpath(i, path),
|
||||
list_images(sync.provider, path)
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
document = {
|
||||
'folder': path,
|
||||
'date': date,
|
||||
'title': title,
|
||||
'post': create_post_from_image_list(images),
|
||||
'images': images
|
||||
}
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
manifest = sync.provider.type_filter(path, '.md')
|
||||
manifest = manifest[0] if len(manifest) else None
|
||||
manifest = get_manifest(sync.provider, manifest) if manifest else {}
|
||||
manifest = manifest if manifest else {}
|
||||
|
||||
document = {
|
||||
**document,
|
||||
**manifest
|
||||
}
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
document = sync.create_id(document)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
document = sync.create_hash(document)
|
||||
|
||||
return document
|
||||
|
||||
|
||||
def main(sync, do_update, do_delete):
|
||||
# GET ALBUMS FOLDERS
|
||||
documents = lmap(
|
||||
lambda i: i,
|
||||
filter(
|
||||
lambda i: sync.provider.is_dir(i),
|
||||
sync.provider.scan('.')
|
||||
)
|
||||
)
|
||||
|
||||
documents = lmap(
|
||||
lambda folder: album_from_folder(sync, folder),
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmapfn(documents)(
|
||||
lambda i: i['id']
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
documents = lmap(
|
||||
sync.create_album,
|
||||
documents
|
||||
)
|
||||
|
||||
# # FILTER INCORRECT EVENTS
|
||||
documents = list(filter(None, documents))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('images', synced_images_ids),
|
||||
documents
|
||||
)
|
||||
documents = lmap(
|
||||
key_mapper('preview', synced_image_id),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC OBJECT WITH DB
|
||||
if do_update:
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
|
||||
if do_delete:
|
||||
documents_to_remove = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_to_remove
|
||||
|
||||
|
||||
def sync_albums(provider, collection, update=True, delete=True):
|
||||
"""
|
||||
:param collection:
|
||||
:param provider:
|
||||
:param update:
|
||||
:param delete:
|
||||
:return:
|
||||
"""
|
||||
|
||||
return main(
|
||||
SyncAlbum(
|
||||
collection,
|
||||
provider,
|
||||
image_creator=ImageCreator(provider, cache=None),
|
||||
images_dir=settings.dir_static_images
|
||||
),
|
||||
do_update=update,
|
||||
do_delete=delete
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.models.Album import Album
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Album')
|
||||
|
||||
|
||||
class SyncAlbum(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Album.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build()
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,139 +0,0 @@
|
||||
import os
|
||||
from lxml import etree
|
||||
|
||||
import lxml.html
|
||||
import settings
|
||||
from sync import create_post
|
||||
from sync.core import Sync
|
||||
from sync.models.product import create_product
|
||||
from utils.fn import lmap, iterate_iter_over_fns, combine
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
def create_url(a, i, s, e):
|
||||
return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
|
||||
album=a,
|
||||
id=i,
|
||||
size=s,
|
||||
ext=e
|
||||
)
|
||||
|
||||
|
||||
def url_creator(aid, pid):
|
||||
return lambda size, ext: create_url(aid, pid, size, ext.lower())
|
||||
|
||||
|
||||
class SyncAlbum(Sync):
|
||||
@staticmethod
|
||||
def id_from_file(i):
|
||||
return url_encode_text(
|
||||
os.path.splitext(i)[0]
|
||||
)
|
||||
|
||||
def __init__(self, collection, provider, image_creator, images_dir):
|
||||
super().__init__()
|
||||
self.collection = collection
|
||||
self.provider = provider
|
||||
self.image_creator = image_creator
|
||||
self.skip_image_processing = False
|
||||
self.images_dir = images_dir
|
||||
|
||||
def create_id(self, document):
|
||||
if 'id' in document:
|
||||
return document
|
||||
|
||||
y = str(document['date'].year)
|
||||
document['id'] = url_encode_text('-'.join([y, document['title']]))
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
files = sorted(document['images'])
|
||||
hashes = lmap(
|
||||
self.provider.hash,
|
||||
lmap(
|
||||
lambda i: os.path.join(document['folder'], i),
|
||||
files
|
||||
)
|
||||
)
|
||||
|
||||
document['hash'] = hash_str(
|
||||
combine([hash_str(document)] + hashes)
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def create_image(self, file, url_fn):
|
||||
ss = settings.album_image_sizes
|
||||
od = self.images_dir
|
||||
sp = self.skip_image_processing
|
||||
|
||||
return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
|
||||
|
||||
def create_album(self, doc):
|
||||
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
|
||||
|
||||
image_id = SyncAlbum.id_from_file
|
||||
image_path = lambda i: os.path.join(doc['folder'], i)
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
return None
|
||||
|
||||
create_album_id = lambda date, title: '{date}-{title}'.format(
|
||||
date=date.strftime('%Y'),
|
||||
title=url_encode_text(title)
|
||||
)
|
||||
|
||||
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
|
||||
|
||||
if 'album' in doc:
|
||||
doc['album'] = self.create_products(doc, image_path)
|
||||
|
||||
def process_post_image(src):
|
||||
path = image_path(src)
|
||||
|
||||
if self.provider.exists(path):
|
||||
url = url_creator(
|
||||
doc['id'],
|
||||
image_id(src)
|
||||
)
|
||||
|
||||
image = self.create_image(path, url)
|
||||
if image:
|
||||
images.append(image)
|
||||
return image['data']['big']['url']
|
||||
return None
|
||||
|
||||
images = []
|
||||
post_html = create_post(doc['post'], process_post_image)
|
||||
|
||||
if 'preview' in doc:
|
||||
preview = os.path.join(doc['folder'], doc['preview'])
|
||||
doc['preview'] = self.create_image(preview, url_creator(
|
||||
doc['id'],
|
||||
image_id(doc['preview'])
|
||||
))
|
||||
|
||||
doc['post'] = post_html
|
||||
doc['images'] = images
|
||||
return doc
|
||||
|
||||
def create_products(self, doc, image_file_fn):
|
||||
image_id = SyncAlbum.id_from_file
|
||||
iter_album = iterate_iter_over_fns([
|
||||
lambda product: {
|
||||
**product,
|
||||
'image': self.create_image(
|
||||
image_file_fn(product['file']),
|
||||
url_creator(
|
||||
doc['id'],
|
||||
image_id(product['file'])
|
||||
)
|
||||
)
|
||||
}
|
||||
])
|
||||
|
||||
return lmap(
|
||||
create_product,
|
||||
iter_album(doc['album'])
|
||||
)
|
||||
+247
-17
@@ -1,21 +1,251 @@
|
||||
__author__ = 'Roman Timashev'
|
||||
import logging
|
||||
|
||||
import os
|
||||
|
||||
import lxml.html
|
||||
|
||||
import kazimir
|
||||
import settings
|
||||
from db import collection
|
||||
from kazimir import kazimir_to_html
|
||||
from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \
|
||||
create_post
|
||||
from sync.data import Provider, list_images
|
||||
from sync.models import Model
|
||||
from sync.models.Image import Image
|
||||
|
||||
from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns
|
||||
from utils.hash import hash_str
|
||||
from utils.io import read_yaml_md, parse_yaml_front_matter
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Album')
|
||||
store = collection(settings.collection_albums)
|
||||
|
||||
|
||||
class Album:
|
||||
def __init__(self, type, title, uri, course, course_uri, date, teacher, comment):
|
||||
super().__init__()
|
||||
self.products = []
|
||||
self.type = type
|
||||
self.title = title
|
||||
self.uri = uri
|
||||
self.course = course
|
||||
self.course_uri = course_uri
|
||||
self.date = date
|
||||
self.teacher = teacher
|
||||
self.comment = comment
|
||||
def find(item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
def add_product(self, product):
|
||||
self.products.append(product)
|
||||
|
||||
def __repr__(self, *args, **kwargs):
|
||||
return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products))
|
||||
class Album(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
documents = [i for i in provider.scan('.') if provider.is_dir(i)]
|
||||
documents = [Album.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
date, title = create_date_and_title_from_folder_name(
|
||||
os.path.basename(path)
|
||||
)
|
||||
|
||||
images = list_images(provider, path)
|
||||
images = [os.path.relpath(i, path) for i in images]
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
document = {
|
||||
'folder': path,
|
||||
'date': date,
|
||||
'title': title,
|
||||
'post': create_post_from_image_list(images),
|
||||
'images': images
|
||||
}
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
manifest_file = provider.type_filter(path, '.md')
|
||||
manifest_file = manifest_file[0] if len(manifest_file) else None
|
||||
manifest = get_manifest(provider, manifest_file) if manifest_file else {}
|
||||
manifest = manifest if manifest else {}
|
||||
|
||||
file = manifest_file if manifest_file else path
|
||||
return Album(
|
||||
provider,
|
||||
file,
|
||||
params={
|
||||
**document,
|
||||
**manifest,
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
# self.__id_template: str = '{category}-{file}'
|
||||
# self.__url_template: str = settings.document_url_template
|
||||
# self.__url_preview_template: str = settings.document_url_preview_template
|
||||
|
||||
self.sizes = settings.image_sizes
|
||||
self.version = '1'
|
||||
|
||||
self.origin = settings.origin
|
||||
self.post = None
|
||||
self.images = None
|
||||
super().__init__(provider, store, file, params=params)
|
||||
|
||||
def init(self):
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
def validate(self):
|
||||
document = self.params
|
||||
if 'date' not in document or not document['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in document or not document['title']:
|
||||
return None
|
||||
|
||||
async def build(self, **kwargs):
|
||||
params = self.params
|
||||
image_id = id_from_file
|
||||
images = []
|
||||
|
||||
create_album_id = lambda date, title: '{date}-{title}'.format(
|
||||
date=date.strftime('%Y'),
|
||||
title=url_encode_text(title)
|
||||
)
|
||||
|
||||
# params['id'] = params['id'] if 'id' in params else create_album_id(params['date'], params['title'])
|
||||
|
||||
# if 'album' in params:
|
||||
# params['album'] = self.create_products(params, image_path)
|
||||
|
||||
async def process_post_image(src):
|
||||
url = url_creator(
|
||||
self.id,
|
||||
id_from_file(src)
|
||||
)
|
||||
|
||||
image_path = os.path.join(params['folder'], src)
|
||||
image = await Image.new(self.provider, image_path, self.sizes, url)
|
||||
if image:
|
||||
images.append(image)
|
||||
return image.get_size('big')['url']
|
||||
return None
|
||||
|
||||
post_html = await create_post(params['post'], process_post_image)
|
||||
|
||||
if self.has_param('preview'):
|
||||
preview = self.get_param('preview')
|
||||
preview = self._get_relpath(preview)
|
||||
preview = await Image.new(self.provider, preview, sizes=self.sizes)
|
||||
else:
|
||||
# preview = 'https://art.shlisselburg.org/entrance.jpg'
|
||||
preview = None
|
||||
|
||||
self.post = post_html
|
||||
self.images = images
|
||||
self.preview = preview
|
||||
|
||||
# sizes = kwargs['sizes']
|
||||
# post, images = await create_post(self.provider, self.id, self.params, sizes)
|
||||
# self.post = post
|
||||
# self.images = images
|
||||
#
|
||||
# post, images = await create_post(self.provider, self.id, self.params, sizes)
|
||||
# self.post = kazimir.html_from_tree(post)
|
||||
# self.images = images
|
||||
|
||||
def create_products(self, doc, image_file_fn):
|
||||
pass
|
||||
# image_id = id_from_file
|
||||
# iter_album = iterate_iter_over_fns([
|
||||
# lambda product: {
|
||||
# **product,
|
||||
# 'image': self.create_image(
|
||||
# image_file_fn(product['file']),
|
||||
# url_creator(
|
||||
# doc['id'],
|
||||
# image_id(product['file'])
|
||||
# )
|
||||
# )
|
||||
# }
|
||||
# ])
|
||||
#
|
||||
# return lmap(
|
||||
# create_product,
|
||||
# iter_album(doc['album'])
|
||||
# )
|
||||
|
||||
def bake(self):
|
||||
preview = None if not self.preview else self.preview.id
|
||||
|
||||
return {
|
||||
**self.params,
|
||||
'images': [i.id for i in self.images],
|
||||
'preview': preview,
|
||||
'post': self.post,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
'origin': self.origin,
|
||||
'version': self.version,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
if 'id' in self.params:
|
||||
self.id = self.params['id']
|
||||
else:
|
||||
year = str(self.params['date'].year)
|
||||
title = self.params['title']
|
||||
self.id = url_encode_text('%s-%s' % (year, title))
|
||||
|
||||
def __set_hash(self):
|
||||
files = self.get_param('images')
|
||||
files = [self._get_relpath(i) for i in files]
|
||||
|
||||
if not self.provider.is_dir(self.file):
|
||||
files.append(self.file)
|
||||
|
||||
files = sorted(files)
|
||||
files = [self.provider.hash(i) for i in files]
|
||||
|
||||
self.hash = hash_str(files)
|
||||
self.hash = hash_str(
|
||||
combine([hash_str(self.params)] + files)
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return '<Album hash={} id={} file={}>'.format(self.hash, self.id, self.file)
|
||||
|
||||
|
||||
def get_manifest(provider, filepath):
|
||||
data = provider.read(filepath).read().decode('utf-8')
|
||||
y, m = read_yaml_md(data)
|
||||
|
||||
data = {**y} if y else {}
|
||||
_ = lambda key, d=None: data[key] if key in data else d
|
||||
|
||||
date = create_date(str(_('date')), settings.date_formats_direct)
|
||||
if date:
|
||||
data['date'] = date
|
||||
|
||||
album_id = _('id')
|
||||
if album_id:
|
||||
data['id'] = str(album_id)
|
||||
|
||||
post = {'post': m} if m else {}
|
||||
return {**data, **post}
|
||||
|
||||
|
||||
def create_url(a, i, s, e):
|
||||
return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
|
||||
album=a,
|
||||
id=i,
|
||||
size=s,
|
||||
ext=e
|
||||
)
|
||||
|
||||
|
||||
def url_creator(aid, pid):
|
||||
return lambda file, size, ext: create_url(aid, pid, size, ext.lower())
|
||||
|
||||
|
||||
def id_from_file(i):
|
||||
return url_encode_text(
|
||||
os.path.splitext(i)[0]
|
||||
)
|
||||
Reference in new issue
Block a user