diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 02a40d8b..d8ad6de9 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -138,7 +138,7 @@ dir_awards = f('Awards') dir_schedules = f('Schedules') dir_articles = f('Articles') dir_collective = f('Collective') -dir_gallery = f('Gallery') +dir_albums = f('Gallery') dir_pages = f('Pages') page_url_base = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}' diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 2dcdeff5..4c371fe8 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,6 +1,7 @@ import settings from db import collection from sync.core import Sync +from sync.core.SyncAlbum import SyncAlbum from sync.core.SyncArticle import SyncArticle from sync.core.SyncDocument import SyncDocument from sync.core.SyncPage import SyncPage @@ -68,16 +69,19 @@ async def run(run_interval=0): provider=io(settings.dir_schedules), collection=c(settings.collection_schedules), ) + albums = SyncAlbum( + provider=io(settings.dir_albums), + collection=c(settings.collection_albums), + ) while True: - # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) - await asyncio.wait([ main(documents), main(pages), main(persons), main(articles), main(schedules), + main(albums), ]) if run_interval == 0: diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index cbe3196a..2cb7c6c1 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -89,12 +89,12 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None): return date, title -def create_post(md, image_path_fn): +async def create_post(md, image_path_fn): html = lxml.html.fromstring(md) for img in html.cssselect('img'): src = img.get('src') - path = image_path_fn(src) + path = await image_path_fn(src) if path: img.set('src', path) diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py deleted file mode 100644 index 71a97d41..00000000 --- a/utils/Sync/sync/album.py +++ /dev/null @@ -1,169 +0,0 @@ -import os - -import settings -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \ - synced_images_ids, synced_image_id -from sync.core.album import SyncAlbum -from sync.data import list_images -from utils.fn import lmap, lmapfn, key_mapper -from utils.image import create_image -from utils.io import read_yaml_md - - -class ImageCreator: - def __init__(self, provider, cache): - super().__init__() - self.provider = provider - self.cache = cache if cache else set() - - def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False): - filehash = self.provider.hash(file) - cached = self.cached(filehash) - return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, - skip_processing=skip_processing) if not cached else cached - - def cached(self, name): - return self.cache[name] if name in self.cache else None - - -def get_manifest(provider, filepath): - data = provider.read(filepath).read().decode('utf-8') - y, m = read_yaml_md(data) - - data = {**y} if y else {} - _ = lambda key, d=None: data[key] if key in data else d - - date = create_date(str(_('date')), settings.date_formats_direct) - if date: - data['date'] = date - - album_id = _('id') - if album_id: - data['id'] = str(album_id) - - post = {'post': m} if m else {} - return {**data, **post} - - -def album_from_folder(sync, path): - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - date, title = create_date_and_title_from_folder_name( - os.path.basename(path) - ) - - images = lmap( - lambda i: os.path.relpath(i, path), - list_images(sync.provider, path) - ) - - # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - document = { - 'folder': path, - 'date': date, - 'title': title, - 'post': create_post_from_image_list(images), - 'images': images - } - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - manifest = sync.provider.type_filter(path, '.md') - manifest = manifest[0] if len(manifest) else None - manifest = get_manifest(sync.provider, manifest) if manifest else {} - manifest = manifest if manifest else {} - - document = { - **document, - **manifest - } - - # CREATE DOCUMENT IDENTITY - document = sync.create_id(document) - - # CREATE HASH OF DOCUMENT FILE - document = sync.create_hash(document) - - return document - - -def main(sync, do_update, do_delete): - # GET ALBUMS FOLDERS - documents = lmap( - lambda i: i, - filter( - lambda i: sync.provider.is_dir(i), - sync.provider.scan('.') - ) - ) - - documents = lmap( - lambda folder: album_from_folder(sync, folder), - documents - ) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmapfn(documents)( - lambda i: i['id'] - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # MAP EVENT MANIFEST -> EVENT_OBJECT - documents = lmap( - sync.create_album, - documents - ) - - # # FILTER INCORRECT EVENTS - documents = list(filter(None, documents)) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) - documents = lmap( - key_mapper('preview', synced_image_id), - documents - ) - - # SYNC OBJECT WITH DB - if do_update: - documents = lmap( - sync.update, - documents - ) - - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - - if do_delete: - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove - ) - ) - - return documents, documents_to_remove - - -def sync_albums(provider, collection, update=True, delete=True): - """ - :param collection: - :param provider: - :param update: - :param delete: - :return: - """ - - return main( - SyncAlbum( - collection, - provider, - image_creator=ImageCreator(provider, cache=None), - images_dir=settings.dir_static_images - ), - do_update=update, - do_delete=delete - ) diff --git a/utils/Sync/sync/core/SyncAlbum.py b/utils/Sync/sync/core/SyncAlbum.py new file mode 100644 index 00000000..d15bec7c --- /dev/null +++ b/utils/Sync/sync/core/SyncAlbum.py @@ -0,0 +1,51 @@ +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.models.Album import Album + +logger = logging.getLogger(settings.name + '.Album') + + +class SyncAlbum(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Album.scan(self.provider) + + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build() + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() diff --git a/utils/Sync/sync/core/album.py b/utils/Sync/sync/core/album.py deleted file mode 100644 index 6941656b..00000000 --- a/utils/Sync/sync/core/album.py +++ /dev/null @@ -1,139 +0,0 @@ -import os -from lxml import etree - -import lxml.html -import settings -from sync import create_post -from sync.core import Sync -from sync.models.product import create_product -from utils.fn import lmap, iterate_iter_over_fns, combine -from utils.hash import hash_file, hash_str -from utils.text.transform import url_encode_text - - -def create_url(a, i, s, e): - return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( - album=a, - id=i, - size=s, - ext=e - ) - - -def url_creator(aid, pid): - return lambda size, ext: create_url(aid, pid, size, ext.lower()) - - -class SyncAlbum(Sync): - @staticmethod - def id_from_file(i): - return url_encode_text( - os.path.splitext(i)[0] - ) - - def __init__(self, collection, provider, image_creator, images_dir): - super().__init__() - self.collection = collection - self.provider = provider - self.image_creator = image_creator - self.skip_image_processing = False - self.images_dir = images_dir - - def create_id(self, document): - if 'id' in document: - return document - - y = str(document['date'].year) - document['id'] = url_encode_text('-'.join([y, document['title']])) - return document - - def create_hash(self, document): - files = sorted(document['images']) - hashes = lmap( - self.provider.hash, - lmap( - lambda i: os.path.join(document['folder'], i), - files - ) - ) - - document['hash'] = hash_str( - combine([hash_str(document)] + hashes) - ) - - return document - - def create_image(self, file, url_fn): - ss = settings.album_image_sizes - od = self.images_dir - sp = self.skip_image_processing - - return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp) - - def create_album(self, doc): - # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' - - image_id = SyncAlbum.id_from_file - image_path = lambda i: os.path.join(doc['folder'], i) - - if 'title' not in doc or not doc['title']: - return None - - create_album_id = lambda date, title: '{date}-{title}'.format( - date=date.strftime('%Y'), - title=url_encode_text(title) - ) - - doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) - - if 'album' in doc: - doc['album'] = self.create_products(doc, image_path) - - def process_post_image(src): - path = image_path(src) - - if self.provider.exists(path): - url = url_creator( - doc['id'], - image_id(src) - ) - - image = self.create_image(path, url) - if image: - images.append(image) - return image['data']['big']['url'] - return None - - images = [] - post_html = create_post(doc['post'], process_post_image) - - if 'preview' in doc: - preview = os.path.join(doc['folder'], doc['preview']) - doc['preview'] = self.create_image(preview, url_creator( - doc['id'], - image_id(doc['preview']) - )) - - doc['post'] = post_html - doc['images'] = images - return doc - - def create_products(self, doc, image_file_fn): - image_id = SyncAlbum.id_from_file - iter_album = iterate_iter_over_fns([ - lambda product: { - **product, - 'image': self.create_image( - image_file_fn(product['file']), - url_creator( - doc['id'], - image_id(product['file']) - ) - ) - } - ]) - - return lmap( - create_product, - iter_album(doc['album']) - ) diff --git a/utils/Sync/sync/models/Album.py b/utils/Sync/sync/models/Album.py index f30f6348..3c8fa145 100644 --- a/utils/Sync/sync/models/Album.py +++ b/utils/Sync/sync/models/Album.py @@ -1,21 +1,251 @@ -__author__ = 'Roman Timashev' +import logging + +import os + +import lxml.html + +import kazimir +import settings +from db import collection +from kazimir import kazimir_to_html +from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \ + create_post +from sync.data import Provider, list_images +from sync.models import Model +from sync.models.Image import Image + +from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns +from utils.hash import hash_str +from utils.io import read_yaml_md, parse_yaml_front_matter +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Album') +store = collection(settings.collection_albums) -class Album: - def __init__(self, type, title, uri, course, course_uri, date, teacher, comment): - super().__init__() - self.products = [] - self.type = type - self.title = title - self.uri = uri - self.course = course - self.course_uri = course_uri - self.date = date - self.teacher = teacher - self.comment = comment +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None - def add_product(self, product): - self.products.append(product) - def __repr__(self, *args, **kwargs): - return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products)) +class Album(Model): + @staticmethod + async def scan(provider): + documents = [i for i in provider.scan('.') if provider.is_dir(i)] + documents = [Album.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + date, title = create_date_and_title_from_folder_name( + os.path.basename(path) + ) + + images = list_images(provider, path) + images = [os.path.relpath(i, path) for i in images] + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + document = { + 'folder': path, + 'date': date, + 'title': title, + 'post': create_post_from_image_list(images), + 'images': images + } + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + manifest_file = provider.type_filter(path, '.md') + manifest_file = manifest_file[0] if len(manifest_file) else None + manifest = get_manifest(provider, manifest_file) if manifest_file else {} + manifest = manifest if manifest else {} + + file = manifest_file if manifest_file else path + return Album( + provider, + file, + params={ + **document, + **manifest, + }, + ) + + def __init__(self, provider, file, params=None): + # self.__id_template: str = '{category}-{file}' + # self.__url_template: str = settings.document_url_template + # self.__url_preview_template: str = settings.document_url_preview_template + + self.sizes = settings.image_sizes + self.version = '1' + + self.origin = settings.origin + self.post = None + self.images = None + super().__init__(provider, store, file, params=params) + + def init(self): + self.__set_id() + self.__set_hash() + + def validate(self): + document = self.params + if 'date' not in document or not document['date']: + return None + + if 'title' not in document or not document['title']: + return None + + async def build(self, **kwargs): + params = self.params + image_id = id_from_file + images = [] + + create_album_id = lambda date, title: '{date}-{title}'.format( + date=date.strftime('%Y'), + title=url_encode_text(title) + ) + + # params['id'] = params['id'] if 'id' in params else create_album_id(params['date'], params['title']) + + # if 'album' in params: + # params['album'] = self.create_products(params, image_path) + + async def process_post_image(src): + url = url_creator( + self.id, + id_from_file(src) + ) + + image_path = os.path.join(params['folder'], src) + image = await Image.new(self.provider, image_path, self.sizes, url) + if image: + images.append(image) + return image.get_size('big')['url'] + return None + + post_html = await create_post(params['post'], process_post_image) + + if self.has_param('preview'): + preview = self.get_param('preview') + preview = self._get_relpath(preview) + preview = await Image.new(self.provider, preview, sizes=self.sizes) + else: + # preview = 'https://art.shlisselburg.org/entrance.jpg' + preview = None + + self.post = post_html + self.images = images + self.preview = preview + + # sizes = kwargs['sizes'] + # post, images = await create_post(self.provider, self.id, self.params, sizes) + # self.post = post + # self.images = images + # + # post, images = await create_post(self.provider, self.id, self.params, sizes) + # self.post = kazimir.html_from_tree(post) + # self.images = images + + def create_products(self, doc, image_file_fn): + pass + # image_id = id_from_file + # iter_album = iterate_iter_over_fns([ + # lambda product: { + # **product, + # 'image': self.create_image( + # image_file_fn(product['file']), + # url_creator( + # doc['id'], + # image_id(product['file']) + # ) + # ) + # } + # ]) + # + # return lmap( + # create_product, + # iter_album(doc['album']) + # ) + + def bake(self): + preview = None if not self.preview else self.preview.id + + return { + **self.params, + 'images': [i.id for i in self.images], + 'preview': preview, + 'post': self.post, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + 'origin': self.origin, + 'version': self.version, + } + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + year = str(self.params['date'].year) + title = self.params['title'] + self.id = url_encode_text('%s-%s' % (year, title)) + + def __set_hash(self): + files = self.get_param('images') + files = [self._get_relpath(i) for i in files] + + if not self.provider.is_dir(self.file): + files.append(self.file) + + files = sorted(files) + files = [self.provider.hash(i) for i in files] + + self.hash = hash_str(files) + self.hash = hash_str( + combine([hash_str(self.params)] + files) + ) + + def __str__(self): + return ''.format(self.hash, self.id, self.file) + + +def get_manifest(provider, filepath): + data = provider.read(filepath).read().decode('utf-8') + y, m = read_yaml_md(data) + + data = {**y} if y else {} + _ = lambda key, d=None: data[key] if key in data else d + + date = create_date(str(_('date')), settings.date_formats_direct) + if date: + data['date'] = date + + album_id = _('id') + if album_id: + data['id'] = str(album_id) + + post = {'post': m} if m else {} + return {**data, **post} + + +def create_url(a, i, s, e): + return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( + album=a, + id=i, + size=s, + ext=e + ) + + +def url_creator(aid, pid): + return lambda file, size, ext: create_url(aid, pid, size, ext.lower()) + + +def id_from_file(i): + return url_encode_text( + os.path.splitext(i)[0] + )