diff --git a/utils/Sync/config.yaml b/utils/Sync/config.yaml new file mode 100644 index 00000000..6a4e6ff9 --- /dev/null +++ b/utils/Sync/config.yaml @@ -0,0 +1,23 @@ +--- +database_uri: 'mongodb://localhost:27017/hudozka' + +env: 'production' +origin: 'Yandex.Disk' +yandex_disk_access_token: '' +interval: 0 + +skip_unchanged: True + +image_processing_enabled: True + +upload_enabled: True +upload_auth: + login: '' + password: '' + +provider: + name: 'fs' + root: '' + +update_enabled: True +delete_enabled: True diff --git a/utils/Sync/kazimir/__init__.py b/utils/Sync/kazimir/__init__.py index 8f71421e..81d330b3 100644 --- a/utils/Sync/kazimir/__init__.py +++ b/utils/Sync/kazimir/__init__.py @@ -16,7 +16,7 @@ def markdown_to_html(text): def kazimir_to_html(text): - html = markdown_to_html(text) + html = make(text) return typo(html) @@ -41,15 +41,209 @@ def typo(html): return html +def make(text): + tokens = text.split('\n') + tokens = markup_tokens(tokens) + tokens = join_tokens(tokens) + tokens = merge_tokens(tokens) + + return compile_tokens(tokens) + # return tokens + + +def join_tokens(tokens): + buffer_tag = None + buffer = [] + ts = [] + for token in tokens: + tag = token['tag'] + changed = buffer_tag is not None and tag != buffer_tag + + flush = is_tag_split(tag) + if changed: + flush = True + + if not is_tag_joinable(tag): + flush = True + + if flush: + if buffer: + ts.append(buffer) + if is_tag_split(tag): + buffer = [] + buffer_tag = None + else: + buffer = [token] + buffer_tag = tag + else: + buffer_tag = tag + buffer.append(token) + + if len(buffer): + ts.append(buffer) + return ts + + +def merge_tokens(tokens: list): + tokens_out = [] + for ts in tokens: + tag = ts[0]['tag'] + fn = None + if tag == 'image': + fn = merge_image + elif tag == 'text': + fn = merge_text + else: + fn = merge_any + tokens_out.append(fn(ts)) + return tokens_out + + +def compile_tokens(tokens: list): + compiled = [] + for ts in tokens: + tag = ts['tag'] + fn = None + if tag == 'text': + fn = compile_text + elif tag == 'image_collection': + fn = compile_image_collection + elif tag == 'image': + fn = compile_image + else: + fn = compile_any + compiled.append(fn(ts)) + return '\n'.join(compiled) + + +def markup_tokens(tokens: list): + marked_tokens = [] + for token in tokens: + tag = 'text' + if is_url(token): + tag = 'url' + elif is_image(token): + tag = 'image' + elif not token: + tag = 'split' + token = None + + marked_tokens.append({ + 'tag': tag, + 'data': token, + }) + return marked_tokens + + +def is_tag_split(tag): + return tag == 'split' + + +def is_tag_joinable(tag: str): + return tag not in ['url'] + + +def is_url(sample: str): + from urllib.parse import urlparse + url = urlparse(sample) + + return url.scheme and url.netloc + + +def is_image(sample: str): + return sample[:2] == '![' and sample[-1:] == ')' + + +def merge_any(tokens): + return tokens[0] + + +def merge_text(tokens): + data = '' + + for t in tokens: + data += t['data'] + data += '\n' + + return { + 'tag': 'text', + 'data': data.strip(), + } + + +def merge_image(tokens): + if len(tokens) == 1: + return tokens[0] + else: + data = [t['data'] for t in tokens] + return { + 'tag': 'image_collection', + 'data': data, + } + + +def compile_text(token): + return markdown_to_html(token['data']) + + +def compile_image_collection(token): + file = lambda i: i[4:][:-1] + imgs = [''.format(file(i)) for i in token['data']] + return ''' +
+
+ {} +
+
+ '''.format('\n'.join(imgs)) + + +def compile_image(token): + img = markdown_to_html(token['data']) + return ''' +
+ {} +
+ '''.format(img) + + +def compile_any(token): + return markdown_to_html(token['data']) + + if __name__ == '__main__': sample_text = ''' Hello! https://www.instagram.com/p/BUelE04l1kU +ku! https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka +https://www.youtube.com/watch?v=qnguOqWpraI +![](1.jpg) +![](2.jpg) +![](3.jpg) + +LOL NOOB +![](12.jpg) +![](22.jpg) +WTF OMG Bye! +Fye! + +![](1.jpg) + +More text ''' - print(markdown_to_html(sample_text)) + ts = make(sample_text) + if isinstance(ts, list): + for i in ts: + print(i) + else: + print(ts) + diff --git a/utils/Sync/kazimir/instagram.py b/utils/Sync/kazimir/instagram.py index a1cc4d06..ee5f8c0c 100644 --- a/utils/Sync/kazimir/instagram.py +++ b/utils/Sync/kazimir/instagram.py @@ -55,7 +55,7 @@ def embed_instagram(media): i = etree.fromstring(html) obj = etree.Element('div') - obj.set('class', "kazimir-embed") + obj.set('class', "kazimir__embed") obj.append(i) return obj else: diff --git a/utils/Sync/kazimir/video.py b/utils/Sync/kazimir/video.py index 4f7060bd..8cc2df47 100644 --- a/utils/Sync/kazimir/video.py +++ b/utils/Sync/kazimir/video.py @@ -33,6 +33,6 @@ def flex_video(url): iframe.set('allowfullscreen', '') obj = etree.Element('div') - obj.set('class', "kazimir-video") + obj.set('class', "kazimir__video") obj.append(iframe) return obj diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 882aa061..d8ad6de9 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -1,7 +1,22 @@ import os +import yaml +import argparse + __author__ = 'Roman Timashev' +name = 'HudozkaSync' + +_parser = argparse.ArgumentParser(description='Hudozka Sync Daemon') +_parser.add_argument('--config', dest='config', default='config.yaml', help='path to config.yaml') +_args = _parser.parse_args() + +config = yaml.load(open(_args.config, 'r')) + + +def value(key, default=None): + return config[key] if key in config else default + def absolute(path, ensure=True): a = os.path.expanduser(path) @@ -20,9 +35,11 @@ def env(param, default=None): return default -database_uri = env('MONGO_URI') +interval = value('interval', 0) -skip_unchanged = False +database_uri = value('database_uri') # env('MONGO_URI') + +skip_unchanged = value('skip_unchanged') person_uri = { 'Н.В.Андреева': 'nv-andreeva', @@ -33,18 +50,35 @@ person_uri = { 'А.С.Тимашева': 'as-timasheva', } -image_processing = True +upload_enabled = value('upload_enabled') +upload_url_image = 'https://static.shlisselburg.org/upload/art/images/{}' +upload_auth = ( + value('upload_auth')['login'], + value('upload_auth')['password'], +) + +image_processing_enabled = value('image_processing') album_html_img_class = 'hudozka-product' image_ext = '.jpg' image_sizes = [ ('original', None, None), + ('big', 1500, 667), + ('big@2', 3000, 1334), + ('medium', 400, 300), + ('medium@2', 800, 600), + ('small', 250, 175), - ('little', 100, 100), - ('preview', 50, 50), + ('small@2', 500, 350), + + ('og', 968, 504), + ('og@2', 1936, 1008), + + ('fb', 1200, 630), + ('fb@2', 2400, 1260), ] awards_image_sizes = [ @@ -57,14 +91,6 @@ event_image_sizes = [ ('big', 1200, 800) ] -teacher_image_url_base = 'https://static.shlisselburg.org/art/images/teacher-{id}-{size}{ext}' -teacher_image_output = '/Users/tmshv/Desktop/Hudozka Static/images' -teacher_image_sizes = [ - ('big', 1200, 800), - ('medium', 640, 480), - ('preview', 50, 50), -] - album_image_sizes = [ ('big', 1200, 800), ('medium', 640, 480), @@ -91,8 +117,9 @@ date_formats_reverse = [ '%Y.%m.%d' ] -origin = 'Yandex.Disk' +origin = value('origin') +collection_images = 'images' collection_documents = 'documents' collection_awards = 'awards' collection_schedules = 'schedules' @@ -101,34 +128,32 @@ collection_collective = 'collective' collection_albums = 'albums' collection_pages = 'pages' -sync_provider_type = env('SYNC_PROVIDER', 'fs') +provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs') +provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') -_providers_roots = { - 'fs': abs_fn(env('SYNC_LOCAL_PATH')), - 'yd': abs_fn('/'), -} -f = _providers_roots[sync_provider_type] +f = abs_fn(provider_root) -dir_documents = f('Hudozka/Site/Documents') -dir_awards = f('Hudozka/Site/Awards') -dir_schedules = f('Hudozka/Site/Schedules') -dir_articles = f('Hudozka/Site/Articles') -dir_collective = f('Hudozka/Site/Collective') -dir_gallery = f('Hudozka/Site/Gallery') -dir_pages = f('Hudozka/Site/Pages') +dir_documents = f('Documents') +dir_awards = f('Awards') +dir_schedules = f('Schedules') +dir_articles = f('Articles') +dir_collective = f('Collective') +dir_albums = f('Gallery') +dir_pages = f('Pages') -dir_static = env('SYNC_STATIC', '~/Hudozka Static') -dir_static_uploads = absolute(dir_static + '/uploads') -dir_static_images = absolute(dir_static + '/images') +page_url_base = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}' -url_page_base_preview = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}' -url_base_preview = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' -url_base_document = 'https://static.shlisselburg.org/art/uploads/{file}' +document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}' +document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploads/{file}' +document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' -image_base_url = 'https://static.shlisselburg.org/art/images/' +image_url_upload = 'https://static.shlisselburg.org/upload/art/images/' +image_url_base = 'https://static.shlisselburg.org/art/images/' image_name_format = '{type}-{id}-{img}-{size}{ext}' -yandex_disk_access_token = env('YANDEX_DISK_ACCESS_TOKEN') +person_image_url_template = 'https://static.shlisselburg.org/art/images/person-{id}-{size}{ext}' -do_update = env('SYNC_ENV', 'production') == 'production' -do_delete = env('SYNC_ENV', 'production') == 'production' +yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN') + +update_enabled = value('update_enabled') +delete_enabled = value('delete_enabled') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index ad7fd908..4c371fe8 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,15 +1,93 @@ import settings -from db import db, collection -from sync.album import sync_albums +from db import collection +from sync.core import Sync +from sync.core.SyncAlbum import SyncAlbum +from sync.core.SyncArticle import SyncArticle +from sync.core.SyncDocument import SyncDocument +from sync.core.SyncPage import SyncPage +from sync.core.SyncPerson import SyncPerson +from sync.core.SyncSchedule import SyncSchedule from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider -from sync.document import sync_documents -from sync.page import sync_pages -from sync.person import sync_persons -from sync.post import sync_posts -from sync.schedule import sync_schedules -from utils.fn import lprint_json -from utils.image import thumbnail + +import asyncio +import logging + +# create logger with 'spam_application' +logger = logging.getLogger(settings.name) +logger.setLevel(logging.DEBUG) + +# create file handler which logs even debug messages +fh = logging.FileHandler('hudozka.log') +fh.setLevel(logging.DEBUG) + +# create console handler with a higher log level +ch = logging.StreamHandler() +# ch.setLevel(logging.ERROR) +ch.setLevel(logging.DEBUG) + +# create formatter and add it to the handlers +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) + +# add the handlers to the logger +logger.addHandler(fh) +logger.addHandler(ch) + + +async def main(sync: Sync): + await sync.run() + + +async def run(run_interval=0): + io = lambda root: get_provider(settings.provider_name, root) + c = collection + + documents = SyncDocument( + provider=io(settings.dir_documents), + collection=c(settings.collection_documents), + sizes=settings.image_sizes, + ) + + pages = SyncPage( + provider=io(settings.dir_pages), + collection=c(settings.collection_pages), + ) + + persons = SyncPerson( + provider=io(settings.dir_collective), + collection=c(settings.collection_collective), + ) + + articles = SyncArticle( + provider=io(settings.dir_articles), + collection=c(settings.collection_articles), + ) + + schedules = SyncSchedule( + provider=io(settings.dir_schedules), + collection=c(settings.collection_schedules), + ) + albums = SyncAlbum( + provider=io(settings.dir_albums), + collection=c(settings.collection_albums), + ) + + while True: + await asyncio.wait([ + main(documents), + main(pages), + main(persons), + main(articles), + main(schedules), + main(albums), + ]) + + if run_interval == 0: + break + else: + await asyncio.sleep(run_interval) def get_provider(provider_type, root): @@ -20,27 +98,12 @@ def get_provider(provider_type, root): return None -def main(provider, fn, collection_name): - if provider: - print('SYNC: %s' % (collection_name.upper())) - u, d = fn(provider, collection(collection_name), update=settings.do_update, delete=settings.do_update) - - print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d)))) - lprint_json(d) - - print('UPDATE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_update else str(len(u)))) - lprint_json(u) - - print('') - - if __name__ == '__main__': - io = lambda root: get_provider(settings.sync_provider_type, root) + interval = settings.interval - main(io(settings.dir_documents), sync_documents, settings.collection_documents) - main(io(settings.dir_awards), sync_documents, settings.collection_awards) - main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) - main(io(settings.dir_articles), sync_posts, settings.collection_articles) - main(io(settings.dir_collective), sync_persons, settings.collection_collective) - main(io(settings.dir_gallery), sync_albums, settings.collection_albums) - main(io(settings.dir_pages), sync_pages, settings.collection_pages) + loop = asyncio.get_event_loop() + loop.run_until_complete(run(interval)) + loop.close() + + # Да, 968×504 пискеля это меньше минимально рекомендованных Фейсбуком 1200×630. + # Зато при таком размере и ратио картинку нигде не кропят, и выглядит она отлично. diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index 10357e94..2cb7c6c1 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -9,6 +9,7 @@ from markdown import markdown import settings from sync.image import sync_image +from sync.models import Model from utils.fn import lmap, combine @@ -43,26 +44,30 @@ def create_date(date_str, date_formats=None): folder_name_pattern = re.compile('([\d.]+)(.*)') -def untouched(documents, store): +async def untouched(items: [Model]) -> [Model]: if settings.skip_unchanged: - return documents - return lmap( - lambda i: i[0], - filter( - lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), - map( - lambda i: (i, store.read(i)), - documents - ) - )) + return items + + result = [] + for item in items: + status = await item.is_changed() + if status: + result.append(item) + return result + + # def is_equals(e, n): + # return (n is None) or ('hash' not in n) or (e.hash != n['hash']) + # + # doc = lambda i: {'id': i.id, 'hash': i.hash} + # + # stored_documents = [(i, store.read(doc(i))) for i in documents] + # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)] + + # return filtered_documents def synced_images_ids(images): - ids = lmap( - lambda image: image['_id'], - sync_image(images) - ) - return ids + return [i['_id'] for i in sync_image(images)] def synced_image_id(image): @@ -84,12 +89,12 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None): return date, title -def create_post(md, image_path_fn): +async def create_post(md, image_path_fn): html = lxml.html.fromstring(md) for img in html.cssselect('img'): src = img.get('src') - path = image_path_fn(src) + path = await image_path_fn(src) if path: img.set('src', path) diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py deleted file mode 100644 index 71a97d41..00000000 --- a/utils/Sync/sync/album.py +++ /dev/null @@ -1,169 +0,0 @@ -import os - -import settings -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \ - synced_images_ids, synced_image_id -from sync.core.album import SyncAlbum -from sync.data import list_images -from utils.fn import lmap, lmapfn, key_mapper -from utils.image import create_image -from utils.io import read_yaml_md - - -class ImageCreator: - def __init__(self, provider, cache): - super().__init__() - self.provider = provider - self.cache = cache if cache else set() - - def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False): - filehash = self.provider.hash(file) - cached = self.cached(filehash) - return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, - skip_processing=skip_processing) if not cached else cached - - def cached(self, name): - return self.cache[name] if name in self.cache else None - - -def get_manifest(provider, filepath): - data = provider.read(filepath).read().decode('utf-8') - y, m = read_yaml_md(data) - - data = {**y} if y else {} - _ = lambda key, d=None: data[key] if key in data else d - - date = create_date(str(_('date')), settings.date_formats_direct) - if date: - data['date'] = date - - album_id = _('id') - if album_id: - data['id'] = str(album_id) - - post = {'post': m} if m else {} - return {**data, **post} - - -def album_from_folder(sync, path): - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - date, title = create_date_and_title_from_folder_name( - os.path.basename(path) - ) - - images = lmap( - lambda i: os.path.relpath(i, path), - list_images(sync.provider, path) - ) - - # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - document = { - 'folder': path, - 'date': date, - 'title': title, - 'post': create_post_from_image_list(images), - 'images': images - } - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - manifest = sync.provider.type_filter(path, '.md') - manifest = manifest[0] if len(manifest) else None - manifest = get_manifest(sync.provider, manifest) if manifest else {} - manifest = manifest if manifest else {} - - document = { - **document, - **manifest - } - - # CREATE DOCUMENT IDENTITY - document = sync.create_id(document) - - # CREATE HASH OF DOCUMENT FILE - document = sync.create_hash(document) - - return document - - -def main(sync, do_update, do_delete): - # GET ALBUMS FOLDERS - documents = lmap( - lambda i: i, - filter( - lambda i: sync.provider.is_dir(i), - sync.provider.scan('.') - ) - ) - - documents = lmap( - lambda folder: album_from_folder(sync, folder), - documents - ) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmapfn(documents)( - lambda i: i['id'] - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # MAP EVENT MANIFEST -> EVENT_OBJECT - documents = lmap( - sync.create_album, - documents - ) - - # # FILTER INCORRECT EVENTS - documents = list(filter(None, documents)) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) - documents = lmap( - key_mapper('preview', synced_image_id), - documents - ) - - # SYNC OBJECT WITH DB - if do_update: - documents = lmap( - sync.update, - documents - ) - - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - - if do_delete: - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove - ) - ) - - return documents, documents_to_remove - - -def sync_albums(provider, collection, update=True, delete=True): - """ - :param collection: - :param provider: - :param update: - :param delete: - :return: - """ - - return main( - SyncAlbum( - collection, - provider, - image_creator=ImageCreator(provider, cache=None), - images_dir=settings.dir_static_images - ), - do_update=update, - do_delete=delete - ) diff --git a/utils/Sync/sync/compile.py b/utils/Sync/sync/compile.py deleted file mode 100644 index c9da54d8..00000000 --- a/utils/Sync/sync/compile.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -from tempfile import mkstemp - -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text - - -class Preview: - def __init__(self, provider, url_base, output, sizes): - super().__init__() - self.provider = provider - self.url_base = url_base - self.output = output - self.sizes = sizes - - def create(self, file): - def url(size, ext): - return self.url_base.format(id=file, size=size, ext=ext) - - temp_preview_path = self.pdf_to_jpg(file) - file = os.path.basename(file) - file = url_encode_text(file) - img = create_image(temp_preview_path, self.sizes, url, self.output) - os.remove(temp_preview_path) - return img - - def pdf_to_jpg(self, pdf): - _, temp_in = mkstemp(suffix='.pdf') - _, temp_out = mkstemp(suffix='.jpg') - - abspdf = self.provider.copy(pdf, temp_in) - image_magick_pdf_to_img(abspdf, temp_out) - - os.remove(temp_in) - return temp_out diff --git a/utils/Sync/sync/core/SyncAlbum.py b/utils/Sync/sync/core/SyncAlbum.py new file mode 100644 index 00000000..d15bec7c --- /dev/null +++ b/utils/Sync/sync/core/SyncAlbum.py @@ -0,0 +1,51 @@ +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.models.Album import Album + +logger = logging.getLogger(settings.name + '.Album') + + +class SyncAlbum(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Album.scan(self.provider) + + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build() + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() diff --git a/utils/Sync/sync/core/SyncArticle.py b/utils/Sync/sync/core/SyncArticle.py new file mode 100644 index 00000000..666079a5 --- /dev/null +++ b/utils/Sync/sync/core/SyncArticle.py @@ -0,0 +1,58 @@ +import os + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import Provider +from sync.models.Article import Article +from sync.models.Person import Person + +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Article') + + +class SyncArticle(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Article.scan(self.provider) + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build( + sizes=settings.image_sizes, + ) + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() \ No newline at end of file diff --git a/utils/Sync/sync/core/SyncDocument.py b/utils/Sync/sync/core/SyncDocument.py new file mode 100644 index 00000000..704b493a --- /dev/null +++ b/utils/Sync/sync/core/SyncDocument.py @@ -0,0 +1,186 @@ +import os +from tempfile import mkstemp + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import scan_subdirs, Provider +from sync.models.Document import Document + +from sync.models import Model +from utils.fn import last_good, lmap +from utils.hash import hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + +logger = logging.getLogger(settings.name + '.Document') + + +class SyncDocument(Sync): + def __init__(self, provider: Provider, collection, sizes): + super().__init__(provider, collection) + self.sizes = sizes + self.dir_static_previews = '' + + self.temp_dir = '/Users/tmshv/HudozkaSyncTemp' + + async def clean(self): + pass + + async def run(self): + logger.info('Checking documents for update') + + documents = await Document.scan(self.provider) + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Document(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + # documents = untouched(documents, self) + documents = await untouched(documents) + logger.info('Changed {} Document(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Documents to update') + + for document in documents: + logger.info('Updating Document {}'.format(document.id)) + + await document.upload() + await document.create_preview(self.sizes) + await document.save() + + logger.info('Updated Document {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Documents to delete') + + for document in documents_delete: + logger.info('Deleting {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + await self.clean() + + +async def get_assets(self, document) -> []: + return [] + + +def validate(self, documents): + files = [doc['file'] for doc in documents] + file_names = [os.path.basename(f) for f in files] + file_names = set(file_names) + + if len(file_names) != len(documents): + raise Exception('File names should be unique') + + +def create(self, document): + file = document['file'] + filename = os.path.basename(file) + + document['type'] = document_type(document) + document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) + document['title'] = last_good([ + document['title'], + get_pdf_title(self.provider, document['file']) + ]) + document['file'] = { + 'name': filename, + 'size': self.provider.size(file) + } + + return document + + +def create_id(self, document: dict) -> dict: + filepath = document['file'] + filename = os.path.basename(filepath) + + return { + **document, + 'id': self.id_template.format( + category=url_encode_text(document['category']), + file=url_encode_text(filename) + ), + } + + +def create_url(self, document: dict) -> dict: + filename = os.path.basename(document['file']) + + return { + **document, + 'url': self.url_template.format( + file=url_encode_file(filename) + ), + } + + +def create_hash(self, document: dict) -> dict: + hash_doc = [document[key] for key in self.hash_keys] + + return { + **document, + 'hash': hash_str( + hash_str(hash_doc) + self.provider.hash(document['file']) + ), + } + + +def create_preview(self, file, sizes, preview_dir): + def url(size, ext): + return self.url_base_preview.format(id=file, size=size, ext=ext) + + temp_preview_path = pdf_to_jpg(self.provider, file) + file = os.path.basename(file) + file = url_encode_text(file) + img = create_image(temp_preview_path, sizes, url, preview_dir) + os.remove(temp_preview_path) + return img + + +def document_type(document): + if document['category'] == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None diff --git a/utils/Sync/sync/core/SyncPage.py b/utils/Sync/sync/core/SyncPage.py new file mode 100644 index 00000000..705c42ee --- /dev/null +++ b/utils/Sync/sync/core/SyncPage.py @@ -0,0 +1,77 @@ +import os + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import Provider +from sync.models.Page import Page + +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Page') + + +class SyncPage(Sync): + def __init__(self, provider: Provider, collection): + super().__init__(provider, collection) + + async def clean(self): + pass + + async def run(self): + logger.info('Checking for update') + + documents = await Page.scan(self.provider) + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Page(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Pages(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Pages to update') + + for document in documents: + logger.info('Updating Page {}'.format(document.id)) + + await document.setup_images(settings.image_sizes, get_image_url_fn(document.id)) + await document.save() + + logger.info('Updated Page {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Pages to delete') + else: + for document in documents_delete: + logger.info('Deleting Page {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + await self.clean() + + +def get_image_url_fn(page_id: str): + image_id = lambda filename: url_encode_text( + os.path.splitext(os.path.basename(filename))[0] + ) + + def img_url_fn(file, size, ext): + return settings.page_url_base.format( + page=page_id, + id=image_id(file), + size=size, + ext=ext + ) + + return img_url_fn diff --git a/utils/Sync/sync/core/SyncPerson.py b/utils/Sync/sync/core/SyncPerson.py new file mode 100644 index 00000000..9b5d758b --- /dev/null +++ b/utils/Sync/sync/core/SyncPerson.py @@ -0,0 +1,75 @@ +import os + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import Provider +from sync.models.Person import Person + +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Person') + + +class SyncPerson(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Person.scan(self.provider) + + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build( + sizes=settings.image_sizes, + url_factory=get_image_url_fn(document.id), + ) + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() + + +def get_image_url_fn(page_id: str): + image_id = lambda filename: url_encode_text( + os.path.splitext(os.path.basename(filename))[0] + ) + + def img_url_fn(file, size, ext): + return settings.page_url_base.format( + page=page_id, + id=image_id(file), + size=size, + ext=ext + ) + + return img_url_fn diff --git a/utils/Sync/sync/core/SyncSchedule.py b/utils/Sync/sync/core/SyncSchedule.py new file mode 100644 index 00000000..39579bb4 --- /dev/null +++ b/utils/Sync/sync/core/SyncSchedule.py @@ -0,0 +1,51 @@ +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.models.Schedule import Schedule + +logger = logging.getLogger(settings.name + '.Schedule') + + +class SyncSchedule(Sync): + async def run(self): + logger.info('Checking for update') + + documents = await Schedule.scan(self.provider) + + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Item(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + documents = await untouched(documents) + logger.info('Changed {} Items(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Items to update') + + for document in documents: + logger.info('Updating Item {}'.format(document.id)) + + await document.build() + await document.save() + + logger.info('Updated Item {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Items to delete') + else: + for document in documents_delete: + logger.info('Deleting Item {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + # await self.clean() diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py index 14e73d43..2df6a9d6 100644 --- a/utils/Sync/sync/core/__init__.py +++ b/utils/Sync/sync/core/__init__.py @@ -1,21 +1,43 @@ +from sync.data import Provider from sync.models import Model from utils.fn import lmap +def get_id(item): + try: + return item.id + except AttributeError: + return item['id'] + + class Sync: @staticmethod def compile(document): return document.bake() - def __init__(self): + def __init__(self, provider: Provider, collection): super().__init__() - self.collection = None + self.provider = provider + self.collection = collection - def read(self, document): - if isinstance(document, list): - return lmap(self.read, document) + async def run(self): + """ + # Get scope files + # Validate these files. Raise an error + # calc_hashes + # make diff with previous run + # compile objects + # upload + :return: + """ + return None, None + + async def upload(self): + pass + + def read(self, document: dict): q = {'id': document['id']} try: return self.collection.find_one(q) @@ -23,16 +45,10 @@ class Sync: pass return None - def update(self, document): - if isinstance(document, list): - return lmap(self.update, document) - - if isinstance(document, Model): - return self.update(self.compile(document)) - - q = {'id': document['id']} + def update(self, document: dict): + query = {'id': document['id']} try: - self.collection.update_one(q, {'$set': document}, upsert=True) + self.collection.update_one(query, {'$set': document}, upsert=True) return self.read(document) except ValueError: pass diff --git a/utils/Sync/sync/core/album.py b/utils/Sync/sync/core/album.py deleted file mode 100644 index 6941656b..00000000 --- a/utils/Sync/sync/core/album.py +++ /dev/null @@ -1,139 +0,0 @@ -import os -from lxml import etree - -import lxml.html -import settings -from sync import create_post -from sync.core import Sync -from sync.models.product import create_product -from utils.fn import lmap, iterate_iter_over_fns, combine -from utils.hash import hash_file, hash_str -from utils.text.transform import url_encode_text - - -def create_url(a, i, s, e): - return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( - album=a, - id=i, - size=s, - ext=e - ) - - -def url_creator(aid, pid): - return lambda size, ext: create_url(aid, pid, size, ext.lower()) - - -class SyncAlbum(Sync): - @staticmethod - def id_from_file(i): - return url_encode_text( - os.path.splitext(i)[0] - ) - - def __init__(self, collection, provider, image_creator, images_dir): - super().__init__() - self.collection = collection - self.provider = provider - self.image_creator = image_creator - self.skip_image_processing = False - self.images_dir = images_dir - - def create_id(self, document): - if 'id' in document: - return document - - y = str(document['date'].year) - document['id'] = url_encode_text('-'.join([y, document['title']])) - return document - - def create_hash(self, document): - files = sorted(document['images']) - hashes = lmap( - self.provider.hash, - lmap( - lambda i: os.path.join(document['folder'], i), - files - ) - ) - - document['hash'] = hash_str( - combine([hash_str(document)] + hashes) - ) - - return document - - def create_image(self, file, url_fn): - ss = settings.album_image_sizes - od = self.images_dir - sp = self.skip_image_processing - - return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp) - - def create_album(self, doc): - # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' - - image_id = SyncAlbum.id_from_file - image_path = lambda i: os.path.join(doc['folder'], i) - - if 'title' not in doc or not doc['title']: - return None - - create_album_id = lambda date, title: '{date}-{title}'.format( - date=date.strftime('%Y'), - title=url_encode_text(title) - ) - - doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) - - if 'album' in doc: - doc['album'] = self.create_products(doc, image_path) - - def process_post_image(src): - path = image_path(src) - - if self.provider.exists(path): - url = url_creator( - doc['id'], - image_id(src) - ) - - image = self.create_image(path, url) - if image: - images.append(image) - return image['data']['big']['url'] - return None - - images = [] - post_html = create_post(doc['post'], process_post_image) - - if 'preview' in doc: - preview = os.path.join(doc['folder'], doc['preview']) - doc['preview'] = self.create_image(preview, url_creator( - doc['id'], - image_id(doc['preview']) - )) - - doc['post'] = post_html - doc['images'] = images - return doc - - def create_products(self, doc, image_file_fn): - image_id = SyncAlbum.id_from_file - iter_album = iterate_iter_over_fns([ - lambda product: { - **product, - 'image': self.create_image( - image_file_fn(product['file']), - url_creator( - doc['id'], - image_id(product['file']) - ) - ) - } - ]) - - return lmap( - create_product, - iter_album(doc['album']) - ) diff --git a/utils/Sync/sync/core/document.py b/utils/Sync/sync/core/document.py deleted file mode 100644 index e541619d..00000000 --- a/utils/Sync/sync/core/document.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -from tempfile import mkstemp - -from sync.core import Sync -from utils.fn import last_good, lmap -from utils.hash import hash_str -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class SyncDocument(Sync): - def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document): - super().__init__() - self.collection = collection - self.provider = provider - self.sizes = sizes - self.dir_static_previews = dir_static_previews - self.url_base_preview = url_base_preview - self.url_base_document = url_base_document - - def create(self, document): - file = document['file'] - filename = os.path.basename(file) - - document['type'] = document_type(document) - document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) - document['title'] = last_good([ - document['title'], - get_pdf_title(self.provider, document['file']) - ]) - document['file'] = { - 'name': filename, - 'size': self.provider.size(file) - } - - return document - - def create_id(self, document): - bn = os.path.basename(document['file']) - document['id'] = url_encode_text('{category}-{file}'.format( - category=document['category'], - file=bn - )) - return document - - def create_url(self, document): - filename = os.path.basename(document['file']) - document['url'] = self.url_base_document.format(file=url_encode_file(filename)) - return document - - def create_hash(self, document): - hash_keys = ['id', 'url'] - hash_doc = lmap( - lambda key: document[key], - hash_keys - ) - document['hash'] = hash_str( - hash_str(hash_doc) + self.provider.hash(document['file']) - ) - return document - - def create_preview(self, file, sizes, preview_dir): - def url(size, ext): - return self.url_base_preview.format(id=file, size=size, ext=ext) - - temp_preview_path = pdf_to_jpg(self.provider, file) - file = os.path.basename(file) - file = url_encode_text(file) - img = create_image(temp_preview_path, sizes, url, preview_dir) - os.remove(temp_preview_path) - return img - - -def document_type(document): - if document['category'] == 'Награды': - return 'award' - return 'document' - - -def pdf_to_jpg(provider, pdf): - _, temp_in = mkstemp(suffix='.pdf') - _, temp_out = mkstemp(suffix='.jpg') - - abspdf = provider.copy(pdf, temp_in) - image_magick_pdf_to_img(abspdf, temp_out) - - os.remove(temp_in) - return temp_out - - -def get_pdf_title(provider, file): - from PyPDF2 import PdfFileReader - from PyPDF2.generic import TextStringObject - from PyPDF2.generic import IndirectObject - - pdf = PdfFileReader(provider.read(file)) - info = pdf.getDocumentInfo() - - if info: - if type(info.title) == TextStringObject: - return str(info.title) - - if type(info.title_raw) == IndirectObject: - o = pdf.getObject(info.title_raw) - return str(o) - return None diff --git a/utils/Sync/sync/core/person.py b/utils/Sync/sync/core/person.py deleted file mode 100644 index cf3e2728..00000000 --- a/utils/Sync/sync/core/person.py +++ /dev/null @@ -1,39 +0,0 @@ -import settings -from sync.core import Sync -from utils.hash import hash_str -from utils.image import create_image -from utils.text.transform import url_encode_text - -url_base = settings.teacher_image_url_base -sizes = settings.teacher_image_sizes - - -class SyncPerson(Sync): - def __init__(self, collection, provider, dir_local_images): - super().__init__() - self.collection = collection - self.provider = provider - self.dir_local_images = dir_local_images - - def compile(self, profile): - image_path = self.provider.get_abs(profile['picture']) - profile['picture'] = create_image( - image_path, - sizes, - lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), - self.dir_local_images - ) - return profile - - def create_id(self, document): - if 'id' in document and document['id']: - return document - - document['id'] = url_encode_text(document['name']) - return document - - def create_hash(self, document): - document['hash'] = hash_str( - self.provider.hash(document['file']) + self.provider.hash(document['picture']) - ) - return document diff --git a/utils/Sync/sync/core/post.py b/utils/Sync/sync/core/post.py deleted file mode 100644 index 51f8f094..00000000 --- a/utils/Sync/sync/core/post.py +++ /dev/null @@ -1,92 +0,0 @@ -import os -from tempfile import mkstemp - -import lxml.html - -import kazimir -from sync.core import Sync -from utils.fn import last_good, lmap, combine -from utils.hash import hash_str, hash_file -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class SyncPost(Sync): - def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin): - super().__init__() - self.collection = collection - self.provider = provider - self.sizes = sizes - self.type = document_type - self.image_url_base = image_url_base - self.dir_local_images = dir_local_images - self.origin = origin - - def create_id(self, document): - new_id = url_encode_text(document['title']) - document['id'] = document['id'] if 'id' in document else new_id - return document - - def create_hash(self, document): - files = sorted(document['images']) - hashes = lmap( - self.provider.hash, - lmap( - lambda i: os.path.join(document['folder'], i), - files - ) - ) - - document['hash'] = hash_str( - combine([hash_str(document)] + hashes) - ) - - return document - - def create(self, document): - if 'date' not in document or not document['date']: - return None - - if 'title' not in document or not document['title']: - return None - - if self.type: - document['type'] = self.type - - post_html = lxml.html.fromstring(document['post']) - images = [] - for img in post_html.cssselect('img'): - src = img.get('src') - relative_image_path = os.path.join(document['folder'], src) - - img_path = self.provider.get_local(relative_image_path) - if os.path.exists(img_path): - img_id = url_encode_text(os.path.splitext(src)[0]) - url_fn = lambda size, ext: self.image_url_base.format( - id=document['id'], - img=img_id, - size=size, - ext=ext.lower() - ) - - image = create_image(img_path, self.sizes, url_fn, self.dir_local_images) - if image: - images.append(image) - img.set('src', image['data']['big']['url']) - else: - print('fail: ', document['folder'], src) - - document['post'] = kazimir.html_from_tree(post_html) - document['images'] = images - document['origin'] = self.origin - return document - - def create_remove_query(self, query): - if self.type: - query = { - **query, - 'type': self.type, - 'origin': self.origin - } - return query diff --git a/utils/Sync/sync/core/schedule.py b/utils/Sync/sync/core/schedule.py deleted file mode 100644 index dd402ce9..00000000 --- a/utils/Sync/sync/core/schedule.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -from tempfile import mkstemp - -from sync.core import Sync -from utils.fn import last_good, lmap -from utils.hash import hash_str, hash_file -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class SyncSchedule(Sync): - def __init__(self, collection, provider): - super().__init__() - self.collection = collection - self.provider = provider - - def create_id(self, document): - period = get_period(document['period']) - doc_id = document['id'] if 'id' in document and document['id'] else None - if not doc_id: - doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=document['semester']) - document['id'] = doc_id - return document - - def create_hash(self, document): - return { - **document, - 'hash': self.provider.hash(document['file']) - } - - -def get_period(value): - if isinstance(value, str): - return value - - if isinstance(value, list): - return '-'.join(lmap(str, value)) diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index d42cdf42..e1ede610 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -3,6 +3,7 @@ from io import BytesIO import requests +import settings from utils.fn import combine, lmap @@ -85,23 +86,3 @@ def list_images(provider, path): lambda ext: provider.type_filter(path, ext), image_types )) - - -def upload(file_path, relative_path): - """ - /path/to/file, /url/path -> https://static.shlisselburg.org/art/url/path - :param file_path: - :param relative_path: - :return: - """ - files = {'file': open(file_path, 'rb')} - url = 'https://static.shlisselburg.org/upload/art{}'.format(relative_path) - r = requests.put( - url=url, - files=files, - auth=('uploader', '1207690962638465') - ) - if r.status_code == 200: - return url - else: - return None diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py index 5e8e74cd..d203ce29 100644 --- a/utils/Sync/sync/data/fs.py +++ b/utils/Sync/sync/data/fs.py @@ -13,13 +13,9 @@ class FSProvider(Provider): return open(self.get_abs(path), 'rb') def scan(self, path): - return lmap( - self.get_rel, - map( - lambda i: i.path, - os.scandir(self.get_abs(path)) - ) - ) + paths = [i.path for i in os.scandir(self.get_abs(path))] + + return [self.get_rel(path) for path in paths] def glob(self, pattr): return lmap(self.get_rel, glob(self.get_abs(pattr))) diff --git a/utils/Sync/sync/data/request.py b/utils/Sync/sync/data/request.py new file mode 100644 index 00000000..d7601262 --- /dev/null +++ b/utils/Sync/sync/data/request.py @@ -0,0 +1,41 @@ +import aiohttp +import requests + +import settings + + +async def upload(url, path): + """ + :param url: + :param path: + :return: + """ + + if not settings.upload_enabled: + return url + + # urlf = 'https://static.shlisselburg.org/upload/art{}' + + files = {'file': open(path, 'rb')} + r = requests.put( + url=url, + files=files, + # auth=settings.upload_auth + auth=('uploader', '1207690962638465') + ) + if r.status_code == 200: + return url + else: + return None + + +async def get(url): + async with aiohttp.ClientSession as session: + res = session.get(url) + return await res.text() + + +async def put(url, data): + async with aiohttp.ClientSession as session: + res = session.put(url, data=data) + return await res.text() diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py deleted file mode 100644 index 35a4c40c..00000000 --- a/utils/Sync/sync/document.py +++ /dev/null @@ -1,206 +0,0 @@ -import os - -import requests - -import settings -from sync import synced_image_id, untouched -from sync.core.document import SyncDocument -from sync.data import scan_subdirs -from sync.models.document import Document -from utils.fn import combine, lmap, key_mapper - - -def sync_documents(provider, collection, update=True, delete=True): - """ - DOCUMENT_OBJECT SAMPLE - { - "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0", - "_id": { - "$oid": "56d08411ace9573958e5e497" - }, - "type": "document", - "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf", - "category": "Для поступающих", - "file": { - "size": 53088, - "name": "Договор о пожертвовании.pdf" - }, - "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf", - "preview": { - "$oid": "56d08007ace9573958e5e48f" - }, - "title": "Договор о пожертвовании" - } - - :param provider: - :param update: - :param delete: - :return: - """ - - static_path = static_path_fn(settings.dir_static_uploads) - - return main( - SyncDocument( - collection, - provider, - sizes=settings.image_sizes, - dir_static_previews=settings.dir_static_images, - url_base_preview=settings.url_base_preview, - url_base_document=settings.url_base_document, - ), - static_path, - update_documents=update, - delete_documents=delete - ) - - -def upload(provider, document): - f = document['file'] - p = provider.get_local(f) - n = os.path.basename(document['url']) - - files = {'file': open(p, 'rb')} - # values = {'DB': 'photcat', 'OUT': 'csv', 'SHORT': 'short'} - - r = requests.put( - url='https://static.shlisselburg.org/upload/art/uploads/{}'.format(n), - files=files, - auth=('uploader', '1207690962638465') - ) - if r.status_code == 200: - print('Document {} uploaded'.format(document['url'])) - else: - print('Failed to upload document {}'.format(document['url'])) - print('https://static.shlisselburg.org/upload/art/uploads/{}'.format(n)) - print(r.request.headers) - print(r.headers) - print(r.text) - print() - - -def main(sync, static_path, update_documents, delete_documents): - documents = scan_subdirs(sync.provider, '.pdf') - documents = lmap( - lambda document: read_document(sync, document), - documents - ) - documents_id = lmap( - lambda document: document['id'], - documents - ) - - # CHECKING - file_names = set(lmap( - lambda document: document['file'], - documents - )) - if len(file_names) != len(documents): - raise Exception('File names should be unique') - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # COPY FILE -> STATIC_DIR/URL_FILENAME - for i in documents: - sync.provider.copy( - i['file'], - static_path(i) - ) - - for i in documents: - upload(sync.provider, i) - - # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT - documents = lmap( - sync.create, - documents - ) - - if update_documents: - # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('preview', lambda img: synced_image_id(img)), - documents - ) - - # SYNC DOCUMENT_OBJECT WITH DB - documents = lmap( - sync.update, - documents - ) - - # DELETING - documents_delete = sync.query({'id': {'$nin': documents_id}}) - if delete_documents: - documents_delete = lmap( - sync.delete, - map( - lambda document: {'_id': document['_id']}, - documents_delete - ) - ) - - return documents, documents_delete - - -def read_document(sync, path): - name = os.path.basename(path) - - document = { - 'file': path, - 'category': os.path.dirname(path), - 'title': os.path.splitext(name)[0] - } - - document = sync.create_id(document) - document = sync.create_url(document) - document = sync.create_hash(document) - - return document - - -def create_document(provider, m): - # dir_static_previews = settings.dir_static_images, - # url_base_preview = settings.url_base_preview, - # url_base_document = settings.url_base_document - return Document(provider, m, {}) - - -def static_path_fn(dir_path): - return lambda document: os.path.join(dir_path, os.path.basename(document['url'])) - - -def categorize_files_list(files, title): - """ - Adds specified category to each file in files list - :param files: - :param title: - :return: - """ - return map( - lambda profile: { - **profile, - 'category': title - }, - files - ) - - -def unwrap_manifest(param): - if isinstance(param, list): - return combine(map( - unwrap_manifest, - param - )) - - if 'files' in param: - return lmap( - unwrap_manifest, - categorize_files_list(param['files'], param['title']) - ) - - if 'file' not in param: - return None - - return param diff --git a/utils/Sync/sync/image.py b/utils/Sync/sync/image.py index 3f3a9967..c4fdfec0 100644 --- a/utils/Sync/sync/image.py +++ b/utils/Sync/sync/image.py @@ -1,48 +1,7 @@ from db import db -from settings import image_name_format, image_base_url from utils.fn import lmap -# class SyncImage(Sync): -# def __init__(self, sizes, save_path, create_url_fn): -# super().__init__() -# self.sizes = sizes -# self.save_path = save_path -# self.create_url_fn = create_url_fn -# self.collection = db().images -# -# def create_id(self, document): -# id = document['id'] if 'id' in document and document['id'] else None -# if not id: -# id = 'schedule-{period}-{semester}'.format(**document) -# document['id'] = id -# return document -# -# def read(self, document): -# return super().read(document) -# -# def create_hash(self, document): -# return { -# **document, -# 'hash': hash_file(document) -# } -# -# def create(self, document): -# image = create_image(document, self.sizes, self.create_url_fn, self.save_path) -# -# return { -# 'file': document, -# 'hash': self.create_hash(document), -# 'data': image['data'] -# } -# -# def find_or_create(self, document): -# i = self.read(document) -# if not i: -# return self.create(document) -# return i - - def sync_image(record): if isinstance(record, list): return lmap( @@ -59,8 +18,3 @@ def sync_image(record): pass return None - - -def image_create_url(file): - url = image_base_url + image_name_format - return url.format(file) diff --git a/utils/Sync/sync/models/Album.py b/utils/Sync/sync/models/Album.py index f30f6348..3c8fa145 100644 --- a/utils/Sync/sync/models/Album.py +++ b/utils/Sync/sync/models/Album.py @@ -1,21 +1,251 @@ -__author__ = 'Roman Timashev' +import logging + +import os + +import lxml.html + +import kazimir +import settings +from db import collection +from kazimir import kazimir_to_html +from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date, \ + create_post +from sync.data import Provider, list_images +from sync.models import Model +from sync.models.Image import Image + +from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns +from utils.hash import hash_str +from utils.io import read_yaml_md, parse_yaml_front_matter +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Album') +store = collection(settings.collection_albums) -class Album: - def __init__(self, type, title, uri, course, course_uri, date, teacher, comment): - super().__init__() - self.products = [] - self.type = type - self.title = title - self.uri = uri - self.course = course - self.course_uri = course_uri - self.date = date - self.teacher = teacher - self.comment = comment +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None - def add_product(self, product): - self.products.append(product) - def __repr__(self, *args, **kwargs): - return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products)) +class Album(Model): + @staticmethod + async def scan(provider): + documents = [i for i in provider.scan('.') if provider.is_dir(i)] + documents = [Album.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + date, title = create_date_and_title_from_folder_name( + os.path.basename(path) + ) + + images = list_images(provider, path) + images = [os.path.relpath(i, path) for i in images] + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + document = { + 'folder': path, + 'date': date, + 'title': title, + 'post': create_post_from_image_list(images), + 'images': images + } + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + manifest_file = provider.type_filter(path, '.md') + manifest_file = manifest_file[0] if len(manifest_file) else None + manifest = get_manifest(provider, manifest_file) if manifest_file else {} + manifest = manifest if manifest else {} + + file = manifest_file if manifest_file else path + return Album( + provider, + file, + params={ + **document, + **manifest, + }, + ) + + def __init__(self, provider, file, params=None): + # self.__id_template: str = '{category}-{file}' + # self.__url_template: str = settings.document_url_template + # self.__url_preview_template: str = settings.document_url_preview_template + + self.sizes = settings.image_sizes + self.version = '1' + + self.origin = settings.origin + self.post = None + self.images = None + super().__init__(provider, store, file, params=params) + + def init(self): + self.__set_id() + self.__set_hash() + + def validate(self): + document = self.params + if 'date' not in document or not document['date']: + return None + + if 'title' not in document or not document['title']: + return None + + async def build(self, **kwargs): + params = self.params + image_id = id_from_file + images = [] + + create_album_id = lambda date, title: '{date}-{title}'.format( + date=date.strftime('%Y'), + title=url_encode_text(title) + ) + + # params['id'] = params['id'] if 'id' in params else create_album_id(params['date'], params['title']) + + # if 'album' in params: + # params['album'] = self.create_products(params, image_path) + + async def process_post_image(src): + url = url_creator( + self.id, + id_from_file(src) + ) + + image_path = os.path.join(params['folder'], src) + image = await Image.new(self.provider, image_path, self.sizes, url) + if image: + images.append(image) + return image.get_size('big')['url'] + return None + + post_html = await create_post(params['post'], process_post_image) + + if self.has_param('preview'): + preview = self.get_param('preview') + preview = self._get_relpath(preview) + preview = await Image.new(self.provider, preview, sizes=self.sizes) + else: + # preview = 'https://art.shlisselburg.org/entrance.jpg' + preview = None + + self.post = post_html + self.images = images + self.preview = preview + + # sizes = kwargs['sizes'] + # post, images = await create_post(self.provider, self.id, self.params, sizes) + # self.post = post + # self.images = images + # + # post, images = await create_post(self.provider, self.id, self.params, sizes) + # self.post = kazimir.html_from_tree(post) + # self.images = images + + def create_products(self, doc, image_file_fn): + pass + # image_id = id_from_file + # iter_album = iterate_iter_over_fns([ + # lambda product: { + # **product, + # 'image': self.create_image( + # image_file_fn(product['file']), + # url_creator( + # doc['id'], + # image_id(product['file']) + # ) + # ) + # } + # ]) + # + # return lmap( + # create_product, + # iter_album(doc['album']) + # ) + + def bake(self): + preview = None if not self.preview else self.preview.id + + return { + **self.params, + 'images': [i.id for i in self.images], + 'preview': preview, + 'post': self.post, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + 'origin': self.origin, + 'version': self.version, + } + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + year = str(self.params['date'].year) + title = self.params['title'] + self.id = url_encode_text('%s-%s' % (year, title)) + + def __set_hash(self): + files = self.get_param('images') + files = [self._get_relpath(i) for i in files] + + if not self.provider.is_dir(self.file): + files.append(self.file) + + files = sorted(files) + files = [self.provider.hash(i) for i in files] + + self.hash = hash_str(files) + self.hash = hash_str( + combine([hash_str(self.params)] + files) + ) + + def __str__(self): + return ''.format(self.hash, self.id, self.file) + + +def get_manifest(provider, filepath): + data = provider.read(filepath).read().decode('utf-8') + y, m = read_yaml_md(data) + + data = {**y} if y else {} + _ = lambda key, d=None: data[key] if key in data else d + + date = create_date(str(_('date')), settings.date_formats_direct) + if date: + data['date'] = date + + album_id = _('id') + if album_id: + data['id'] = str(album_id) + + post = {'post': m} if m else {} + return {**data, **post} + + +def create_url(a, i, s, e): + return 'https://static.shlisselburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( + album=a, + id=i, + size=s, + ext=e + ) + + +def url_creator(aid, pid): + return lambda file, size, ext: create_url(aid, pid, size, ext.lower()) + + +def id_from_file(i): + return url_encode_text( + os.path.splitext(i)[0] + ) diff --git a/utils/Sync/sync/models/Article.py b/utils/Sync/sync/models/Article.py new file mode 100644 index 00000000..b1ba8768 --- /dev/null +++ b/utils/Sync/sync/models/Article.py @@ -0,0 +1,202 @@ +import logging + +import os + +import lxml.html + +import kazimir +import settings +from db import collection +from kazimir import kazimir_to_html +from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date +from sync.data import Provider, list_images +from sync.models import Model +from sync.models.Image import Image + +from utils.fn import swap_ext, first, map_cases, constant, combine +from utils.hash import hash_str +from utils.io import read_yaml_md, parse_yaml_front_matter +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Person') +store = collection(settings.collection_articles) + + +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Article(Model): + @staticmethod + async def scan(provider): + documents = [i for i in provider.scan('.') if provider.is_dir(i)] + documents = [Article.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + file_time_formats = ['%Y.%m.%d'] + + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats) + document = { + 'folder': path, + 'date': doc_date, + 'title': doc_title, + } + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + images = list_images(provider, path) + images = [os.path.relpath(i, path) for i in images] + document['images'] = images + document['post'] = create_post_from_image_list(images) + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + manifest_file = md_from_folder(provider, path) + if manifest_file: + manifest = get_manifest(provider, manifest_file) + else: + manifest = {} + + return Article( + provider, + path, + params={ + **document, + **manifest, + }, + ) + + def __init__(self, provider, file, params=None): + # self.__id_template: str = '{category}-{file}' + # self.__url_template: str = settings.document_url_template + # self.__url_preview_template: str = settings.document_url_preview_template + + self.origin = settings.origin + self.version = '2' + self.post = None + self.images = None + super().__init__(provider, store, file, params=params) + + def init(self): + self.__set_id() + self.__set_hash() + + def validate(self): + document = self.params + if 'date' not in document or not document['date']: + return None + + if 'title' not in document or not document['title']: + return None + + async def build(self, **kwargs): + sizes = kwargs['sizes'] + post, images = await create_post(self.provider, self.id, self.params, sizes) + self.post = post + self.images = images + + post, images = await create_post(self.provider, self.id, self.params, sizes) + self.post = kazimir.html_from_tree(post) + self.images = images + + if 'preview' in self.params: + preview = self.get_param('preview') + preview = self._get_relpath(preview) + preview = await Image.new(self.provider, preview, sizes) + self.preview = preview + elif len(self.images): + self.preview = self.images[0] + else: + self.preview = 'https://art.shlisselburg.org/entrance.jpg' + + def bake(self): + return { + **self.params, + 'images': [i.id for i in self.images], + 'post': self.post, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + 'origin': self.origin, + 'version': self.version, + 'preview': self.preview if isinstance(self.preview, str) else self.preview.id, + } + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + self.id = url_encode_text(self.params['title']) + + def __set_hash(self): + f = self.params['folder'] + images = sorted(self.params['images']) + images = [os.path.join(f, i) for i in images] + images = [self.provider.hash(i) for i in images] + + self.hash = hash_str( + combine([hash_str(self.params)] + images) + ) + + def __str__(self): + return '
'.format(self.hash, self.file, self.id) + + +def md_from_folder(provider, i): + return first(provider.type_filter(i, '.md')) + + +def get_manifest(provider, path): + data = provider.read(path).read().decode('utf-8') + manifest = parse_yaml_front_matter(data) + + if 'date' in manifest: + manifest['date'] = create_date(manifest['date'], settings.date_formats) + + if 'until' in manifest: + manifest['until'] = create_date(manifest['until'], settings.date_formats) + + if 'id' in manifest: + manifest['id'] = str(manifest['id']) + + html = kazimir_to_html(manifest['content']) + return { + **manifest, + 'post': html, + 'images': images_from_html(html) # Add images for hashing + } + + +async def create_post(provider, doc_id, document, sizes): + image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}' + + post_html = lxml.html.fromstring(document['post']) + images = [] + for img in post_html.cssselect('img'): + src = img.get('src') + relative_image_path = os.path.join(document['folder'], src) + + img_path = provider.get_local(relative_image_path) + if os.path.exists(img_path): + img_id = url_encode_text(os.path.splitext(src)[0]) + url_fn = lambda file, size, ext: image_url_base.format( + id=doc_id, + img=img_id, + size=size, + ext=ext.lower() + ) + + image = await Image.new(provider, img_path, sizes, url_fn) + if image: + url = image.get_size('big')['url'] + images.append(image) + img.set('src', url) + else: + logger.error('Fail to get Image', document['folder'], src) + return post_html, images diff --git a/utils/Sync/sync/models/Document.py b/utils/Sync/sync/models/Document.py new file mode 100644 index 00000000..d426f2d9 --- /dev/null +++ b/utils/Sync/sync/models/Document.py @@ -0,0 +1,188 @@ +import os +from tempfile import mkstemp + +import requests + +import settings +from db import collection +from sync.data import scan_subdirs, request +from sync.models import Model +from sync.models.Image import Image +from utils.fn import last_good, lmap +from utils.hash import hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + + +def find(store, item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Document(Model): + @staticmethod + async def scan(provider): + paths = scan_subdirs(provider, '.pdf') + return [Document.read_path(provider, path) for path in paths] + + @staticmethod + def read_path(provider, path): + name = os.path.basename(path) + + document = { + 'category': os.path.dirname(path), + 'title': os.path.splitext(name)[0] + } + + return Document( + provider, + path, + **document + ) + + def __init__(self, provider, file, category=None, title=None): + store = collection(settings.collection_documents) + + self.__hash_keys = ['id', 'url'] + + self.__id_template: str = '{category}-{file}' + self.__url_template: str = settings.document_url_template + self.__url_preview_template: str = settings.document_url_preview_template + + self.category = category + self.title = title + self.type = None + self.preview = None + + super().__init__(provider, store, file) + + def init(self): + self.__set_id() + self.__set_url() + self.__set_hash() + + async def is_changed(self): + i = find(self.store, self.id) + if not i: + return True + + if 'hash' not in i: + return True + + return not (self.hash == i['hash']) + + async def save(self): + document = self.bake() + + query = {'id': document['id']} + try: + self.store.update_one(query, {'$set': document}, upsert=True) + # return self.read(document) + return self + except ValueError: + pass + return None + + async def upload(self): + filepath = self.provider.get_local(self.file) + name = os.path.basename(self.url) + url = settings.document_url_upload_template.format(file=name) + + await request.upload(url, filepath) + + def bake(self): + file = self.file + filename = os.path.basename(file) + + return { + 'id': self.id, + 'hash': self.hash, + 'url': self.url, + 'type': document_type(self.type), + 'file': { + 'name': filename, + 'size': self.provider.size(file) + }, + 'preview': self.preview, + 'title': last_good([ + self.title, + get_pdf_title(self.provider, self.file) + ]), + } + + def __filename(self): + return os.path.basename(self.file) + + def __set_id(self): + self.id = self.__id_template.format( + category=url_encode_text(self.category), + file=url_encode_text(self.__filename()) + ) + + def __set_url(self): + self.url = self.__url_template.format( + file=url_encode_file(self.__filename()) + ) + + def __set_hash(self): + doc = [getattr(self, key) for key in self.__hash_keys] + + self.hash = hash_str( + hash_str(doc) + self.provider.hash(self.file) + ) + + async def create_preview(self, sizes): + def url(_, size, ext): + file = url_encode_text(self.__filename()) + return self.__url_preview_template.format(id=file, size=size, ext=ext) + + temp_preview_path = pdf_to_jpg(self.provider, self.file) + preview = await Image.new(self.provider, temp_preview_path, sizes, url) + + if preview: + self.preview = preview.id + + os.remove(temp_preview_path) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def document_type(doctype): + if doctype == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None diff --git a/utils/Sync/sync/models/Image.py b/utils/Sync/sync/models/Image.py new file mode 100644 index 00000000..fcfaf650 --- /dev/null +++ b/utils/Sync/sync/models/Image.py @@ -0,0 +1,231 @@ +import tempfile +from tempfile import mkstemp +from typing import Optional, List + +from db import collection +from sync.data import request +from sync.models import Model +from utils.hash import md5 +from utils.image.resize import image_magick_pdf_to_img +import os + +import logging + +import settings +from utils.image.resize import optimize, thumbnail + +logger = logging.getLogger(settings.name + '.Image') + +store = collection(settings.collection_images) + + +def get_upload_url(url): + return os.path.join(settings.image_url_upload, os.path.basename(url)) + + +def default_url_factory(file, size, ext): + url = settings.image_url_base + '{}-{}{}' + return url.format(md5(file), size, ext) + + +class Image(Model): + @staticmethod + async def find_one(query): + document = store.find_one(query) + return document + + @staticmethod + async def new(provider, file, sizes, url_factory=None): + url_factory = url_factory if url_factory else default_url_factory + + img = Image(provider, file, data=None, url_factory=url_factory) + changed = await img.is_changed() + if changed: + await img.compile(sizes) + await img.upload() + await img.save() + else: + doc = await Image.find_one({'file': file}) + img = Image(provider, file, doc['data'], url_factory) + img.__set_id(doc['_id']) + + return img + + def __init__(self, provider, file, data, url_factory): + self.data = data + self.url_factory = url_factory + + super().__init__(provider, store, file) + + def init(self): + self.hash = self.__get_hash() + + async def is_changed(self): + i = await Image.find_one({'file': self.file}) + return self._is_changed_hash(i) + + async def compile(self, sizes): + img_in = self.provider.get_abs(self.file) + img_out = tempfile.mkdtemp() + + images = create_image(img_in, sizes, self.url_factory, img_out) + data = {} + if images: + for i in images: + size_name = i['size'] + data[size_name] = { + **i, + } + self.data = data + + async def upload(self): + logger.info('Uploading Image {}'.format(self.hash)) + + if self.data: + for i in self.data.values(): + url = get_upload_url(i['url']) + file = i['file'] + + await request.upload(url, file) + + async def save(self): + c = sync_image(self.bake()) + self.__set_id(c['_id']) + + def bake(self): + return { + 'file': self.file, + 'hash': self.hash, + 'data': self.data, + } + + def get_size(self, size): + if size in self.data: + return self.data[size] + return None + + def __filename(self): + return os.path.basename(self.file) + + def __set_id(self, value): + self.id = value + + def __get_hash(self): + return self.provider.hash(self.file) + + def __str__(self): + return ''.format(self.hash, self.file) + + +def sync_image(record): + from db import db + + q = {'hash': record['hash']} + try: + db().images.update_one(q, {'$set': record}, upsert=True) + i = db().images.find_one({'hash': record['hash']}) + return i + except ValueError: + pass + + return None + + +def document_type(doctype): + if doctype == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None + + +def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]: + """ + + :param file: Image path to process + :param sizes: list of sizes to generate image (, , ) + :param url_fn: function (size_name, ext) for creating image url + :param output_dir: path to local folder for thumbs storing + :return: + """ + if not os.path.exists(file): + return None + + _, ext = os.path.splitext(file) + + def fn(size): + size_name, width, height = size + + image_url = url_fn(file, size_name, ext) + image_filename = os.path.basename(image_url) + local_image_path = os.path.join(output_dir, image_filename) + + if settings.image_processing_enabled: + image = process_image(file, local_image_path, size) + if not image: + logger.warning('Failed to process image {}'.format(file)) + return None + width, height = image.size + + return { + 'file': local_image_path, + 'url': image_url, + 'size': size_name, + 'width': width, + 'height': height + } + + return [fn(s) for s in sizes] + + +def process_image(input_file, output_file, size): + size_name, width, height = size + + if size_name == 'original': + return optimize(input_file, output_file, quality=90) + else: + image = read_image(input_file) + if not image: + return None + image_width, image_height = image.size + if image_height > image_width: + width, height = height, width + return thumbnail(input_file, output_file, (width, height)) + + +def read_image(src): + from PIL import Image + + try: + i = Image.open(src) + return i + except Exception as e: + print(src, e) + return None diff --git a/utils/Sync/sync/models/Page.py b/utils/Sync/sync/models/Page.py new file mode 100644 index 00000000..1c4c1802 --- /dev/null +++ b/utils/Sync/sync/models/Page.py @@ -0,0 +1,203 @@ +import os + +import logging + +from lxml import etree +import lxml.html + +import settings +from db import collection +from sync.data import request +from sync.models import Model + +from sync import create_date, images_from_html, title_from_html +from sync.models.Image import Image +from utils.hash import hash_str +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Page') + + +def find(store, item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Page(Model): + @staticmethod + async def scan(provider): + documents = [i for i in provider.scan('.') if provider.is_dir(i)] + documents = [provider.type_filter(i, '.md') for i in documents] + documents = [i[0] for i in documents if len(i)] + documents = [Page.read_path(provider, i) for i in documents] + documents = [i for i in documents if i] + + return documents + + @staticmethod + def read_path(provider, path): + params = { + 'file': os.path.basename(path), + 'folder': os.path.dirname(path), + 'title': os.path.basename(path), + } + + manifest = get_manifest(provider, path) + params = { + **params, + **manifest, + } + + if 'url' not in params: + logger.warning('URL not specified for Page {}'.format(path)) + return None + + return Page( + provider, + path, + params=params, + ) + + def __init__(self, provider, file, params=None): + store = collection(settings.collection_pages) + + self.__hash_keys = ['id', 'url'] + + self.__id_template: str = '{category}-{file}' + self.__url_template: str = settings.document_url_template + self.__url_preview_template: str = settings.document_url_preview_template + + self.params = params if params else {} + + super().__init__(provider, store, file) + + def init(self): + self.__set_id() + self.__set_url() + self.__set_hash() + + async def is_changed(self): + i = find(self.store, self.id) + if not i: + return True + + if 'hash' not in i: + return True + + return not (self.hash == i['hash']) + + async def save(self): + document = self.bake() + + query = {'id': document['id']} + try: + self.store.update_one(query, {'$set': document}, upsert=True) + return self + except ValueError: + pass + return None + + async def setup_images(self, sizes, url_factory): + folder = self.params['folder'] + images = [] + for i in self.params['images']: + img_file = os.path.join(folder, i) + + img = await Image.new(self.provider, img_file, sizes, url_factory) + if img: + images.append(img) + self.images = images + + def bake(self): + images = [image.id for image in self.images] + + data = create_post(self.params['data'], lambda src: self.__image_url_by_src(src)) + + return { + **self.params, + 'id': self.id, + 'hash': self.hash, + 'url': self.url, + 'file': self.file, + 'data': data, + 'images': images, + } + + def __image_url_by_src(self, src): + for img in self.images: + if img.file.endswith(src): + s = img.get_size('big') + if s: + return s['url'] + else: + return None + + return None + + def __filename(self): + return os.path.basename(self.file) + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + self.id = url_encode_text(self.params['url']) + + def __set_url(self): + self.url = self.params['url'] + + def __set_hash(self): + images = [os.path.join(self.params['folder'], i) for i in self.params['images']] + files = sorted([self.file] + images) + hashes = [self.provider.hash(i) for i in files] + + self.hash = hash_str(''.join(hashes)) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def create_post(md, image_path_fn): + html = lxml.html.fromstring(md) + + for img in html.cssselect('img'): + src = img.get('src') + path = image_path_fn(src) + + if path: + img.set('src', path) + img.set('class', settings.album_html_img_class) + img.set('data-file', src) + return etree.tounicode(html) + + +def get_manifest(provider, path): + dirname = os.path.dirname(path) + data = provider.read(path).read().decode('utf-8') + manifest, body = read_yaml_md(data) + manifest = manifest if manifest else {} + + if 'date' in manifest: + manifest['date'] = create_date(manifest['date'], settings.date_formats) + + if 'until' in manifest: + manifest['until'] = create_date(manifest['until'], settings.date_formats) + + if 'id' in manifest: + manifest['id'] = str(manifest['id']) + + if 'content' in manifest: + body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8') + del manifest['content'] + + return { + **manifest, + 'data': body, + 'title': title_from_html(body), + 'images': images_from_html(body) + } diff --git a/utils/Sync/sync/models/Person.py b/utils/Sync/sync/models/Person.py new file mode 100644 index 00000000..099fe305 --- /dev/null +++ b/utils/Sync/sync/models/Person.py @@ -0,0 +1,109 @@ +import logging + +import settings +from db import collection +from sync.data import Provider +from sync.models import Model +from sync.models.Image import Image + +from utils.fn import swap_ext +from utils.hash import hash_str +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + +logger = logging.getLogger(settings.name + '.Person') +store = collection(settings.collection_collective) + + +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Person(Model): + @staticmethod + async def scan(provider): + documents = provider.type_filter('', '.md') + documents = [Person.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + params = get_manifest(provider, path) + + return Person( + provider, + path, + params=params, + ) + + def __init__(self, provider, file, params=None): + self.__hash_keys = ['id', 'url'] + + self.__id_template: str = '{category}-{file}' + self.__url_template: str = settings.document_url_template + self.__url_preview_template: str = settings.document_url_preview_template + + self.params = params if params else {} + self.picture = None + + super().__init__(provider, store, file) + + def init(self): + pass + self.__set_id() + self.__set_hash() + + async def build(self, **kwargs): + sizes = kwargs['sizes'] + image_path = self.provider.get_abs(self.params['picture']) + + self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id)) + + def bake(self): + return { + **self.params, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + 'picture': self.picture.id, + } + + def __set_id(self): + if 'id' in self.params: + self.id = self.params['id'] + else: + self.id = url_encode_text(self.params['name']) + + def __set_hash(self): + self.hash = hash_str( + self.provider.hash(self.file) + self.provider.hash(self.params['picture']) + ) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def get_manifest(provider: Provider, path: str) -> dict: + data = provider.read(path).read().decode('utf-8') + + manifest, body = read_yaml_md(data) + manifest = { + **manifest, + 'file': path, + 'biography': body, + 'picture': manifest['picture'] if 'picture' in manifest else swap_ext('.jpg')(path), + } + + return manifest + + +def url_factory(id): + def url(file, size, ext): + return settings.person_image_url_template.format(id=id, size=size, ext=ext) + + return url diff --git a/utils/Sync/sync/models/Schedule.py b/utils/Sync/sync/models/Schedule.py new file mode 100644 index 00000000..2bb6ad83 --- /dev/null +++ b/utils/Sync/sync/models/Schedule.py @@ -0,0 +1,109 @@ +import logging + +from toml import loads as toml_str +from yaml import load as yaml + +import settings +from db import collection +from sync.data import Provider, get_data +from sync.models import Model + +from utils.fn import constant + +logger = logging.getLogger(settings.name + '.Person') +store = collection(settings.collection_schedules) + + +def find(item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Schedule(Model): + @staticmethod + async def scan(provider): + documents = provider.type_filter('.', '.yaml') + documents += provider.type_filter('.', '.toml') + + documents = [Schedule.read(provider, i) for i in documents] + return documents + + @staticmethod + def read(provider, path): + params = get_manifest(provider, path) + + return Schedule( + provider, + path, + params=params, + ) + + def __init__(self, provider, file, params=None): + self.params = params if params else {} + super().__init__(provider, store, file) + + def init(self): + pass + self.__set_id() + self.__set_hash() + + def bake(self): + return { + **self.params, + 'id': self.id, + 'hash': self.hash, + 'file': self.file, + } + + def __set_id(self): + doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None + if not doc_id: + period = self.__get_period() + doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester']) + self.id = doc_id + + def __set_hash(self): + self.hash = self.provider.hash(self.file) + + def __get_period(self): + value = self.params['period'] + if isinstance(value, str): + return value + + if isinstance(value, list): + return '-'.join(list(map(str, value))) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def combine(fns): + def call(value): + data = value.read() + for f in fns: + try: + return f(data) + except Exception as e: + pass + return None + + return call + + +def toml(bytes_value): + s = bytes_value.decode('utf-8') + return toml_str(s) + + +def get_manifest(provider: Provider, path: str) -> dict: + transform = combine([yaml, toml, constant({})]) + manifest = get_data(provider, path, transform) + + return { + 'file': path, + **manifest, + } diff --git a/utils/Sync/sync/models/__init__.py b/utils/Sync/sync/models/__init__.py index 5c6d8dea..5a843a89 100644 --- a/utils/Sync/sync/models/__init__.py +++ b/utils/Sync/sync/models/__init__.py @@ -1,6 +1,78 @@ +import os + + +def find(store, item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + class Model: - def __init__(self): + def __init__(self, provider, store, file, params=None): super().__init__() + self.provider = provider + self.store = store + self.file = file + + self.id = None + self.url = None + self.hash = None + self.params = params if params else {} + + self.init() + + def init(self): + pass + + async def build(self, **kwargs): + pass + def bake(self): return None + + async def is_changed(self): + i = find(self.store, self.id) + return self._is_changed_hash(i) + + def has_param(self, name): + if not self.params: + return False + return name in self.params + + def get_param(self, name, default_value=None): + if not self.params: + return default_value + if not (name in self.params): + return default_value + return self.params[name] + + def _is_changed_hash(self, item): + if not item: + return True + + if 'hash' not in item: + return True + + return not (self.hash == item['hash']) + + def _filename(self): + return os.path.basename(self.file) + + def _get_relpath(self, path): + folder = self.get_param('folder', self.file) + return os.path.join(folder, path) + + async def save(self): + document = self.bake() + + query = {'id': document['id']} + try: + self.store.update_one(query, {'$set': document}, upsert=True) + return self + except ValueError: + pass + return None diff --git a/utils/Sync/sync/models/document.py b/utils/Sync/sync/models/document.py deleted file mode 100644 index 29724b4a..00000000 --- a/utils/Sync/sync/models/document.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -from tempfile import mkstemp - -import settings -from sync.models import Model -from utils.fn import last_good, lmap -from utils.hash import hash_str -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class Document(Model): - @staticmethod - def read(path): - name = os.path.basename(path) - - document = { - 'file': path, - 'category': os.path.dirname(path), - 'title': os.path.splitext(name)[0] - } - - return document - - def __init__(self, provider, path, url_base, preview_compiler): - super().__init__() - - self.path = path - - self.category = None - self.file = None - self.id = None - self.type = None - self.url = None - self.title = None - self.preview = None - - for k, v in m.items(): - setattr(self, k, v) - - self.provider = provider - - def bake(self): - file = self.file - filename = os.path.basename(file) - - self.type = document_type(self.type) - self.preview = self.create_preview(file) - self.title = last_good([ - self.title, - get_pdf_title(self.provider, self.file) - ]) - self.file = { - 'name': filename, - 'size': self.provider.size(file) - } - - def __set_id(self): - bn = os.path.basename(self.file) - self.id = url_encode_text('{category}-{file}'.format( - category=self.category, - file=bn - )) - - def __set_url(self): - filename = os.path.basename(self.file) - self.url = self.url_base_document.format(file=url_encode_file(filename)) - - def __set_hash(self): - hash_keys = ['id', 'url'] - hash_doc = lmap( - lambda key: getattr(self, key), - hash_keys - ) - self.hash = hash_str( - hash_str(hash_doc) + self.provider.hash(self.file) - ) - - def create_preview(self, file): - def url(size, ext): - return self.url_base_preview.format(id=file, size=size, ext=ext) - - temp_preview_path = pdf_to_jpg(self.provider, file) - file = os.path.basename(file) - file = url_encode_text(file) - img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images) - os.remove(temp_preview_path) - return img - - -def document_type(doctype): - if doctype == 'Награды': - return 'award' - return 'document' - - -def pdf_to_jpg(provider, pdf): - _, temp_in = mkstemp(suffix='.pdf') - _, temp_out = mkstemp(suffix='.jpg') - - abspdf = provider.copy(pdf, temp_in) - image_magick_pdf_to_img(abspdf, temp_out) - - os.remove(temp_in) - return temp_out - - -def get_pdf_title(provider, file): - from PyPDF2 import PdfFileReader - from PyPDF2.generic import TextStringObject - from PyPDF2.generic import IndirectObject - - pdf = PdfFileReader(provider.read(file)) - info = pdf.getDocumentInfo() - - if info: - if type(info.title) == TextStringObject: - return str(info.title) - - if type(info.title_raw) == IndirectObject: - o = pdf.getObject(info.title_raw) - return str(o) - return None diff --git a/utils/Sync/sync/page.py b/utils/Sync/sync/page.py deleted file mode 100644 index 0705e1fd..00000000 --- a/utils/Sync/sync/page.py +++ /dev/null @@ -1,213 +0,0 @@ -import os - -import settings -from sync.core import Sync -from sync.data import list_images, Provider -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ - synced_images_ids, untouched, title_from_html, create_post -from sync.core.post import SyncPost -from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant -from utils.hash import hash_str -from utils.image import create_image -from utils.io import read_yaml_md -from utils.text.transform import url_encode_text - - -class ImageCreator: - def __init__(self, provider: Provider, sizes: list, output_dir: str, cache: dict): - super().__init__() - self.provider = provider - self.sizes = sizes - self.output_dir = output_dir - self.cache = cache if cache else set() - - def create(self, file, name_fn): - filehash = self.provider.hash(file) - cached = self.cached(filehash) - if cached: - return cached - return create_image(self.provider.get_abs(file), self.sizes, name_fn, self.output_dir, skip_processing=False) - - def cached(self, name): - return self.cache[name] if name in self.cache else None - - -def create_document_from_folder(provider, path): - document = { - 'file': os.path.basename(path), - 'folder': os.path.dirname(path), - 'title': os.path.basename(path), - } - - manifest = get_manifest(provider, path) - document = { - **document, - **manifest, - } - - if 'url' not in document: - return None - - document = create_id(document) - document = create_hash(provider, document) - return document - - -def get_manifest(provider, path): - dirname = os.path.dirname(path) - data = provider.read(path).read().decode('utf-8') - manifest, body = read_yaml_md(data) - manifest = manifest if manifest else {} - - if 'date' in manifest: - manifest['date'] = create_date(manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date(manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - if 'content' in manifest: - body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8') - del manifest['content'] - - return { - **manifest, - 'data': body, - 'title': title_from_html(body), - 'images': images_from_html(body) - } - - -def create_id(document): - new_id = url_encode_text(document['url']) - return { - **document, - 'id': document['id'] if 'id' in document else new_id - } - - -def create_hash(provider, document): - files = sorted([document['file']] + document['images']) - hashes = lmap( - provider.hash, - lmap( - lambda x: os.path.join(document['folder'], x), - files - ) - ) - - return { - **document, - 'hash': hash_str(''.join(hashes)) - } - - -def create_page(provider: Provider, img: ImageCreator, document): - image_path = lambda x: os.path.join(document['folder'], x) - image_id = lambda filename: url_encode_text( - os.path.splitext(filename)[0] - ) - - def process_image(src: str): - path = image_path(src) - - if provider.exists(path): - url = lambda s, e: settings.url_page_base_preview.format( - page=document['id'], - id=image_id(src), - size=s, - ext=e - ) - - image = img.create(path, url) - if image: - images.append(image) - return image['data']['big']['url'] - return None - - images = [] - post_html = create_post(document['data'], process_image) - - document['data'] = post_html - document['images'] = images - return document - - -def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True): - img_sizes = settings.album_image_sizes - output_dir = settings.dir_static_images - - sync = Sync() - sync.collection = collection - image_creator = ImageCreator(provider, img_sizes, output_dir, {}) - - documents = list(filter( - lambda x: provider.is_dir(x), - provider.scan('.') - )) - - documents = lmap( - lambda x: provider.type_filter(x, '.md'), - documents - ) - - documents = list(filter( - lambda x: len(x), - documents - )) - - documents = lmap( - lambda x: create_document_from_folder(provider, x[0]), - documents - ) - - documents = list(filter(None, documents)) - - # CREATE SCOPE OF CURRENT SESSION - documents_ids = lmapfn(documents)( - lambda i: i['id'] - ) - - # SKIP UNTOUCHED DOCUMENTS - if skip_unchanged: - documents = untouched(documents, sync) - - # # MAP EVENT MANIFEST -> EVENT_OBJECT - # documents = lmap( - # sync.create, - # documents - # ) - - documents = lmap( - lambda document: create_page(provider, image_creator, document), - documents - ) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) - - # SYNC EVENT_OBJECT WITH DB - if update: - documents = lmap( - sync.update, - documents - ) - - documents_to_delete = [] - if delete: - remove_query = sync.create_remove_query({'id': {'$nin': documents_ids}}) - documents_to_delete = sync.query(remove_query) - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - - return documents, documents_to_delete diff --git a/utils/Sync/sync/person.py b/utils/Sync/sync/person.py deleted file mode 100644 index f42e0b13..00000000 --- a/utils/Sync/sync/person.py +++ /dev/null @@ -1,119 +0,0 @@ -import os -from glob import glob - -import settings -from sync import untouched -from sync.core.person import SyncPerson -from sync.image import sync_image -from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn -from utils.io import read_yaml_md - -url_base = settings.teacher_image_url_base -dir_local_images = settings.teacher_image_output -sizes = settings.teacher_image_sizes - - -def sync_persons(provider, collection, update=True, delete=True): - """ - :param collection: - :param provider: - :param update: - :param delete: - :return: - """ - - return main( - SyncPerson( - collection, - provider, - dir_local_images - ), - update_documents=update, - delete_documents=delete - ) - - -def make_person(provider, file): - pass - - -def main(sync, update_documents=False, delete_documents=False): - # GET TEACHER YAML/MD MANIFEST FILES - documents = sync.provider.type_filter('', '.md') - - # READ TEACHER YAML/MD MANIFEST - documents = lmap( - lambda i: (i,) + get_manifest(sync.provider, i), - documents - ) - - # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY - documents = lmap( - lambda i: { - **i[1], - 'file': i[0], - 'biography': i[2] - }, - documents - ) - - # DEFINE PROFILE PICTURE - documents = lmap( - lambda i: { - **i, - 'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']), - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmapfn(documents)(sync.create_id) - - # CREATE HASH OF DOCUMENT FILES - documents = lmapfn(documents)(sync.create_hash) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # DO HEAVY PROCESS WITH DOCUMENTS - documents = lmap( - sync.compile, - documents - ) - - # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB - sync_picture = lambda i: None if i is None else sync_image(i)['_id'] - documents = lmap( - key_mapper('picture', sync_picture), - documents - ) - - # SYNC TEACHER_PROFILE - if update_documents: - documents = lmap( - sync.update, - documents - ) - - documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}}) - if delete_documents: - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - - return documents, documents_to_delete - - -def get_manifest(provider, path): - data = provider.read(path).read().decode('utf-8') - return read_yaml_md(data) \ No newline at end of file diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py deleted file mode 100644 index 9e8a9f5a..00000000 --- a/utils/Sync/sync/post.py +++ /dev/null @@ -1,192 +0,0 @@ -import os - -import settings -from kazimir import markdown_to_html -from sync.data import list_images -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ - synced_images_ids, untouched -from sync.core.post import SyncPost -from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant -from utils.io import read_yaml_md, parse_yaml_front_matter - - -def create_document_from_folder(sync, path, file_time_formats): - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats) - document = { - 'folder': path, - 'date': doc_date, - 'title': doc_title, - } - - # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - images = lmap( - lambda img: os.path.relpath(img, path), - list_images(sync.provider, path) - ) - post = create_post_from_image_list(images) - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - params = map_cases( - document, - [( - lambda doc: md_from_folder(sync.provider, path), - lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)), - )], - constant({}) - ) - - document = { - 'images': images, - 'post': post, - **document, - **params, - } - - document = sync.create_id(document) - document = sync.create_hash(document) - return document - - -def get_manifest(provider, path): - data = provider.read(path).read().decode('utf-8') - manifest = parse_yaml_front_matter(data) - - if 'date' in manifest: - manifest['date'] = create_date(manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date(manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - html = markdown_to_html(manifest['content']) - return { - **manifest, - 'post': html, - 'images': images_from_html(html) # Add images for hashing - } - - -def md_from_folder(provider, i): - return first(provider.type_filter(i, '.md')) - - -def get_folder_documents(sync, file_time_formats): - # GET EVENT FOLDERS - documents = lmap( - lambda i: i, - filter( - lambda i: sync.provider.is_dir(i), - sync.provider.scan('.') - ) - ) - - return lmap( - lambda doc: create_document_from_folder(sync, doc, file_time_formats), - documents - ) - - -# GET EVENT FILES -def get_file_documents(sync, file_time_formats): - documents = sync.provider.type_filter('.', '.md') - documents = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats), - documents - ) - - documents = lmap( - lambda i: { - 'file': i[0], - 'date': i[1], - 'title': i[2], - }, - documents - ) - - # OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST - documents = lmap( - lambda i: { - **i, - **get_manifest(sync.provider, i['file']) - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) - - # CREATE HASH OF DOCUMENT FILE - documents = lmap(sync.create_hash, documents) - - return documents - - -def main(sync, file_time_formats, update_documents=True, delete_documents=True): - documents = get_folder_documents(sync, file_time_formats) - documents += get_file_documents(sync, file_time_formats) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmapfn(documents)( - lambda i: i['id'] - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # MAP EVENT MANIFEST -> EVENT_OBJECT - documents = lmap( - sync.create, - documents - ) - - # FILTER INCORRECT EVENTS - documents = list(filter(None, documents)) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('images', synced_images_ids), - documents - ) - - # SYNC EVENT_OBJECT WITH DB - if update_documents: - documents = lmap( - sync.update, - documents - ) - - documents_to_delete = [] - if delete_documents: - remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}}) - documents_to_delete = sync.query(remove_query) - documents_to_delete = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_delete - ) - ) - - return documents, documents_to_delete - - -def sync_posts(provider, collection, update=True, delete=True): - image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' - - return main( - SyncPost( - collection, - provider, - 'post', - image_url_base, - settings.dir_static_images, - settings.event_image_sizes, - origin=settings.origin - ), - file_time_formats=['%Y.%m.%d'], - update_documents=update, - delete_documents=delete - ) diff --git a/utils/Sync/sync/schedule.py b/utils/Sync/sync/schedule.py deleted file mode 100644 index 05737efb..00000000 --- a/utils/Sync/sync/schedule.py +++ /dev/null @@ -1,97 +0,0 @@ -from toml import loads as toml_str -from yaml import load as yaml - -from sync import untouched -from sync.core.schedule import SyncSchedule -from sync.data import get_data -from utils.fn import lmap, constant - -dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules' - - -def sync_schedules(provider, collection, update=True, delete=True): - """ - :param collection: - :param provider: - :param update: - :param delete: - :return: - """ - - return main( - SyncSchedule( - collection, - provider - ), - update_documents=update, - delete_documents=delete - ) - - -def combine(fns): - def call(value): - data = value.read() - for f in fns: - try: - return f(data) - except Exception as e: - pass - return None - return call - - -def toml(bytes_value): - s = bytes_value.decode('utf-8') - return toml_str(s) - - -def main(sync, update_documents=False, delete_documents=False): - # COLLECT YAML/MD/JPG/PDF MANIFEST FILES - documents = sync.provider.type_filter('.', '.yaml') - documents += sync.provider.type_filter('.', '.toml') - - # READ MANIFEST - documents = lmap( - lambda i: { - 'file': i, - **get_data(sync.provider, i, combine([yaml, toml, constant({})])) - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) - - # CREATE DOCUMENT HASH - documents = lmap(sync.create_hash, documents) - - # CREATE SCOPE OF CURRENT SESSION - # ALL DOCUMENTS IDS FOUNDED - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # DO HEAVY PROCESS WITH DOCUMENTS - - # SYNC SCHEDULE - if update_documents: - documents = lmap( - sync.update, - documents - ) - - documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - if delete_documents: - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove - ) - ) - - return documents, documents_to_remove diff --git a/utils/Sync/test/sync/test_document.py b/utils/Sync/test/sync/test_document.py index b31b9d65..9c13dc39 100644 --- a/utils/Sync/test/sync/test_document.py +++ b/utils/Sync/test/sync/test_document.py @@ -3,8 +3,7 @@ import unittest import settings from sync.data import scan_subdirs from sync.data.fs import FSProvider -from sync.document import create_document -from sync.models.document import Document +from sync.models.Document import Document from utils.fn import lmap diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index 7cd0b53a..c1fd0ca6 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -49,8 +49,8 @@ lprint_json = lambda i: lprint(lmap(to_json, i)) first = lambda i: i[0] if len(i) > 0 else None -def ext(ext): - return lambda i: os.path.splitext(i)[0] + ext +def swap_ext(extension: str): + return lambda i: os.path.splitext(i)[0] + extension def map_item_key(item, key, map_fn): diff --git a/utils/Sync/utils/hash.py b/utils/Sync/utils/hash.py index 87891e57..ffef5494 100644 --- a/utils/Sync/utils/hash.py +++ b/utils/Sync/utils/hash.py @@ -26,3 +26,8 @@ def hash_str(data): data = to_json(data) return hash_data(str(data).encode()) + + +def md5(data: str) -> str: + i = hashlib.md5(data.encode('utf-8')) + return i.hexdigest() diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py index dba3a52e..6aac3f27 100644 --- a/utils/Sync/utils/image/__init__.py +++ b/utils/Sync/utils/image/__init__.py @@ -1,12 +1,17 @@ import os +import logging +from typing import Optional + from PIL import Image -from sync.data import upload +import settings from utils.hash import hash_file # from utils.image import resize from utils.image.resize import optimize, thumbnail, orient +logger = logging.getLogger(settings.name) + def read_image(src): try: @@ -17,19 +22,15 @@ def read_image(src): return None -def create_image(file, sizes, url_fn, output_dir, skip_processing=False): +def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[dict]: """ :param file: Image path to process :param sizes: list of sizes to generate image (, , ) :param url_fn: function (size_name, ext) for creating image url :param output_dir: path to local folder for thumbs storing - :param skip_processing: True/False to skip thumbs generation :return: """ - # image = read_image(file) - # if not image: - # return None if not os.path.exists(file): return None @@ -38,18 +39,20 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): for size in sizes: size_name, width, height = size - image_url = url_fn(size_name, ext) + image_url = url_fn(file, size_name, ext) image_filename = os.path.basename(image_url) local_image_path = os.path.join(output_dir, image_filename) - if not skip_processing: + if settings.image_processing_enabled: image = process_image(file, local_image_path, size) if not image: + logger.warning('Failed to process image {}'.format(file)) continue + width, height = image.size - upload_url = upload(local_image_path, '/images/{}'.format(image_filename)) - if not upload_url: - print('Image {} not uploaded'.format(file)) + # upload_url = upload(local_image_path, '/images/{}'.format(image_filename)) + # if not upload_url: + # print('Image {} not uploaded'.format(file)) result[size_name] = { 'url': image_url, @@ -58,11 +61,7 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): 'height': height } - return { - 'file': os.path.basename(file), - 'hash': hash_file(file), - 'data': result - } + return result def process_image(input_file, output_file, size):