diff --git a/utils/Sync/config.yaml b/utils/Sync/config.yaml new file mode 100644 index 00000000..e9e94466 --- /dev/null +++ b/utils/Sync/config.yaml @@ -0,0 +1,21 @@ +--- +database_uri: 'mongodb://localhost:27017/hudozka' + +env: 'production' + +skip_unchanged: True +image_processing: True +upload_enabled: True +origin: 'Yandex.Disk' +upload_auth: + login: '' + password: '' + +provider: + name: 'fs' + root: '' + +yandex_disk_access_token: '' + +update_enabled: True +delete_enabled: True diff --git a/utils/Sync/kazimir/__init__.py b/utils/Sync/kazimir/__init__.py index 5eee70ae..07b52c2d 100644 --- a/utils/Sync/kazimir/__init__.py +++ b/utils/Sync/kazimir/__init__.py @@ -14,7 +14,7 @@ def markdown_to_html(text): def kazimir_to_html(text): - html = markdown_to_html(text) + html = make(text) return typo(html) @@ -39,15 +39,209 @@ def typo(html): return html +def make(text): + tokens = text.split('\n') + tokens = markup_tokens(tokens) + tokens = join_tokens(tokens) + tokens = merge_tokens(tokens) + + return compile_tokens(tokens) + # return tokens + + +def join_tokens(tokens): + buffer_tag = None + buffer = [] + ts = [] + for token in tokens: + tag = token['tag'] + changed = buffer_tag is not None and tag != buffer_tag + + flush = is_tag_split(tag) + if changed: + flush = True + + if not is_tag_joinable(tag): + flush = True + + if flush: + if buffer: + ts.append(buffer) + if is_tag_split(tag): + buffer = [] + buffer_tag = None + else: + buffer = [token] + buffer_tag = tag + else: + buffer_tag = tag + buffer.append(token) + + if len(buffer): + ts.append(buffer) + return ts + + +def merge_tokens(tokens: list): + tokens_out = [] + for ts in tokens: + tag = ts[0]['tag'] + fn = None + if tag == 'image': + fn = merge_image + elif tag == 'text': + fn = merge_text + else: + fn = merge_any + tokens_out.append(fn(ts)) + return tokens_out + + +def compile_tokens(tokens: list): + compiled = [] + for ts in tokens: + tag = ts['tag'] + fn = None + if tag == 'text': + fn = compile_text + elif tag == 'image_collection': + fn = compile_image_collection + elif tag == 'image': + fn = compile_image + else: + fn = compile_any + compiled.append(fn(ts)) + return '\n'.join(compiled) + + +def markup_tokens(tokens: list): + marked_tokens = [] + for token in tokens: + tag = 'text' + if is_url(token): + tag = 'url' + elif is_image(token): + tag = 'image' + elif not token: + tag = 'split' + token = None + + marked_tokens.append({ + 'tag': tag, + 'data': token, + }) + return marked_tokens + + +def is_tag_split(tag): + return tag == 'split' + + +def is_tag_joinable(tag: str): + return tag not in ['url'] + + +def is_url(sample: str): + from urllib.parse import urlparse + url = urlparse(sample) + + return url.scheme and url.netloc + + +def is_image(sample: str): + return sample[:2] == '![' and sample[-1:] == ')' + + +def merge_any(tokens): + return tokens[0] + + +def merge_text(tokens): + data = '' + + for t in tokens: + data += t['data'] + data += '\n' + + return { + 'tag': 'text', + 'data': data.strip(), + } + + +def merge_image(tokens): + if len(tokens) == 1: + return tokens[0] + else: + data = [t['data'] for t in tokens] + return { + 'tag': 'image_collection', + 'data': data, + } + + +def compile_text(token): + return markdown_to_html(token['data']) + + +def compile_image_collection(token): + file = lambda i: i[4:][:-1] + imgs = [''.format(file(i)) for i in token['data']] + return ''' +
+
+ {} +
+
+ '''.format('\n'.join(imgs)) + + +def compile_image(token): + img = markdown_to_html(token['data']) + return ''' +
+ {} +
+ '''.format(img) + + +def compile_any(token): + return markdown_to_html(token['data']) + + if __name__ == '__main__': sample_text = ''' Hello! https://www.instagram.com/p/BUelE04l1kU +ku! https://www.instagram.com/p/BUrOXoBFayG/?taken-by=hudozka +https://www.youtube.com/watch?v=qnguOqWpraI +![](1.jpg) +![](2.jpg) +![](3.jpg) + +LOL NOOB +![](12.jpg) +![](22.jpg) +WTF OMG Bye! +Fye! + +![](1.jpg) + +More text ''' - print(markdown_to_html(sample_text)) + ts = make(sample_text) + if isinstance(ts, list): + for i in ts: + print(i) + else: + print(ts) + diff --git a/utils/Sync/kazimir/instagram.py b/utils/Sync/kazimir/instagram.py index a1cc4d06..ee5f8c0c 100644 --- a/utils/Sync/kazimir/instagram.py +++ b/utils/Sync/kazimir/instagram.py @@ -55,7 +55,7 @@ def embed_instagram(media): i = etree.fromstring(html) obj = etree.Element('div') - obj.set('class', "kazimir-embed") + obj.set('class', "kazimir__embed") obj.append(i) return obj else: diff --git a/utils/Sync/kazimir/video.py b/utils/Sync/kazimir/video.py index 4f7060bd..8cc2df47 100644 --- a/utils/Sync/kazimir/video.py +++ b/utils/Sync/kazimir/video.py @@ -33,6 +33,6 @@ def flex_video(url): iframe.set('allowfullscreen', '') obj = etree.Element('div') - obj.set('class', "kazimir-video") + obj.set('class', "kazimir__video") obj.append(iframe) return obj diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 882aa061..e4e71f01 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -1,7 +1,13 @@ import os +import yaml + __author__ = 'Roman Timashev' +name = 'HudozkaSync' + +config = yaml.load(open('config.yaml', 'r')) + def absolute(path, ensure=True): a = os.path.expanduser(path) @@ -20,9 +26,9 @@ def env(param, default=None): return default -database_uri = env('MONGO_URI') +database_uri = config['database_uri'] # env('MONGO_URI') -skip_unchanged = False +skip_unchanged = config['skip_unchanged'] person_uri = { 'Н.В.Андреева': 'nv-andreeva', @@ -33,18 +39,32 @@ person_uri = { 'А.С.Тимашева': 'as-timasheva', } -image_processing = True +upload_enabled = config['upload_enabled'] +upload_url_image = 'https://static.shlisselburg.org/upload/art/images/{}' +upload_auth = ( + config['upload_auth']['login'], + config['upload_auth']['password'], +) + +image_processing_enabled = config['image_processing'] album_html_img_class = 'hudozka-product' image_ext = '.jpg' image_sizes = [ ('original', None, None), + ('big', 1500, 667), + ('big@2', 3000, 1334), + ('medium', 400, 300), + ('medium@2', 800, 600), + ('small', 250, 175), + ('small@2', 500, 350), + ('little', 100, 100), - ('preview', 50, 50), + ('little@2', 200, 200), ] awards_image_sizes = [ @@ -91,8 +111,9 @@ date_formats_reverse = [ '%Y.%m.%d' ] -origin = 'Yandex.Disk' +origin = config['origin'] +collection_images = 'images' collection_documents = 'documents' collection_awards = 'awards' collection_schedules = 'schedules' @@ -101,34 +122,37 @@ collection_collective = 'collective' collection_albums = 'albums' collection_pages = 'pages' -sync_provider_type = env('SYNC_PROVIDER', 'fs') +provider_name = config['provider']['name'] # env('SYNC_PROVIDER', 'fs') +provider_root = config['provider']['root'] # env('SYNC_PROVIDER', 'fs') -_providers_roots = { - 'fs': abs_fn(env('SYNC_LOCAL_PATH')), - 'yd': abs_fn('/'), -} -f = _providers_roots[sync_provider_type] +# _providers_roots = { +# 'fs': abs_fn(env('SYNC_LOCAL_PATH')), +# 'yd': abs_fn('/'), +# } +# f = _providers_roots[provider_name] +f = abs_fn(provider_root) -dir_documents = f('Hudozka/Site/Documents') -dir_awards = f('Hudozka/Site/Awards') -dir_schedules = f('Hudozka/Site/Schedules') -dir_articles = f('Hudozka/Site/Articles') -dir_collective = f('Hudozka/Site/Collective') -dir_gallery = f('Hudozka/Site/Gallery') -dir_pages = f('Hudozka/Site/Pages') - -dir_static = env('SYNC_STATIC', '~/Hudozka Static') -dir_static_uploads = absolute(dir_static + '/uploads') -dir_static_images = absolute(dir_static + '/images') +dir_documents = f('Documents') +dir_awards = f('Awards') +dir_schedules = f('Schedules') +dir_articles = f('Articles') +dir_collective = f('Collective') +dir_gallery = f('Gallery') +dir_pages = f('Pages') url_page_base_preview = 'https://static.shlisselburg.org/art/images/page-{page}-{id}-{size}{ext}' -url_base_preview = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' -url_base_document = 'https://static.shlisselburg.org/art/uploads/{file}' -image_base_url = 'https://static.shlisselburg.org/art/images/' +document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}' +document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploads/{file}' +document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' + +image_url_upload = 'https://static.shlisselburg.org/upload/art/images/' +image_url_base = 'https://static.shlisselburg.org/art/images/' image_name_format = '{type}-{id}-{img}-{size}{ext}' -yandex_disk_access_token = env('YANDEX_DISK_ACCESS_TOKEN') +yandex_disk_access_token = config['yandex_disk_access_token'] # env('YANDEX_DISK_ACCESS_TOKEN') -do_update = env('SYNC_ENV', 'production') == 'production' -do_delete = env('SYNC_ENV', 'production') == 'production' +# update_enabled = env('SYNC_ENV', 'production') == 'production' +# delete_enabled = env('SYNC_ENV', 'production') == 'production' +update_enabled = config['update_enabled'] +delete_enabled = config['delete_enabled'] diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index ad7fd908..15816dec 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,6 +1,9 @@ import settings from db import db, collection from sync.album import sync_albums +from sync.core import Sync +from sync.core.SyncDocument import SyncDocument +from sync.data import Provider from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider from sync.document import sync_documents @@ -11,6 +14,65 @@ from sync.schedule import sync_schedules from utils.fn import lprint_json from utils.image import thumbnail +import asyncio +import datetime +import logging + +# create logger with 'spam_application' +logger = logging.getLogger(settings.name) +logger.setLevel(logging.DEBUG) + +# create file handler which logs even debug messages +fh = logging.FileHandler('hudozka.log') +fh.setLevel(logging.DEBUG) + +# create console handler with a higher log level +ch = logging.StreamHandler() +# ch.setLevel(logging.ERROR) +ch.setLevel(logging.DEBUG) + +# create formatter and add it to the handlers +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) + +# add the handlers to the logger +logger.addHandler(fh) +logger.addHandler(ch) + + +async def main(sync: Sync): + await sync.run() + + +async def run(run_interval=0): + io = lambda root: get_provider(settings.provider_name, root) + c = collection + + documents = SyncDocument( + provider=io(settings.dir_documents), + collection=c(settings.collection_documents), + sizes=settings.image_sizes, + ) + + while True: + # main(io(settings.dir_documents), sync_documents, settings.collection_documents) + # main(io(settings.dir_awards), sync_documents, settings.collection_awards) + # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) + # main(io(settings.dir_articles), sync_posts, settings.collection_articles) + # main(io(settings.dir_collective), sync_persons, settings.collection_collective) + # main(io(settings.dir_gallery), sync_albums, settings.collection_albums) + # main(io(settings.dir_pages), sync_pages, settings.collection_pages) + + await asyncio.wait([ + main(documents), + ]) + + if run_interval == 0: + break + else: + await asyncio.sleep(run_interval) + def get_provider(provider_type, root): if provider_type == 'fs': @@ -20,27 +82,13 @@ def get_provider(provider_type, root): return None -def main(provider, fn, collection_name): - if provider: - print('SYNC: %s' % (collection_name.upper())) - u, d = fn(provider, collection(collection_name), update=settings.do_update, delete=settings.do_update) - - print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d)))) - lprint_json(d) - - print('UPDATE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_update else str(len(u)))) - lprint_json(u) - - print('') - - if __name__ == '__main__': - io = lambda root: get_provider(settings.sync_provider_type, root) + interval = 0 + # interval = 5 - main(io(settings.dir_documents), sync_documents, settings.collection_documents) - main(io(settings.dir_awards), sync_documents, settings.collection_awards) - main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules) - main(io(settings.dir_articles), sync_posts, settings.collection_articles) - main(io(settings.dir_collective), sync_persons, settings.collection_collective) - main(io(settings.dir_gallery), sync_albums, settings.collection_albums) - main(io(settings.dir_pages), sync_pages, settings.collection_pages) + loop = asyncio.get_event_loop() + loop.run_until_complete(run(interval)) + loop.close() + + # Да, 968×504 пискеля это меньше минимально рекомендованных Фейсбуком 1200×630. + # Зато при таком размере и ратио картинку нигде не кропят, и выглядит она отлично. diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index 10357e94..cbe3196a 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -9,6 +9,7 @@ from markdown import markdown import settings from sync.image import sync_image +from sync.models import Model from utils.fn import lmap, combine @@ -43,26 +44,30 @@ def create_date(date_str, date_formats=None): folder_name_pattern = re.compile('([\d.]+)(.*)') -def untouched(documents, store): +async def untouched(items: [Model]) -> [Model]: if settings.skip_unchanged: - return documents - return lmap( - lambda i: i[0], - filter( - lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), - map( - lambda i: (i, store.read(i)), - documents - ) - )) + return items + + result = [] + for item in items: + status = await item.is_changed() + if status: + result.append(item) + return result + + # def is_equals(e, n): + # return (n is None) or ('hash' not in n) or (e.hash != n['hash']) + # + # doc = lambda i: {'id': i.id, 'hash': i.hash} + # + # stored_documents = [(i, store.read(doc(i))) for i in documents] + # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)] + + # return filtered_documents def synced_images_ids(images): - ids = lmap( - lambda image: image['_id'], - sync_image(images) - ) - return ids + return [i['_id'] for i in sync_image(images)] def synced_image_id(image): diff --git a/utils/Sync/sync/core/SyncDocument.py b/utils/Sync/sync/core/SyncDocument.py new file mode 100644 index 00000000..90c641e6 --- /dev/null +++ b/utils/Sync/sync/core/SyncDocument.py @@ -0,0 +1,188 @@ +import os +from tempfile import mkstemp + +import logging + +import settings +from sync import untouched +from sync.core import Sync +from sync.data import scan_subdirs, Provider +from sync.models.Document import Document + +from sync.models import Model +from utils.fn import last_good, lmap +from utils.hash import hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + +logger = logging.getLogger(settings.name + '.Document') + + +class SyncDocument(Sync): + def __init__(self, provider: Provider, collection, sizes): + super().__init__() + self.collection = collection + self.provider = provider + self.sizes = sizes + self.dir_static_previews = '' + + self.temp_dir = '/Users/tmshv/HudozkaSyncTemp' + + async def clean(self): + pass + + async def run(self) -> ([Model], [Model]): + logger.info('Checking documents for update') + + documents = await Document.scan(self.provider) + documents_id = [doc.id for doc in documents] + # self.validate(documents) + logger.info('Found {} Document(s)'.format(len(documents))) + + # SKIP UNTOUCHED DOCUMENTS + # documents = untouched(documents, self) + documents = await untouched(documents) + logger.info('Changed {} Document(s)'.format(len(documents))) + + # UPDATING + if settings.update_enabled: + if len(documents) == 0: + logger.info('No Documents to update') + + for document in documents: + logger.info('Updating Document {}'.format(document.id)) + + await document.upload() + await document.create_preview(self.sizes) + await document.save() + + logger.info('Updated Document {}'.format(document.id)) + + # DELETING + if settings.delete_enabled: + documents_delete = list(self.query({'id': {'$nin': documents_id}})) + + if len(documents_delete) == 0: + logger.info('No Documents to delete') + + for document in documents_delete: + logger.info('Deleting {}'.format(document['id'])) + + q = {'_id': document['_id']} + self.delete(q) + + await self.clean() + + +async def get_assets(self, document) -> []: + return [] + + +def validate(self, documents): + files = [doc['file'] for doc in documents] + file_names = [os.path.basename(f) for f in files] + file_names = set(file_names) + + if len(file_names) != len(documents): + raise Exception('File names should be unique') + + +def create(self, document): + file = document['file'] + filename = os.path.basename(file) + + document['type'] = document_type(document) + document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) + document['title'] = last_good([ + document['title'], + get_pdf_title(self.provider, document['file']) + ]) + document['file'] = { + 'name': filename, + 'size': self.provider.size(file) + } + + return document + + +def create_id(self, document: dict) -> dict: + filepath = document['file'] + filename = os.path.basename(filepath) + + return { + **document, + 'id': self.id_template.format( + category=url_encode_text(document['category']), + file=url_encode_text(filename) + ), + } + + +def create_url(self, document: dict) -> dict: + filename = os.path.basename(document['file']) + + return { + **document, + 'url': self.url_template.format( + file=url_encode_file(filename) + ), + } + + +def create_hash(self, document: dict) -> dict: + hash_doc = [document[key] for key in self.hash_keys] + + return { + **document, + 'hash': hash_str( + hash_str(hash_doc) + self.provider.hash(document['file']) + ), + } + + +def create_preview(self, file, sizes, preview_dir): + def url(size, ext): + return self.url_base_preview.format(id=file, size=size, ext=ext) + + temp_preview_path = pdf_to_jpg(self.provider, file) + file = os.path.basename(file) + file = url_encode_text(file) + img = create_image(temp_preview_path, sizes, url, preview_dir) + os.remove(temp_preview_path) + return img + + +def document_type(document): + if document['category'] == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py index 14e73d43..b7f29b27 100644 --- a/utils/Sync/sync/core/__init__.py +++ b/utils/Sync/sync/core/__init__.py @@ -2,6 +2,13 @@ from sync.models import Model from utils.fn import lmap +def get_id(item): + try: + return item.id + except AttributeError: + return item['id'] + + class Sync: @staticmethod def compile(document): @@ -12,10 +19,23 @@ class Sync: self.collection = None - def read(self, document): - if isinstance(document, list): - return lmap(self.read, document) + async def run(self) -> ([Model], [Model]): + """ + # Get scope files + # Validate these files. Raise an error + # calc_hashes + # make diff with previous run + # compile objects + # upload + :return: + """ + return None, None + + async def upload(self): + pass + + def read(self, document: dict): q = {'id': document['id']} try: return self.collection.find_one(q) @@ -23,16 +43,10 @@ class Sync: pass return None - def update(self, document): - if isinstance(document, list): - return lmap(self.update, document) - - if isinstance(document, Model): - return self.update(self.compile(document)) - - q = {'id': document['id']} + def update(self, document: dict): + query = {'id': document['id']} try: - self.collection.update_one(q, {'$set': document}, upsert=True) + self.collection.update_one(query, {'$set': document}, upsert=True) return self.read(document) except ValueError: pass diff --git a/utils/Sync/sync/core/document.py b/utils/Sync/sync/core/document.py deleted file mode 100644 index e541619d..00000000 --- a/utils/Sync/sync/core/document.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -from tempfile import mkstemp - -from sync.core import Sync -from utils.fn import last_good, lmap -from utils.hash import hash_str -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class SyncDocument(Sync): - def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document): - super().__init__() - self.collection = collection - self.provider = provider - self.sizes = sizes - self.dir_static_previews = dir_static_previews - self.url_base_preview = url_base_preview - self.url_base_document = url_base_document - - def create(self, document): - file = document['file'] - filename = os.path.basename(file) - - document['type'] = document_type(document) - document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) - document['title'] = last_good([ - document['title'], - get_pdf_title(self.provider, document['file']) - ]) - document['file'] = { - 'name': filename, - 'size': self.provider.size(file) - } - - return document - - def create_id(self, document): - bn = os.path.basename(document['file']) - document['id'] = url_encode_text('{category}-{file}'.format( - category=document['category'], - file=bn - )) - return document - - def create_url(self, document): - filename = os.path.basename(document['file']) - document['url'] = self.url_base_document.format(file=url_encode_file(filename)) - return document - - def create_hash(self, document): - hash_keys = ['id', 'url'] - hash_doc = lmap( - lambda key: document[key], - hash_keys - ) - document['hash'] = hash_str( - hash_str(hash_doc) + self.provider.hash(document['file']) - ) - return document - - def create_preview(self, file, sizes, preview_dir): - def url(size, ext): - return self.url_base_preview.format(id=file, size=size, ext=ext) - - temp_preview_path = pdf_to_jpg(self.provider, file) - file = os.path.basename(file) - file = url_encode_text(file) - img = create_image(temp_preview_path, sizes, url, preview_dir) - os.remove(temp_preview_path) - return img - - -def document_type(document): - if document['category'] == 'Награды': - return 'award' - return 'document' - - -def pdf_to_jpg(provider, pdf): - _, temp_in = mkstemp(suffix='.pdf') - _, temp_out = mkstemp(suffix='.jpg') - - abspdf = provider.copy(pdf, temp_in) - image_magick_pdf_to_img(abspdf, temp_out) - - os.remove(temp_in) - return temp_out - - -def get_pdf_title(provider, file): - from PyPDF2 import PdfFileReader - from PyPDF2.generic import TextStringObject - from PyPDF2.generic import IndirectObject - - pdf = PdfFileReader(provider.read(file)) - info = pdf.getDocumentInfo() - - if info: - if type(info.title) == TextStringObject: - return str(info.title) - - if type(info.title_raw) == IndirectObject: - o = pdf.getObject(info.title_raw) - return str(o) - return None diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index d42cdf42..f96373d4 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -3,6 +3,7 @@ from io import BytesIO import requests +import settings from utils.fn import combine, lmap @@ -96,6 +97,10 @@ def upload(file_path, relative_path): """ files = {'file': open(file_path, 'rb')} url = 'https://static.shlisselburg.org/upload/art{}'.format(relative_path) + + if not settings.upload_enabled: + return url + r = requests.put( url=url, files=files, diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py index 5e8e74cd..d203ce29 100644 --- a/utils/Sync/sync/data/fs.py +++ b/utils/Sync/sync/data/fs.py @@ -13,13 +13,9 @@ class FSProvider(Provider): return open(self.get_abs(path), 'rb') def scan(self, path): - return lmap( - self.get_rel, - map( - lambda i: i.path, - os.scandir(self.get_abs(path)) - ) - ) + paths = [i.path for i in os.scandir(self.get_abs(path))] + + return [self.get_rel(path) for path in paths] def glob(self, pattr): return lmap(self.get_rel, glob(self.get_abs(pattr))) diff --git a/utils/Sync/sync/data/request.py b/utils/Sync/sync/data/request.py new file mode 100644 index 00000000..d7601262 --- /dev/null +++ b/utils/Sync/sync/data/request.py @@ -0,0 +1,41 @@ +import aiohttp +import requests + +import settings + + +async def upload(url, path): + """ + :param url: + :param path: + :return: + """ + + if not settings.upload_enabled: + return url + + # urlf = 'https://static.shlisselburg.org/upload/art{}' + + files = {'file': open(path, 'rb')} + r = requests.put( + url=url, + files=files, + # auth=settings.upload_auth + auth=('uploader', '1207690962638465') + ) + if r.status_code == 200: + return url + else: + return None + + +async def get(url): + async with aiohttp.ClientSession as session: + res = session.get(url) + return await res.text() + + +async def put(url, data): + async with aiohttp.ClientSession as session: + res = session.put(url, data=data) + return await res.text() diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 35a4c40c..426cd4fb 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -4,9 +4,9 @@ import requests import settings from sync import synced_image_id, untouched -from sync.core.document import SyncDocument +from sync.core.SyncDocument import SyncDocument from sync.data import scan_subdirs -from sync.models.document import Document +from sync.models.Document import Document from utils.fn import combine, lmap, key_mapper @@ -47,7 +47,7 @@ def sync_documents(provider, collection, update=True, delete=True): sizes=settings.image_sizes, dir_static_previews=settings.dir_static_images, url_base_preview=settings.url_base_preview, - url_base_document=settings.url_base_document, + url_base_document=settings.document_url_template, ), static_path, update_documents=update, diff --git a/utils/Sync/sync/image.py b/utils/Sync/sync/image.py index 3f3a9967..3ee75eca 100644 --- a/utils/Sync/sync/image.py +++ b/utils/Sync/sync/image.py @@ -1,5 +1,5 @@ from db import db -from settings import image_name_format, image_base_url +from settings import image_name_format, image_url_base from utils.fn import lmap @@ -62,5 +62,5 @@ def sync_image(record): def image_create_url(file): - url = image_base_url + image_name_format + url = image_url_base + image_name_format return url.format(file) diff --git a/utils/Sync/sync/models/Document.py b/utils/Sync/sync/models/Document.py new file mode 100644 index 00000000..d426f2d9 --- /dev/null +++ b/utils/Sync/sync/models/Document.py @@ -0,0 +1,188 @@ +import os +from tempfile import mkstemp + +import requests + +import settings +from db import collection +from sync.data import scan_subdirs, request +from sync.models import Model +from sync.models.Image import Image +from utils.fn import last_good, lmap +from utils.hash import hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + + +def find(store, item_id: str): + q = {'id': item_id} + try: + return store.find_one(q) + except ValueError: + pass + return None + + +class Document(Model): + @staticmethod + async def scan(provider): + paths = scan_subdirs(provider, '.pdf') + return [Document.read_path(provider, path) for path in paths] + + @staticmethod + def read_path(provider, path): + name = os.path.basename(path) + + document = { + 'category': os.path.dirname(path), + 'title': os.path.splitext(name)[0] + } + + return Document( + provider, + path, + **document + ) + + def __init__(self, provider, file, category=None, title=None): + store = collection(settings.collection_documents) + + self.__hash_keys = ['id', 'url'] + + self.__id_template: str = '{category}-{file}' + self.__url_template: str = settings.document_url_template + self.__url_preview_template: str = settings.document_url_preview_template + + self.category = category + self.title = title + self.type = None + self.preview = None + + super().__init__(provider, store, file) + + def init(self): + self.__set_id() + self.__set_url() + self.__set_hash() + + async def is_changed(self): + i = find(self.store, self.id) + if not i: + return True + + if 'hash' not in i: + return True + + return not (self.hash == i['hash']) + + async def save(self): + document = self.bake() + + query = {'id': document['id']} + try: + self.store.update_one(query, {'$set': document}, upsert=True) + # return self.read(document) + return self + except ValueError: + pass + return None + + async def upload(self): + filepath = self.provider.get_local(self.file) + name = os.path.basename(self.url) + url = settings.document_url_upload_template.format(file=name) + + await request.upload(url, filepath) + + def bake(self): + file = self.file + filename = os.path.basename(file) + + return { + 'id': self.id, + 'hash': self.hash, + 'url': self.url, + 'type': document_type(self.type), + 'file': { + 'name': filename, + 'size': self.provider.size(file) + }, + 'preview': self.preview, + 'title': last_good([ + self.title, + get_pdf_title(self.provider, self.file) + ]), + } + + def __filename(self): + return os.path.basename(self.file) + + def __set_id(self): + self.id = self.__id_template.format( + category=url_encode_text(self.category), + file=url_encode_text(self.__filename()) + ) + + def __set_url(self): + self.url = self.__url_template.format( + file=url_encode_file(self.__filename()) + ) + + def __set_hash(self): + doc = [getattr(self, key) for key in self.__hash_keys] + + self.hash = hash_str( + hash_str(doc) + self.provider.hash(self.file) + ) + + async def create_preview(self, sizes): + def url(_, size, ext): + file = url_encode_text(self.__filename()) + return self.__url_preview_template.format(id=file, size=size, ext=ext) + + temp_preview_path = pdf_to_jpg(self.provider, self.file) + preview = await Image.new(self.provider, temp_preview_path, sizes, url) + + if preview: + self.preview = preview.id + + os.remove(temp_preview_path) + + def __str__(self): + return ''.format(self.hash, self.file, self.id) + + +def document_type(doctype): + if doctype == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None diff --git a/utils/Sync/sync/models/Image.py b/utils/Sync/sync/models/Image.py new file mode 100644 index 00000000..c57be58f --- /dev/null +++ b/utils/Sync/sync/models/Image.py @@ -0,0 +1,207 @@ +import os +import tempfile +from tempfile import mkstemp +from typing import Optional, List + +import requests + +import settings +from db import collection +from sync.data import scan_subdirs, request +from sync.models import Model +from utils.fn import last_good, lmap +from utils.hash import hash_str +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file +import os + +import logging + +import settings +from sync.data import upload +from utils.hash import hash_file +from utils.image.resize import optimize, thumbnail, orient + +logger = logging.getLogger(settings.name) + + +def get_upload_url(url): + return os.path.join(settings.image_url_upload, os.path.basename(url)) + + +class Image(Model): + @staticmethod + async def new(provider, file, sizes, url_factory): + img = Image(provider, file, url_factory) + await img.compile(sizes) + await img.upload() + await img.sync() + return img + + def __init__(self, provider, file, url_factory): + store = collection(settings.collection_images) + self.url_factory = url_factory + + super().__init__(provider, store, file) + + def init(self): + self.hash = self.__get_hash() + + async def compile(self, sizes): + img_out = tempfile.mkdtemp() + self.image = create_image(self.file, sizes, self.url_factory, img_out) + + async def upload(self): + if self.image: + for i in self.image: + url = get_upload_url(i['url']) + file = i['file'] + + logger.info('Uploading image {}'.format(file)) + + await request.upload(url, file) + + async def sync(self): + c = sync_image(self.bake()) + self.id = c['_id'] + + def bake(self): + data = {} + if self.image: + for i in self.image: + size_name = i['size'] + data[size_name] = { + 'url': i['url'], + 'size': size_name, + 'width': i['width'], + 'height': i['height'], + } + + return { + 'file': self.file, + 'hash': self.hash, + 'data': data, + } + + def __filename(self): + return os.path.basename(self.file) + + def __get_hash(self): + return self.provider.hash(self.file) + + def __str__(self): + return ''.format(self.hash, self.file) + + +def sync_image(record): + from db import db + + q = {'hash': record['hash']} + try: + db().images.update_one(q, {'$set': record}, upsert=True) + i = db().images.find_one({'hash': record['hash']}) + return i + except ValueError: + pass + + return None + + +def document_type(doctype): + if doctype == 'Награды': + return 'award' + return 'document' + + +def pdf_to_jpg(provider, pdf): + _, temp_in = mkstemp(suffix='.pdf') + _, temp_out = mkstemp(suffix='.jpg') + + abspdf = provider.copy(pdf, temp_in) + image_magick_pdf_to_img(abspdf, temp_out) + + os.remove(temp_in) + return temp_out + + +def get_pdf_title(provider, file): + from PyPDF2 import PdfFileReader + from PyPDF2.generic import TextStringObject + from PyPDF2.generic import IndirectObject + + pdf = PdfFileReader(provider.read(file)) + info = pdf.getDocumentInfo() + + if info: + if type(info.title) == TextStringObject: + return str(info.title) + + if type(info.title_raw) == IndirectObject: + o = pdf.getObject(info.title_raw) + return str(o) + return None + + +def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]: + """ + + :param file: Image path to process + :param sizes: list of sizes to generate image (, , ) + :param url_fn: function (size_name, ext) for creating image url + :param output_dir: path to local folder for thumbs storing + :return: + """ + if not os.path.exists(file): + return None + + _, ext = os.path.splitext(file) + + def fn(size): + size_name, width, height = size + + image_url = url_fn(file, size_name, ext) + image_filename = os.path.basename(image_url) + local_image_path = os.path.join(output_dir, image_filename) + + if settings.image_processing_enabled: + image = process_image(file, local_image_path, size) + if not image: + logger.warning('Failed to process image {}'.format(file)) + return None + width, height = image.size + + return { + 'file': local_image_path, + 'url': image_url, + 'size': size_name, + 'width': width, + 'height': height + } + + return [fn(s) for s in sizes] + + +def process_image(input_file, output_file, size): + size_name, width, height = size + + if size_name == 'original': + return optimize(input_file, output_file, quality=90) + else: + image = read_image(input_file) + if not image: + return None + image_width, image_height = image.size + if image_height > image_width: + width, height = height, width + return thumbnail(input_file, output_file, (width, height)) + + +def read_image(src): + from PIL import Image + + try: + i = Image.open(src) + return i + except Exception as e: + print(src, e) + return None diff --git a/utils/Sync/sync/models/__init__.py b/utils/Sync/sync/models/__init__.py index 5c6d8dea..1f29260b 100644 --- a/utils/Sync/sync/models/__init__.py +++ b/utils/Sync/sync/models/__init__.py @@ -1,6 +1,25 @@ class Model: - def __init__(self): + def __init__(self, provider, store, file): super().__init__() + self.provider = provider + self.store = store + self.file = file + + self.id = None + self.url = None + self.hash = None + + self.init() + + def init(self): + pass + def bake(self): return None + + async def is_changed(self): + return False + + async def save(self): + pass diff --git a/utils/Sync/sync/models/document.py b/utils/Sync/sync/models/document.py deleted file mode 100644 index 29724b4a..00000000 --- a/utils/Sync/sync/models/document.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -from tempfile import mkstemp - -import settings -from sync.models import Model -from utils.fn import last_good, lmap -from utils.hash import hash_str -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - - -class Document(Model): - @staticmethod - def read(path): - name = os.path.basename(path) - - document = { - 'file': path, - 'category': os.path.dirname(path), - 'title': os.path.splitext(name)[0] - } - - return document - - def __init__(self, provider, path, url_base, preview_compiler): - super().__init__() - - self.path = path - - self.category = None - self.file = None - self.id = None - self.type = None - self.url = None - self.title = None - self.preview = None - - for k, v in m.items(): - setattr(self, k, v) - - self.provider = provider - - def bake(self): - file = self.file - filename = os.path.basename(file) - - self.type = document_type(self.type) - self.preview = self.create_preview(file) - self.title = last_good([ - self.title, - get_pdf_title(self.provider, self.file) - ]) - self.file = { - 'name': filename, - 'size': self.provider.size(file) - } - - def __set_id(self): - bn = os.path.basename(self.file) - self.id = url_encode_text('{category}-{file}'.format( - category=self.category, - file=bn - )) - - def __set_url(self): - filename = os.path.basename(self.file) - self.url = self.url_base_document.format(file=url_encode_file(filename)) - - def __set_hash(self): - hash_keys = ['id', 'url'] - hash_doc = lmap( - lambda key: getattr(self, key), - hash_keys - ) - self.hash = hash_str( - hash_str(hash_doc) + self.provider.hash(self.file) - ) - - def create_preview(self, file): - def url(size, ext): - return self.url_base_preview.format(id=file, size=size, ext=ext) - - temp_preview_path = pdf_to_jpg(self.provider, file) - file = os.path.basename(file) - file = url_encode_text(file) - img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images) - os.remove(temp_preview_path) - return img - - -def document_type(doctype): - if doctype == 'Награды': - return 'award' - return 'document' - - -def pdf_to_jpg(provider, pdf): - _, temp_in = mkstemp(suffix='.pdf') - _, temp_out = mkstemp(suffix='.jpg') - - abspdf = provider.copy(pdf, temp_in) - image_magick_pdf_to_img(abspdf, temp_out) - - os.remove(temp_in) - return temp_out - - -def get_pdf_title(provider, file): - from PyPDF2 import PdfFileReader - from PyPDF2.generic import TextStringObject - from PyPDF2.generic import IndirectObject - - pdf = PdfFileReader(provider.read(file)) - info = pdf.getDocumentInfo() - - if info: - if type(info.title) == TextStringObject: - return str(info.title) - - if type(info.title_raw) == IndirectObject: - o = pdf.getObject(info.title_raw) - return str(o) - return None diff --git a/utils/Sync/sync/page.py b/utils/Sync/sync/page.py index 0705e1fd..bedf8a55 100644 --- a/utils/Sync/sync/page.py +++ b/utils/Sync/sync/page.py @@ -135,6 +135,17 @@ def create_page(provider: Provider, img: ImageCreator, document): return document +def check_url(documents): + urls = list(map( + lambda i: i['url'], + documents + )) + + unique_urls = set(urls) + if len(urls) != len(unique_urls): + raise Exception('URL of pages should be unique') + + def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True): img_sizes = settings.album_image_sizes output_dir = settings.dir_static_images @@ -170,6 +181,8 @@ def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=Tr lambda i: i['id'] ) + check_url(documents) + # SKIP UNTOUCHED DOCUMENTS if skip_unchanged: documents = untouched(documents, sync) diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 9e8a9f5a..e1579bd0 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -1,7 +1,7 @@ import os import settings -from kazimir import markdown_to_html +from kazimir import kazimir_to_html from sync.data import list_images from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ synced_images_ids, untouched @@ -61,7 +61,7 @@ def get_manifest(provider, path): if 'id' in manifest: manifest['id'] = str(manifest['id']) - html = markdown_to_html(manifest['content']) + html = kazimir_to_html(manifest['content']) return { **manifest, 'post': html, @@ -134,7 +134,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True): ) # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) + # documents = untouched(documents, sync) # MAP EVENT MANIFEST -> EVENT_OBJECT documents = lmap( @@ -174,7 +174,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True): def sync_posts(provider, collection, update=True, delete=True): - image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' + image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}' return main( SyncPost( diff --git a/utils/Sync/test/sync/test_document.py b/utils/Sync/test/sync/test_document.py index b31b9d65..b824c1f0 100644 --- a/utils/Sync/test/sync/test_document.py +++ b/utils/Sync/test/sync/test_document.py @@ -4,7 +4,7 @@ import settings from sync.data import scan_subdirs from sync.data.fs import FSProvider from sync.document import create_document -from sync.models.document import Document +from sync.models.Document import Document from utils.fn import lmap diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py index dba3a52e..9ffc6969 100644 --- a/utils/Sync/utils/image/__init__.py +++ b/utils/Sync/utils/image/__init__.py @@ -1,12 +1,18 @@ import os +import logging +from typing import Optional + from PIL import Image +import settings from sync.data import upload from utils.hash import hash_file # from utils.image import resize from utils.image.resize import optimize, thumbnail, orient +logger = logging.getLogger(settings.name) + def read_image(src): try: @@ -17,19 +23,15 @@ def read_image(src): return None -def create_image(file, sizes, url_fn, output_dir, skip_processing=False): +def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[dict]: """ :param file: Image path to process :param sizes: list of sizes to generate image (, , ) :param url_fn: function (size_name, ext) for creating image url :param output_dir: path to local folder for thumbs storing - :param skip_processing: True/False to skip thumbs generation :return: """ - # image = read_image(file) - # if not image: - # return None if not os.path.exists(file): return None @@ -38,18 +40,20 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): for size in sizes: size_name, width, height = size - image_url = url_fn(size_name, ext) + image_url = url_fn(file, size_name, ext) image_filename = os.path.basename(image_url) local_image_path = os.path.join(output_dir, image_filename) - if not skip_processing: + if settings.image_processing_enabled: image = process_image(file, local_image_path, size) if not image: + logger.warning('Failed to process image {}'.format(file)) continue + width, height = image.size - upload_url = upload(local_image_path, '/images/{}'.format(image_filename)) - if not upload_url: - print('Image {} not uploaded'.format(file)) + # upload_url = upload(local_image_path, '/images/{}'.format(image_filename)) + # if not upload_url: + # print('Image {} not uploaded'.format(file)) result[size_name] = { 'url': image_url, @@ -58,11 +62,7 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): 'height': height } - return { - 'file': os.path.basename(file), - 'hash': hash_file(file), - 'data': result - } + return result def process_image(input_file, output_file, size):