diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 4c371fe8..a3e0e0b3 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,12 +1,7 @@ import settings from db import collection from sync.core import Sync -from sync.core.SyncAlbum import SyncAlbum -from sync.core.SyncArticle import SyncArticle from sync.core.SyncDocument import SyncDocument -from sync.core.SyncPage import SyncPage -from sync.core.SyncPerson import SyncPerson -from sync.core.SyncSchedule import SyncSchedule from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider @@ -14,6 +9,12 @@ import asyncio import logging # create logger with 'spam_application' +from sync.models.Album import Album +from sync.models.Article import Article +from sync.models.Page import Page +from sync.models.Person import Person +from sync.models.Schedule import Schedule + logger = logging.getLogger(settings.name) logger.setLevel(logging.DEBUG) @@ -42,36 +43,35 @@ async def main(sync: Sync): async def run(run_interval=0): io = lambda root: get_provider(settings.provider_name, root) - c = collection documents = SyncDocument( provider=io(settings.dir_documents), - collection=c(settings.collection_documents), sizes=settings.image_sizes, ) - pages = SyncPage( + pages = Sync( provider=io(settings.dir_pages), - collection=c(settings.collection_pages), + model=Page, ) - persons = SyncPerson( + persons = Sync( provider=io(settings.dir_collective), - collection=c(settings.collection_collective), + model=Person, ) - articles = SyncArticle( + articles = Sync( provider=io(settings.dir_articles), - collection=c(settings.collection_articles), + model=Article, ) - schedules = SyncSchedule( + schedules = Sync( provider=io(settings.dir_schedules), - collection=c(settings.collection_schedules), + model=Schedule, ) - albums = SyncAlbum( + + albums = Sync( provider=io(settings.dir_albums), - collection=c(settings.collection_albums), + model=Album, ) while True: diff --git a/utils/Sync/sync/core/SyncAlbum.py b/utils/Sync/sync/core/SyncAlbum.py deleted file mode 100644 index d15bec7c..00000000 --- a/utils/Sync/sync/core/SyncAlbum.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging - -import settings -from sync import untouched -from sync.core import Sync -from sync.models.Album import Album - -logger = logging.getLogger(settings.name + '.Album') - - -class SyncAlbum(Sync): - async def run(self): - logger.info('Checking for update') - - documents = await Album.scan(self.provider) - - documents_id = [doc.id for doc in documents] - # self.validate(documents) - logger.info('Found {} Item(s)'.format(len(documents))) - - # SKIP UNTOUCHED DOCUMENTS - documents = await untouched(documents) - logger.info('Changed {} Items(s)'.format(len(documents))) - - # UPDATING - if settings.update_enabled: - if len(documents) == 0: - logger.info('No Items to update') - - for document in documents: - logger.info('Updating Item {}'.format(document.id)) - - await document.build() - await document.save() - - logger.info('Updated Item {}'.format(document.id)) - - # DELETING - if settings.delete_enabled: - documents_delete = list(self.query({'id': {'$nin': documents_id}})) - - if len(documents_delete) == 0: - logger.info('No Items to delete') - else: - for document in documents_delete: - logger.info('Deleting Item {}'.format(document['id'])) - - q = {'_id': document['_id']} - self.delete(q) - - # await self.clean() diff --git a/utils/Sync/sync/core/SyncArticle.py b/utils/Sync/sync/core/SyncArticle.py deleted file mode 100644 index 666079a5..00000000 --- a/utils/Sync/sync/core/SyncArticle.py +++ /dev/null @@ -1,58 +0,0 @@ -import os - -import logging - -import settings -from sync import untouched -from sync.core import Sync -from sync.data import Provider -from sync.models.Article import Article -from sync.models.Person import Person - -from utils.text.transform import url_encode_text - -logger = logging.getLogger(settings.name + '.Article') - - -class SyncArticle(Sync): - async def run(self): - logger.info('Checking for update') - - documents = await Article.scan(self.provider) - documents_id = [doc.id for doc in documents] - # self.validate(documents) - logger.info('Found {} Item(s)'.format(len(documents))) - - # SKIP UNTOUCHED DOCUMENTS - documents = await untouched(documents) - logger.info('Changed {} Items(s)'.format(len(documents))) - - # UPDATING - if settings.update_enabled: - if len(documents) == 0: - logger.info('No Items to update') - - for document in documents: - logger.info('Updating Item {}'.format(document.id)) - - await document.build( - sizes=settings.image_sizes, - ) - await document.save() - - logger.info('Updated Item {}'.format(document.id)) - - # DELETING - if settings.delete_enabled: - documents_delete = list(self.query({'id': {'$nin': documents_id}})) - - if len(documents_delete) == 0: - logger.info('No Items to delete') - else: - for document in documents_delete: - logger.info('Deleting Item {}'.format(document['id'])) - - q = {'_id': document['_id']} - self.delete(q) - - # await self.clean() \ No newline at end of file diff --git a/utils/Sync/sync/core/SyncDocument.py b/utils/Sync/sync/core/SyncDocument.py index 8806130b..7af5bbf0 100644 --- a/utils/Sync/sync/core/SyncDocument.py +++ b/utils/Sync/sync/core/SyncDocument.py @@ -1,8 +1,6 @@ import os from tempfile import mkstemp -import logging - import settings from sync import untouched from sync.core import Sync @@ -15,65 +13,16 @@ from utils.image import create_image from utils.image.resize import image_magick_pdf_to_img from utils.text.transform import url_encode_text, url_encode_file -logger = logging.getLogger(settings.name + '.Document') - class SyncDocument(Sync): - def __init__(self, provider: Provider, collection, sizes): - super().__init__(provider, collection) + def __init__(self, provider: Provider, sizes): + super().__init__(provider, Document) self.sizes = sizes - self.dir_static_previews = '' - self.temp_dir = '/Users/tmshv/HudozkaSyncTemp' - - async def clean(self): - pass - - async def run(self): - logger.info('Checking documents for update') - - documents = await Document.scan(self.provider) - documents_id = [doc.id for doc in documents] - # self.validate(documents) - logger.info('Found {} Document(s)'.format(len(documents))) - - # SKIP UNTOUCHED DOCUMENTS - # documents = untouched(documents, self) - documents = await untouched(documents) - logger.info('Changed {} Document(s)'.format(len(documents))) - - # UPDATING - if settings.update_enabled: - if len(documents) == 0: - logger.info('No Documents to update') - - for document in documents: - logger.info('Updating Document {}'.format(document.id)) - - await document.upload() - await document.create_preview(self.sizes) - await document.save() - - logger.info('Updated Document {}'.format(document.id)) - - # DELETING - if settings.delete_enabled: - documents_delete = list(self.query({'id': {'$nin': documents_id}})) - - if len(documents_delete) == 0: - logger.info('No Documents to delete') - - for document in documents_delete: - logger.info('Deleting {}'.format(document['id'])) - - q = {'_id': document['_id']} - self.delete(q) - - await self.clean() - - -async def get_assets(self, document) -> []: - return [] + def _get_build_args(self): + return { + 'sizes': self.sizes, + } def validate(self, documents): diff --git a/utils/Sync/sync/core/SyncPage.py b/utils/Sync/sync/core/SyncPage.py deleted file mode 100644 index 705c42ee..00000000 --- a/utils/Sync/sync/core/SyncPage.py +++ /dev/null @@ -1,77 +0,0 @@ -import os - -import logging - -import settings -from sync import untouched -from sync.core import Sync -from sync.data import Provider -from sync.models.Page import Page - -from utils.text.transform import url_encode_text - -logger = logging.getLogger(settings.name + '.Page') - - -class SyncPage(Sync): - def __init__(self, provider: Provider, collection): - super().__init__(provider, collection) - - async def clean(self): - pass - - async def run(self): - logger.info('Checking for update') - - documents = await Page.scan(self.provider) - documents_id = [doc.id for doc in documents] - # self.validate(documents) - logger.info('Found {} Page(s)'.format(len(documents))) - - # SKIP UNTOUCHED DOCUMENTS - documents = await untouched(documents) - logger.info('Changed {} Pages(s)'.format(len(documents))) - - # UPDATING - if settings.update_enabled: - if len(documents) == 0: - logger.info('No Pages to update') - - for document in documents: - logger.info('Updating Page {}'.format(document.id)) - - await document.setup_images(settings.image_sizes, get_image_url_fn(document.id)) - await document.save() - - logger.info('Updated Page {}'.format(document.id)) - - # DELETING - if settings.delete_enabled: - documents_delete = list(self.query({'id': {'$nin': documents_id}})) - - if len(documents_delete) == 0: - logger.info('No Pages to delete') - else: - for document in documents_delete: - logger.info('Deleting Page {}'.format(document['id'])) - - q = {'_id': document['_id']} - self.delete(q) - - await self.clean() - - -def get_image_url_fn(page_id: str): - image_id = lambda filename: url_encode_text( - os.path.splitext(os.path.basename(filename))[0] - ) - - def img_url_fn(file, size, ext): - return settings.page_url_base.format( - page=page_id, - id=image_id(file), - size=size, - ext=ext - ) - - return img_url_fn diff --git a/utils/Sync/sync/core/SyncPerson.py b/utils/Sync/sync/core/SyncPerson.py deleted file mode 100644 index 9b5d758b..00000000 --- a/utils/Sync/sync/core/SyncPerson.py +++ /dev/null @@ -1,75 +0,0 @@ -import os - -import logging - -import settings -from sync import untouched -from sync.core import Sync -from sync.data import Provider -from sync.models.Person import Person - -from utils.text.transform import url_encode_text - -logger = logging.getLogger(settings.name + '.Person') - - -class SyncPerson(Sync): - async def run(self): - logger.info('Checking for update') - - documents = await Person.scan(self.provider) - - documents_id = [doc.id for doc in documents] - # self.validate(documents) - logger.info('Found {} Item(s)'.format(len(documents))) - - # SKIP UNTOUCHED DOCUMENTS - documents = await untouched(documents) - logger.info('Changed {} Items(s)'.format(len(documents))) - - # UPDATING - if settings.update_enabled: - if len(documents) == 0: - logger.info('No Items to update') - - for document in documents: - logger.info('Updating Item {}'.format(document.id)) - - await document.build( - sizes=settings.image_sizes, - url_factory=get_image_url_fn(document.id), - ) - await document.save() - - logger.info('Updated Item {}'.format(document.id)) - - # DELETING - if settings.delete_enabled: - documents_delete = list(self.query({'id': {'$nin': documents_id}})) - - if len(documents_delete) == 0: - logger.info('No Items to delete') - else: - for document in documents_delete: - logger.info('Deleting Item {}'.format(document['id'])) - - q = {'_id': document['_id']} - self.delete(q) - - # await self.clean() - - -def get_image_url_fn(page_id: str): - image_id = lambda filename: url_encode_text( - os.path.splitext(os.path.basename(filename))[0] - ) - - def img_url_fn(file, size, ext): - return settings.page_url_base.format( - page=page_id, - id=image_id(file), - size=size, - ext=ext - ) - - return img_url_fn diff --git a/utils/Sync/sync/core/SyncSchedule.py b/utils/Sync/sync/core/SyncSchedule.py deleted file mode 100644 index 39579bb4..00000000 --- a/utils/Sync/sync/core/SyncSchedule.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging - -import settings -from sync import untouched -from sync.core import Sync -from sync.models.Schedule import Schedule - -logger = logging.getLogger(settings.name + '.Schedule') - - -class SyncSchedule(Sync): - async def run(self): - logger.info('Checking for update') - - documents = await Schedule.scan(self.provider) - - documents_id = [doc.id for doc in documents] - # self.validate(documents) - logger.info('Found {} Item(s)'.format(len(documents))) - - # SKIP UNTOUCHED DOCUMENTS - documents = await untouched(documents) - logger.info('Changed {} Items(s)'.format(len(documents))) - - # UPDATING - if settings.update_enabled: - if len(documents) == 0: - logger.info('No Items to update') - - for document in documents: - logger.info('Updating Item {}'.format(document.id)) - - await document.build() - await document.save() - - logger.info('Updated Item {}'.format(document.id)) - - # DELETING - if settings.delete_enabled: - documents_delete = list(self.query({'id': {'$nin': documents_id}})) - - if len(documents_delete) == 0: - logger.info('No Items to delete') - else: - for document in documents_delete: - logger.info('Deleting Item {}'.format(document['id'])) - - q = {'_id': document['_id']} - self.delete(q) - - # await self.clean() diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py index dd44d4d8..515c4478 100644 --- a/utils/Sync/sync/core/__init__.py +++ b/utils/Sync/sync/core/__init__.py @@ -1,3 +1,7 @@ +import logging + +import settings +from sync import untouched, Model from sync.data import Provider @@ -9,15 +13,35 @@ def get_id(item): class Sync: - @staticmethod - def compile(document): - return document.bake() - - def __init__(self, provider: Provider, collection): + def __init__(self, provider: Provider, model): super().__init__() + self.__default_build_args = { + 'sizes': settings.image_sizes, + } + self.__items_to_delete_query = lambda items_id: {'id': {'$nin': items_id}} + self.__delete_query = lambda item: {'_id': item['_id']} + + name = model.__name__ + self.logger = logging.getLogger('%s.%s' % (settings.name, name)) + self.provider = provider - self.collection = collection + self.model = model + + async def clean(self): + pass + + async def update(self, item): + args = self._get_build_args() + + await item.build(**args) + await item.upload() + await item.save() + + def _get_build_args(self): + return { + **self.__default_build_args + } async def run(self): """ @@ -30,33 +54,38 @@ class Sync: :return: """ - return None, None + self.logger.info('Checking for update') + + items = await self.model.scan(self.provider) + items_id = [doc.id for doc in items] + self.logger.info('Found {} Item(s)'.format(len(items))) + + # SKIP UNTOUCHED DOCUMENTS + items = await untouched(items) + self.logger.info('Changed {} Items(s)'.format(len(items))) + + # UPDATING + if settings.update_enabled: + if len(items) == 0: + self.logger.info('No Items to update') + + for item in items: + self.logger.info('Updating Item {}'.format(item.id)) + await self.update(item) + + # DELETING + if settings.delete_enabled: + items = await self.model.find(self.__items_to_delete_query(items_id)) + items = list(items) + + if len(items) == 0: + self.logger.info('No Items to delete') + else: + for item in items: + self.logger.info('Deleting Item {}'.format(item['id'])) + await self.model.delete(self.__delete_query(item)) + + await self.clean() async def upload(self): pass - - def read(self, document: dict): - q = {'id': document['id']} - try: - return self.collection.find_one(q) - except ValueError: - pass - return None - - def update(self, document: dict): - query = {'id': document['id']} - try: - self.collection.update_one(query, {'$set': document}, upsert=True) - return self.read(document) - except ValueError: - pass - return None - - def query(self, q): - return self.collection.find(q) - - def delete(self, q): - return self.collection.find_one_and_delete(q) - - def create_remove_query(self, query): - return query diff --git a/utils/Sync/sync/models/Album.py b/utils/Sync/sync/models/Album.py index 0dd35cb2..8c3be4da 100644 --- a/utils/Sync/sync/models/Album.py +++ b/utils/Sync/sync/models/Album.py @@ -23,16 +23,15 @@ logger = logging.getLogger(settings.name + '.Album') store = collection(settings.collection_albums) -def find(item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None - - class Album(Model): + @staticmethod + async def find(query): + return store.find(query) + + @staticmethod + async def delete(query): + return store.find_one_and_delete(query) + @staticmethod async def scan(provider): documents = [i for i in provider.scan('.') if provider.is_dir(i)] diff --git a/utils/Sync/sync/models/Article.py b/utils/Sync/sync/models/Article.py index b1ba8768..1f6a7dde 100644 --- a/utils/Sync/sync/models/Article.py +++ b/utils/Sync/sync/models/Article.py @@ -18,20 +18,19 @@ from utils.hash import hash_str from utils.io import read_yaml_md, parse_yaml_front_matter from utils.text.transform import url_encode_text -logger = logging.getLogger(settings.name + '.Person') +logger = logging.getLogger(settings.name + '.Article') store = collection(settings.collection_articles) -def find(item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None - - class Article(Model): + @staticmethod + async def find(query): + return store.find(query) + + @staticmethod + async def delete(query): + return store.find_one_and_delete(query) + @staticmethod async def scan(provider): documents = [i for i in provider.scan('.') if provider.is_dir(i)] diff --git a/utils/Sync/sync/models/Document.py b/utils/Sync/sync/models/Document.py index 64d162d0..efc8c83f 100644 --- a/utils/Sync/sync/models/Document.py +++ b/utils/Sync/sync/models/Document.py @@ -11,17 +11,18 @@ from utils.hash import hash_str from utils.image.resize import image_magick_pdf_to_img from utils.text.transform import url_encode_text, url_encode_file - -def find(store, item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None +store = collection(settings.collection_documents) class Document(Model): + @staticmethod + async def find(query): + return store.find(query) + + @staticmethod + async def delete(query): + return store.find_one_and_delete(query) + @staticmethod async def scan(provider): paths = scan_subdirs(provider, '.pdf') @@ -43,8 +44,6 @@ class Document(Model): ) def __init__(self, provider, file, category=None, title=None): - store = collection(settings.collection_documents) - self.__hash_keys = ['id', 'url'] self.__id_template: str = '{category}-{file}' @@ -63,15 +62,9 @@ class Document(Model): self.__set_url() self.__set_hash() - async def is_changed(self): - i = find(self.store, self.id) - if not i: - return True - - if 'hash' not in i: - return True - - return not (self.hash == i['hash']) + async def build(self, **kwargs): + sizes = kwargs['sizes'] + await self.create_preview(sizes) async def save(self): document = self.bake() diff --git a/utils/Sync/sync/models/Page.py b/utils/Sync/sync/models/Page.py index 473bea59..1e0261ac 100644 --- a/utils/Sync/sync/models/Page.py +++ b/utils/Sync/sync/models/Page.py @@ -17,6 +17,7 @@ from utils.io import read_yaml_md from utils.text.transform import url_encode_text logger = logging.getLogger(settings.name + '.Page') +store = collection(settings.collection_pages) def find(store, item_id: str): @@ -29,6 +30,14 @@ def find(store, item_id: str): class Page(Model): + @staticmethod + async def find(query): + return store.find(query) + + @staticmethod + async def delete(query): + return store.find_one_and_delete(query) + @staticmethod async def scan(provider): documents = [i for i in provider.scan('.') if provider.is_dir(i)] @@ -89,6 +98,9 @@ class Page(Model): return not (self.hash == i['hash']) + async def build(self, **kwargs): + await self.setup_images(settings.image_sizes, get_image_url_fn(self.id)) + async def setup_images(self, sizes, url_factory): folder = self.params['folder'] images = [] @@ -189,3 +201,19 @@ def get_manifest(provider, path): 'title': title_from_html(body), 'images': images_from_html(body) } + + +def get_image_url_fn(page_id: str): + image_id = lambda filename: url_encode_text( + os.path.splitext(os.path.basename(filename))[0] + ) + + def img_url_fn(file, size, ext): + return settings.page_url_base.format( + page=page_id, + id=image_id(file), + size=size, + ext=ext + ) + + return img_url_fn diff --git a/utils/Sync/sync/models/Person.py b/utils/Sync/sync/models/Person.py index b37a5971..8ca2b5c2 100644 --- a/utils/Sync/sync/models/Person.py +++ b/utils/Sync/sync/models/Person.py @@ -15,16 +15,15 @@ logger = logging.getLogger(settings.name + '.Person') store = collection(settings.collection_collective) -def find(item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None - - class Person(Model): + @staticmethod + async def find(query): + return store.find(query) + + @staticmethod + async def delete(query): + return store.find_one_and_delete(query) + @staticmethod async def scan(provider): documents = provider.type_filter('', '.md') @@ -59,9 +58,9 @@ class Person(Model): async def build(self, **kwargs): sizes = kwargs['sizes'] - image_path = self.provider.get_abs(self.params['picture']) + image_path = self.provider.get_abs(self.get_param('picture')) - self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id)) + self.picture = await Image.new(self.provider, image_path, sizes) def bake(self): return { diff --git a/utils/Sync/sync/models/Schedule.py b/utils/Sync/sync/models/Schedule.py index 881d722a..bda94d20 100644 --- a/utils/Sync/sync/models/Schedule.py +++ b/utils/Sync/sync/models/Schedule.py @@ -1,5 +1,3 @@ -import logging - from toml import loads as toml_str from yaml import load as yaml @@ -10,7 +8,6 @@ from sync.models import Model from utils.fn import constant -logger = logging.getLogger(settings.name + '.Person') store = collection(settings.collection_schedules) @@ -24,6 +21,14 @@ def find(item_id: str): class Schedule(Model): + @staticmethod + async def find(query): + return store.find(query) + + @staticmethod + async def delete(query): + return store.find_one_and_delete(query) + @staticmethod async def scan(provider): documents = provider.type_filter('.', '.yaml') diff --git a/utils/Sync/sync/models/__init__.py b/utils/Sync/sync/models/__init__.py index 5a843a89..5b06fcb9 100644 --- a/utils/Sync/sync/models/__init__.py +++ b/utils/Sync/sync/models/__init__.py @@ -28,6 +28,9 @@ class Model: def init(self): pass + async def upload(self, **kwargs): + pass + async def build(self, **kwargs): pass