[Sync] update: removed unnecesarry Sync subclasses

utils/Sync/sync.py

@@ -1,12 +1,7 @@
 import settings
 from db import collection
 from sync.core import Sync
-from sync.core.SyncAlbum import SyncAlbum
-from sync.core.SyncArticle import SyncArticle
 from sync.core.SyncDocument import SyncDocument
-from sync.core.SyncPage import SyncPage
-from sync.core.SyncPerson import SyncPerson
-from sync.core.SyncSchedule import SyncSchedule
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
 
@@ -14,6 +9,12 @@ import asyncio
 import logging
 
 # create logger with 'spam_application'
+from sync.models.Album import Album
+from sync.models.Article import Article
+from sync.models.Page import Page
+from sync.models.Person import Person
+from sync.models.Schedule import Schedule
+
 logger = logging.getLogger(settings.name)
 logger.setLevel(logging.DEBUG)
 
@@ -42,36 +43,35 @@ async def main(sync: Sync):
 
 async def run(run_interval=0):
     io = lambda root: get_provider(settings.provider_name, root)
-    c = collection
 
     documents = SyncDocument(
         provider=io(settings.dir_documents),
-        collection=c(settings.collection_documents),
         sizes=settings.image_sizes,
     )
 
-    pages = SyncPage(
+    pages = Sync(
         provider=io(settings.dir_pages),
-        collection=c(settings.collection_pages),
+        model=Page,
     )
 
-    persons = SyncPerson(
+    persons = Sync(
         provider=io(settings.dir_collective),
-        collection=c(settings.collection_collective),
+        model=Person,
     )
 
-    articles = SyncArticle(
+    articles = Sync(
         provider=io(settings.dir_articles),
-        collection=c(settings.collection_articles),
+        model=Article,
     )
 
-    schedules = SyncSchedule(
+    schedules = Sync(
         provider=io(settings.dir_schedules),
-        collection=c(settings.collection_schedules),
+        model=Schedule,
     )
-    albums = SyncAlbum(
+
+    albums = Sync(
         provider=io(settings.dir_albums),
-        collection=c(settings.collection_albums),
+        model=Album,
     )
 
     while True:

utils/Sync/sync/core/SyncAlbum.py

@@ -1,51 +0,0 @@
-import logging
-
-import settings
-from sync import untouched
-from sync.core import Sync
-from sync.models.Album import Album
-
-logger = logging.getLogger(settings.name + '.Album')
-
-
-class SyncAlbum(Sync):
-    async def run(self):
-        logger.info('Checking for update')
-
-        documents = await Album.scan(self.provider)
-
-        documents_id = [doc.id for doc in documents]
-        # self.validate(documents)
-        logger.info('Found {} Item(s)'.format(len(documents)))
-
-        # SKIP UNTOUCHED DOCUMENTS
-        documents = await untouched(documents)
-        logger.info('Changed {} Items(s)'.format(len(documents)))
-
-        # UPDATING
-        if settings.update_enabled:
-            if len(documents) == 0:
-                logger.info('No Items to update')
-
-            for document in documents:
-                logger.info('Updating Item {}'.format(document.id))
-
-                await document.build()
-                await document.save()
-
-                logger.info('Updated Item {}'.format(document.id))
-
-        # DELETING
-        if settings.delete_enabled:
-            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
-
-            if len(documents_delete) == 0:
-                logger.info('No Items to delete')
-            else:
-                for document in documents_delete:
-                    logger.info('Deleting Item {}'.format(document['id']))
-
-                    q = {'_id': document['_id']}
-                    self.delete(q)
-
-        # await self.clean()

utils/Sync/sync/core/SyncArticle.py

@@ -1,58 +0,0 @@
-import os
-
-import logging
-
-import settings
-from sync import untouched
-from sync.core import Sync
-from sync.data import Provider
-from sync.models.Article import Article
-from sync.models.Person import Person
-
-from utils.text.transform import url_encode_text
-
-logger = logging.getLogger(settings.name + '.Article')
-
-
-class SyncArticle(Sync):
-    async def run(self):
-        logger.info('Checking for update')
-
-        documents = await Article.scan(self.provider)
-        documents_id = [doc.id for doc in documents]
-        # self.validate(documents)
-        logger.info('Found {} Item(s)'.format(len(documents)))
-
-        # SKIP UNTOUCHED DOCUMENTS
-        documents = await untouched(documents)
-        logger.info('Changed {} Items(s)'.format(len(documents)))
-
-        # UPDATING
-        if settings.update_enabled:
-            if len(documents) == 0:
-                logger.info('No Items to update')
-
-            for document in documents:
-                logger.info('Updating Item {}'.format(document.id))
-
-                await document.build(
-                    sizes=settings.image_sizes,
-                )
-                await document.save()
-
-                logger.info('Updated Item {}'.format(document.id))
-
-        # DELETING
-        if settings.delete_enabled:
-            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
-
-            if len(documents_delete) == 0:
-                logger.info('No Items to delete')
-            else:
-                for document in documents_delete:
-                    logger.info('Deleting Item {}'.format(document['id']))
-
-                    q = {'_id': document['_id']}
-                    self.delete(q)
-
-        # await self.clean()

utils/Sync/sync/core/SyncDocument.py

@@ -1,8 +1,6 @@
 import os
 from tempfile import mkstemp
 
-import logging
-
 import settings
 from sync import untouched
 from sync.core import Sync
@@ -15,65 +13,16 @@ from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
 from utils.text.transform import url_encode_text, url_encode_file
 
-logger = logging.getLogger(settings.name + '.Document')
-
 
 class SyncDocument(Sync):
-    def __init__(self, provider: Provider, collection, sizes):
-        super().__init__(provider, collection)
+    def __init__(self, provider: Provider, sizes):
+        super().__init__(provider, Document)
         self.sizes = sizes
-        self.dir_static_previews = ''
 
-        self.temp_dir = '/Users/tmshv/HudozkaSyncTemp'
-
-    async def clean(self):
-        pass
-
-    async def run(self):
-        logger.info('Checking documents for update')
-
-        documents = await Document.scan(self.provider)
-        documents_id = [doc.id for doc in documents]
-        # self.validate(documents)
-        logger.info('Found {} Document(s)'.format(len(documents)))
-
-        # SKIP UNTOUCHED DOCUMENTS
-        # documents = untouched(documents, self)
-        documents = await untouched(documents)
-        logger.info('Changed {} Document(s)'.format(len(documents)))
-
-        # UPDATING
-        if settings.update_enabled:
-            if len(documents) == 0:
-                logger.info('No Documents to update')
-
-            for document in documents:
-                logger.info('Updating Document {}'.format(document.id))
-
-                await document.upload()
-                await document.create_preview(self.sizes)
-                await document.save()
-
-                logger.info('Updated Document {}'.format(document.id))
-
-        # DELETING
-        if settings.delete_enabled:
-            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
-
-            if len(documents_delete) == 0:
-                logger.info('No Documents to delete')
-
-            for document in documents_delete:
-                logger.info('Deleting {}'.format(document['id']))
-
-                q = {'_id': document['_id']}
-                self.delete(q)
-
-        await self.clean()
-
-
-async def get_assets(self, document) -> []:
-    return []
+    def _get_build_args(self):
+        return {
+            'sizes': self.sizes,
+        }
 
 
 def validate(self, documents):

utils/Sync/sync/core/SyncPage.py

@@ -1,77 +0,0 @@
-import os
-
-import logging
-
-import settings
-from sync import untouched
-from sync.core import Sync
-from sync.data import Provider
-from sync.models.Page import Page
-
-from utils.text.transform import url_encode_text
-
-logger = logging.getLogger(settings.name + '.Page')
-
-
-class SyncPage(Sync):
-    def __init__(self, provider: Provider, collection):
-        super().__init__(provider, collection)
-
-    async def clean(self):
-        pass
-
-    async def run(self):
-        logger.info('Checking for update')
-
-        documents = await Page.scan(self.provider)
-        documents_id = [doc.id for doc in documents]
-        # self.validate(documents)
-        logger.info('Found {} Page(s)'.format(len(documents)))
-
-        # SKIP UNTOUCHED DOCUMENTS
-        documents = await untouched(documents)
-        logger.info('Changed {} Pages(s)'.format(len(documents)))
-
-        # UPDATING
-        if settings.update_enabled:
-            if len(documents) == 0:
-                logger.info('No Pages to update')
-
-            for document in documents:
-                logger.info('Updating Page {}'.format(document.id))
-
-                await document.setup_images(settings.image_sizes, get_image_url_fn(document.id))
-                await document.save()
-
-                logger.info('Updated Page {}'.format(document.id))
-
-        # DELETING
-        if settings.delete_enabled:
-            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
-
-            if len(documents_delete) == 0:
-                logger.info('No Pages to delete')
-            else:
-                for document in documents_delete:
-                    logger.info('Deleting Page {}'.format(document['id']))
-
-                    q = {'_id': document['_id']}
-                    self.delete(q)
-
-        await self.clean()
-
-
-def get_image_url_fn(page_id: str):
-    image_id = lambda filename: url_encode_text(
-        os.path.splitext(os.path.basename(filename))[0]
-    )
-
-    def img_url_fn(file, size, ext):
-        return settings.page_url_base.format(
-            page=page_id,
-            id=image_id(file),
-            size=size,
-            ext=ext
-        )
-
-    return img_url_fn

utils/Sync/sync/core/SyncPerson.py

@@ -1,75 +0,0 @@
-import os
-
-import logging
-
-import settings
-from sync import untouched
-from sync.core import Sync
-from sync.data import Provider
-from sync.models.Person import Person
-
-from utils.text.transform import url_encode_text
-
-logger = logging.getLogger(settings.name + '.Person')
-
-
-class SyncPerson(Sync):
-    async def run(self):
-        logger.info('Checking for update')
-
-        documents = await Person.scan(self.provider)
-
-        documents_id = [doc.id for doc in documents]
-        # self.validate(documents)
-        logger.info('Found {} Item(s)'.format(len(documents)))
-
-        # SKIP UNTOUCHED DOCUMENTS
-        documents = await untouched(documents)
-        logger.info('Changed {} Items(s)'.format(len(documents)))
-
-        # UPDATING
-        if settings.update_enabled:
-            if len(documents) == 0:
-                logger.info('No Items to update')
-
-            for document in documents:
-                logger.info('Updating Item {}'.format(document.id))
-
-                await document.build(
-                    sizes=settings.image_sizes,
-                    url_factory=get_image_url_fn(document.id),
-                )
-                await document.save()
-
-                logger.info('Updated Item {}'.format(document.id))
-
-        # DELETING
-        if settings.delete_enabled:
-            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
-
-            if len(documents_delete) == 0:
-                logger.info('No Items to delete')
-            else:
-                for document in documents_delete:
-                    logger.info('Deleting Item {}'.format(document['id']))
-
-                    q = {'_id': document['_id']}
-                    self.delete(q)
-
-        # await self.clean()
-
-
-def get_image_url_fn(page_id: str):
-    image_id = lambda filename: url_encode_text(
-        os.path.splitext(os.path.basename(filename))[0]
-    )
-
-    def img_url_fn(file, size, ext):
-        return settings.page_url_base.format(
-            page=page_id,
-            id=image_id(file),
-            size=size,
-            ext=ext
-        )
-
-    return img_url_fn

utils/Sync/sync/core/SyncSchedule.py

@@ -1,51 +0,0 @@
-import logging
-
-import settings
-from sync import untouched
-from sync.core import Sync
-from sync.models.Schedule import Schedule
-
-logger = logging.getLogger(settings.name + '.Schedule')
-
-
-class SyncSchedule(Sync):
-    async def run(self):
-        logger.info('Checking for update')
-
-        documents = await Schedule.scan(self.provider)
-
-        documents_id = [doc.id for doc in documents]
-        # self.validate(documents)
-        logger.info('Found {} Item(s)'.format(len(documents)))
-
-        # SKIP UNTOUCHED DOCUMENTS
-        documents = await untouched(documents)
-        logger.info('Changed {} Items(s)'.format(len(documents)))
-
-        # UPDATING
-        if settings.update_enabled:
-            if len(documents) == 0:
-                logger.info('No Items to update')
-
-            for document in documents:
-                logger.info('Updating Item {}'.format(document.id))
-
-                await document.build()
-                await document.save()
-
-                logger.info('Updated Item {}'.format(document.id))
-
-        # DELETING
-        if settings.delete_enabled:
-            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
-
-            if len(documents_delete) == 0:
-                logger.info('No Items to delete')
-            else:
-                for document in documents_delete:
-                    logger.info('Deleting Item {}'.format(document['id']))
-
-                    q = {'_id': document['_id']}
-                    self.delete(q)
-
-        # await self.clean()

utils/Sync/sync/core/__init__.py

@@ -1,3 +1,7 @@
+import logging
+
+import settings
+from sync import untouched, Model
 from sync.data import Provider
 
 
@@ -9,15 +13,35 @@ def get_id(item):
 
 
 class Sync:
-    @staticmethod
-    def compile(document):
-        return document.bake()
-
-    def __init__(self, provider: Provider, collection):
+    def __init__(self, provider: Provider, model):
         super().__init__()
 
+        self.__default_build_args = {
+            'sizes': settings.image_sizes,
+        }
+        self.__items_to_delete_query = lambda items_id: {'id': {'$nin': items_id}}
+        self.__delete_query = lambda item: {'_id': item['_id']}
+
+        name = model.__name__
+        self.logger = logging.getLogger('%s.%s' % (settings.name, name))
+
         self.provider = provider
-        self.collection = collection
+        self.model = model
+
+    async def clean(self):
+        pass
+
+    async def update(self, item):
+        args = self._get_build_args()
+
+        await item.build(**args)
+        await item.upload()
+        await item.save()
+
+    def _get_build_args(self):
+        return {
+            **self.__default_build_args
+        }
 
     async def run(self):
         """
@@ -30,33 +54,38 @@ class Sync:
 
         :return:
         """
-        return None, None
+        self.logger.info('Checking for update')
+
+        items = await self.model.scan(self.provider)
+        items_id = [doc.id for doc in items]
+        self.logger.info('Found {} Item(s)'.format(len(items)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        items = await untouched(items)
+        self.logger.info('Changed {} Items(s)'.format(len(items)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(items) == 0:
+                self.logger.info('No Items to update')
+
+            for item in items:
+                self.logger.info('Updating Item {}'.format(item.id))
+                await self.update(item)
+
+        # DELETING
+        if settings.delete_enabled:
+            items = await self.model.find(self.__items_to_delete_query(items_id))
+            items = list(items)
+
+            if len(items) == 0:
+                self.logger.info('No Items to delete')
+            else:
+                for item in items:
+                    self.logger.info('Deleting Item {}'.format(item['id']))
+                    await self.model.delete(self.__delete_query(item))
+
+        await self.clean()
 
     async def upload(self):
         pass
-
-    def read(self, document: dict):
-        q = {'id': document['id']}
-        try:
-            return self.collection.find_one(q)
-        except ValueError:
-            pass
-        return None
-
-    def update(self, document: dict):
-        query = {'id': document['id']}
-        try:
-            self.collection.update_one(query, {'$set': document}, upsert=True)
-            return self.read(document)
-        except ValueError:
-            pass
-        return None
-
-    def query(self, q):
-        return self.collection.find(q)
-
-    def delete(self, q):
-        return self.collection.find_one_and_delete(q)
-
-    def create_remove_query(self, query):
-        return query

utils/Sync/sync/models/Album.py

@@ -23,16 +23,15 @@ logger = logging.getLogger(settings.name + '.Album')
 store = collection(settings.collection_albums)
 
 
-def find(item_id: str):
-    q = {'id': item_id}
-    try:
-        return store.find_one(q)
-    except ValueError:
-        pass
-    return None
-
-
 class Album(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
     @staticmethod
     async def scan(provider):
         documents = [i for i in provider.scan('.') if provider.is_dir(i)]

utils/Sync/sync/models/Article.py

@@ -18,20 +18,19 @@ from utils.hash import hash_str
 from utils.io import read_yaml_md, parse_yaml_front_matter
 from utils.text.transform import url_encode_text
 
-logger = logging.getLogger(settings.name + '.Person')
+logger = logging.getLogger(settings.name + '.Article')
 store = collection(settings.collection_articles)
 
 
-def find(item_id: str):
-    q = {'id': item_id}
-    try:
-        return store.find_one(q)
-    except ValueError:
-        pass
-    return None
-
-
 class Article(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
     @staticmethod
     async def scan(provider):
         documents = [i for i in provider.scan('.') if provider.is_dir(i)]

utils/Sync/sync/models/Document.py

@@ -11,17 +11,18 @@ from utils.hash import hash_str
 from utils.image.resize import image_magick_pdf_to_img
 from utils.text.transform import url_encode_text, url_encode_file
 
-
-def find(store, item_id: str):
-    q = {'id': item_id}
-    try:
-        return store.find_one(q)
-    except ValueError:
-        pass
-    return None
+store = collection(settings.collection_documents)
 
 
 class Document(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
     @staticmethod
     async def scan(provider):
         paths = scan_subdirs(provider, '.pdf')
@@ -43,8 +44,6 @@ class Document(Model):
         )
 
     def __init__(self, provider, file, category=None, title=None):
-        store = collection(settings.collection_documents)
-
         self.__hash_keys = ['id', 'url']
 
         self.__id_template: str = '{category}-{file}'
@@ -63,15 +62,9 @@ class Document(Model):
         self.__set_url()
         self.__set_hash()
 
-    async def is_changed(self):
-        i = find(self.store, self.id)
-        if not i:
-            return True
-
-        if 'hash' not in i:
-            return True
-
-        return not (self.hash == i['hash'])
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        await self.create_preview(sizes)
 
     async def save(self):
         document = self.bake()

utils/Sync/sync/models/Page.py

@@ -17,6 +17,7 @@ from utils.io import read_yaml_md
 from utils.text.transform import url_encode_text
 
 logger = logging.getLogger(settings.name + '.Page')
+store = collection(settings.collection_pages)
 
 
 def find(store, item_id: str):
@@ -29,6 +30,14 @@ def find(store, item_id: str):
 
 
 class Page(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
     @staticmethod
     async def scan(provider):
         documents = [i for i in provider.scan('.') if provider.is_dir(i)]
@@ -89,6 +98,9 @@ class Page(Model):
 
         return not (self.hash == i['hash'])
 
+    async def build(self, **kwargs):
+        await self.setup_images(settings.image_sizes, get_image_url_fn(self.id))
+
     async def setup_images(self, sizes, url_factory):
         folder = self.params['folder']
         images = []
@@ -189,3 +201,19 @@ def get_manifest(provider, path):
         'title': title_from_html(body),
         'images': images_from_html(body)
     }
+
+
+def get_image_url_fn(page_id: str):
+    image_id = lambda filename: url_encode_text(
+        os.path.splitext(os.path.basename(filename))[0]
+    )
+
+    def img_url_fn(file, size, ext):
+        return settings.page_url_base.format(
+            page=page_id,
+            id=image_id(file),
+            size=size,
+            ext=ext
+        )
+
+    return img_url_fn

utils/Sync/sync/models/Person.py

@@ -15,16 +15,15 @@ logger = logging.getLogger(settings.name + '.Person')
 store = collection(settings.collection_collective)
 
 
-def find(item_id: str):
-    q = {'id': item_id}
-    try:
-        return store.find_one(q)
-    except ValueError:
-        pass
-    return None
-
-
 class Person(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
     @staticmethod
     async def scan(provider):
         documents = provider.type_filter('', '.md')
@@ -59,9 +58,9 @@ class Person(Model):
 
     async def build(self, **kwargs):
         sizes = kwargs['sizes']
-        image_path = self.provider.get_abs(self.params['picture'])
+        image_path = self.provider.get_abs(self.get_param('picture'))
 
-        self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id))
+        self.picture = await Image.new(self.provider, image_path, sizes)
 
     def bake(self):
         return {

utils/Sync/sync/models/Schedule.py

@@ -1,5 +1,3 @@
-import logging
-
 from toml import loads as toml_str
 from yaml import load as yaml
 
@@ -10,7 +8,6 @@ from sync.models import Model
 
 from utils.fn import constant
 
-logger = logging.getLogger(settings.name + '.Person')
 store = collection(settings.collection_schedules)
 
 
@@ -24,6 +21,14 @@ def find(item_id: str):
 
 
 class Schedule(Model):
+    @staticmethod
+    async def find(query):
+        return store.find(query)
+
+    @staticmethod
+    async def delete(query):
+        return store.find_one_and_delete(query)
+
     @staticmethod
     async def scan(provider):
         documents = provider.type_filter('.', '.yaml')

utils/Sync/sync/models/__init__.py

@@ -28,6 +28,9 @@ class Model:
     def init(self):
         pass
 
+    async def upload(self, **kwargs):
+        pass
+
     async def build(self, **kwargs):
         pass