sync moved to modules

modules/sync/sync/core/SyncDocument.py

@@ -0,0 +1,14 @@
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Document import Document
+
+
+class SyncDocument(Sync):
+    def __init__(self, provider: Provider, sizes):
+        super().__init__(provider, Document)
+        self.sizes = sizes
+
+    def _get_build_args(self):
+        return {
+            'sizes': self.sizes,
+        }

modules/sync/sync/core/__init__.py

@@ -0,0 +1,144 @@
+import logging
+
+import settings
+from sync import untouched
+from sync.data import Provider
+
+
+def get_id(item):
+    try:
+        return item.id
+    except AttributeError:
+        return item['id']
+
+
+class Sync:
+    def __init__(self, provider: Provider, model):
+        super().__init__()
+
+        self.validate_urls = True
+        self.strict_origin = False
+        self.__default_build_args = {
+            'sizes': settings.image_sizes,
+        }
+        self.__delete_query = lambda item: {'_id': item['_id']}
+
+        name = model.__name__
+        self.logger = logging.getLogger('%s.%s' % (settings.name, name))
+
+        self.provider = provider
+        self.model = model
+        self.filter_item_fn = lambda x: True
+
+        self.skip_unchanged = True
+        self.update_enabled = True
+        self.delete_enabled = True
+
+    def set_options(self, su: bool, u: bool, d: bool):
+        self.skip_unchanged = su
+        self.update_enabled = u
+        self.delete_enabled = d
+        return self
+
+    def set_item_filter(self, fn):
+        self.filter_item_fn = fn
+        return self
+
+    async def clean(self):
+        pass
+
+    async def build(self, item):
+        args = self._get_build_args()
+        await item.build(**args)
+        return item
+
+    async def update(self, item):
+        await item.upload()
+        await item.save()
+        return item
+
+    def _get_build_args(self):
+        return {
+            **self.__default_build_args
+        }
+
+    def __items_to_delete_query(self, items):
+        query = {
+            'id': {'$nin': [doc.id for doc in items]},
+        }
+
+        if self.strict_origin:
+            items_origin = list(set([doc.origin for doc in items]))
+
+            query = {
+                **query,
+                'origin': {'$in': items_origin},
+            }
+
+        return query
+
+    async def run(self):
+        """
+        # Get scope files
+        # Validate these files. Raise an error
+        # calc_hashes
+        # make diff with previous run
+        # compile objects
+        # upload
+
+        :return:
+        """
+        self.logger.info('Checking for update')
+
+        items = await self.model.scan(self.provider)
+        self.logger.info('Found {} Item(s)'.format(len(items)))
+        all_items = items
+
+        if self.validate_urls:
+            self.check_urls_uniqueness(all_items)
+
+        # SKIP UNTOUCHED DOCUMENTS
+        if self.skip_unchanged:
+            items = await untouched(all_items)
+
+        # Optionally trim list of models by custom filter
+        items = [
+            x
+            for x in items
+            if self.filter_item_fn(x)
+        ]
+
+        self.logger.info(f'Ready to process {len(items)} item(s)')
+
+        # BUILDING
+        for item in items:
+            self.logger.info(f'Build Item {item}')
+            await self.build(item)
+
+        # UPDATING
+        if self.update_enabled:
+            for item in items:
+                await self.update(item)
+                self.logger.info('Updated Item {}'.format(item))
+
+        # DELETING
+        if self.delete_enabled:
+            obsolete_items = await self.model.find(self.__items_to_delete_query(all_items))
+            obsolete_items = list(obsolete_items)
+
+            for item in obsolete_items:
+                self.logger.info('Deleting Item {}'.format(item['id']))
+                await self.model.delete(self.__delete_query(item))
+
+        await self.clean()
+
+    def check_urls_uniqueness(self, items):
+        urls = [x.url for x in items]
+        unique_urls = set(urls)
+
+        if len(urls) != len(unique_urls):
+            self.logger.error('URLs is not unique')
+            raise Exception('URLs is not unique')
+
+    async def upload(self):
+        pass