[Sync] added url checking

utils/Sync/sync.py

@@ -61,26 +61,31 @@ async def run(run_interval=0):
         provider=io(settings.provider_root),
         model=Person,
     )
+    sync_persons.validate_urls = False
 
     sync_articles = Sync(
         provider=io(settings.provider_root),
         model=Article,
     )
+    sync_articles.validate_urls = False
 
     sync_schedules = Sync(
         provider=io(settings.provider_root),
         model=Schedule,
     )
+    sync_schedules.validate_urls = False
 
     sync_albums = Sync(
         provider=io(settings.provider_root),
         model=Album,
     )
+    sync_albums.validate_urls = False
 
     sync_settings = Sync(
         provider=io(settings.provider_root),
         model=Settings,
     )
+    sync_settings.validate_urls = False
 
     sync_images = Sync(
         provider=io(settings.provider_root),

utils/Sync/sync/core/__init__.py

@@ -16,6 +16,7 @@ class Sync:
     def __init__(self, provider: Provider, model):
         super().__init__()
 
+        self.validate_urls = True
         self.strict_origin = False
         self.__default_build_args = {
             'sizes': settings.image_sizes,
@@ -74,6 +75,9 @@ class Sync:
         items = await self.model.scan(self.provider)
         self.logger.info('Found {} Item(s)'.format(len(items)))
 
+        if self.validate_urls:
+            self.check_urls_uniqueness(items)
+
         # SKIP UNTOUCHED DOCUMENTS
         changed_items = await untouched(items)
         self.logger.info('Changed {} Items(s)'.format(len(changed_items)))
@@ -95,5 +99,13 @@ class Sync:
 
         await self.clean()
 
+    def check_urls_uniqueness(self, items):
+        urls = [x.url for x in items]
+        unique_urls = set(urls)
+
+        if len(urls) != len(unique_urls):
+            self.logger.error('URLs is not unique')
+            raise Exception('URLs is not unique')
+
     async def upload(self):
         pass