diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index ec67f095..69aca868 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -61,26 +61,31 @@ async def run(run_interval=0): provider=io(settings.provider_root), model=Person, ) + sync_persons.validate_urls = False sync_articles = Sync( provider=io(settings.provider_root), model=Article, ) + sync_articles.validate_urls = False sync_schedules = Sync( provider=io(settings.provider_root), model=Schedule, ) + sync_schedules.validate_urls = False sync_albums = Sync( provider=io(settings.provider_root), model=Album, ) + sync_albums.validate_urls = False sync_settings = Sync( provider=io(settings.provider_root), model=Settings, ) + sync_settings.validate_urls = False sync_images = Sync( provider=io(settings.provider_root), diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py index 1afde674..34123478 100644 --- a/utils/Sync/sync/core/__init__.py +++ b/utils/Sync/sync/core/__init__.py @@ -16,6 +16,7 @@ class Sync: def __init__(self, provider: Provider, model): super().__init__() + self.validate_urls = True self.strict_origin = False self.__default_build_args = { 'sizes': settings.image_sizes, @@ -74,6 +75,9 @@ class Sync: items = await self.model.scan(self.provider) self.logger.info('Found {} Item(s)'.format(len(items))) + if self.validate_urls: + self.check_urls_uniqueness(items) + # SKIP UNTOUCHED DOCUMENTS changed_items = await untouched(items) self.logger.info('Changed {} Items(s)'.format(len(changed_items))) @@ -95,5 +99,13 @@ class Sync: await self.clean() + def check_urls_uniqueness(self, items): + urls = [x.url for x in items] + unique_urls = set(urls) + + if len(urls) != len(unique_urls): + self.logger.error('URLs is not unique') + raise Exception('URLs is not unique') + async def upload(self): pass