ya trash

utils/Sync/async_process.py

@@ -0,0 +1,65 @@
+import asyncio
+
+# import os
+
+
+# async def image_magick_resize(input_file, size):
+#     s = '%dx%d' % size
+#     name = '{s}-{name}'.format(s=s, name=os.path.basename(input_file))
+#     output_file = os.path.join('/Users/tmshv/Pyasync/', name)
+#
+#     cmd = [
+#         '/usr/local/bin/convert', '"%s"' % input_file,
+#         '-strip',
+#         '-auto-orient',
+#         '-resize', s,
+#         '-quality', '85',
+#                                   '"%s"' % output_file,
+#     ]
+#
+#     cmd = ' '.join(cmd)
+#     await asyncio.create_subprocess_shell(cmd)
+from sync.models.Image import create_image, default_url_factory
+
+
+async def resize(input_file):
+    # from utils.image import image_magick_resize
+    sizes = [
+        ('a', 1880, 1900),
+        ('b', 10, 10),
+        ('c', 5, 5),
+    ]
+
+    await create_image(input_file, sizes, default_url_factory, '/Users/tmshv')
+
+    # for size in sizes:
+    #     await image_magick_resize(input_file, '/Users/tmshv/AAA-{}.jpg'.format(size), size)
+
+
+async def run():
+    from sync.data import request
+
+    files = {'file': open('/Users/tmshv/Downloads/39174605.png', 'rb')}
+
+    data = {
+        'file': files
+    }
+
+    # await request.put('https://static.shlisselburg.org/upload/art/uploads/test.jpg', data)
+    await request.put('http://localhost:8080/upload/art/uploads/test.jpg', data)
+
+    # await asyncio.wait([
+    #     resize('/Users/tmshv/Yandex.Disk.localized/Hudozka/Trash/Cats/Photo 26-01-15 18 07 39.jpg'),
+    #
+    #     # resize('/Users/tmshv/Yandex.Disk.localized/Hudozka/Trash/Cats/Photo 26-01-15 18 07 54.jpg'),
+    #     # resize('/Users/tmshv/Yandex.Disk.localized/Hudozka/Trash/Cats/Photo 26-01-15 18 08 14.jpg'),
+    #     # resize('/Users/tmshv/Yandex.Disk.localized/Hudozka/Trash/Cats/Photo 26-01-15 18 08 18.jpg'),
+    #     # resize('/Users/tmshv/Yandex.Disk.localized/Hudozka/Trash/Cats/Photo 26-01-15 18 08 30.jpg'),
+    #     # resize('/Users/tmshv/Yandex.Disk.localized/Hudozka/Trash/Cats/Photo 26-01-15 18 09 04.jpg'),
+    # ])
+
+
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run())
+    loop.close()

utils/Sync/config-development-yd.yaml

@@ -0,0 +1,24 @@
+---
+database_uri: 'mongodb://localhost:27017/hudozka'
+
+env: 'development'
+
+interval: 0
+origin: 'Yandex.Disk'
+
+skip_unchanged: False
+image_processing: True
+upload_enabled: True
+
+upload_auth:
+  login: 'uploader'
+  password: '1207690962638465'
+
+provider:
+  name: 'yd'
+  root: '/Hudozka/Site'
+
+yandex_disk_access_token: 'AQAAAAABbOfKAAFEbvckxZKiR01JsTcDniJsi2A'
+
+update_enabled: True
+delete_enabled: False

utils/Sync/config-development.yaml

@@ -0,0 +1,31 @@
+---
+database_uri: 'mongodb://hudozka:h123@localhost:1817/hudozka?authSource=admin'
+database_uri2: 'mongodb://localhost:27017/hudozka'
+
+env: 'development'
+
+interval: 0
+origin: 'Yandex.Disk'
+
+skip_unchanged: False
+image_processing: False
+upload_enabled: False
+
+upload_auth:
+  login: 'uploader'
+  password: '1207690962638465'
+
+provider:
+  name: 'fs'
+  root: '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site'
+
+s3:
+  host: '185.185.68.103:2090'
+  bucket: 'art'
+  access_key: 'CUKJ1K1557N6YJEF3F6K'
+  secret_key: 'WCx52kadWYB3yGAvpPvi+JfzcZkEAQY2P0nelpHq'
+
+yandex_disk_access_token: 'AQAAAAABbOfKAAFEbvckxZKiR01JsTcDniJsi2A'
+
+update_enabled: True
+delete_enabled: True

utils/Sync/config-production.yaml

@@ -0,0 +1,21 @@
+---
+database_uri: 'mongodb://hudozka:R6juwUwx4wz7WYKjQufz@art.shlisselburg.org:1817/hudozka?authSource=admin'
+
+env: 'development'
+
+skip_unchanged: False
+image_processing: True
+upload_enabled: True
+origin: 'Yandex.Disk'
+upload_auth:
+  login: 'uploader'
+  password: '1207690962638465'
+
+provider:
+  name: 'fs'
+  root: '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site'
+
+yandex_disk_access_token: 'AQAAAAABbOfKAAFEblSYy3Q2jkEauWzv6ttyfXM'
+
+update_enabled: True
+delete_enabled: True

utils/Sync/run_prod

@@ -0,0 +1 @@
+python3 sync.py --config config-production.yaml

utils/Sync/settings.py

@@ -116,11 +116,11 @@ collection_albums = 'albums'
 collection_pages = 'pages'
 
 hash_salt_articles = '5'
-hash_salt_pages = '2'
+hash_salt_pages = '3'
 hash_salt_albums = '0'
 hash_salt_documents = '1'
-hash_salt_images = '1'
-hash_salt_person = '2'
+hash_salt_images = '2'
+hash_salt_person = '3'
 
 provider_name = value('provider')['name']  # env('SYNC_PROVIDER', 'fs')
 provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
@@ -141,9 +141,7 @@ document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploa
 document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
 
 image_url_upload = 'https://static.shlisselburg.org/upload/art/images/'
-image_url_base = 'https://static.shlisselburg.org/art/images/'
-
-person_image_url_template = 'https://static.shlisselburg.org/art/images/person-{id}-{size}{ext}'
+image_url_base = 'https://art.shlisselburg.org/data/images/'
 
 yandex_disk_access_token = value('yandex_disk_access_token')  # env('YANDEX_DISK_ACCESS_TOKEN')
 

utils/Sync/sync-smart.py

@@ -0,0 +1,50 @@
+import settings
+from sync.data.fs import FSProvider
+from sync.data.yandexdisk import YDProvider
+
+import asyncio
+
+
+async def run_check(run_interval=0):
+    run_interval = 1
+
+    from yandexdisk import YDClient
+    client = YDClient(settings.yandex_disk_access_token)
+
+    async def need_to_update():
+        res = client.last_uploaded()
+        items = res['items']
+        paths = [i['path'] for i in items]
+
+        for i in paths:
+            if i.startswith('disk:/Hudozka/Site'):
+                return True
+
+        return False
+
+    while True:
+        status = await need_to_update()
+        if status:
+            print('DO SYNC')
+            # await run(0)
+
+        if run_interval == 0:
+            break
+        else:
+            await asyncio.sleep(run_interval)
+
+
+def get_provider(provider_type, root):
+    if provider_type == 'fs':
+        return FSProvider(root)
+    elif provider_type == 'yd':
+        return YDProvider(root, settings.yandex_disk_access_token)
+    return None
+
+
+if __name__ == '__main__':
+    interval = settings.interval
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(run_check(interval))
+    loop.close()

utils/Sync/sync.py

@@ -96,14 +96,14 @@ async def run(run_interval=0):
 
     while True:
         await asyncio.wait([
-            run_sync(sync_documents),
+            # run_sync(sync_documents),
             run_sync(sync_pages),
             run_sync(sync_persons),
             run_sync(sync_articles),
-            run_sync(sync_schedules),
-            run_sync(sync_albums),
-            run_sync(sync_settings),
-            run_sync(sync_images),
+            # run_sync(sync_schedules),
+            # run_sync(sync_albums),
+            # run_sync(sync_settings),
+            # run_sync(sync_images),
         ])
 
         if run_interval == 0:

utils/Sync/sync/data/request.py

@@ -39,6 +39,15 @@ async def post(url, data, params={}, headers={}):
 
 
 async def put(url, data):
-    async with aiohttp.ClientSession as session:
-        res = session.put(url, data=data)
-        return await res.text()
+    # login, password = settings.upload_auth
+    login, password = ('uploader', '207690962638465')
+
+    auth = aiohttp.BasicAuth(login, password, encoding='utf-8')
+
+    async with aiohttp.request('put', url=url, data=data, auth=auth) as resp:
+        print(resp.status)
+        print(await resp.text())
+
+        # async with aiohttp.ClientSession(auth=auth) as session:
+        #     res = session.put(url, data=data)
+        #     return await res.text()

utils/Sync/yandexdisk/__init__.py

@@ -16,6 +16,30 @@ class YDClient:
             Authorization='OAuth %s' % self.access_token,
         )
 
+    async def get_flat(self, limit=20):
+        """
+        https://cloud-api.yandex.net/v1/disk/resources/files?
+        [  limit=<количество файлов в списке>]
+        [& media_type=<тип запрашиваемых файлов>]
+        [& offset=<смещение относительно начала списка>]
+        [& fields=<нужные ключи ответа>]
+        [& preview_size=<размер превью>]
+        [& preview_crop=<признак обрезки превью>]
+        :return:
+        """
+
+        url = '{0}/resources/files'.format(self.base_url)
+        payload = dict(
+            limit=limit,
+            offset=0,
+        )
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        # return Directory(**json_dict)
+        return json_dict
+
     def get_disk_metadata(self):
         """
         :return: disk metadata
@@ -31,6 +55,26 @@ class YDClient:
             system_folders=disk['system_folders']
         )
 
+    def last_uploaded(self):
+        """
+        https://cloud-api.yandex.net/v1/disk/resources/last-uploaded?
+        [  limit=<количество файлов в списке>]
+        [& media_type=<тип запрашиваемых файлов>]
+        [& fields=<нужные ключи ответа>]
+        [& preview_size=<размер превью>]
+        [& preview_crop=<признак обрезки превью>]
+        """
+        url = '{0}/resources/last-uploaded'.format(self.base_url)
+        payload = dict(
+            limit=100,
+        )
+        r = requests.get(url, headers=self.base_headers, params=payload)
+        self.check_code(r)
+
+        json_dict = r.json()
+        # return Directory(**json_dict)
+        return json_dict
+
     def get_content(self, path):
         """
         :param path: path to folder