[sync] upload driver changed to s3

utils/Sync/settings.py

@@ -50,13 +50,6 @@ person_uri = {
     'А.С.Тимашева': 'as-timasheva',
 }
 
-upload_enabled = value('upload_enabled')
-upload_url_image = 'https://static.shlisselburg.org/upload/art/images/{}'
-upload_auth = (
-    value('upload_auth')['login'],
-    value('upload_auth')['password'],
-)
-
 image_processing_enabled = value('image_processing')
 
 album_html_img_class = 'hudozka-product'
@@ -115,12 +108,12 @@ collection_collective = 'collective'
 collection_albums = 'albums'
 collection_pages = 'pages'
 
-hash_salt_articles = '5'
-hash_salt_pages = '3'
-hash_salt_albums = '0'
-hash_salt_documents = '1'
-hash_salt_images = '2'
-hash_salt_person = '3'
+hash_salt_articles = '6'
+hash_salt_pages = '4'
+hash_salt_albums = '1'
+hash_salt_documents = '3'
+hash_salt_images = '4'
+hash_salt_person = '4'
 
 provider_name = value('provider')['name']  # env('SYNC_PROVIDER', 'fs')
 provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
@@ -137,13 +130,18 @@ dir_pages = './Pages'
 dir_images = './Images'
 
 document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
-document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploads/{file}'
-document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
+document_url_preview_template = 'https://art.shlisselburg.org/static/images/{id}-{size}{ext}'
 
-image_url_upload = 'https://static.shlisselburg.org/upload/art/images/'
-image_url_base = 'https://art.shlisselburg.org/data/images/'
+image_url_base = 'https://art.shlisselburg.org/static/images/'
 
 yandex_disk_access_token = value('yandex_disk_access_token')  # env('YANDEX_DISK_ACCESS_TOKEN')
 
 update_enabled = value('update_enabled')
 delete_enabled = value('delete_enabled')
+
+auth_s3 = {
+    'region_name': 'ams3',
+    'endpoint_url': 'https://dir.ams3.digitaloceanspaces.com',
+    'aws_access_key_id': '',
+    'aws_secret_access_key': '',
+}

utils/Sync/sync/data/request.py

@@ -1,44 +1,40 @@
 import aiohttp
-import requests
-
+from boto3 import session
+from mimetypes import MimeTypes
+import logging
 import settings
 
+logger = logging.getLogger(settings.name + '.S3')
 
-async def upload(url, path):
+
+async def s3_put(obj, path):
     """
-    :param url:
-    :param path:
+    :param obj: uploads/file.pdf
+    :param path: to/to/local/file.pdf
     :return:
     """
+    logger.info('Put {}'.format(obj))
 
-    if not settings.upload_enabled:
-        return url
+    s = session.Session()
+    client = s.client('s3', **settings.auth_s3)
 
-    files = {'file': open(path, 'rb')}
-    r = requests.put(
-        url=url,
-        files=files,
-        auth=settings.upload_auth
-    )
-    if r.status_code == 200:
-        return url
-    else:
-        return None
+    mime = MimeTypes()
+    mime_type, enc = mime.guess_type(path)
+
+    # Upload a file to your Space
+    client.upload_file(path, 'hudozka', obj, ExtraArgs={
+        'ACL': 'public-read',
+        'ContentType': mime_type,
+    })
 
 
 async def get(url):
-    async with aiohttp.ClientSession as session:
-        res = session.get(url)
+    async with aiohttp.ClientSession as aio_session:
+        res = aio_session.get(url)
         return await res.text()
 
 
 async def post(url, data, params={}, headers={}):
-    async with aiohttp.ClientSession() as session:
-        async with session.post(url, data=data, params=params, headers=headers) as resp:
+    async with aiohttp.ClientSession() as aio_session:
+        async with aio_session.post(url, data=data, params=params, headers=headers) as resp:
             return await resp.text()
-
-
-async def put(url, data):
-    async with aiohttp.ClientSession as session:
-        res = session.put(url, data=data)
-        return await res.text()

utils/Sync/sync/models/Document.py

@@ -68,7 +68,7 @@ class Document(Model):
         self.__hash_keys = ['id', 'url']
 
         self.__id_template: str = '{category}-{file}'
-        self.__url_template: str = settings.document_url_template
+        self.__url_template: str = 'https://art.shlisselburg.org/static/uploads/{file}'
         self.__url_preview_template: str = settings.document_url_preview_template
 
         self.title = None
@@ -121,11 +121,14 @@ class Document(Model):
         return None
 
     async def upload(self):
+        print('uploading to s3', self.file)
+
         filepath = self.provider.get_local(self.file)
         name = os.path.basename(self.url)
-        url = settings.document_url_upload_template.format(file=name)
 
-        await request.upload(url, filepath)
+        o = 'uploads/{}'.format(name)
+        await request.s3_put(o, filepath)
+        # await request.upload(url, filepath)
 
     def bake(self):
         return {

utils/Sync/sync/models/Image.py

@@ -18,10 +18,6 @@ logger = logging.getLogger(settings.name + '.Image')
 store = collection(settings.collection_images)
 
 
-def get_upload_url(url):
-    return os.path.join(settings.image_url_upload, os.path.basename(url))
-
-
 def default_url_factory(file, size, ext):
     url = settings.image_url_base + '{}-{}{}'
     return url.format(md5(file), size, ext.lower())
@@ -115,11 +111,10 @@ class Image(Model):
     async def upload(self):
         if self.data:
             async def fn(i):
-                url = get_upload_url(i['url'])
                 file = i['file']
+                filename = os.path.basename(file)
 
-                logger.info('Uploading to {}'.format(url))
-                await request.upload(url, file)
+                await request.s3_put('images/'+filename, file)
 
             await asyncio.wait([fn(i) for i in self.data.values()])