diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index ca42f376..9743338c 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -50,13 +50,6 @@ person_uri = { 'А.С.Тимашева': 'as-timasheva', } -upload_enabled = value('upload_enabled') -upload_url_image = 'https://static.shlisselburg.org/upload/art/images/{}' -upload_auth = ( - value('upload_auth')['login'], - value('upload_auth')['password'], -) - image_processing_enabled = value('image_processing') album_html_img_class = 'hudozka-product' @@ -115,12 +108,12 @@ collection_collective = 'collective' collection_albums = 'albums' collection_pages = 'pages' -hash_salt_articles = '5' -hash_salt_pages = '3' -hash_salt_albums = '0' -hash_salt_documents = '1' -hash_salt_images = '2' -hash_salt_person = '3' +hash_salt_articles = '6' +hash_salt_pages = '4' +hash_salt_albums = '1' +hash_salt_documents = '3' +hash_salt_images = '4' +hash_salt_person = '4' provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs') provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') @@ -137,13 +130,18 @@ dir_pages = './Pages' dir_images = './Images' document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}' -document_url_upload_template = 'https://static.shlisselburg.org/upload/art/uploads/{file}' -document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' +document_url_preview_template = 'https://art.shlisselburg.org/static/images/{id}-{size}{ext}' -image_url_upload = 'https://static.shlisselburg.org/upload/art/images/' -image_url_base = 'https://art.shlisselburg.org/data/images/' +image_url_base = 'https://art.shlisselburg.org/static/images/' yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN') update_enabled = value('update_enabled') delete_enabled = value('delete_enabled') + +auth_s3 = { + 'region_name': 'ams3', + 'endpoint_url': 'https://dir.ams3.digitaloceanspaces.com', + 'aws_access_key_id': '', + 'aws_secret_access_key': '', +} diff --git a/utils/Sync/sync/data/request.py b/utils/Sync/sync/data/request.py index a9a6063a..4b2cf544 100644 --- a/utils/Sync/sync/data/request.py +++ b/utils/Sync/sync/data/request.py @@ -1,44 +1,40 @@ import aiohttp -import requests - +from boto3 import session +from mimetypes import MimeTypes +import logging import settings +logger = logging.getLogger(settings.name + '.S3') -async def upload(url, path): + +async def s3_put(obj, path): """ - :param url: - :param path: + :param obj: uploads/file.pdf + :param path: to/to/local/file.pdf :return: """ + logger.info('Put {}'.format(obj)) - if not settings.upload_enabled: - return url + s = session.Session() + client = s.client('s3', **settings.auth_s3) - files = {'file': open(path, 'rb')} - r = requests.put( - url=url, - files=files, - auth=settings.upload_auth - ) - if r.status_code == 200: - return url - else: - return None + mime = MimeTypes() + mime_type, enc = mime.guess_type(path) + + # Upload a file to your Space + client.upload_file(path, 'hudozka', obj, ExtraArgs={ + 'ACL': 'public-read', + 'ContentType': mime_type, + }) async def get(url): - async with aiohttp.ClientSession as session: - res = session.get(url) + async with aiohttp.ClientSession as aio_session: + res = aio_session.get(url) return await res.text() async def post(url, data, params={}, headers={}): - async with aiohttp.ClientSession() as session: - async with session.post(url, data=data, params=params, headers=headers) as resp: + async with aiohttp.ClientSession() as aio_session: + async with aio_session.post(url, data=data, params=params, headers=headers) as resp: return await resp.text() - - -async def put(url, data): - async with aiohttp.ClientSession as session: - res = session.put(url, data=data) - return await res.text() diff --git a/utils/Sync/sync/models/Document.py b/utils/Sync/sync/models/Document.py index 33fd9c02..e76072a8 100644 --- a/utils/Sync/sync/models/Document.py +++ b/utils/Sync/sync/models/Document.py @@ -68,7 +68,7 @@ class Document(Model): self.__hash_keys = ['id', 'url'] self.__id_template: str = '{category}-{file}' - self.__url_template: str = settings.document_url_template + self.__url_template: str = 'https://art.shlisselburg.org/static/uploads/{file}' self.__url_preview_template: str = settings.document_url_preview_template self.title = None @@ -121,11 +121,14 @@ class Document(Model): return None async def upload(self): + print('uploading to s3', self.file) + filepath = self.provider.get_local(self.file) name = os.path.basename(self.url) - url = settings.document_url_upload_template.format(file=name) - await request.upload(url, filepath) + o = 'uploads/{}'.format(name) + await request.s3_put(o, filepath) + # await request.upload(url, filepath) def bake(self): return { diff --git a/utils/Sync/sync/models/Image.py b/utils/Sync/sync/models/Image.py index 57efa69e..565c573d 100644 --- a/utils/Sync/sync/models/Image.py +++ b/utils/Sync/sync/models/Image.py @@ -18,10 +18,6 @@ logger = logging.getLogger(settings.name + '.Image') store = collection(settings.collection_images) -def get_upload_url(url): - return os.path.join(settings.image_url_upload, os.path.basename(url)) - - def default_url_factory(file, size, ext): url = settings.image_url_base + '{}-{}{}' return url.format(md5(file), size, ext.lower()) @@ -115,11 +111,10 @@ class Image(Model): async def upload(self): if self.data: async def fn(i): - url = get_upload_url(i['url']) file = i['file'] + filename = os.path.basename(file) - logger.info('Uploading to {}'.format(url)) - await request.upload(url, file) + await request.s3_put('images/'+filename, file) await asyncio.wait([fn(i) for i in self.data.values()])