[sync] upload driver changed to s3

This commit is contained in:
tmshv committed 2019-05-29 23:06:57 +03:00
1 parent dbadbdafc3
commit 1bca661e13
4 files changed
+46 -54

No files matched your search

+23 -27
View File
@@ -1,44 +1,40 @@
import aiohttp
import requests
from boto3 import session
from mimetypes import MimeTypes
import logging
import settings
logger = logging.getLogger(settings.name + '.S3')
async def upload(url, path):
async def s3_put(obj, path):
"""
:param url:
:param path:
:param obj: uploads/file.pdf
:param path: to/to/local/file.pdf
:return:
"""
logger.info('Put {}'.format(obj))
if not settings.upload_enabled:
return url
s = session.Session()
client = s.client('s3', **settings.auth_s3)
files = {'file': open(path, 'rb')}
r = requests.put(
url=url,
files=files,
auth=settings.upload_auth
)
if r.status_code == 200:
return url
else:
return None
mime = MimeTypes()
mime_type, enc = mime.guess_type(path)
# Upload a file to your Space
client.upload_file(path, 'hudozka', obj, ExtraArgs={
'ACL': 'public-read',
'ContentType': mime_type,
})
async def get(url):
async with aiohttp.ClientSession as session:
res = session.get(url)
async with aiohttp.ClientSession as aio_session:
res = aio_session.get(url)
return await res.text()
async def post(url, data, params={}, headers={}):
async with aiohttp.ClientSession() as session:
async with session.post(url, data=data, params=params, headers=headers) as resp:
async with aiohttp.ClientSession() as aio_session:
async with aio_session.post(url, data=data, params=params, headers=headers) as resp:
return await resp.text()
async def put(url, data):
async with aiohttp.ClientSession as session:
res = session.put(url, data=data)
return await res.text()
+6 -3
View File
@@ -68,7 +68,7 @@ class Document(Model):
self.__hash_keys = ['id', 'url']
self.__id_template: str = '{category}-{file}'
self.__url_template: str = settings.document_url_template
self.__url_template: str = 'https://art.shlisselburg.org/static/uploads/{file}'
self.__url_preview_template: str = settings.document_url_preview_template
self.title = None
@@ -121,11 +121,14 @@ class Document(Model):
return None
async def upload(self):
print('uploading to s3', self.file)
filepath = self.provider.get_local(self.file)
name = os.path.basename(self.url)
url = settings.document_url_upload_template.format(file=name)
await request.upload(url, filepath)
o = 'uploads/{}'.format(name)
await request.s3_put(o, filepath)
# await request.upload(url, filepath)
def bake(self):
return {
+2 -7
View File
@@ -18,10 +18,6 @@ logger = logging.getLogger(settings.name + '.Image')
store = collection(settings.collection_images)
def get_upload_url(url):
return os.path.join(settings.image_url_upload, os.path.basename(url))
def default_url_factory(file, size, ext):
url = settings.image_url_base + '{}-{}{}'
return url.format(md5(file), size, ext.lower())
@@ -115,11 +111,10 @@ class Image(Model):
async def upload(self):
if self.data:
async def fn(i):
url = get_upload_url(i['url'])
file = i['file']
filename = os.path.basename(file)
logger.info('Uploading to {}'.format(url))
await request.upload(url, file)
await request.s3_put('images/'+filename, file)
await asyncio.wait([fn(i) for i in self.data.values()])