tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

Merge branch 'dev' into dependabot/pip/utils/Sync/pillow-6.2.0

Browse Source
This commit is contained in:
tmshv authored and GitHub committed 2020-02-12 22:32:14 +03:00
2 parents a52e25ee53 48bbce1e91
commit e6fbda5ae7
153 files changed
+9626 -6467
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/Sync/requirements.txt
+2 -2
View File
@@ -5,8 +5,8 @@ Pillow==6.2.0
pymongo==3.3.0
PyPDF2==1.26.0
python-frontmatter==0.3.1
PyYAML==3.11
requests==2.10.0
PyYAML==5.1
requests==2.20.0
six==1.10.0
smartcrop==0.1
toml==0.9.2
utils/Sync/settings.py
+4 -4
View File
@@ -111,8 +111,8 @@ collection_pages = 'pages'
hash_salt_articles = '6'
hash_salt_pages = '4'
hash_salt_albums = '1'
hash_salt_documents = '3'
hash_salt_images = '4'
hash_salt_documents = '4'
hash_salt_images = '5'
hash_salt_person = '4'
provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs')
@@ -130,9 +130,9 @@ dir_pages = './Pages'
dir_images = './Images'
document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
document_url_preview_template = 'https://art.shlisselburg.org/static/images/{id}-{size}{ext}'
document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
image_url_base = 'https://art.shlisselburg.org/static/images/'
image_url_base = 'https://static.shlisselburg.org/art/images/'
yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN')
utils/Sync/sync.py
+15 -8
View File
@@ -56,54 +56,61 @@ async def run(run_interval=0):
provider=io(settings.provider_root),
model=Page,
)
sync_pages.set_options(True, True, True)
sync_persons = Sync(
provider=io(settings.provider_root),
model=Person,
)
sync_persons.set_options(True, True, True)
sync_persons.validate_urls = False
sync_articles = Sync(
provider=io(settings.provider_root),
model=Article,
)
sync_articles.set_options(True, True, True)
sync_articles.validate_urls = False
sync_schedules = Sync(
provider=io(settings.provider_root),
model=Schedule,
)
sync_schedules.set_options(True, True, True)
sync_schedules.validate_urls = False
sync_albums = Sync(
provider=io(settings.provider_root),
model=Album,
)
sync_albums.set_options(True, True, True)
sync_albums.validate_urls = False
sync_settings = Sync(
provider=io(settings.provider_root),
model=Settings,
)
sync_settings.set_options(True, True, True)
sync_settings.validate_urls = False
sync_images = Sync(
provider=io(settings.provider_root),
model=Image,
)
sync_images.set_options(True, True, True)
sync_images.strict_origin = True
sync_images.validate_urls = False
while True:
await asyncio.wait([
run_sync(sync_documents, skip_changed=False, update=True, remove=True),
run_sync(sync_pages, skip_changed=False, update=True, remove=True),
run_sync(sync_persons, skip_changed=False, update=True, remove=True),
run_sync(sync_articles, skip_changed=False, update=True, remove=True),
run_sync(sync_schedules, skip_changed=False, update=True, remove=True),
run_sync(sync_albums, skip_changed=False, update=True, remove=True),
run_sync(sync_settings, skip_changed=False, update=True, remove=True),
run_sync(sync_images, skip_changed=False, update=True, remove=True),
run_sync(sync_documents),
run_sync(sync_pages),
run_sync(sync_persons),
run_sync(sync_articles),
run_sync(sync_schedules),
run_sync(sync_albums),
run_sync(sync_settings),
run_sync(sync_images),
])
if run_interval == 0:
utils/Sync/sync/__init__.py
+2 -2
View File
@@ -155,9 +155,9 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
# m.add_token_factory(TokenFactory(FileToken))
m.add_token_factory(TokenFactory(TextToken))
m.add_tree_middleware(fix_links_quotes)
html = await m.create_tree(md)
tree = await m.create_tree(md)
post = kazimir.html_from_tree(html)
post = kazimir.html_from_tree(tree)
post = await kazimir.typo(post)
return post, images, documents
utils/Sync/sync/core/__init__.py
+43 -12
View File
@@ -28,16 +28,34 @@ class Sync:
self.provider = provider
self.model = model
self.filter_item_fn = lambda x: True
self.skip_unchanged = True
self.update_enabled = True
self.delete_enabled = True
def set_options(self, su: bool, u: bool, d: bool):
self.skip_unchanged = su
self.update_enabled = u
self.delete_enabled = d
return self
def set_item_filter(self, fn):
self.filter_item_fn = fn
return self
async def clean(self):
pass
async def update(self, item):
async def build(self, item):
args = self._get_build_args()
await item.build(**args)
return item
async def update(self, item):
await item.upload()
await item.save()
return item
def _get_build_args(self):
return {
@@ -59,7 +77,7 @@ class Sync:
return query
async def run(self, skip_changed=False, update=True, remove=True):
async def run(self):
"""
# Get scope files
# Validate these files. Raise an error
@@ -74,25 +92,38 @@ class Sync:
items = await self.model.scan(self.provider)
self.logger.info('Found {} Item(s)'.format(len(items)))
all_items = items
if self.validate_urls:
self.check_urls_uniqueness(items)
self.check_urls_uniqueness(all_items)
# SKIP UNTOUCHED DOCUMENTS
changed_items = items
if not skip_changed:
changed_items = await untouched(items)
self.logger.info('Changed {} Items(s)'.format(len(changed_items)))
if self.skip_unchanged:
items = await untouched(all_items)
# Optionally trim list of models by custom filter
items = [
x
for x in items
if self.filter_item_fn(x)
]
self.logger.info(f'Ready to process {len(items)} item(s)')
# BUILDING
for item in items:
self.logger.info(f'Build Item {item}')
await self.build(item)
# UPDATING
if update:
for item in changed_items:
if self.update_enabled:
for item in items:
await self.update(item)
self.logger.info('Updated Item {}'.format(item))
# DELETING
if remove:
obsolete_items = await self.model.find(self.__items_to_delete_query(items))
if self.delete_enabled:
obsolete_items = await self.model.find(self.__items_to_delete_query(all_items))
obsolete_items = list(obsolete_items)
for item in obsolete_items:
utils/Sync/sync/models/Document.py
+2 -6
View File
@@ -68,7 +68,7 @@ class Document(Model):
self.__hash_keys = ['id', 'url']
self.__id_template: str = '{category}-{file}'
self.__url_template: str = 'https://art.shlisselburg.org/static/uploads/{file}'
self.__url_template: str = 'https://static.shlisselburg.org/art/uploads/{file}'
self.__url_preview_template: str = settings.document_url_preview_template
self.title = None
@@ -121,14 +121,10 @@ class Document(Model):
return None
async def upload(self):
print('uploading to s3', self.file)
filepath = self.provider.get_local(self.file)
name = os.path.basename(self.url)
o = 'uploads/{}'.format(name)
await request.s3_put(o, filepath)
# await request.upload(url, filepath)
await request.s3_put(f'uploads/{name}', filepath)
def bake(self):
return {
utils/Sync/sync/models/Image.py
+13 -6
View File
@@ -114,7 +114,7 @@ class Image(Model):
file = i['file']
filename = os.path.basename(file)
await request.s3_put('images/'+filename, file)
await request.s3_put(f'images/{filename}', file)
await asyncio.wait([fn(i) for i in self.data.values()])
@@ -156,10 +156,10 @@ class Image(Model):
def sync_image(record):
from db import db
q = {'hash': record['hash']}
q = {'file': record['file']}
try:
db().images.update_one(q, {'$set': record}, upsert=True)
i = db().images.find_one({'hash': record['hash']})
i = db().images.find_one({'file': record['file']})
return i
except ValueError:
pass
@@ -182,6 +182,11 @@ async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optio
_, ext = os.path.splitext(file)
images = []
if not settings.image_processing_enabled:
logger.warning(
f'Image processing disabled {file}. Size of image will be set to original'
)
async def fn(size):
size_name, width, height = size
@@ -189,10 +194,12 @@ async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optio
image_filename = os.path.basename(image_url)
local_image_path = os.path.join(output_dir, image_filename)
# if settings.image_processing_enabled:
image = await process_image(file, local_image_path, size)
if settings.image_processing_enabled:
image = await process_image(file, local_image_path, size)
else:
image = await read_image(file)
if not image:
logger.warning('Failed to process image {}'.format(file))
logger.warning(f'Failed to process image {file}')
return None
width, height = image.size
utils/Sync/sync/models/Page.py
+7 -5
View File
@@ -42,12 +42,14 @@ class Page(Model):
@staticmethod
async def scan(provider):
scan_path = settings.dir_pages
documents = [i for i in provider.scan(scan_path) if provider.is_dir(i)]
items = [
Page.read(provider, path)
for path in provider.scan(scan_path)
if provider.is_dir(path)
]
items = [i for i in items if i]
documents = [Page.read(provider, i) for i in documents]
documents = [i for i in documents if i]
return documents
return items
@staticmethod
def read(provider, path):