sync enhancements & update

This commit is contained in:
tmshv committed 2016-01-16 23:10:33 +03:00
1 parent 5b6b09b66b
commit 3b612ab4e7
5 files changed
+150 -144

No files matched your search

+81 -138
View File
@@ -1,12 +1,12 @@
from functools import reduce
from glob import glob
import os
import re
from pymongo import MongoClient
import img
import img as img_util
from read_gallery_1 import GalleryReader
import settings
import validation
__author__ = 'Roman Timashev'
@@ -30,60 +30,16 @@ def get_description_reader(version):
return None
def process_image(name, filename, sizes):
ext = settings.image_ext
thumbs = {}
for size, width, height in sizes:
source_image = img.read_image(filename)
sw, sh = source_image.size
if sw < sh:
width, height = height, width
out_image_filename = '{name}-{size}{ext}'.format(name=name, size=size, ext=ext)
out_image_local_path = os.path.join(settings.image_save_dir, out_image_filename)
out_image_url = os.path.join(settings.image_base_url, out_image_filename)
if width and height:
tw = width
th = height
else:
tw = sw
th = sh
thumbs[size] = {
'url': out_image_url,
'size': size,
'width': sw,
'height': sh
}
# print('generate %s for %s' % (size, filename), out_image_url)
if settings.image_processing:
if width and height:
img.minify(source_image, out_image_local_path, (width, height))
else:
img.optimize(source_image, out_image_local_path)
return thumbs
def process_product(album, product):
tr = {ord(a):ord(b) for a, b in zip(*settings.translit_table)}
author_name, author_age, filename, product_title = product
author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr)
image_id = '%s-%s-%s-%s' % (
album.date.year,
re.sub(r'[/]', '', album.uri),
author_name_encoded,
os.path.splitext(filename)[0]
)
image = process_image(image_id, filename, settings.image_sizes)
return (author_name, author_age, image, product_title)
def process_product_image(product, save_dir):
for k in product.image.keys():
thumb = product.image[k]
filename = os.path.basename(thumb['url'])
outpath = os.path.join(save_dir, filename)
img_util.image_magick_resize(
product.image_original_path,
outpath,
(thumb['width'], thumb['height'])
)
def get_db():
@@ -99,60 +55,46 @@ def clear_db():
def sync_product(product, type):
db = get_db()
author_name, author_age, image, product_title = product
author_name, author_age, image, product_title, original_path = product.to_tuple()
product_record = db['products'].insert({
'author': author_name,
'authorAge': author_age,
'title': product_title,
'type': type,
'content': image
})
print('inserted', product_record)
return product_record
def sync_album(album):
print('syncing %s' % album.title)
db = get_db()
product_ids = list(map(
lambda product: sync_product(product, album.type),
album.products
return db['products'].insert(dict(
author=author_name,
authorAge=author_age,
title=product_title,
type=type,
content=image
))
album_title = album.title
album_uri = album.uri
album_teacher = album.teacher
album_date = album.date
album_comment = album.comment
album_course = album.course
album_course_uri = album.course_uri
db['albums'].insert({
'title': album_title,
'uri': album_uri,
'date': album_date,
'description': album_comment,
'course': album_course,
'course_uri': album_course_uri,
'teacher': album_teacher,
'content': product_ids
})
def sync_album(album):
db = get_db()
product_ids = list(map(
lambda product: sync_product(product, album.type),
album.products
))
return db['albums'].insert(dict(
title=album.title,
uri=album.uri,
date=album.date,
description=album.comment,
course=album.course,
course_uri=album.course_uri,
teacher=album.teacher,
content=product_ids
))
def read_gallery_item(dir_name, sync=True):
def read_gallery_item(dir_name):
"""
Read directory with .md and images and return an instance of Album.
None if reads fails.
:param dir_name:
:param sync:
:return:
"""
set_path = os.path.join(dir_gallery, i)
set_path = os.path.join(dir_gallery, dir_name)
os.chdir(set_path)
md = glob('*.md')
@@ -160,51 +102,52 @@ def read_gallery_item(dir_name, sync=True):
description_filename = md[0]
description_path = os.path.join(set_path, description_filename)
#if validation.check_description_filename(description_filename):
if True:
#v = get_description_version(description_filename)
v = '1'
DescriptionReader = get_description_reader(v)
if DescriptionReader is not None:
r = DescriptionReader(description_path)
album = None
try:
album = r.read()
v = '1'
DescriptionReader = get_description_reader(v)
if DescriptionReader is not None:
r = DescriptionReader(set_path, description_path)
try:
return r.read()
except Exception as e:
print('Error with reading description file %s' % (description_path))
print(e)
return
try:
album.products = list(map(
lambda p : process_product(album, p),
album.products
))
except Exception as e:
print('Error with image processing %s' % (description_path))
print(e)
if sync:
try:
sync_album(album)
except Exception as e:
print('Cannot sync %s' % (description_path))
print(e)
else:
print('No reader for description file %s' % (description_path))
else:
new_name = validation.suggest_description_filename(description_filename)
os.rename(description_filename, new_name)
read_gallery_item(dir_name)
except Exception as e:
print('Error with reading description file %s' % (description_path))
print(e)
return None
if __name__ == '__main__':
# clear_db()
for i in catalog_files:
read_gallery_item(i, sync=False)
print()
print('--> READING GALLERY CATALOGS')
albums = list(map(read_gallery_item, catalog_files))
print('--> GALLERY TO SYNC')
albums = list(filter(lambda a: a is not None, albums))
for a in albums:
print(a)
img = False
save_dir = settings.image_save_dir
print('--> PROCESSING IMAGES to %s (%s)' % (save_dir, img))
if img:
products = reduce(lambda p, a: p + a.products, albums, [])
print('Products image to process %s' % len(products))
for p in products:
print('Process %s' % p.image_original_path)
process_product_image(p, save_dir)
sync = True
print('--> SYNCING ALBUMS (%s)' % sync)
if sync:
print('--> DROP DB')
clear_db()
for a in albums:
try:
a_id = sync_album(a)
print('SYNC DONE (%s: %s)' % a_id, a.title)
except Exception as e:
print('Cannot sync %s' % (a))
print(e)
exit()