From 3b612ab4e7464a45a5e3fed81443eade571a5c5f Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sat, 16 Jan 2016 23:10:33 +0300 Subject: [PATCH] sync enhancements & update --- utils/Sync/Album.py | 1 - utils/Sync/Product.py | 14 +++ utils/Sync/img.py | 2 +- utils/Sync/read_gallery_1.py | 58 +++++++++- utils/Sync/sync_gallery.py | 219 +++++++++++++---------------------- 5 files changed, 150 insertions(+), 144 deletions(-) create mode 100644 utils/Sync/Product.py diff --git a/utils/Sync/Album.py b/utils/Sync/Album.py index 59cec1f2..f30f6348 100644 --- a/utils/Sync/Album.py +++ b/utils/Sync/Album.py @@ -19,4 +19,3 @@ class Album: def __repr__(self, *args, **kwargs): return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products)) - diff --git a/utils/Sync/Product.py b/utils/Sync/Product.py new file mode 100644 index 00000000..e88ec302 --- /dev/null +++ b/utils/Sync/Product.py @@ -0,0 +1,14 @@ +__author__ = 'Roman Timashev' + + +class Product: + def __init__(self, author_name, author_age, image, product_title): + super().__init__() + self.author_name = author_name + self.author_age = author_age + self.image = image + self.product_title = product_title + self.image_original_path = None + + def to_tuple(self): + return self.author_name, self.author_age, self.image, self.product_title, self.image_original_path diff --git a/utils/Sync/img.py b/utils/Sync/img.py index 0ea192f4..ad88fcc1 100644 --- a/utils/Sync/img.py +++ b/utils/Sync/img.py @@ -11,7 +11,7 @@ def read_image(src): return Image.open(src) -def minify(image, dest, size, ext=None): +def thumbnail(image, dest, size, ext=None): image.thumbnail(size) image.save(override_extension(dest, ext)) diff --git a/utils/Sync/read_gallery_1.py b/utils/Sync/read_gallery_1.py index a010c4f1..327769d3 100644 --- a/utils/Sync/read_gallery_1.py +++ b/utils/Sync/read_gallery_1.py @@ -1,23 +1,33 @@ +import os from datetime import datetime import re import lxml.html from markdown import markdown import time + +import img from Album import Album import lxml_utils import settings +from Product import Product __author__ = 'Roman Timashev' -artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER+settings.CYR_UPPER)) +artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER + settings.CYR_UPPER)) # artist_name_exp = re.compile('(.+)(\s\(\d+ лет\))?$') artist_age_exp = re.compile('.+\((\d+) лет\)$') + class GalleryReader: - def __init__(self, path): + def __init__(self, dir, path): super().__init__() self.html = None self.description_path = path + self.dir_path = dir + + self.image_sizes = settings.image_sizes + self.image_ext = settings.image_ext + self.image_base_url = settings.image_base_url def read(self): with open(self.description_path, 'r', encoding='utf-8') as dfile: @@ -54,9 +64,13 @@ class GalleryReader: # PRODUCT content = self.get_drawings() - a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date, teacher=teacher_name, comment=None) + a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date, + teacher=teacher_name, comment=None) for p in content: - a.add_product(p) + author_name, author_age, filename, product_title = p + product = self.create_product(a, author_name, author_age, filename, product_title) + product.image_original_path = os.path.join(self.dir_path, filename) + a.add_product(product) return a def get_drawings(self): @@ -98,6 +112,42 @@ class GalleryReader: return drawings + def create_product(self, album, author_name, author_age, filename, product_title): + tr = {ord(a): ord(b) for a, b in zip(*settings.translit_table)} + + author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr) + image_id = '%s-%s-%s-%s' % ( + album.date.year, + re.sub(r'[/]', '', album.uri), + author_name_encoded, + os.path.splitext(filename)[0] + ) + + image = self.create_image(image_id, filename, self.image_sizes, self.image_ext, self.image_base_url) + return Product(author_name, author_age, image, product_title) + + def create_image(self, name, filename, sizes, ext, base_url): + thumbs = {} + for size, width, height in sizes: + source_image = img.read_image(filename) + sw, sh = source_image.size + if sw < sh: + width, height = height, width + + width = width if width else sw + height = height if height else sh + + out_filename = '{name}-{w}x{h}{ext}'.format(name=name, w=width, h=height, ext=ext) + out_url = os.path.join(base_url, out_filename) + + thumbs[size] = { + 'url': out_url, + 'size': size, + 'width': width, + 'height': height + } + return thumbs + def get_date(self): string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата") if string is None: diff --git a/utils/Sync/sync_gallery.py b/utils/Sync/sync_gallery.py index 35b944a6..7512f88a 100644 --- a/utils/Sync/sync_gallery.py +++ b/utils/Sync/sync_gallery.py @@ -1,12 +1,12 @@ +from functools import reduce from glob import glob import os import re from pymongo import MongoClient -import img +import img as img_util from read_gallery_1 import GalleryReader import settings -import validation __author__ = 'Roman Timashev' @@ -30,60 +30,16 @@ def get_description_reader(version): return None -def process_image(name, filename, sizes): - ext = settings.image_ext - - thumbs = {} - - for size, width, height in sizes: - source_image = img.read_image(filename) - sw, sh = source_image.size - if sw < sh: - width, height = height, width - - out_image_filename = '{name}-{size}{ext}'.format(name=name, size=size, ext=ext) - out_image_local_path = os.path.join(settings.image_save_dir, out_image_filename) - out_image_url = os.path.join(settings.image_base_url, out_image_filename) - - if width and height: - tw = width - th = height - else: - tw = sw - th = sh - - thumbs[size] = { - 'url': out_image_url, - 'size': size, - 'width': sw, - 'height': sh - } - - # print('generate %s for %s' % (size, filename), out_image_url) - - if settings.image_processing: - if width and height: - img.minify(source_image, out_image_local_path, (width, height)) - else: - img.optimize(source_image, out_image_local_path) - - return thumbs - - -def process_product(album, product): - tr = {ord(a):ord(b) for a, b in zip(*settings.translit_table)} - - author_name, author_age, filename, product_title = product - author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr) - image_id = '%s-%s-%s-%s' % ( - album.date.year, - re.sub(r'[/]', '', album.uri), - author_name_encoded, - os.path.splitext(filename)[0] - ) - - image = process_image(image_id, filename, settings.image_sizes) - return (author_name, author_age, image, product_title) +def process_product_image(product, save_dir): + for k in product.image.keys(): + thumb = product.image[k] + filename = os.path.basename(thumb['url']) + outpath = os.path.join(save_dir, filename) + img_util.image_magick_resize( + product.image_original_path, + outpath, + (thumb['width'], thumb['height']) + ) def get_db(): @@ -99,60 +55,46 @@ def clear_db(): def sync_product(product, type): db = get_db() - author_name, author_age, image, product_title = product + author_name, author_age, image, product_title, original_path = product.to_tuple() - product_record = db['products'].insert({ - 'author': author_name, - 'authorAge': author_age, - 'title': product_title, - 'type': type, - 'content': image - }) - - print('inserted', product_record) - - return product_record - -def sync_album(album): - print('syncing %s' % album.title) - db = get_db() - - product_ids = list(map( - lambda product: sync_product(product, album.type), - album.products + return db['products'].insert(dict( + author=author_name, + authorAge=author_age, + title=product_title, + type=type, + content=image )) - album_title = album.title - album_uri = album.uri - album_teacher = album.teacher - album_date = album.date - album_comment = album.comment - album_course = album.course - album_course_uri = album.course_uri - db['albums'].insert({ - 'title': album_title, - 'uri': album_uri, - 'date': album_date, - 'description': album_comment, - 'course': album_course, - 'course_uri': album_course_uri, - 'teacher': album_teacher, - 'content': product_ids - }) +def sync_album(album): + db = get_db() + product_ids = list(map( + lambda product: sync_product(product, album.type), + album.products + )) + + return db['albums'].insert(dict( + title=album.title, + uri=album.uri, + date=album.date, + description=album.comment, + course=album.course, + course_uri=album.course_uri, + teacher=album.teacher, + content=product_ids + )) -def read_gallery_item(dir_name, sync=True): +def read_gallery_item(dir_name): """ Read directory with .md and images and return an instance of Album. None if reads fails. :param dir_name: - :param sync: :return: """ - set_path = os.path.join(dir_gallery, i) + set_path = os.path.join(dir_gallery, dir_name) os.chdir(set_path) md = glob('*.md') @@ -160,51 +102,52 @@ def read_gallery_item(dir_name, sync=True): description_filename = md[0] description_path = os.path.join(set_path, description_filename) - #if validation.check_description_filename(description_filename): - if True: - #v = get_description_version(description_filename) - v = '1' - DescriptionReader = get_description_reader(v) - if DescriptionReader is not None: - r = DescriptionReader(description_path) - album = None - try: - album = r.read() + v = '1' + DescriptionReader = get_description_reader(v) + if DescriptionReader is not None: + r = DescriptionReader(set_path, description_path) + try: + return r.read() - except Exception as e: - print('Error with reading description file %s' % (description_path)) - print(e) - return - - try: - album.products = list(map( - lambda p : process_product(album, p), - album.products - )) - except Exception as e: - print('Error with image processing %s' % (description_path)) - print(e) - - if sync: - try: - sync_album(album) - except Exception as e: - print('Cannot sync %s' % (description_path)) - print(e) - - else: - print('No reader for description file %s' % (description_path)) - - else: - new_name = validation.suggest_description_filename(description_filename) - os.rename(description_filename, new_name) - read_gallery_item(dir_name) + except Exception as e: + print('Error with reading description file %s' % (description_path)) + print(e) + return None if __name__ == '__main__': - # clear_db() - for i in catalog_files: - read_gallery_item(i, sync=False) - print() + print('--> READING GALLERY CATALOGS') + albums = list(map(read_gallery_item, catalog_files)) + + print('--> GALLERY TO SYNC') + albums = list(filter(lambda a: a is not None, albums)) + + for a in albums: + print(a) + + img = False + save_dir = settings.image_save_dir + print('--> PROCESSING IMAGES to %s (%s)' % (save_dir, img)) + if img: + products = reduce(lambda p, a: p + a.products, albums, []) + print('Products image to process %s' % len(products)) + for p in products: + print('Process %s' % p.image_original_path) + process_product_image(p, save_dir) + + sync = True + print('--> SYNCING ALBUMS (%s)' % sync) + if sync: + print('--> DROP DB') + clear_db() + + for a in albums: + try: + a_id = sync_album(a) + print('SYNC DONE (%s: %s)' % a_id, a.title) + + except Exception as e: + print('Cannot sync %s' % (a)) + print(e) exit()