tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

Merge branch 'sync' into dev

Browse Source
This commit is contained in:
tmshv committed 2016-01-16 23:12:32 +03:00
2 parents 7bdd23d2aa 3b612ab4e7
commit e295dd5863
6 files changed
+159 -145
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+1
View File
@@ -10,3 +10,4 @@ private*.json
/public
/build
**/__pycache__
**/*.pyc
utils/Sync/Album.py
-1
View File
@@ -19,4 +19,3 @@ class Album:
def __repr__(self, *args, **kwargs):
return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products))
utils/Sync/Product.py
+14
View File
@@ -0,0 +1,14 @@
__author__ = 'Roman Timashev'
class Product:
def __init__(self, author_name, author_age, image, product_title):
super().__init__()
self.author_name = author_name
self.author_age = author_age
self.image = image
self.product_title = product_title
self.image_original_path = None
def to_tuple(self):
return self.author_name, self.author_age, self.image, self.product_title, self.image_original_path
utils/Sync/img.py
+9 -2
View File
@@ -1,5 +1,7 @@
import os
from subprocess import call
__author__ = 'Roman Timashev'
from PIL import Image
@@ -9,7 +11,7 @@ def read_image(src):
return Image.open(src)
def minify(image, dest, size, ext=None):
def thumbnail(image, dest, size, ext=None):
image.thumbnail(size)
image.save(override_extension(dest, ext))
@@ -22,4 +24,9 @@ def override_extension(dest, ext=None):
if ext is not None:
dest = os.path.splitext(dest)[0] + ext
return dest
return dest
def image_magick_resize(input_file, output_file, size, quality=85):
s = '%dx%d' % size
call(['convert', input_file, '-resize', s, '-quality', str(quality), output_file])
utils/Sync/read_gallery_1.py
+54 -4
View File
@@ -1,23 +1,33 @@
import os
from datetime import datetime
import re
import lxml.html
from markdown import markdown
import time
import img
from Album import Album
import lxml_utils
import settings
from Product import Product
__author__ = 'Roman Timashev'
artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER+settings.CYR_UPPER))
artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER + settings.CYR_UPPER))
# artist_name_exp = re.compile('(.+)(\s\(\d+ лет\))?$')
artist_age_exp = re.compile('.+\((\d+) лет\)$')
class GalleryReader:
def __init__(self, path):
def __init__(self, dir, path):
super().__init__()
self.html = None
self.description_path = path
self.dir_path = dir
self.image_sizes = settings.image_sizes
self.image_ext = settings.image_ext
self.image_base_url = settings.image_base_url
def read(self):
with open(self.description_path, 'r', encoding='utf-8') as dfile:
@@ -54,9 +64,13 @@ class GalleryReader:
# PRODUCT
content = self.get_drawings()
a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date, teacher=teacher_name, comment=None)
a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date,
teacher=teacher_name, comment=None)
for p in content:
a.add_product(p)
author_name, author_age, filename, product_title = p
product = self.create_product(a, author_name, author_age, filename, product_title)
product.image_original_path = os.path.join(self.dir_path, filename)
a.add_product(product)
return a
def get_drawings(self):
@@ -98,6 +112,42 @@ class GalleryReader:
return drawings
def create_product(self, album, author_name, author_age, filename, product_title):
tr = {ord(a): ord(b) for a, b in zip(*settings.translit_table)}
author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr)
image_id = '%s-%s-%s-%s' % (
album.date.year,
re.sub(r'[/]', '', album.uri),
author_name_encoded,
os.path.splitext(filename)[0]
)
image = self.create_image(image_id, filename, self.image_sizes, self.image_ext, self.image_base_url)
return Product(author_name, author_age, image, product_title)
def create_image(self, name, filename, sizes, ext, base_url):
thumbs = {}
for size, width, height in sizes:
source_image = img.read_image(filename)
sw, sh = source_image.size
if sw < sh:
width, height = height, width
width = width if width else sw
height = height if height else sh
out_filename = '{name}-{w}x{h}{ext}'.format(name=name, w=width, h=height, ext=ext)
out_url = os.path.join(base_url, out_filename)
thumbs[size] = {
'url': out_url,
'size': size,
'width': width,
'height': height
}
return thumbs
def get_date(self):
string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата")
if string is None:
utils/Sync/sync_gallery.py
+81 -138
View File
@@ -1,12 +1,12 @@
from functools import reduce
from glob import glob
import os
import re
from pymongo import MongoClient
import img
import img as img_util
from read_gallery_1 import GalleryReader
import settings
import validation
__author__ = 'Roman Timashev'
@@ -30,60 +30,16 @@ def get_description_reader(version):
return None
def process_image(name, filename, sizes):
ext = settings.image_ext
thumbs = {}
for size, width, height in sizes:
source_image = img.read_image(filename)
sw, sh = source_image.size
if sw < sh:
width, height = height, width
out_image_filename = '{name}-{size}{ext}'.format(name=name, size=size, ext=ext)
out_image_local_path = os.path.join(settings.image_save_dir, out_image_filename)
out_image_url = os.path.join(settings.image_base_url, out_image_filename)
if width and height:
tw = width
th = height
else:
tw = sw
th = sh
thumbs[size] = {
'url': out_image_url,
'size': size,
'width': sw,
'height': sh
}
# print('generate %s for %s' % (size, filename), out_image_url)
if settings.image_processing:
if width and height:
img.minify(source_image, out_image_local_path, (width, height))
else:
img.optimize(source_image, out_image_local_path)
return thumbs
def process_product(album, product):
tr = {ord(a):ord(b) for a, b in zip(*settings.translit_table)}
author_name, author_age, filename, product_title = product
author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr)
image_id = '%s-%s-%s-%s' % (
album.date.year,
re.sub(r'[/]', '', album.uri),
author_name_encoded,
os.path.splitext(filename)[0]
)
image = process_image(image_id, filename, settings.image_sizes)
return (author_name, author_age, image, product_title)
def process_product_image(product, save_dir):
for k in product.image.keys():
thumb = product.image[k]
filename = os.path.basename(thumb['url'])
outpath = os.path.join(save_dir, filename)
img_util.image_magick_resize(
product.image_original_path,
outpath,
(thumb['width'], thumb['height'])
)
def get_db():
@@ -99,60 +55,46 @@ def clear_db():
def sync_product(product, type):
db = get_db()
author_name, author_age, image, product_title = product
author_name, author_age, image, product_title, original_path = product.to_tuple()
product_record = db['products'].insert({
'author': author_name,
'authorAge': author_age,
'title': product_title,
'type': type,
'content': image
})
print('inserted', product_record)
return product_record
def sync_album(album):
print('syncing %s' % album.title)
db = get_db()
product_ids = list(map(
lambda product: sync_product(product, album.type),
album.products
return db['products'].insert(dict(
author=author_name,
authorAge=author_age,
title=product_title,
type=type,
content=image
))
album_title = album.title
album_uri = album.uri
album_teacher = album.teacher
album_date = album.date
album_comment = album.comment
album_course = album.course
album_course_uri = album.course_uri
db['albums'].insert({
'title': album_title,
'uri': album_uri,
'date': album_date,
'description': album_comment,
'course': album_course,
'course_uri': album_course_uri,
'teacher': album_teacher,
'content': product_ids
})
def sync_album(album):
db = get_db()
product_ids = list(map(
lambda product: sync_product(product, album.type),
album.products
))
return db['albums'].insert(dict(
title=album.title,
uri=album.uri,
date=album.date,
description=album.comment,
course=album.course,
course_uri=album.course_uri,
teacher=album.teacher,
content=product_ids
))
def read_gallery_item(dir_name, sync=True):
def read_gallery_item(dir_name):
"""
Read directory with .md and images and return an instance of Album.
None if reads fails.
:param dir_name:
:param sync:
:return:
"""
set_path = os.path.join(dir_gallery, i)
set_path = os.path.join(dir_gallery, dir_name)
os.chdir(set_path)
md = glob('*.md')
@@ -160,51 +102,52 @@ def read_gallery_item(dir_name, sync=True):
description_filename = md[0]
description_path = os.path.join(set_path, description_filename)
#if validation.check_description_filename(description_filename):
if True:
#v = get_description_version(description_filename)
v = '1'
DescriptionReader = get_description_reader(v)
if DescriptionReader is not None:
r = DescriptionReader(description_path)
album = None
try:
album = r.read()
v = '1'
DescriptionReader = get_description_reader(v)
if DescriptionReader is not None:
r = DescriptionReader(set_path, description_path)
try:
return r.read()
except Exception as e:
print('Error with reading description file %s' % (description_path))
print(e)
return
try:
album.products = list(map(
lambda p : process_product(album, p),
album.products
))
except Exception as e:
print('Error with image processing %s' % (description_path))
print(e)
if sync:
try:
sync_album(album)
except Exception as e:
print('Cannot sync %s' % (description_path))
print(e)
else:
print('No reader for description file %s' % (description_path))
else:
new_name = validation.suggest_description_filename(description_filename)
os.rename(description_filename, new_name)
read_gallery_item(dir_name)
except Exception as e:
print('Error with reading description file %s' % (description_path))
print(e)
return None
if __name__ == '__main__':
# clear_db()
for i in catalog_files:
read_gallery_item(i, sync=False)
print()
print('--> READING GALLERY CATALOGS')
albums = list(map(read_gallery_item, catalog_files))
print('--> GALLERY TO SYNC')
albums = list(filter(lambda a: a is not None, albums))
for a in albums:
print(a)
img = False
save_dir = settings.image_save_dir
print('--> PROCESSING IMAGES to %s (%s)' % (save_dir, img))
if img:
products = reduce(lambda p, a: p + a.products, albums, [])
print('Products image to process %s' % len(products))
for p in products:
print('Process %s' % p.image_original_path)
process_product_image(p, save_dir)
sync = True
print('--> SYNCING ALBUMS (%s)' % sync)
if sync:
print('--> DROP DB')
clear_db()
for a in albums:
try:
a_id = sync_album(a)
print('SYNC DONE (%s: %s)' % a_id, a.title)
except Exception as e:
print('Cannot sync %s' % (a))
print(e)
exit()