diff --git a/utils/Sync/Product.py b/utils/Sync/Product.py deleted file mode 100644 index e88ec302..00000000 --- a/utils/Sync/Product.py +++ /dev/null @@ -1,14 +0,0 @@ -__author__ = 'Roman Timashev' - - -class Product: - def __init__(self, author_name, author_age, image, product_title): - super().__init__() - self.author_name = author_name - self.author_age = author_age - self.image = image - self.product_title = product_title - self.image_original_path = None - - def to_tuple(self): - return self.author_name, self.author_age, self.image, self.product_title, self.image_original_path diff --git a/utils/Sync/db-test.py b/utils/Sync/db-test.py deleted file mode 100644 index 2c8a1a95..00000000 --- a/utils/Sync/db-test.py +++ /dev/null @@ -1,13 +0,0 @@ -from pymongo import MongoClient -import settings - -__author__ = 'Roman Timashev' - - -client = MongoClient(settings.database_uri) -p = client.hudozka.posts - -print(list(p.find())) - -# print(dir(p)) -# print(p.insert({'a': 1})) \ No newline at end of file diff --git a/utils/Sync/db.py b/utils/Sync/db.py new file mode 100644 index 00000000..3c0ad0c7 --- /dev/null +++ b/utils/Sync/db.py @@ -0,0 +1,8 @@ +from pymongo import MongoClient + +import settings + + +def db(): + client = MongoClient(settings.database_uri) + return client.hudozka diff --git a/utils/Sync/image/image.py b/utils/Sync/image/image.py deleted file mode 100644 index 37752dce..00000000 --- a/utils/Sync/image/image.py +++ /dev/null @@ -1,8 +0,0 @@ -from PIL import Image - - -def read_image(src): - try: - return Image.open(src) - except FileNotFoundError: - return None \ No newline at end of file diff --git a/utils/Sync/image/image_processor.py b/utils/Sync/image/image_processor.py deleted file mode 100644 index 2af1724d..00000000 --- a/utils/Sync/image/image_processor.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -from image.image import read_image -from image.resize import optimize, thumbnail - - -def image_processor(images_dir, public_base_url, sizes, name_generator): - def process(file): - image = read_image(file) - if not image: - return None - - basename = os.path.basename(file) - uri, ext = os.path.splitext(basename) - - image_dict = {} - for size in sizes: - size_name, width, height = size - - image_filename = name_generator(uri, size_name, ext) - d = os.path.join(images_dir, image_filename) - image_url = os.path.join(public_base_url, image_filename) - - do_optimize = size_name == 'original' - thumb = optimize(image, d, quality=90) if do_optimize else thumbnail(image, d, (width, height)) - width, height = thumb.size - image_dict[size_name] = { - 'url': image_url, - 'size': size_name, - 'width': width, - 'height': height - } - - return image_dict - return process \ No newline at end of file diff --git a/utils/Sync/Album.py b/utils/Sync/models/Album.py similarity index 100% rename from utils/Sync/Album.py rename to utils/Sync/models/Album.py diff --git a/utils/Sync/models/award.py b/utils/Sync/models/award.py new file mode 100644 index 00000000..c7d17197 --- /dev/null +++ b/utils/Sync/models/award.py @@ -0,0 +1,8 @@ +# JSON +# { +# "assigned": "\\(author)", +# "file": "1 diploma.jpg" +# } +def create_award(doc): + return {**doc} + diff --git a/utils/Sync/models/product.py b/utils/Sync/models/product.py new file mode 100644 index 00000000..40df3931 --- /dev/null +++ b/utils/Sync/models/product.py @@ -0,0 +1,14 @@ +from sync.image import sync_image +from utils.fn import key_mapper + + +unpop_image = key_mapper('image', lambda i: sync_image(i)['_id']) + + +def create_product(doc): + # author, image, title, awards, teacher, date + + product = {**doc} + unpop_image(product) + + return product diff --git a/utils/Sync/read_gallery_1.py b/utils/Sync/read_gallery_1.py index c0cc23ba..546cde6d 100644 --- a/utils/Sync/read_gallery_1.py +++ b/utils/Sync/read_gallery_1.py @@ -6,11 +6,11 @@ from datetime import datetime import lxml.html from markdown import markdown -import lxml_utils import settings -from Album import Album -from Product import Product -from image import resize +from models.Album import Album +from models.product import Product +from utils import lxml_utils +from utils.image import resize __author__ = 'Roman Timashev' @@ -128,9 +128,11 @@ class GalleryReader: return Product(author_name, author_age, image, product_title) def create_image(self, name, filename, sizes, ext, base_url): + print('DEPRECATED: use utils.image.__init__.create_image') + thumbs = {} for size, width, height in sizes: - source_image = resize.read_image(filename) + source_image = read_image(filename) sw, sh = source_image.size if sw < sh: width, height = height, width @@ -150,6 +152,8 @@ class GalleryReader: return thumbs def get_date(self): + print('DEPRECATED: use sync.__init__.create_date') + string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата") if string is None: return None diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 25bada8a..c4b437d7 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -14,9 +14,10 @@ EN_UPPER = 'QWERTYUIOPASDFGHJKLZXCVBNM' DIGITS = '0123456789' description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-' + translit_table = ( - 'абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ', - 'abvgdeejzijklmnoprstufhzcss_y_euaABVGDEEJZIJKLMNOPRSTUFHZCSS_Y_EUA' + ['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'], + ['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', '_'] ) teachers_names = { @@ -34,7 +35,9 @@ image_save_dir = './static_images' image_base_url = 'https://static.shburg.org/art/image/' collective_image_base_url = 'https://static.shburg.org/art/collective/' -collective_image_output = '/Users/tmshv/Dropbox/Dev/Hud School/Static/collective' +collective_image_output = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' + +album_html_img_class = 'hudozka-product' image_ext = '.jpg' image_sizes = [ @@ -46,6 +49,10 @@ image_sizes = [ ('preview', 50, 50), ] +event_image_sizes = [ + ('big', 1200, 800) +] + date_formats = [ '%Y', '%m.%Y', @@ -53,3 +60,15 @@ date_formats = [ '%d.%m.%Y %H:%M', '%d.%m.%Y %H:%M:%S' ] + +date_formats_direct = [ + '%Y', + '%m.%Y', + '%d.%m.%Y', +] + +date_formats_reverse = [ + '%Y', + '%Y.%m', + '%Y.%m.%d' +] \ No newline at end of file diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py new file mode 100644 index 00000000..c315d39f --- /dev/null +++ b/utils/Sync/sync/__init__.py @@ -0,0 +1,59 @@ +import re + +from datetime import datetime + +from markdown import markdown + + +def create_date(date_str, date_formats=None): + """ + 2016 + 2016.10 + 2016.10.10 + -> datetime + + :param date_str: + :return: + """ + if not date_formats: + date_formats = [ + '%Y', + '%Y.%m', + '%Y.%m.%d' + ] + + for format in date_formats: + try: + return datetime.strptime(date_str, format) + except: + continue + return None + + +# 2016 На крыльях бабочек +# 2016.10 На крыльях бабочек +# 2016.10.10 На крыльях бабочек +folder_name_pattern = re.compile('([\d.]+)(.*)') + + +def create_date_and_title_from_folder_name(folder_name, date_formats=None): + m = folder_name_pattern.findall(folder_name) + if not m: + return None, None + + date_str, title = m[0] + title = title.strip() + date = create_date(date_str, date_formats) + + if not date: + return None, title + + return date, title + + +create_post_from_image_list = lambda images: markdown( + '\n'.join(map( + lambda i: '![]({img})'.format(img=i), + images + )) +) \ No newline at end of file diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py new file mode 100644 index 00000000..cf8f946d --- /dev/null +++ b/utils/Sync/sync/album.py @@ -0,0 +1,230 @@ +import os +from glob import glob + +import lxml.html + +import settings +from db import db +from models.product import create_product +from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date +from sync.image import sync_image +from utils.fn import lmap, first, map_cases, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns +from utils.image import create_image +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text, interpolate, interpolate_swift + + +def sync_album(record): + if isinstance(record, list): + return lmap( + sync_album, + record + ) + + q = {'id': record['id']} + try: + collection = db().events + update_result = collection.update_one(q, {'$set': record}, upsert=True) + i = collection.find_one({'id': record['id']}) + return i + except ValueError: + pass + + return None + + +def create_album(doc): + sizes = settings.event_image_sizes + # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' + images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' + url = lambda a, i, s, e: 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format( + album=a, + id=i, + size=s, + ext=e + ) + create_url = lambda aid, pid: lambda size, ext: url(aid, pid, size, ext.lower()) + image_id = lambda i: url_encode_text( + os.path.splitext(i)[0] + ) + image_path = lambda i: os.path.join(doc['folder'], i) + + if 'title' not in doc or not doc['title']: + return None + + create_album_id = lambda date, title: '{date}-{title}'.format( + date=date.strftime('%Y'), + title=url_encode_text(title) + ) + + images = [] + + doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) + + if 'album' in doc: + # album_context = + iter_album = iterate_iter_over_fns([ + lambda product: { + **product, + 'image': create_image( + image_path(product['file']), + sizes, + create_url( + doc['id'], + image_id(product['file']) + ), + images_dir, + skip_processing=True + ) + }, + lambda product: { + **product, + 'awards': lmap( + lambda award: { + **award, + 'file': os.path.join(doc['folder'], award['file']), + 'assigned': interpolate(award['assigned'], product, interpolate_swift), + # 'awardFor': product + }, + product['awards'] + ) + } + ]) + + doc['album'] = lmap( + create_product, + iter_album(doc['album']) + ) + + post_html = lxml.html.fromstring(doc['post']) + + for img in post_html.cssselect('img'): + src = img.get('src') + img_path = image_path(src) + + if os.path.exists(img_path): + image = create_image( + img_path, + sizes, + create_url( + doc['id'], + image_id(src) + ), + images_dir, + skip_processing=True + ) + images.append(image) + + img.set('src', image['data']['big']['url']) + img.set('class', settings.album_html_img_class) + img.set('data-file', src) + + doc['post'] = lxml.html.tostring(post_html).decode('utf-8') + doc['images'] = images + return doc + + +def get_manifest(filepath): + y, m = read_yaml_md(filepath) + + data = {**y} if y else {} + _ = lambda key, d=None: data[key] if key in data else d + date = create_date(_('date'), settings.date_formats_direct) + if date: + data['date'] = date + + id = _('id') + if id: + data['id'] = str(id) + + post = {'post': m} if m else {} + return {**data, **post} + + +if __name__ == '__main__': + albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' + os.chdir(albums_dir) + + # GET ALBUMS FOLDERS + albums = lmap( + lambda i: i.path, + filter( + lambda i: i.is_dir(), + os.scandir('.') + ) + ) + + # print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md')) + # print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md')) + # exit() + + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + albums = lmap( + lambda i: (i,) + create_date_and_title_from_folder_name( + os.path.basename(i) + ), + albums + ) + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG']) + albums = lmap( + lambda i: { + 'folder': i[0], + 'date': i[1], + 'title': i[2], + 'post': create_post_from_image_list(lmap( + lambda path: os.path.relpath(path, i[0]), + glob_in(i[0]) + )) + }, + albums + ) + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + md_glob = lambda i: first(glob(i + '/*.md')) + albums = lmap( + lambda album: { + **album, + **map_cases( + album, + [( + lambda album: md_glob(album['folder']), + lambda album: get_manifest(md_glob(album['folder'])), + )], + lambda i: {} + ) + }, + albums + ) + + # MAP EVENT MANIFEST -> EVENT_OBJECT + albums = lmap( + create_album, + albums + ) + + # FILTER INCORRECT EVENTS + albums = list(filter(None, albums)) + + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB + def sync_images(i): + return lmap( + lambda image: image['_id'], + sync_image(i) + ) + + + albums = lmap( + key_mapper('images', sync_images), + albums + ) + + # SYNC EVENT_OBJECT WITH DB + # albums = lmap( + # sync_album, + # albums + # ) + + lprint_json(albums) + print('SYNC ALBUMS DONE') diff --git a/utils/Sync/sync/collective.py b/utils/Sync/sync/collective.py new file mode 100644 index 00000000..82e6a1ee --- /dev/null +++ b/utils/Sync/sync/collective.py @@ -0,0 +1,75 @@ +import os +from glob import glob + +import settings +from db import db +from sync.image import sync_image +from utils.fn import lmap, lprint, key_mapper +from utils.image import create_image +from utils.io import read_yaml_md + + +def sync_collective(record): + if isinstance(record, list): + return lmap( + sync_collective, + record + ) + + q = {'id': record['id']} + try: + collection = db().collective + update_result = collection.update_one(q, {'$set': record}, upsert=True) + i = collection.find_one({'id': record['id']}) + return i + except ValueError: + pass + + return None + + +def create_profile(profile): + sizes = settings.image_sizes + images_dir = settings.collective_image_output + + url_base = 'https://static.shburg.org/art/image/teacher-{id}-{size}{ext}' + url_fn = lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext) + + profile['picture'] = create_image(profile['picture'], sizes, url_fn, images_dir) + return profile + + +if __name__ == '__main__': + dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective' + os.chdir(dir) + + # GET TEACHER YAML/MD MANIFEST FILES + files = glob('*.md') + + # READ TEACHER YAML/MD MANIFEST + collective = lmap( + read_yaml_md, + files + ) + + # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY + collective = lmap( + lambda i: {'biography': i[1], **i[0]}, + collective + ) + + # MAP TEACHER_MANIFEST -> TEACHER_PROFILE + collective = lmap( + create_profile, + collective + ) + + # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB + sync_picture = lambda i: None if i is None else sync_image(i)['_id'] + collective = lmap( + key_mapper('picture', sync_picture), + collective + ) + + lprint(collective) + print('SYNC COLLECTIVE DONE') diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py new file mode 100644 index 00000000..0c22b0e9 --- /dev/null +++ b/utils/Sync/sync/document.py @@ -0,0 +1,139 @@ +import os +from glob import glob + +from tempfile import mkstemp + +from shutil import copyfile + +import settings +from db import db +from sync.image import sync_image +from utils.fn import combine, lmap, lprint, key_mapper +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.io import read_yaml +from utils.text.transform import url_encode_text, url_encode_file + +# Adds specified category to each file in files list +unwrap_file_list = lambda files, title: map( + lambda profile: {'category': title, **profile}, + files +) + + +def sync_document(record): + if isinstance(record, list): + return lmap( + sync_document, + record + ) + + q = {'url': record['url']} + try: + documents = db().documents + update_result = documents.update_one(q, {'$set': record}, upsert=True) + i = documents.find_one({'url': record['url']}) + return i + except ValueError: + pass + + return None + + +def pdf_to_jpg(pdf): + cwd = os.getcwd() + abspdf = os.path.join(cwd, pdf) + _, temp = mkstemp('.jpg') + image_magick_pdf_to_img(abspdf, temp) + return temp + + +def create_preview(pdf, sizes, preview_dir): + temp_preview_path = pdf_to_jpg(pdf) + + id = url_encode_text(pdf) + url_base = 'https://static.shburg.org/art/image/document-{id}-{size}{ext}' + url = lambda size, ext: url_base.format(id=id, size=size, ext=ext) + + img = create_image(temp_preview_path, sizes, url, preview_dir) + os.remove(temp_preview_path) + return img + + +def create_document(doc): + sizes = settings.image_sizes + preview_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' + + doc['url'] = 'https://static.shburg.org/art/uploads/' + url_encode_file(doc['file']) + doc['preview'] = create_preview(doc['file'], sizes=sizes, preview_dir=preview_dir) + + return doc + + +def unwrap_manifest(param): + if isinstance(param, list): + return combine(map( + unwrap_manifest, + param + )) + + if 'files' in param: + return lmap( + unwrap_manifest, + unwrap_file_list(param['files'], param['title']) + ) + + if 'file' not in param: + return None + + return param + + +if __name__ == '__main__': + dir = '/Users/tmshv/Dropbox/Dev/Hud school/Documents' + + os.chdir(dir) + + # GET DOCUMENT YAML_MANIFEST FILES + files = glob(dir + '/*.yaml') + + # READ YAML_MANIFEST FILES + files_data = lmap( + read_yaml, + files + ) + + # GET FLAT LIST OF DOCUMENTS + documents = unwrap_manifest(files_data) + + # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT + documents = lmap( + create_document, + documents + ) + + # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB + sync_preview = lambda i: sync_image(i)['_id'] + documents = lmap( + key_mapper('preview', sync_preview), + documents + ) + + # COPY FILE -> STATIC_DIR/URL_FILENAME + out_path = lambda doc: os.path.join('/Users/tmshv/Dropbox/Dev/Hud school/Static/uploads', os.path.basename(doc['url'])) + lmap( + lambda doc : copyfile( + doc['file'], + out_path(doc) + ), + documents + ) + + # SYNC DOCUMENT_OBJECT WITH DB + documents = lmap( + sync_document, + documents + ) + + lprint(documents) + print('SYNC DOCUMENTS DONE') \ No newline at end of file diff --git a/utils/Sync/sync/event.py b/utils/Sync/sync/event.py new file mode 100644 index 00000000..d48245cf --- /dev/null +++ b/utils/Sync/sync/event.py @@ -0,0 +1,169 @@ +import os +import re +from datetime import datetime +from glob import glob + +import lxml.html +from markdown import markdown + +import settings +from db import db +from sync import create_date_and_title_from_folder_name, create_post_from_image_list +from sync.image import sync_image +from utils.fn import combine, lmap, lprint, first, map_cases, key_mapper +from utils.image import create_image +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + + +def sync_event(record): + if isinstance(record, list): + return lmap( + sync_event, + record + ) + + q = {'id': record['id']} + try: + collection = db().events + update_result = collection.update_one(q, {'$set': record}, upsert=True) + i = collection.find_one({'id': record['id']}) + return i + except ValueError: + pass + + return None + + +def create_event(doc): + if 'date' not in doc or not doc['date']: + return None + + if 'title' not in doc or not doc['title']: + return None + + doc['id'] = doc['id'] if 'id' in doc else url_encode_text(doc['title']) + + url_base = 'https://static.shburg.org/art/image/event-{id}-{img}-{size}{ext}' + images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' + sizes = settings.event_image_sizes + + post_html = lxml.html.fromstring(doc['post']) + + images = [] + for img in post_html.cssselect('img'): + src = img.get('src') + img_path = os.path.join(doc['folder'], src) + + if os.path.exists(img_path): + img_id = url_encode_text(os.path.splitext(src)[0]) + url_fn = lambda size, ext: url_base.format(id=doc['id'], img=img_id, size=size, ext=ext.lower()) + + image = create_image(img_path, sizes, url_fn, images_dir) + images.append(image) + img.set('src', image['data']['big']['url']) + + doc['post'] = lxml.html.tostring(post_html).decode('utf-8') + doc['images'] = images + + return doc + + +def get_md_manifest(md_path): + y, m = read_yaml_md(md_path) + + y['date'] = datetime.strptime(y['date'], '%d.%m.%Y') + y['id'] = str(y['id']) + + return { + **y, + 'post': m + } + + +if __name__ == '__main__': + dir = '/Users/tmshv/Dropbox/Dev/Hud school/Events' + + os.chdir(dir) + + # GET EVENT FOLDERS + events = lmap( + lambda i: i.path, + filter( + lambda i: i.is_dir(), + os.scandir('.') + ) + ) + + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + events = lmap( + lambda i: (i,) + create_date_and_title_from_folder_name( + os.path.basename(i), ['%Y.%m.%d'] + ), + events + ) + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG'] + glob_in = lambda dir: combine(map( + lambda type: glob(os.path.join(dir, type)), + image_types + )) + events = lmap( + lambda i: { + 'folder': i[0], + 'date': i[1], + 'title': i[2], + 'post': create_post_from_image_list(lmap( + lambda path: os.path.relpath(path, i[0]), + glob_in(i[0]) + )) + }, + events + ) + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + md_glob = lambda i : first(glob(i + '/*.md')) + events = lmap( + lambda i: { + **i, + **map_cases( + i, + [( + lambda i: md_glob(i['folder']), + lambda i: get_md_manifest(md_glob(i['folder'])), + )], + + lambda i: {} + ) + }, + events + ) + + # MAP EVENT MANIFEST -> EVENT_OBJECT + events = lmap( + create_event, + events + ) + + # FILTER INCORRECT EVENTS + events = list(filter(None, events)) + + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB + sync_images = lambda i: lmap( + lambda image: image['_id'], + sync_image(i) + ) + events = lmap( + key_mapper('images', sync_images), + events + ) + + # SYNC EVENT_OBJECT WITH DB + events = lmap( + sync_event, + events + ) + + lprint(events) + print('SYNC EVENTS DONE') diff --git a/utils/Sync/sync/image.py b/utils/Sync/sync/image.py new file mode 100644 index 00000000..c4fdfec0 --- /dev/null +++ b/utils/Sync/sync/image.py @@ -0,0 +1,20 @@ +from db import db +from utils.fn import lmap + + +def sync_image(record): + if isinstance(record, list): + return lmap( + sync_image, + record + ) + + q = {'hash': record['hash']} + try: + db().images.update_one(q, {'$set': record}, upsert=True) + i = db().images.find_one({'hash': record['hash']}) + return i + except ValueError: + pass + + return None diff --git a/utils/Sync/sync_awards.py b/utils/Sync/sync_awards.py index 281f7c9d..357a54ec 100644 --- a/utils/Sync/sync_awards.py +++ b/utils/Sync/sync_awards.py @@ -4,7 +4,7 @@ from glob import glob from pymongo import MongoClient import settings -from image import resize +from utils.image import resize def generate_image_filename(image, size_name, ext): diff --git a/utils/Sync/sync_collective.py b/utils/Sync/sync_collective.py deleted file mode 100644 index f0b77354..00000000 --- a/utils/Sync/sync_collective.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -from glob import glob - -import yaml -from markdown import markdown -from pymongo import MongoClient - -import settings -from image.image_processor import image_processor -from text.split import split - - -def get_db(): - client = MongoClient(settings.database_uri) - return client.hudozka - - -def sync(collective): - clear_db() - db = get_db() - for a in collective: - db.collective.insert(a) - - -def clear_db(): - db = get_db() - db.collective.drop() - - -def process_profile(md_path): - sizes = settings.image_sizes - images_dir = settings.collective_image_output - public_base_url = settings.collective_image_base_url - - generate_image_filename = lambda image, size_name, ext: '{image}-{size}{ext}'.format( - image=image, - size=size_name, - ext=ext - ) - - ip = image_processor(images_dir, public_base_url, sizes, generate_image_filename) - - with open(md_path, 'rb') as f: - data = f.read().decode('utf-8') - yaml_data, md_data = split(data) - - profile = yaml.load(yaml_data) - text = markdown(md_data) if md_data else None - - img = os.path.join( - os.path.dirname(md_path), - profile['picture'] - ) - profile['picture'] = ip(img) - profile['biography'] = text - - return profile - - -if __name__ == '__main__': - dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective' - files = glob(dir + '/*.md') - - collective = list(map( - process_profile, - files - )) - - list(map(print, collective)) - sync(collective) diff --git a/utils/Sync/sync_gallery.py b/utils/Sync/sync_gallery.py index fdb449ab..bda83df1 100644 --- a/utils/Sync/sync_gallery.py +++ b/utils/Sync/sync_gallery.py @@ -6,8 +6,8 @@ from glob import glob from pymongo import MongoClient import settings -from image import resize as img_util from read_gallery_1 import GalleryReader +from utils.image import resize as img_util __author__ = 'Roman Timashev' diff --git a/utils/Sync/sync.py b/utils/Sync/sync_old.py similarity index 97% rename from utils/Sync/sync.py rename to utils/Sync/sync_old.py index b131282e..f13a6518 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync_old.py @@ -5,15 +5,15 @@ import lxml.html as html import markdown as md from lxml.etree import tostring -import lxml_utils -from image import resize +from utils import lxml_utils +from utils.image import resize __author__ = 'tmshv' # db = pymongo.MongoClient('mongodb://localhost:27017')['stickers'] # images_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Static/images') -catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Hudozka/Gallery') +catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud School/Gallery') catalog_files = os.listdir(catalog_dir) @@ -79,6 +79,8 @@ def generate_image_filename(src, course, year, theme, author, size, i,ext): for cf in catalog_files: + break + os.chdir(catalog_dir) if os.path.isdir(cf): print('======================================================================================') diff --git a/utils/Sync/text/split.py b/utils/Sync/text/split.py deleted file mode 100644 index 2af53f93..00000000 --- a/utils/Sync/text/split.py +++ /dev/null @@ -1,6 +0,0 @@ -def split(data, separator='\n\n'): - try: - s = data.index(separator) - return data[:s], data[s:] - except ValueError: - return data, None \ No newline at end of file diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py new file mode 100644 index 00000000..4273db5b --- /dev/null +++ b/utils/Sync/utils/fn.py @@ -0,0 +1,68 @@ +import json +import os +from functools import reduce +from glob import glob + +from bson import json_util + +combine = lambda i: reduce( + lambda s, i: s + i, + i +) + +iterate_over_fns = lambda fns: lambda i: reduce( + lambda out, fn: fn(out), + fns, i +) + +iterate_iter_over_fns = lambda fns: lambda i: lmap( + iterate_over_fns(fns), + i +) + +lmap = lambda fn, ls: list(map(fn, ls)) +lprint = lambda i: lmap(print, i) +lprint_json = lambda s: lprint(lmap( + lambda i: json.dumps(i, ensure_ascii=False, indent=4, default=json_util.default), + s +)) + +first = lambda i: i[0] if len(i) > 0 else None + + +def map_item_key(item, key, map_fn): + """ + + :param item: + :param key: + :param map_fn: + :return: + """ + item[key] = map_fn(item[key]) + return item + + +key_mapper = lambda key, fn: lambda i: map_item_key(i, key, fn) + + +def map_cases(param, cases, default_fn=None): + for c, fn in cases: + if c(param): + return fn(param) + return param if not default_fn else default_fn(param) + + +def until(condition_fn, iter_fn, items, default_value=None): + for i in items: + result = iter_fn(i) + if condition_fn(result): + return result + return default_value + + +# glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG']) +# glob_in('./images') +dir_globber = lambda mask_list: lambda dir: combine(map( + lambda mask: glob(os.path.join(dir, mask)), + mask_list +)) diff --git a/utils/Sync/utils/hash.py b/utils/Sync/utils/hash.py new file mode 100644 index 00000000..1c673817 --- /dev/null +++ b/utils/Sync/utils/hash.py @@ -0,0 +1,9 @@ +import hashlib + + +def file_hash(file): + with open(file, 'rb') as f: + data = f.read() + sha = hashlib.sha256(data) + f.close() + return sha.hexdigest() \ No newline at end of file diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py new file mode 100644 index 00000000..3775f017 --- /dev/null +++ b/utils/Sync/utils/image/__init__.py @@ -0,0 +1,55 @@ +import os + +from PIL import Image + +from utils.hash import file_hash +from utils.image import resize +from utils.image.resize import optimize, thumbnail + + +def read_image(src): + try: + return Image.open(src) + except FileNotFoundError: + return None + + +def create_image(file, sizes, url_fn, output_dir, skip_processing=False): + """ + + :param file: Image path to process + :param sizes: list of sizes to generate image (, , ) + :param url_fn: function (size_name, ext) for creating image url + :param output_dir: path to local folder for thumbs storing + :param skip_processing: True/False to skip thumbs generation + :return: + """ + image = read_image(file) + if not image: + return None + + result = {} + _, ext = os.path.splitext(file) + for size in sizes: + size_name, width, height = size + + image_url = url_fn(size_name, ext) + image_filename = os.path.basename(image_url) + d = os.path.join(output_dir, image_filename) + + if not skip_processing: + thumb = optimize(image, d, quality=90) if size_name == 'original' else thumbnail(image, d, (width, height)) + width, height = thumb.size + + result[size_name] = { + 'url': image_url, + 'size': size_name, + 'width': width, + 'height': height + } + + return { + 'file': os.path.basename(file), + 'hash': file_hash(file), + 'data': result + } diff --git a/utils/Sync/image/resize.py b/utils/Sync/utils/image/resize.py similarity index 81% rename from utils/Sync/image/resize.py rename to utils/Sync/utils/image/resize.py index 084a32c2..49f55e4c 100644 --- a/utils/Sync/image/resize.py +++ b/utils/Sync/utils/image/resize.py @@ -24,3 +24,8 @@ def override_extension(dest, ext=None): def image_magick_resize(input_file, output_file, size, quality=85): s = '%dx%d' % size call(['convert', input_file, '-resize', s, '-quality', str(quality), output_file]) + + +def image_magick_pdf_to_img(input_file, output_file): + i = "{image}[0]".format(image=input_file) + call(['convert', i, output_file]) diff --git a/utils/Sync/utils/io.py b/utils/Sync/utils/io.py new file mode 100644 index 00000000..7e02007a --- /dev/null +++ b/utils/Sync/utils/io.py @@ -0,0 +1,36 @@ +import yaml +from markdown import markdown + +from utils.text.split import split, split_with + + +def read_file(file): + with open(file, 'rb') as f: + data = f.read().decode('utf-8') + return data + + +def read_yaml(file): + with open(file, 'rb') as f: + data = f.read().decode('utf-8') + return yaml.load(data) + + +def read_md(file): + with open(file, 'rb') as f: + data = f.read().decode('utf-8') + return markdown(data) + + +def read_yaml_md(file): + data = read_file(file) + yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data) + + try: + y = yaml.load(yaml_data) + except: + y = None + + m = markdown(md_data) if md_data else None + + return y, m diff --git a/utils/Sync/lxml_utils.py b/utils/Sync/utils/lxml_utils.py similarity index 100% rename from utils/Sync/lxml_utils.py rename to utils/Sync/utils/lxml_utils.py diff --git a/utils/Sync/utils/text/split.py b/utils/Sync/utils/text/split.py new file mode 100644 index 00000000..02e4a69a --- /dev/null +++ b/utils/Sync/utils/text/split.py @@ -0,0 +1,18 @@ +from utils.fn import lmap, until + + +def split(data, separator=' '): + try: + s = data.index(separator) + return data[:s], data[s:] + except ValueError: + return None, None + + +def split_with(separators): + return lambda data: until( + lambda i: i[0] and i[1], + lambda s: split(data, s), + separators, + (None, None) + ) diff --git a/utils/Sync/utils/text/transform.py b/utils/Sync/utils/text/transform.py new file mode 100644 index 00000000..d672b95c --- /dev/null +++ b/utils/Sync/utils/text/transform.py @@ -0,0 +1,50 @@ +import re +from functools import reduce + +import settings +from utils.fn import iterate_over_fns + + +def translit(text): + """ + RUSSIAN TEXT => ENGLISH TEXT + based on table like: + ( + ['а', 'б', 'в'], + ['a', 'b', 'v'] + ) + Leave if character is not founded + """ + table = settings.translit_table + ins, outs = table + out = lambda i: outs[ins.index(i)] + replace = lambda i: out(i) if i in ins else i + + return reduce( + lambda t, i: t + replace(i), + list(text), + '' + ) + + +def interpolate(text, storage, key_fn=None): + if not key_fn: + key_fn = lambda i: i + + key = key_fn(text) + return storage[key] if key in storage else text + + +interpolate_swift = lambda i: re.search("([\d\w-]+)", i).group(1) + +remove_pattr = re.compile('[«»,!&]') +remove_punctuation = lambda i: remove_pattr.sub('', i) + +space_to_dash = lambda i: i.replace(' ', '-') +dot_to_dash = lambda i: i.replace('.', '-') +remove_underscores = lambda i: i.replace('_', '') +text_lower = lambda i: i.lower() + +url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower]) +url_encode_text = iterate_over_fns( + [translit, space_to_dash, dot_to_dash, remove_underscores, remove_punctuation, text_lower]) diff --git a/utils/Sync/validation.py b/utils/Sync/utils/validation.py similarity index 100% rename from utils/Sync/validation.py rename to utils/Sync/utils/validation.py