From fe409c04a4a4b4d86d5b5108e2c60dec022f5c23 Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sun, 6 Mar 2016 02:24:24 +0300 Subject: [PATCH] sync update: caching, refactoring, other --- utils/Sync/crop_files.py | 22 -- utils/Sync/models/product.py | 5 +- utils/Sync/read_gallery_1.py | 167 --------------- utils/Sync/settings.py | 1 + utils/Sync/sync/__init__.py | 39 +++- utils/Sync/sync/album.py | 187 ++++++++++------- utils/Sync/sync/award.py | 196 ------------------ utils/Sync/sync/collective.py | 91 --------- utils/Sync/sync/document.py | 282 ++++++++++++++------------ utils/Sync/sync/event.py | 174 ---------------- utils/Sync/sync/image.py | 40 ++++ utils/Sync/sync/news.py | 97 --------- utils/Sync/sync/post.py | 312 +++++++++++++++++++++++++++++ utils/Sync/sync/schedule.py | 42 ---- utils/Sync/sync/teacher.py | 104 +++------- utils/Sync/sync_awards.py | 90 --------- utils/Sync/sync_gallery.py | 154 -------------- utils/Sync/sync_old.py | 224 --------------------- utils/Sync/sync_schedules.py | 38 ---- utils/Sync/utils/fn.py | 31 ++- utils/Sync/utils/hash.py | 5 +- utils/Sync/utils/image/__init__.py | 4 +- utils/Sync/utils/image/resize.py | 41 ++++ 23 files changed, 758 insertions(+), 1588 deletions(-) delete mode 100644 utils/Sync/crop_files.py delete mode 100644 utils/Sync/read_gallery_1.py delete mode 100644 utils/Sync/sync/award.py delete mode 100644 utils/Sync/sync/collective.py delete mode 100644 utils/Sync/sync/event.py delete mode 100644 utils/Sync/sync/news.py create mode 100644 utils/Sync/sync/post.py delete mode 100644 utils/Sync/sync_awards.py delete mode 100644 utils/Sync/sync_gallery.py delete mode 100644 utils/Sync/sync_old.py delete mode 100644 utils/Sync/sync_schedules.py diff --git a/utils/Sync/crop_files.py b/utils/Sync/crop_files.py deleted file mode 100644 index ebc6be10..00000000 --- a/utils/Sync/crop_files.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -from glob import glob - -from utils.fn import lprint, lmap -from utils.image.resize import crop - -size = (400, 400) - -# files = glob('/Users/tmshv/Dropbox/Dev/Hud School/Static/images/product*medium.jpg') -# files = files[:3] - -files = ['product-2015-maketirovanie-doma-peterburga-img0298-medium.jpg','product-2015-zhivopis-img0308-medium.jpg','product-2015-maketirovanie-img0240-medium.jpg','product-2015-podvodnyj-mir-stankovaya-kompoziziya-img0346-medium.jpg','product-2015-risunok-img0037-medium.jpg','product-2015-skazki-mg1037-medium.jpg','product-2015-stankovaya-kompoziziya-img0350-medium.jpg','product-2015-zvety-stankovaya-kompoziziya-img0348-medium.jpg','product-2015-gravyura-mg1014-medium.jpg','product-2015-guash-mg1024-medium.jpg','product-2015-shlisselburg-1-medium.jpg','product-2015-dpk-portret-v-tehnike-applikaziya-img0356-medium.jpg','product-70-img3869-medium.jpg','product-2015-domiki-20150406-1613-medium.jpg','product-2015-skulptura-vov-20150406-1591-medium.jpg','product-2015-history-competition-2-medium.jpg','product-2015-rozhdestvenskaya-skazka-2-medium.jpg','product-2016-zimnie-zabavy-1-medium.jpg','product-2016-na-krylyah-babochek-1-medium.jpg','product-2016-narodnye-motivy-1-medium.jpg','product-around-the-world-1-medium.jpg',] - - -src = lambda i: '/Users/tmshv/Dropbox/Dev/Hud School/Static/images/' + os.path.basename(i) -dest = lambda i: '/Users/tmshv/Desktop/smartcrop_sandbox/' + os.path.basename(i) - -for i in files: - try: - crop(src(i), dest(i), size) - except: - print(i) diff --git a/utils/Sync/models/product.py b/utils/Sync/models/product.py index 70ff4a0b..c7abf5ad 100644 --- a/utils/Sync/models/product.py +++ b/utils/Sync/models/product.py @@ -1,5 +1,5 @@ -from sync.teacher import read_teacher_by_id from sync.image import sync_image +from sync.teacher import Teacher from utils.fn import key_mapper from utils.text.transform import interpolate_swift @@ -20,7 +20,8 @@ unpop_image = key_mapper('image', lambda i: sync_image(i)['_id']) def resolve_teacher(i): - return resolve_value(i, read_teacher_by_id) + sync = Teacher() + return resolve_value(i, lambda i: sync.read({'id': i})) def create_product(doc): diff --git a/utils/Sync/read_gallery_1.py b/utils/Sync/read_gallery_1.py deleted file mode 100644 index 546cde6d..00000000 --- a/utils/Sync/read_gallery_1.py +++ /dev/null @@ -1,167 +0,0 @@ -import os -import re -import time -from datetime import datetime - -import lxml.html -from markdown import markdown - -import settings -from models.Album import Album -from models.product import Product -from utils import lxml_utils -from utils.image import resize - -__author__ = 'Roman Timashev' - -artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER + settings.CYR_UPPER)) -# artist_name_exp = re.compile('(.+)(\s\(\d+ лет\))?$') -artist_age_exp = re.compile('.+\((\d+) лет\)$') - - -class GalleryReader: - def __init__(self, dir, path): - super().__init__() - self.html = None - self.description_path = path - self.dir_path = dir - - self.image_sizes = settings.image_sizes - self.image_ext = settings.image_ext - self.image_base_url = settings.image_base_url - - def read(self): - with open(self.description_path, 'r', encoding='utf-8') as dfile: - data = dfile.read() - md = markdown(data) - self.html = lxml.html.fromstring(md) - - # ALBUM - try: - album_title = self.html.xpath('.//h1/a/text()')[0] - album_uri = self.html.xpath('.//h1/a/@href')[0] - except: - raise Exception('Cannot read album title/uri') - - # COURSE - try: - course_title = self.html.xpath('.//p/a/text()')[0] - course_uri = self.html.xpath('.//p/a/@href')[0] - except: - raise Exception('Cannot read course title/uri') - - # TEACHER - teacher = lxml_utils.get_text_following_by_tag(self.html, 'em', "Преподаватель") - teacher = re.sub(r'\s', '', teacher) - if teacher not in settings.teachers_names: - raise Exception('Teacher %s is not found' % teacher) - teacher_name = settings.teachers_names[teacher] - - # DATE - date = self.get_date() - if date is None: - raise Exception('Date field is empty') - - # PRODUCT - content = self.get_drawings() - - a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date, - teacher=teacher_name, comment=None) - for p in content: - author_name, author_age, filename, product_title = p - product = self.create_product(a, author_name, author_age, filename, product_title) - product.image_original_path = os.path.join(self.dir_path, filename) - a.add_product(product) - return a - - def get_drawings(self): - drawings = [] - h3_list = self.html.xpath('.//h3') - for h3 in h3_list: - artist_name_age = h3.xpath('./a') - if len(artist_name_age) == 0: - raise Exception('Artist name should be a link') - artist_name_age = artist_name_age[0].text - - # ARTIST NAME [required] - if artist_name_age is None: - raise Exception('Artist name section is empty') - - exp = artist_name_exp.search(artist_name_age) - if exp is None: - raise Exception('Artist name is not specified') - artist_name = exp.group(1).strip() - - # ARTIST AGE [optional] - exp = artist_age_exp.search(artist_name_age) - if exp is not None and len(exp.groups()) > 0: - artist_age = int(exp.group(1)) - else: - artist_age = None - - image_filename = h3.xpath('./a/@href')[0] - - # PRODUCT TITLE [optional] - drawing_data = lxml_utils.get_following_tags(h3, 'h3') - if len(drawing_data) == 0: - product_title = None - # raise Exception('Product title not specified') - else: - product_title = drawing_data[0].text[1:-1] - - drawings.append((artist_name, artist_age, image_filename, product_title)) - - return drawings - - def create_product(self, album, author_name, author_age, filename, product_title): - tr = {ord(a): ord(b) for a, b in zip(*settings.translit_table)} - - author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr) - image_id = '%s-%s-%s-%s' % ( - album.date.year, - re.sub(r'[/]', '', album.uri), - author_name_encoded, - os.path.splitext(filename)[0] - ) - - image = self.create_image(image_id, filename, self.image_sizes, self.image_ext, self.image_base_url) - return Product(author_name, author_age, image, product_title) - - def create_image(self, name, filename, sizes, ext, base_url): - print('DEPRECATED: use utils.image.__init__.create_image') - - thumbs = {} - for size, width, height in sizes: - source_image = read_image(filename) - sw, sh = source_image.size - if sw < sh: - width, height = height, width - - width = width if width else sw - height = height if height else sh - - out_filename = '{name}-{w}x{h}{ext}'.format(name=name, w=width, h=height, ext=ext) - out_url = os.path.join(base_url, out_filename) - - thumbs[size] = { - 'url': out_url, - 'size': size, - 'width': width, - 'height': height - } - return thumbs - - def get_date(self): - print('DEPRECATED: use sync.__init__.create_date') - - string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата") - if string is None: - return None - - for f in settings.date_formats: - try: - date = time.strptime(string, f) - return datetime.fromtimestamp(time.mktime(date)) - except ValueError: - continue - return None diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index e4970a6c..e7793a94 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -34,6 +34,7 @@ image_processing = True image_save_dir = './static_images' image_base_url = 'https://static.shburg.org/art/images/' +image_output = '/Users/tmshv/Desktop/Hudozka Static/images' album_html_img_class = 'hudozka-product' diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index f59b8dbd..89d01182 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -1,10 +1,21 @@ +import os import re from datetime import datetime +from glob import glob +import lxml.html from markdown import markdown -from db import db -from utils.fn import lmap, key_mapper +from sync.image import sync_image +from utils.fn import lmap, combine + + +def list_images(dir): + image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG'] + return combine(map( + lambda type: glob(os.path.join(dir, type)), + image_types + )) def create_date(date_str, date_formats=None): @@ -50,13 +61,25 @@ def untouched(documents, store): )) +def synced_images_ids(images): + ids = lmap( + lambda image: image['_id'], + sync_image(images) + ) + return ids + + +def synced_image_id(image): + return sync_image(image)['_id'] + + def create_date_and_title_from_folder_name(folder_name, date_formats=None): m = folder_name_pattern.findall(folder_name) if not m: return None, None date_str, title = m[0] - title = title.strip() + title = os.path.splitext(title.strip())[0] date = create_date(date_str, date_formats) if not date: @@ -65,6 +88,16 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None): return date, title +def images_from_html(md): + post_html = lxml.html.fromstring(md) + + images = [] + for img in post_html.cssselect('img'): + src = img.get('src') + images.append(src) + return images + + create_post_from_image_list = lambda images: markdown( '\n'.join(map( lambda i: '![]({img})'.format(img=i), diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py index 64870f84..2a70b2b6 100644 --- a/utils/Sync/sync/album.py +++ b/utils/Sync/sync/album.py @@ -1,16 +1,17 @@ import os from glob import glob -from lxml import etree import lxml.html +from lxml import etree import settings from db import db from models.product import create_product -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date -from sync.image import sync_image -from utils.fn import lmap, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns, first, map_cases -from utils.hash import file_hash +from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \ + untouched, synced_images_ids +from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \ + key_mapper, lprint +from utils.hash import hash_file, hash_str from utils.image import create_image from utils.io import read_yaml_md from utils.text.transform import url_encode_text @@ -22,7 +23,7 @@ class ImageCreator: self.cache = cache if cache else set() def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False): - hash = file_hash(file) + hash = hash_file(file) cached = self.cached(hash) return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached @@ -30,25 +31,6 @@ class ImageCreator: return self.cache[hash] if hash in self.cache else None -def sync_album(record): - if isinstance(record, list): - return lmap( - sync_album, - record - ) - - q = {'id': record['id']} - try: - collection = db().events - collection.update_one(q, {'$set': record}, upsert=True) - i = collection.find_one({'id': record['id']}) - return i - except ValueError: - pass - - return None - - def create_post(md, image_path_fn): html = lxml.html.fromstring(md) @@ -64,7 +46,7 @@ def create_post(md, image_path_fn): def create_url(a, i, s, e): - return 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format( + return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( album=a, id=i, size=s, @@ -76,18 +58,41 @@ def url_creator(aid, pid): return lambda size, ext: create_url(aid, pid, size, ext.lower()) -class Album: +class SyncAlbum(Sync): @staticmethod def id_from_file(i): return url_encode_text( os.path.splitext(i)[0] ) - def __init__(self, image_creator): + def __init__(self, image_creator, images_dir): super().__init__() + self.collection = db().albums self.image_creator = image_creator self.skip_image_processing = False - self.images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' + self.images_dir = images_dir + + def create_id(self, document): + if 'id' in document: + return document + + y = str(document['date'].year) + document['id'] = url_encode_text('-'.join([y, document['title']])) + return document + + def create_hash(self, document): + hash_images = lmap( + hash_file, + lmap( + lambda i: os.path.join(document['folder'], i), + document['images'] + ) + ) + document['hash'] = hash_str( + combine([hash_str(document)] + hash_images) + ) + + return document def create_image(self, file, url_fn): ss = settings.album_image_sizes @@ -99,7 +104,7 @@ class Album: def create_album(self, doc): # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' - image_id = Album.id_from_file + image_id = SyncAlbum.id_from_file image_path = lambda i: os.path.join(doc['folder'], i) if 'title' not in doc or not doc['title']: @@ -138,7 +143,7 @@ class Album: return doc def create_products(self, doc, image_file_fn): - image_id = Album.id_from_file + image_id = SyncAlbum.id_from_file iter_album = iterate_iter_over_fns([ lambda product: { **product, @@ -188,16 +193,11 @@ def get_manifest(filepath): return {**data, **post} -if __name__ == '__main__': - image_creator = ImageCreator(cache=None) - alb = Album(image_creator=image_creator) - # alb.skip_image_processing = True - - albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' - os.chdir(albums_dir) +def main(dir_documents, sync): + os.chdir(dir_documents) # GET ALBUMS FOLDERS - albums = lmap( + documents = lmap( lambda i: i.path, filter( lambda i: i.is_dir(), @@ -205,22 +205,20 @@ if __name__ == '__main__': ) ) - # print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md')) - # print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md')) - # exit() - # albums = [albums[5]] + # documents = [documents[3]] + # documents = documents[:15] # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - albums = lmap( + documents = lmap( lambda i: (i,) + create_date_and_title_from_folder_name( os.path.basename(i) ), - albums + documents ) # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG']) - albums = lmap( + documents = lmap( lambda i: { 'folder': i[0], 'date': i[1], @@ -230,12 +228,12 @@ if __name__ == '__main__': glob_in(i[0]) )) }, - albums + documents ) # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST md_glob = lambda i: first(glob(i + '/*.md')) - albums = lmap( + documents = lmap( lambda album: { **album, **map_cases( @@ -247,37 +245,88 @@ if __name__ == '__main__': lambda i: {} ) }, - albums + documents + ) + documents = lmap( + lambda i: { + **i, + 'images': lmap( + lambda path: os.path.relpath(path, i['folder']), + list_images(i['folder']) + ) + }, + documents ) + # CREATE DOCUMENT IDENTITY + documents = lmap(sync.create_id, documents) + + # CREATE HASH OF DOCUMENT FILE + documents = lmap(sync.create_hash, documents) + + # CREATE SCOPE OF CURRENT SESSION + scope_documents_ids = lmapfn(documents)( + lambda i: i['id'] + ) + + # SKIP UNTOUCHED DOCUMENTS + documents = untouched(documents, sync) + # MAP EVENT MANIFEST -> EVENT_OBJECT - albums = lmap( + documents = lmap( alb.create_album, - albums + documents ) # FILTER INCORRECT EVENTS - albums = list(filter(None, albums)) - + documents = list(filter(None, documents)) # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - def sync_images(i): - return lmap( - lambda image: image['_id'], - sync_image(i) + documents = lmap( + key_mapper('images', synced_images_ids), + documents + ) + + # SYNC OBJECT WITH DB + documents = lmap( + sync.update, + documents + ) + + documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove ) - - - albums = lmap( - key_mapper('images', sync_images), - albums ) - # SYNC EVENT_OBJECT WITH DB - albums = lmap( - sync_album, - albums - ) + # SCOPE + print('SCOPE:') + lprint(scope_documents_ids) - lprint_json(albums) - print('SYNC ALBUMS DONE') + # DELETE + print('DELETE DOCUMENTS:') + lprint_json(lmap( + lambda i: i['id'], + documents_to_remove + )) + + # DELETE + print('UPDATE DOCUMENTS:') + lprint_json(lmap( + lambda i: i['id'], + documents + )) + # lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') + + +if __name__ == '__main__': + image_creator = ImageCreator(cache=None) + alb = SyncAlbum(image_creator=image_creator, images_dir=settings.image_output) + alb.skip_image_processing = True + + albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' + main(dir_documents=albums_dir, sync=alb) diff --git a/utils/Sync/sync/award.py b/utils/Sync/sync/award.py deleted file mode 100644 index 8c590241..00000000 --- a/utils/Sync/sync/award.py +++ /dev/null @@ -1,196 +0,0 @@ -import os -from glob import glob -from shutil import copyfile -from tempfile import mkstemp - -import settings -from db import db -from sync.image import sync_image -from utils.fn import lmap, lprint_json, key_mapper, lprint -from utils.hash import hash_file, hash_str -from utils.image import create_image -from utils.image.resize import image_magick_pdf_to_img -from utils.text.transform import url_encode_text, url_encode_file - -prefix = 'award' -dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Awards' -dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' -dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' -url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' -url_base_document = 'https://static.shburg.org/art/uploads/{file}' - - -def collection(): - return db().awards - - -def pdf_to_jpg(pdf): - cwd = os.getcwd() - abspdf = os.path.join(cwd, pdf) - _, temp = mkstemp('.jpg') - image_magick_pdf_to_img(abspdf, temp) - return temp - - -def sync_document(record): - if isinstance(record, list): - return lmap( - sync_document, - record - ) - - q = {'id': record['id']} - try: - collection().update_one(q, {'$set': record}, upsert=True) - return collection().find_one(q) - except ValueError: - pass - - return None - - -def read_document(i, query_fn=None): - q = query_fn(i) if query_fn else {'id': i['id']} - try: - return collection().find_one(q) - except ValueError: - pass - - return None - - -def query_documents(q): - return collection().find(q) - - -def delete_document(q): - return collection().find_one_and_delete(q) - - -def create_preview(pdf, sizes, preview_dir): - temp_preview_path = pdf_to_jpg(pdf) - - id = url_encode_text(pdf) - url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext) - - img = create_image(temp_preview_path, sizes, url, preview_dir) - os.remove(temp_preview_path) - return img - - -def create_document(doc): - sizes = settings.awards_image_sizes - file = doc['file'] - - doc['type'] = prefix - doc['url'] = url_base_document.format(file=url_encode_file(file)) - doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews) - doc['file'] = { - 'name': file, - 'size': os.path.getsize((file)) - } - - return doc - - -def create_id(doc): - return url_encode_text('{type}-{file}'.format( - type=prefix, - file=doc['file'] - )) - - -if __name__ == '__main__': - # ALL DOCUMENTS IDS FOUNDED - scope_documents_ids = [] - - # GET DOCUMENT YAML_MANIFEST FILES - os.chdir(dir_documents) - documents = glob('*.pdf') - - # READ YAML_MANIFEST FILES - documents = lmap( - lambda i: { - 'file': i - }, - documents - ) - - # CREATE DOCUMENT IDENTITY - documents = lmap( - lambda i: {**i, 'id': create_id(i)}, - documents - ) - - # CREATE HASH OF DOCUMENT FILE - documents = lmap( - lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])}, - documents - ) - - # CREATE SCOPE OF CURRENT SESSION - scope_documents_ids = lmap( - lambda i: i['id'], - documents - ) - - # SKIP UNTOUCHED DOCUMENTS - documents = lmap( - lambda i: i[0], - filter( - lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']), - lmap( - lambda document: (document, read_document(document)), - documents - ) - )) - - # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT - documents = lmap( - create_document, - documents - ) - - # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('preview', lambda i: sync_image(i)['_id']), - documents - ) - - # COPY FILE -> STATIC_DIR/URL_FILENAME - out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url'])) - lmap( - lambda doc: copyfile( - doc['file']['name'], - out_path(doc) - ), - documents - ) - - # SYNC DOCUMENT_OBJECT WITH DB - documents = lmap( - sync_document, - documents - ) - - documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}}) - documents_to_remove = lmap( - delete_document, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove - ) - ) - - # SCOPE - print('SCOPE:') - lprint(scope_documents_ids) - - # DELETE - print('DELETE DOCUMENTS:') - lprint_json(documents_to_remove) - - # DELETE - print('UPDATE DOCUMENTS:') - lprint_json(documents) - print('[SYNC DOCUMENTS DONE]') diff --git a/utils/Sync/sync/collective.py b/utils/Sync/sync/collective.py deleted file mode 100644 index 585a4935..00000000 --- a/utils/Sync/sync/collective.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -from glob import glob - -import settings -from db import db -from sync.image import sync_image -from utils.fn import lmap, lprint, key_mapper -from utils.image import create_image -from utils.io import read_yaml_md - - -def read_teacher_by_id(record): - if isinstance(record, list): - return lmap( - read_teacher_by_id, - record - ) - - q = {'id': record} - try: - return db().collective.find_one(q) - except ValueError: - pass - - return None - - -def sync_collective(record): - if isinstance(record, list): - return lmap( - sync_collective, - record - ) - - q = {'id': record['id']} - try: - collection = db().collective - update_result = collection.update_one(q, {'$set': record}, upsert=True) - i = collection.find_one({'id': record['id']}) - return i - except ValueError: - pass - - return None - - -def create_profile(profile): - sizes = settings.image_sizes - images_dir = settings.collective_image_output - - url_base = 'https://static.shburg.org/art/image/teacher-{id}-{size}{ext}' - url_fn = lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext) - - profile['picture'] = create_image(profile['picture'], sizes, url_fn, images_dir) - return profile - - -if __name__ == '__main__': - dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective' - os.chdir(dir) - - # GET TEACHER YAML/MD MANIFEST FILES - files = glob('*.md') - - # READ TEACHER YAML/MD MANIFEST - collective = lmap( - read_yaml_md, - files - ) - - # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY - collective = lmap( - lambda i: {'biography': i[1], **i[0]}, - collective - ) - - # MAP TEACHER_MANIFEST -> TEACHER_PROFILE - collective = lmap( - create_profile, - collective - ) - - # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB - sync_picture = lambda i: None if i is None else sync_image(i)['_id'] - collective = lmap( - key_mapper('picture', sync_picture), - collective - ) - - lprint(collective) - print('SYNC COLLECTIVE DONE') diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 3427a4fc..2b760f80 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -5,26 +5,67 @@ from tempfile import mkstemp import settings from db import db -from sync.image import sync_image -from utils.fn import combine, lmap, lprint_json, key_mapper, lprint +from sync import Sync, untouched, synced_image_id +from utils.fn import combine, lmap, lprint_json, lprint, key_mapper from utils.hash import hash_file, hash_str from utils.image import create_image from utils.image.resize import image_magick_pdf_to_img from utils.io import read_yaml from utils.text.transform import url_encode_text, url_encode_file -prefix = 'document' -dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents' -dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' -dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' -url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' -url_base_document = 'https://static.shburg.org/art/uploads/{file}' -# Adds specified category to each file in files list -unwrap_file_list = lambda files, title: map( - lambda profile: {'category': title, **profile}, - files -) +class SyncDocument(Sync): + def __init__(self, collection_name, document_type, sizes): + super().__init__() + self.collection = db()[collection_name] + self.document_type = document_type + self.sizes = sizes + self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' + self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' + self.url_base_document = 'https://static.shburg.org/art/uploads/{file}' + + def create(self, document): + file = document['file'] + filename = os.path.basename(document['file']) + + document['type'] = self.document_type + document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews) + document['file'] = { + 'name': filename, + 'size': os.path.getsize((file)) + } + + return document + + def create_id(self, document): + bn = os.path.basename(document['file']) + document['id'] = url_encode_text('{type}-{category}-{file}'.format( + type=self.document_type, + category=document['category'], + file=bn + )) + return document + + def create_url(self, document): + filename = os.path.basename(document['file']) + document['url'] = self.url_base_document.format(file=url_encode_file(filename)) + return document + + def create_hash(self, document): + document['hash'] = hash_str( + hash_str(document) + hash_file(document['file']) + ) + return document + + def create_preview(self, pdf, sizes, preview_dir): + temp_preview_path = pdf_to_jpg(pdf) + + id = url_encode_text(pdf) + url = lambda size, ext: self.url_base_preview.format(id=id, size=size, ext=ext) + + img = create_image(temp_preview_path, sizes, url, preview_dir) + os.remove(temp_preview_path) + return img def pdf_to_jpg(pdf): @@ -35,75 +76,20 @@ def pdf_to_jpg(pdf): return temp -def sync_document(record): - if isinstance(record, list): - return lmap( - sync_document, - record - ) - - q = {'id': record['id']} - try: - documents = db().documents - update_result = documents.update_one(q, {'$set': record}, upsert=True) - i = documents.find_one({'id': record['id']}) - return i - except ValueError: - pass - - return None - - -def read_document(i, query_fn=None): - q = query_fn(i) if query_fn else {'id': i['id']} - try: - return db().documents.find_one(q) - except ValueError: - pass - - return None - - -def query_documents(q): - return db().documents.find(q) - - -def delete_document(q): - return db().documents.find_one_and_delete(q) - - -def create_preview(pdf, sizes, preview_dir): - temp_preview_path = pdf_to_jpg(pdf) - - id = url_encode_text(pdf) - url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext) - - img = create_image(temp_preview_path, sizes, url, preview_dir) - os.remove(temp_preview_path) - return img - - -def create_document(doc): - sizes = settings.image_sizes - file = doc['file'] - - doc['type'] = prefix - doc['url'] = url_base_document.format(file=url_encode_file(file)) - doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews) - doc['file'] = { - 'name': file, - 'size': os.path.getsize((file)) - } - - return doc - - -def create_id(doc): - return url_encode_text('{type}-{category}-{file}'.format( - type=prefix, - category=doc['category'], - file=doc['file'] - )) +def categorize_files_list(files, title): + """ + Adds specified category to each file in files list + :param files: + :param title: + :return: + """ + return map( + lambda profile: { + **profile, + 'category': title + }, + files + ) def unwrap_manifest(param): @@ -116,7 +102,7 @@ def unwrap_manifest(param): if 'files' in param: return lmap( unwrap_manifest, - unwrap_file_list(param['files'], param['title']) + categorize_files_list(param['files'], param['title']) ) if 'file' not in param: @@ -125,12 +111,9 @@ def unwrap_manifest(param): return param -if __name__ == '__main__': - # ALL DOCUMENTS IDS FOUNDED - scope_documents_ids = [] - +def documents_from_yaml(): # GET DOCUMENT YAML_MANIFEST FILES - os.chdir(dir_documents) + documents = glob(dir_documents + '/*.yaml') # READ YAML_MANIFEST FILES @@ -142,85 +125,128 @@ if __name__ == '__main__': # GET FLAT LIST OF DOCUMENTS documents = unwrap_manifest(documents) + return documents + + +def documents_from_subdirs(): + documents = lmap( + lambda i: i.path, + filter( + lambda i: i.is_dir(), + os.scandir('.') + ) + ) + + documents = lmap( + lambda folder: lmap( + lambda path: { + 'file': path, + 'title': os.path.splitext(os.path.basename(os.path.relpath(path, folder)))[0], + 'category': os.path.basename(folder) + }, + glob(folder + '/*.pdf') + ), + documents + ) + + documents = combine(documents) + return documents + + +def main(dir_documents, dir_static_uploads, sync): + os.chdir(dir_documents) + + # documents = documents_from_yaml() + documents = documents_from_subdirs() + # CREATE DOCUMENT IDENTITY - documents = lmap( - lambda i: {**i, 'id': create_id(i)}, - documents - ) + documents = lmap(sync.create_id, documents) # CREATE HASH OF DOCUMENT FILE - documents = lmap( - lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])}, - documents - ) + documents = lmap(sync.create_hash, documents) - # CREATE HASH OF DOCUMENT FILE + # CREATE SCOPE scope_documents_ids = lmap( lambda i: i['id'], documents ) + file_names = set(lmap( + lambda i: i['file'], + documents + )) + if len(file_names) != len(documents): + raise Exception('File names should be unique') + # SKIP UNTOUCHED DOCUMENTS - documents = lmap( - lambda i: i[0], - filter( - lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']), - lmap( - lambda document: (document, read_document(document)), - documents - ) - )) + documents = untouched(documents, sync) - # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT - documents = lmap( - create_document, - documents - ) - - # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - documents = lmap( - key_mapper('preview', lambda i: sync_image(i)['_id']), - documents - ) + # URL + documents = lmap(sync.create_url, documents) # COPY FILE -> STATIC_DIR/URL_FILENAME out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url'])) lmap( lambda doc: copyfile( - doc['file']['name'], + doc['file'], out_path(doc) ), documents ) - # SYNC DOCUMENT_OBJECT WITH DB + # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT documents = lmap( - sync_document, + sync.create, documents ) - documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}}) + # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('preview', lambda i: synced_image_id(i)), + documents + ) + + # SYNC DOCUMENT_OBJECT WITH DB + documents = lmap( + sync.update, + documents + ) + + documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) documents_to_remove = lmap( - delete_document, + sync.delete, map( lambda i: {'_id': i['_id']}, documents_to_remove ) ) - # SCOPE - print('SCOPE:') + print('SCOPE:', len(scope_documents_ids)) lprint(scope_documents_ids) - # DELETE - print('DELETE DOCUMENTS:') + print('DELETE DOCUMENTS:', len(documents_to_remove)) lprint_json(documents_to_remove) - # DELETE - print('UPDATE DOCUMENTS:') + print('UPDATE DOCUMENTS:', len(documents)) lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') + +if __name__ == '__main__': + dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents' + dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' + + main( + dir_documents, + dir_static_uploads, + SyncDocument( + 'documents', + 'document', + sizes=settings.image_sizes + ) + ) + # DOCUMENT_OBJECT SAMPLE # { # "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0", diff --git a/utils/Sync/sync/event.py b/utils/Sync/sync/event.py deleted file mode 100644 index cb51a5fb..00000000 --- a/utils/Sync/sync/event.py +++ /dev/null @@ -1,174 +0,0 @@ -import os -from datetime import datetime -from glob import glob - -import lxml.html - -import settings -from db import db -from sync import create_date_and_title_from_folder_name, create_post_from_image_list -from sync.image import sync_image -from utils.fn import combine, lmap, lprint, first, map_cases, key_mapper -from utils.image import create_image -from utils.io import read_yaml_md -from utils.text.transform import url_encode_text - - -def sync_event(record): - if isinstance(record, list): - return lmap( - sync_event, - record - ) - - q = {'id': record['id']} - try: - collection = db().events - update_result = collection.update_one(q, {'$set': record}, upsert=True) - i = collection.find_one({'id': record['id']}) - return i - except ValueError: - pass - - return None - - -def create_event(doc): - if 'date' not in doc or not doc['date']: - return None - - if 'title' not in doc or not doc['title']: - return None - - doc['id'] = doc['id'] if 'id' in doc else url_encode_text(doc['title']) - - url_base = 'https://static.shburg.org/art/images/event-{id}-{img}-{size}{ext}' - images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' - sizes = settings.event_image_sizes - - post_html = lxml.html.fromstring(doc['post']) - - images = [] - for img in post_html.cssselect('img'): - src = img.get('src') - img_path = os.path.join(doc['folder'], src) - - if os.path.exists(img_path): - img_id = url_encode_text(os.path.splitext(src)[0]) - url_fn = lambda size, ext: url_base.format(id=doc['id'], img=img_id, size=size, ext=ext.lower()) - - image = create_image(img_path, sizes, url_fn, images_dir) - if image: - images.append(image) - img.set('src', image['data']['big']['url']) - else: - print('fail: ', doc['folder'], src) - - doc['post'] = lxml.html.tostring(post_html).decode('utf-8') - doc['images'] = images - - return doc - - -def get_manifest(md_path): - y, m = read_yaml_md(md_path) - y = y if y else {} - - if 'date' in y: - y['date'] = datetime.strptime(y['date'], '%d.%m.%Y') - - if 'id' in y: - y['id'] = str(y['id']) - - return { - **y, - 'post': m - } - - -if __name__ == '__main__': - dir = '/Users/tmshv/Dropbox/Dev/Hud school/Events' - - os.chdir(dir) - - # GET EVENT FOLDERS - events = lmap( - lambda i: i.path, - filter( - lambda i: i.is_dir(), - os.scandir('.') - ) - ) - - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - events = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name( - os.path.basename(i), ['%Y.%m.%d'] - ), - events - ) - - # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG'] - glob_in = lambda dir: combine(map( - lambda type: glob(os.path.join(dir, type)), - image_types - )) - events = lmap( - lambda i: { - 'folder': i[0], - 'date': i[1], - 'title': i[2], - 'post': create_post_from_image_list(lmap( - lambda path: os.path.relpath(path, i[0]), - glob_in(i[0]) - )) - }, - events - ) - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - md_glob = lambda i : first(glob(i + '/*.md')) - events = lmap( - lambda event: { - **event, - **map_cases( - event, - [( - lambda i: md_glob(i['folder']), - lambda i: get_manifest(md_glob(i['folder'])), - )], - - lambda i: {} - ) - }, - events - ) - - # MAP EVENT MANIFEST -> EVENT_OBJECT - events = lmap( - create_event, - events - ) - - # FILTER INCORRECT EVENTS - events = list(filter(None, events)) - - # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - sync_images = lambda i: lmap( - lambda image: image['_id'], - sync_image(i) - ) - events = lmap( - key_mapper('images', sync_images), - events - ) - - # SYNC EVENT_OBJECT WITH DB - events = lmap( - sync_event, - events - ) - - lprint(events) - print('SYNC EVENTS DONE') diff --git a/utils/Sync/sync/image.py b/utils/Sync/sync/image.py index c4fdfec0..81f210a7 100644 --- a/utils/Sync/sync/image.py +++ b/utils/Sync/sync/image.py @@ -2,6 +2,46 @@ from db import db from utils.fn import lmap +# class SyncImage(Sync): +# def __init__(self, sizes, save_path, create_url_fn): +# super().__init__() +# self.sizes = sizes +# self.save_path = save_path +# self.create_url_fn = create_url_fn +# self.collection = db().images +# +# def create_id(self, document): +# id = document['id'] if 'id' in document and document['id'] else None +# if not id: +# id = 'schedule-{period}-{semester}'.format(**document) +# document['id'] = id +# return document +# +# def read(self, document): +# return super().read(document) +# +# def create_hash(self, document): +# return { +# **document, +# 'hash': hash_file(document) +# } +# +# def create(self, document): +# image = create_image(document, self.sizes, self.create_url_fn, self.save_path) +# +# return { +# 'file': document, +# 'hash': self.create_hash(document), +# 'data': image['data'] +# } +# +# def find_or_create(self, document): +# i = self.read(document) +# if not i: +# return self.create(document) +# return i + + def sync_image(record): if isinstance(record, list): return lmap( diff --git a/utils/Sync/sync/news.py b/utils/Sync/sync/news.py deleted file mode 100644 index a5e7cfbb..00000000 --- a/utils/Sync/sync/news.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -from glob import glob - -from db import db -from sync import create_date -from utils.fn import lmap, lprint_json -from utils.io import read_yaml_md -from utils.text.transform import url_encode_text - - -def sync_news(record): - if isinstance(record, list): - return lmap( - sync_news, - record - ) - - q = {'id': record['id']} - try: - collection = db().timeline - update_result = collection.update_one(q, {'$set': record}, upsert=True) - i = collection.find_one({'id': record['id']}) - return i - except ValueError: - pass - - return None - - -def create_news(profile): - formats = [ - '%d.%m.%Y', - '%d.%m.%Y %H:%M', - '%d.%m.%Y %H:%M:%S' - ] - - if 'title' not in profile: - return None - - if 'date' in profile: - profile['date'] = create_date(profile['date'], formats) - - if 'until' in profile: - profile['until'] = create_date(profile['until'], formats) - - if 'id' not in profile: - id = lambda date, title: '{date}-{title}'.format( - date=date.strftime('%Y-%m-%d'), - title=url_encode_text(title) - ) - profile['id'] = id(profile['date'], profile['title']) - - return profile - - -if __name__ == '__main__': - news_dir = '/Users/tmshv/Dropbox/Dev/Hud school/News' - os.chdir(news_dir) - - # GET NEWS FILES - file_glob = os.path.join(news_dir, '*.md') - news = glob(file_glob) - - # READ MANIFEST FILES -> NEWS_MANIFEST - news = lmap( - lambda i: (os.path.basename(i),) + read_yaml_md(i), - news - ) - news = lmap( - lambda i: { - **i[1], - 'file': i[0], - 'message': i[2] - }, - news - ) - - # NEWS_MANIFEST -> NEWS_OBJECT - news = lmap( - create_news, - news - ) - - # FILTER BROKEN NEWS_OBJECT - news = list(filter( - None, - news - )) - - # SYNC EVENT_OBJECT WITH DB - events = lmap( - sync_news, - news - ) - - lprint_json(news) - print('SYNC NEWS DONE') diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py new file mode 100644 index 00000000..104c0e50 --- /dev/null +++ b/utils/Sync/sync/post.py @@ -0,0 +1,312 @@ +import os +from datetime import datetime +from glob import glob + +import lxml.html + +import settings +from db import db +from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \ + images_from_html, list_images, synced_images_ids +from sync.image import sync_image +from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn +from utils.hash import hash_file, hash_str +from utils.image import create_image +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + + +class SyncPost(Sync): + def __init__(self, collection_name, document_type, image_url_base, dir_local_images, sizes): + super().__init__() + self.collection = db()[collection_name] + self.sizes = sizes + self.type = document_type + self.image_url_base = image_url_base + self.dir_local_images = dir_local_images + + def create_id(self, document): + document['id'] = document['id'] if 'id' in document else url_encode_text(document['title']) + return document + + def create_hash(self, document): + hash_images = lmap( + hash_file, + lmap( + lambda i: os.path.join(document['folder'], i), + document['images'] + ) + ) + + document['hash'] = hash_str( + combine([hash_str(document)] + hash_images) + ) + + return document + + def create(self, document): + if 'date' not in document or not document['date']: + return None + + if 'title' not in document or not document['title']: + return None + + if self.type: + document['type'] = self.type + + post_html = lxml.html.fromstring(document['post']) + images = [] + for img in post_html.cssselect('img'): + src = img.get('src') + img_path = os.path.abspath( + os.path.join(document['folder'], src) + ) + + if os.path.exists(img_path): + img_id = url_encode_text(os.path.splitext(src)[0]) + url_fn = lambda size, ext: self.image_url_base.format(id=document['id'], img=img_id, size=size, + ext=ext.lower()) + + image = create_image(img_path, self.sizes, url_fn, self.dir_local_images) + if image: + images.append(image) + img.set('src', image['data']['big']['url']) + else: + print('fail: ', document['folder'], src) + + document['post'] = lxml.html.tostring(post_html).decode('utf-8') + document['images'] = images + return document + + +def get_manifest(path): + y, m = read_yaml_md(path) + y = y if y else {} + + if 'date' in y: + y['date'] = create_date(y['date'], settings.date_formats) + + if 'until' in y: + y['until'] = create_date(y['until'], settings.date_formats) + + if 'id' in y: + y['id'] = str(y['id']) + + return { + **y, + 'post': m, + 'images': images_from_html(m) + } + + +def md_from_folder(i): + return first(glob(i + '/*.md')) + + +def get_folder_documents(sync, file_time_formats): + # GET EVENT FOLDERS + documents = lmap( + lambda i: i.path, + filter( + lambda i: i.is_dir(), + os.scandir('.') + ) + ) + + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + documents = lmap( + lambda i: (i,) + create_date_and_title_from_folder_name( + os.path.basename(i), file_time_formats + ), + documents + ) + documents = lmap( + lambda i: { + 'folder': i[0], + 'date': i[1], + 'title': i[2], + }, + documents + ) + + # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT + documents = lmap( + lambda i: { + **i, + 'images': lmap( + lambda path: os.path.relpath(path, i['folder']), + list_images(i['folder']) + ) + }, + documents + ) + documents = lmap( + lambda i: { + **i, + 'post': create_post_from_image_list(i['images']) + }, + documents + ) + + # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST + documents = lmap( + lambda i: { + **i, + **map_cases( + i, + [( + lambda i: md_from_folder(i['folder']), + lambda i: get_manifest(md_from_folder(i['folder'])), + )], + lambda i: {} + ) + }, + documents + ) + + # CREATE DOCUMENT IDENTITY + documents = lmap(sync.create_id, documents) + + # CREATE HASH OF DOCUMENT FILE + documents = lmap(sync.create_hash, documents) + + return documents + + +# GET EVENT FILES +def get_file_documents(sync, file_time_formats): + documents = glob('*.md') + documents = lmap( + lambda i: (i,) + create_date_and_title_from_folder_name( + os.path.basename(i), file_time_formats + ), + documents + ) + + documents = lmap( + lambda i: { + 'file': i[0], + 'date': i[1], + 'title': i[2], + }, + documents + ) + + # OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST + documents = lmap( + lambda i: { + **i, + **get_manifest(i['file']) + }, + documents + ) + + # CREATE DOCUMENT IDENTITY + documents = lmap(sync.create_id, documents) + + # CREATE HASH OF DOCUMENT FILE + documents = lmap(sync.create_hash, documents) + + return documents + + +def main(dir_documents, sync, file_time_formats): + # INIT + os.chdir(dir_documents) + + documents = get_folder_documents(sync, file_time_formats) + documents += get_file_documents(sync, file_time_formats) + + # CREATE SCOPE OF CURRENT SESSION + scope_documents_ids = lmapfn(documents)( + lambda i: i['id'] + ) + + # SKIP UNTOUCHED DOCUMENTS + documents = lmap( + lambda i: i[0], + filter( + lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), + lmap( + lambda document: (document, sync.read(document)), + documents + ) + )) + + # MAP EVENT MANIFEST -> EVENT_OBJECT + documents = lmap( + sync.create, + documents + ) + + # FILTER INCORRECT EVENTS + documents = list(filter(None, documents)) + + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('images', synced_images_ids), + documents + ) + + # SYNC EVENT_OBJECT WITH DB + documents = lmap( + sync.update, + documents + ) + + documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) + ) + + # SCOPE + print('SCOPE:') + lprint(scope_documents_ids) + + # DELETE + print('DELETE DOCUMENTS:') + lprint_json(lmap( + lambda i: i['id'], + documents_to_remove + )) + + # DELETE + print('UPDATE DOCUMENTS:') + lprint_json(lmap( + lambda i: i['id'], + documents + )) + lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') + + +if __name__ == '__main__': + image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' + main( + '/Users/tmshv/Dropbox/Dev/Hud school/Events', + SyncPost( + 'events', + None, + image_url_base, + settings.image_output, + settings.event_image_sizes + ), + file_time_formats=['%Y.%m.%d'] + ) + + image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}' + main( + '/Users/tmshv/Dropbox/Dev/Hud school/News', + SyncPost( + 'timeline', + 'post', + image_url_base, + settings.image_output, + settings.event_image_sizes + ), + file_time_formats=settings.date_formats_reverse + ) diff --git a/utils/Sync/sync/schedule.py b/utils/Sync/sync/schedule.py index af136e56..2b83aa66 100644 --- a/utils/Sync/sync/schedule.py +++ b/utils/Sync/sync/schedule.py @@ -15,48 +15,6 @@ class SyncSchedule(Sync): super().__init__() self.collection = db().schedules - # def sync_collective(self, record): - # if isinstance(record, list): - # return lmap( - # self.sync_collective, - # record - # ) - # - # q = {'id': record['id']} - # try: - # collection = db().collective - # update_result = collection.update_one(q, {'$set': record}, upsert=True) - # i = collection.find_one({'id': record['id']}) - # return i - # except ValueError: - # pass - # - # return None - # - # def compile(self, profile): - # profile['picture'] = create_image( - # profile['picture'], - # sizes, - # lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), - # dir_local_images - # ) - # return profile - # - # def read_document(self, i, query_fn=None): - # q = query_fn(i) if query_fn else {'id': i['id']} - # try: - # return self.collection().find_one(q) - # except ValueError: - # pass - # - # return None - # - # def query_documents(self, q): - # return self.collection().find(q) - # - # def delete_document(self, q): - # return self.collection().find_one_and_delete(q) - def create_id(self, document): id = document['id'] if 'id' in document and document['id'] else None if not id: diff --git a/utils/Sync/sync/teacher.py b/utils/Sync/sync/teacher.py index e84e3456..63e1f571 100644 --- a/utils/Sync/sync/teacher.py +++ b/utils/Sync/sync/teacher.py @@ -3,8 +3,9 @@ from glob import glob import settings from db import db +from sync import Sync from sync.image import sync_image -from utils.fn import lmap, lprint, key_mapper, lprint_json, ext +from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn from utils.hash import hash_str, hash_file from utils.image import create_image from utils.io import read_yaml_md @@ -15,42 +16,11 @@ url_base = settings.teacher_image_url_base dir_local_images = settings.teacher_image_output sizes = settings.teacher_image_sizes -class Teacher: - def collection(self): - return db().collective - def read_teacher_by_id(self, record): - if isinstance(record, list): - return lmap( - self.read_teacher_by_id, - record - ) - - q = {'id': record} - try: - return db().collective.find_one(q) - except ValueError: - pass - - return None - - def sync_collective(self, record): - if isinstance(record, list): - return lmap( - self.sync_collective, - record - ) - - q = {'id': record['id']} - try: - collection = db().collective - update_result = collection.update_one(q, {'$set': record}, upsert=True) - i = collection.find_one({'id': record['id']}) - return i - except ValueError: - pass - - return None +class Teacher(Sync): + def __init__(self): + super().__init__() + self.collection = db().collective def compile(self, profile): profile['picture'] = create_image( @@ -61,39 +31,27 @@ class Teacher: ) return profile - def read_document(self, i, query_fn=None): - q = query_fn(i) if query_fn else {'id': i['id']} - try: - return self.collection().find_one(q) - except ValueError: - pass - - return None - - def query_documents(self, q): - return self.collection().find(q) - - def delete_document(self, q): - return self.collection().find_one_and_delete(q) - def create_id(self, document): if 'id' in document and document['id']: - return document['id'] + return document document['id'] = url_encode_text(document['name']) + return document + + def create_hash(self, document): + document['hash'] = hash_str( + hash_file(document['file']) + hash_file(document['picture']) + ) + return document -if __name__ == '__main__': - # ALL DOCUMENTS IDS FOUNDED - scope_documents_ids = [] - +def main(dir_teachers, sync): os.chdir(dir_teachers) - sync = Teacher() # GET TEACHER YAML/MD MANIFEST FILES documents = glob('*.md') - # # READ TEACHER YAML/MD MANIFEST + # READ TEACHER YAML/MD MANIFEST documents = lmap( lambda i: (i,) + read_yaml_md(i), documents @@ -119,20 +77,10 @@ if __name__ == '__main__': ) # CREATE DOCUMENT IDENTITY - documents = lmap( - lambda i: {**i, 'id': sync.create_id(i)}, - documents - ) + documents = lmapfn(documents)(sync.create_id) # CREATE HASH OF DOCUMENT FILES - documents = lmap( - lambda i: { - **i, 'hash': hash_str( - hash_file(i['file']) + hash_file(i['picture']) - ) - }, - documents - ) + documents = lmapfn(documents)(sync.create_hash) # CREATE SCOPE OF CURRENT SESSION scope_documents_ids = lmap( @@ -146,7 +94,7 @@ if __name__ == '__main__': filter( lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), lmap( - lambda document: (document, sync.read_document(document)), + lambda document: (document, sync.read(document)), documents ) )) @@ -157,22 +105,22 @@ if __name__ == '__main__': documents ) - # # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB + # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB sync_picture = lambda i: None if i is None else sync_image(i)['_id'] documents = lmap( key_mapper('picture', sync_picture), documents ) - # # SYNC TEACHER_PROFILE + # SYNC TEACHER_PROFILE documents = lmap( - sync.sync_collective, + sync.update, documents ) - documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) documents_to_remove = lmap( - sync.delete_document, + sync.delete, map( lambda i: {'_id': i['_id']}, documents_to_remove @@ -191,3 +139,7 @@ if __name__ == '__main__': print('UPDATE DOCUMENTS:') lprint_json(documents) print('[SYNC DOCUMENTS DONE]') + + +if __name__ == '__main__': + main(dir_teachers, Teacher()) diff --git a/utils/Sync/sync_awards.py b/utils/Sync/sync_awards.py deleted file mode 100644 index 357a54ec..00000000 --- a/utils/Sync/sync_awards.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -from glob import glob - -from pymongo import MongoClient - -import settings -from utils.image import resize - - -def generate_image_filename(image, size_name, ext): - """ - Example: 2015-history-competition-1-small.jpg - :param image: - :param size_name: - :param ext: - :return: - """ - return '{image}-{size}{ext}'.format( - image=image, - size=size_name, - ext=ext - ) - - -def process_file(file): - sizes = settings.image_sizes - image = resize.read_image(file) - images_dir = '/Users/tmshv/Desktop/Hudozka Awards' - public_base_url = 'https://static.shlisselburg.org/art/awards/' - - basename = os.path.basename(file) - uri, ext = os.path.splitext(basename) - - award = { - 'uri': uri, - 'thumbs': {} - } - - for size in sizes: - size_name, width, height = size - - image_filename = generate_image_filename(uri, size_name, ext) - d = os.path.join(images_dir, image_filename) - image_url = os.path.join(public_base_url, image_filename) - - thumb = resize.optimize(image, d, quality=90) if size_name == 'original' else resize.thumbnail(image, d, (width, height)) - - width, height = thumb.size - award['thumbs'][size_name] = { - 'url': image_url, - 'size': size_name, - 'width': width, - 'height': height - } - - return award - - -def get_db(): - client = MongoClient(settings.database_uri) - return client.hudozka - - -def sync(awards): - clear_db() - db = get_db() - for a in awards: - db.awards.insert(a) - - -def clear_db(): - db = get_db() - db.awards.drop() - - -if __name__ == '__main__': - dir = '/Users/tmshv/Dropbox/Dev/Hud school/Awards' - files = glob(dir + '/*') - - awards = list(map( - process_file, - files - # map( - # lambda f : os.path.join(dir, f), - # files - # ) - )) - - print(awards) - sync(awards) diff --git a/utils/Sync/sync_gallery.py b/utils/Sync/sync_gallery.py deleted file mode 100644 index bda83df1..00000000 --- a/utils/Sync/sync_gallery.py +++ /dev/null @@ -1,154 +0,0 @@ -import os -import re -from functools import reduce -from glob import glob - -from pymongo import MongoClient - -import settings -from read_gallery_1 import GalleryReader -from utils.image import resize as img_util - -__author__ = 'Roman Timashev' - -description_version_exp = re.compile('\[(\d+)\]\.md$') - -dir_gallery = settings.root_dir_gallery -catalog_dir = os.path.expanduser(dir_gallery) -catalog_files = os.listdir(catalog_dir) -os.makedirs(settings.image_save_dir, exist_ok=True) - - -def get_description_version(filename): - v = description_version_exp.search(filename) - return v.group(1) - - -def get_description_reader(version): - if version == '1': - return GalleryReader - - return None - - -def process_product_image(product, save_dir): - for k in product.image.keys(): - thumb = product.image[k] - filename = os.path.basename(thumb['url']) - outpath = os.path.join(save_dir, filename) - img_util.image_magick_resize( - product.image_original_path, - outpath, - (thumb['width'], thumb['height']) - ) - - -def get_db(): - client = MongoClient(settings.database_uri) - return client.hudozka - - -def clear_db(): - db = get_db() - db.products.drop() - db.albums.drop() - - -def sync_product(product, type): - db = get_db() - author_name, author_age, image, product_title, original_path = product.to_tuple() - - return db['products'].insert(dict( - author=author_name, - authorAge=author_age, - title=product_title, - type=type, - content=image - )) - - -def sync_album(album): - db = get_db() - product_ids = list(map( - lambda product: sync_product(product, album.type), - album.products - )) - - return db['albums'].insert(dict( - title=album.title, - uri=album.uri, - date=album.date, - description=album.comment, - course=album.course, - course_uri=album.course_uri, - teacher=album.teacher, - content=product_ids - )) - - -def read_gallery_item(dir_name): - """ - Read directory with .md and images and return an instance of Album. - None if reads fails. - - :param dir_name: - :return: - """ - - set_path = os.path.join(dir_gallery, dir_name) - os.chdir(set_path) - - md = glob('*.md') - if len(md) > 0: - description_filename = md[0] - description_path = os.path.join(set_path, description_filename) - - v = '1' - DescriptionReader = get_description_reader(v) - if DescriptionReader is not None: - r = DescriptionReader(set_path, description_path) - try: - return r.read() - - except Exception as e: - print('Error with reading description file %s' % (description_path)) - print(e) - return None - - -if __name__ == '__main__': - print('--> READING GALLERY CATALOGS') - albums = list(map(read_gallery_item, catalog_files)) - - print('--> GALLERY TO SYNC') - albums = list(filter(lambda a: a is not None, albums)) - - for a in albums: - print(a) - - img = False - save_dir = settings.image_save_dir - print('--> PROCESSING IMAGES to %s (%s)' % (save_dir, img)) - if img: - products = reduce(lambda p, a: p + a.products, albums, []) - print('Products image to process %s' % len(products)) - for p in products: - print('Process %s' % p.image_original_path) - process_product_image(p, save_dir) - - sync = True - print('--> SYNCING ALBUMS (%s)' % sync) - if sync: - print('--> DROP DB') - clear_db() - - for a in albums: - try: - a_id = sync_album(a) - print('SYNC DONE (%s: %s)' % a_id, a.title) - - except Exception as e: - print('Cannot sync %s' % (a)) - print(e) - - exit() diff --git a/utils/Sync/sync_old.py b/utils/Sync/sync_old.py deleted file mode 100644 index f13a6518..00000000 --- a/utils/Sync/sync_old.py +++ /dev/null @@ -1,224 +0,0 @@ -import os -from glob import glob - -import lxml.html as html -import markdown as md -from lxml.etree import tostring - -from utils import lxml_utils -from utils.image import resize - -__author__ = 'tmshv' - -# db = pymongo.MongoClient('mongodb://localhost:27017')['stickers'] - -# images_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Static/images') -catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud School/Gallery') -catalog_files = os.listdir(catalog_dir) - - -def lesson_info(page): - lesson = list(filter(lambda h2: not h2.xpath('./a'), page.xpath('//h2')))[0] - content = lxml_utils.get_following_tags(lesson, 'h2') - for p in lxml_utils.get_following_tags(lesson, 'h2'): - print(p, tostring(p, encoding='utf-8').decode()) - - # print(content[0].xpath('./em/text()')[0].strip()) - # print(content[0].xpath('./text()')[0].strip()) - - # print(content[1].xpath('./text()')) - # return content - - date_str = '12.10.2013' - - #todo: make as `date` - date = date_str - - return { - 'name': 'Collage', - 'course': 'painting', - 'technology': ['pastel', 'huel', 'glue'], - 'date': date, - } - - -def drawings_info(page): - out = [] - pics = list(filter(lambda h2: h2.xpath('./a'), page.xpath('//h2'))) - for p in pics: - content = lxml_utils.get_following_tags(p, 'h2')[0] - author_info = content.xpath('./text()')[0].split('\n') - out.append({ - 'name': p.xpath('./a/text()')[0], - 'image': p.xpath('./a/@href')[0], - 'author': author_info[0], - 'author_age': int(author_info[1].split(' ')[0]) - }) - return out - - -def generate_image_filename(src, course, year, theme, author, size, i,ext): - if ext is None: - ext = os.path.splitext(src)[1] - - if i == 0: - filename = '{course}-{year}-{theme}-{author}-{size}.{ext}' - else: - filename = '{course}-{year}-{theme}-{author}-{index}-{size}.{ext}'\ - - filename = filename.format( - course=course, - year=year, - theme=theme, - author=author, - size=size, - index=i, - ext=ext - ) - return filename - - -for cf in catalog_files: - break - - os.chdir(catalog_dir) - if os.path.isdir(cf): - print('======================================================================================') - print(cf) - - product_dir = os.path.join(catalog_dir, cf) - os.chdir(product_dir) - - def matching_files(param): - if isinstance(param, list): - return list(map(matching_files, param)) - else: - return glob(param) - - - product_file = matching_files('*.md')[0] - - def from_file(path): - f = open(path, 'r') - data = f.read() - f.close() - return data - - markdown_text = from_file(product_file) - html_text = md.markdown(markdown_text) - page = html.fromstring(html_text) - print(html_text) - - # name - name = page.xpath('//h1/text()')[0] - - # lesson - # filtering paragraph with no link in h2 title - lesson = lesson_info(page) - print(lesson) - - drawings = drawings_info(page) - for d in drawings: - print(d) - - products = list(map( - lambda drawing: { - 'image': drawing['image'], - 'name': drawing['name'], - 'author': drawing['author'], - 'authorAge': drawing['author_age'], - 'course': lesson['course'], - 'theme': lesson['name'], - 'rating': 10, - 'date': lesson['date'] - }, - drawings - )) - - for p in products: - print(p) - - # # price - # def str_price_to_int(text): - # items = text.split('\n') - # out = list(map(lambda p: int(re.sub('[^\d]+', '', p)), items)) - # return out - # - # - # price_text = page.xpath('//h2[contains(text(), "Цена")]/following-sibling::p/text()')[0] - # product_price = str_price_to_int(price_text) - # - # # colors - # def str_colors_to_code_list(text): - # cl = re.split('\s+', text) - # return ttcolor.color_by_code(cl) - # - # color_text = page.xpath('//h2[contains(text(), "Цвета")]/following-sibling::p/text()')[0] - # product_colors = str_colors_to_code_list(color_text) - # - # # text - # def format_text(text): - # try: - # res = requests.post("http://mdash.ru/api.v1.php", { - # 'text': text - # }) - # return res.json()['result'] - # - # except Exception as e: - # - # return text - # - # product_text = page.xpath('//h2[contains(text(), "Текст")]/following-sibling::p/text()')[0] - # # product_text = format_text(product_text) - # - # - # # qualities - # def get_quality(li): - # q = { - # 'image': li.xpath('./img/@src')[0], - # 'text': li.xpath('text()')[0].strip() - # } - # return q - # - # - # def qualities_list(elements): - # return list(map(get_quality, elements)) - # - # - # qualities_elements = page.xpath('//h2[contains(text(), "Свойства")]/following-sibling::ul/li') - # qualities = qualities_list(qualities_elements) - # - # - - product_images = [p['image']] - - image_sizes = [ - (50, 50, 'small'), - (300, 300, 'preview'), - (800, 800, 'medium'), - (2000, 2000, 'big'), - (0, 0, 'original') - ] - - images = [] - i = 0 - for image in product_images: - pi = {'source': image} - - for size in image_sizes: - size_name = size[2] - image_filename = generate_image_filename(image, size_name, product_sku, i, 'png') - d = os.path.join(images_dir, image_filename) - pi[size_name] = 'http://static.touchtwin.ru/images/{img}'.format(img=image_filename) - if size_name == 'original': - resize.optimize(image, d, quality=90) - else: - resize.minify(image, d, size[:2]) - - images.append(pi) - i += 1 - - p['images'] = images - del p['image'] - - # db['products'].update({'uri': product_uri}, p, upsert=True) \ No newline at end of file diff --git a/utils/Sync/sync_schedules.py b/utils/Sync/sync_schedules.py deleted file mode 100644 index 91a8d895..00000000 --- a/utils/Sync/sync_schedules.py +++ /dev/null @@ -1,38 +0,0 @@ -from glob import glob -import json -import os -import re -from pymongo import MongoClient -import settings - -__author__ = 'tmshv' - - -def process_file(schedule_path): - with open(schedule_path, 'rb') as schedule_file: - data = schedule_file.read().decode('utf-8') - try: - return json.loads(data) - except json.decoder.JSONDecodeError: - print(schedule_path) - - -if __name__ == '__main__': - db = MongoClient(settings.database_uri).hudozka - db.schedules.drop() - - # dir_schedule = settings.root_dir_schedule - dir_schedule = '/Users/tmshv/Dropbox/Dev/Hud School/Schedules/' - files = glob(dir_schedule + '*.json') - ss = list(map(process_file, files)) - - list(map(print, ss)) - - list(map( - db.schedules.insert, - filter( - lambda i : i is not None, - ss - ) - )) - print('schedule sync complete') \ No newline at end of file diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index e73e0a9e..4b7e751d 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -3,12 +3,19 @@ import os from functools import reduce from glob import glob -from bson import json_util +import bson.json_util + + +def combine(items): + return reduce( + lambda total, i: total + i, + items + ) + + +def to_json(i): + return json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default) -combine = lambda i: reduce( - lambda s, i: s + i, - i -) iterate_over_fns = lambda fns: lambda i: reduce( lambda out, fn: fn(out), @@ -20,12 +27,16 @@ iterate_iter_over_fns = lambda fns: lambda i: lmap( i ) -lmap = lambda fn, ls: list(map(fn, ls)) + +def lmap(fn, ls): + return list(map(fn, ls)) + +def lmapfn(ls): + return lambda fn: list(map(fn, ls)) + + lprint = lambda i: lmap(print, i) -lprint_json = lambda s: lprint(lmap( - lambda i: json.dumps(i, ensure_ascii=False, indent=4, default=json_util.default), - s -)) +lprint_json = lambda i: lprint(lmap(to_json, i)) first = lambda i: i[0] if len(i) > 0 else None diff --git a/utils/Sync/utils/hash.py b/utils/Sync/utils/hash.py index f192e463..564e89c2 100644 --- a/utils/Sync/utils/hash.py +++ b/utils/Sync/utils/hash.py @@ -1,7 +1,7 @@ import hashlib import json -from utils.fn import map_cases +from utils.fn import map_cases, to_json def hash_file(file): @@ -16,7 +16,6 @@ def hash_data(data): def hash_str(data): - to_json = lambda i: json.dumps(i, sort_keys=True) data = map_cases(data, [ (lambda i: isinstance(i, dict), to_json), (lambda i: isinstance(i, tuple), to_json), @@ -25,6 +24,6 @@ def hash_str(data): (lambda i: isinstance(i, float), float), ]) if isinstance(data, dict): - data = json.dumps(data, sort_keys=True) + data = to_json(data) return hash_data(str(data).encode()) diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py index 6e5244c4..e3c9ca35 100644 --- a/utils/Sync/utils/image/__init__.py +++ b/utils/Sync/utils/image/__init__.py @@ -2,7 +2,7 @@ import os from PIL import Image -from utils.hash import file_hash +from utils.hash import hash_file from utils.image import resize from utils.image.resize import optimize, thumbnail, orient @@ -54,7 +54,7 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): return { 'file': os.path.basename(file), - 'hash': file_hash(file), + 'hash': hash_file(file), 'data': result } diff --git a/utils/Sync/utils/image/resize.py b/utils/Sync/utils/image/resize.py index 2ea50f52..169c9186 100644 --- a/utils/Sync/utils/image/resize.py +++ b/utils/Sync/utils/image/resize.py @@ -2,7 +2,9 @@ import os from subprocess import call +import smartcrop from PIL import Image +from smartcrop import SmartCrop import utils.image @@ -30,6 +32,29 @@ def override_extension(dest, ext=None): return dest +def crop(src, dest, size): + width, height = size + image = utils.image.read_image(src) + crop_options = smartcrop.DEFAULTS + crop_options['width'] = width + crop_options['height'] = height + + crop = SmartCrop() + result = crop.crop(image.copy(), crop_options) + + box = (result['topCrop']['x'], + result['topCrop']['y'], + result['topCrop']['x'] + result['topCrop']['width'], + result['topCrop']['y'] + result['topCrop']['height']) + + i = utils.image\ + .read_image(src)\ + .crop(box) + i.thumbnail(size, Image.ANTIALIAS) + i.save(dest) + + + def orient(image): orientation = 0x0112 rotations = { @@ -52,6 +77,22 @@ def image_magick_resize(input_file, output_file, size, quality=85): call(['convert', input_file, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), output_file]) +# def image_magick_resize(input_file, output_file, size, quality=85): +# from wand.image import Image +# +# w, h = size +# s = '{w}x{h}>'.format(w=w, h=h) +# +# with Image(filename=input_file) as img: +# # print(img.size) +# # print(list(img.metadata.items())) +# with img.clone() as i: +# i.transform(resize=s) +# i.auto_orient() +# i.strip() +# i.save(filename=output_file) + + def image_magick_pdf_to_img(input_file, output_file): i = "{image}[0]".format(image=input_file) call(['convert', i, output_file])