diff --git a/utils/Sync/crop_files.py b/utils/Sync/crop_files.py new file mode 100644 index 00000000..ebc6be10 --- /dev/null +++ b/utils/Sync/crop_files.py @@ -0,0 +1,22 @@ +import os +from glob import glob + +from utils.fn import lprint, lmap +from utils.image.resize import crop + +size = (400, 400) + +# files = glob('/Users/tmshv/Dropbox/Dev/Hud School/Static/images/product*medium.jpg') +# files = files[:3] + +files = ['product-2015-maketirovanie-doma-peterburga-img0298-medium.jpg','product-2015-zhivopis-img0308-medium.jpg','product-2015-maketirovanie-img0240-medium.jpg','product-2015-podvodnyj-mir-stankovaya-kompoziziya-img0346-medium.jpg','product-2015-risunok-img0037-medium.jpg','product-2015-skazki-mg1037-medium.jpg','product-2015-stankovaya-kompoziziya-img0350-medium.jpg','product-2015-zvety-stankovaya-kompoziziya-img0348-medium.jpg','product-2015-gravyura-mg1014-medium.jpg','product-2015-guash-mg1024-medium.jpg','product-2015-shlisselburg-1-medium.jpg','product-2015-dpk-portret-v-tehnike-applikaziya-img0356-medium.jpg','product-70-img3869-medium.jpg','product-2015-domiki-20150406-1613-medium.jpg','product-2015-skulptura-vov-20150406-1591-medium.jpg','product-2015-history-competition-2-medium.jpg','product-2015-rozhdestvenskaya-skazka-2-medium.jpg','product-2016-zimnie-zabavy-1-medium.jpg','product-2016-na-krylyah-babochek-1-medium.jpg','product-2016-narodnye-motivy-1-medium.jpg','product-around-the-world-1-medium.jpg',] + + +src = lambda i: '/Users/tmshv/Dropbox/Dev/Hud School/Static/images/' + os.path.basename(i) +dest = lambda i: '/Users/tmshv/Desktop/smartcrop_sandbox/' + os.path.basename(i) + +for i in files: + try: + crop(src(i), dest(i), size) + except: + print(i) diff --git a/utils/Sync/models/Document.py b/utils/Sync/models/Document.py new file mode 100644 index 00000000..61952f99 --- /dev/null +++ b/utils/Sync/models/Document.py @@ -0,0 +1,4 @@ +class Document: + def __init__(self): + super().__init__() + diff --git a/utils/Sync/models/Model.py b/utils/Sync/models/Model.py new file mode 100644 index 00000000..e4306c05 --- /dev/null +++ b/utils/Sync/models/Model.py @@ -0,0 +1,19 @@ +class Model: + def __init__(self, create_fn, read_fn, update_fn, delete_fn): + super().__init__() + self.update_fn = update_fn + self.create_fn = create_fn + self.read_fn = read_fn + self.delete_fn = delete_fn + + def create(self, document): + return self.create_fn(document) + + def read(self, query): + return self.read_fn(query) + + def update(self, query, record): + return self.update_fn(query) + + def delete(self, query): + return self.delete_fn(query) \ No newline at end of file diff --git a/utils/Sync/models/product.py b/utils/Sync/models/product.py index 40df3931..70ff4a0b 100644 --- a/utils/Sync/models/product.py +++ b/utils/Sync/models/product.py @@ -1,14 +1,38 @@ +from sync.teacher import read_teacher_by_id from sync.image import sync_image from utils.fn import key_mapper +from utils.text.transform import interpolate_swift + + +def resolve_value(param, store): + """ + + :param param: \(nv-andreeva) + :param store: function should return value by key nv-andreeva + :return: + """ + i = interpolate_swift(param) + return store(i) unpop_image = key_mapper('image', lambda i: sync_image(i)['_id']) + +def resolve_teacher(i): + return resolve_value(i, read_teacher_by_id) + + def create_product(doc): # author, image, title, awards, teacher, date + unpop_teacher = key_mapper('teacher', lambda i: i['_id']) + product = {**doc} unpop_image(product) + product['teacher'] = resolve_teacher(doc['teacher']) + if(product['teacher']): + unpop_teacher(product) + return product diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index c4b437d7..e4970a6c 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -33,9 +33,7 @@ teachers_names = { image_processing = True image_save_dir = './static_images' -image_base_url = 'https://static.shburg.org/art/image/' -collective_image_base_url = 'https://static.shburg.org/art/collective/' -collective_image_output = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' +image_base_url = 'https://static.shburg.org/art/images/' album_html_img_class = 'hudozka-product' @@ -49,10 +47,30 @@ image_sizes = [ ('preview', 50, 50), ] +awards_image_sizes = [ + ('medium', 640, 480), + ('small', 300, 200), + ('preview', 50, 50), +] + event_image_sizes = [ ('big', 1200, 800) ] +teacher_image_url_base = 'https://static.shburg.org/art/images/teacher-{id}-{size}{ext}' +teacher_image_output = '/Users/tmshv/Desktop/Hudozka Static/images' +teacher_image_sizes = [ + ('big', 1200, 800), + ('medium', 640, 480), + ('preview', 50, 50), +] + +album_image_sizes = [ + ('big', 1200, 800), + ('medium', 640, 480), + ('small', 250, 175) +] + date_formats = [ '%Y', '%m.%Y', diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index c315d39f..f59b8dbd 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -1,9 +1,11 @@ import re - from datetime import datetime from markdown import markdown +from db import db +from utils.fn import lmap, key_mapper + def create_date(date_str, date_formats=None): """ @@ -36,6 +38,18 @@ def create_date(date_str, date_formats=None): folder_name_pattern = re.compile('([\d.]+)(.*)') +def untouched(documents, store): + return lmap( + lambda i: i[0], + filter( + lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), + map( + lambda i: (i, store.read(i)), + documents + ) + )) + + def create_date_and_title_from_folder_name(folder_name, date_formats=None): m = folder_name_pattern.findall(folder_name) if not m: @@ -56,4 +70,49 @@ create_post_from_image_list = lambda images: markdown( lambda i: '![]({img})'.format(img=i), images )) -) \ No newline at end of file +) + + +class Sync: + def __init__(self): + super().__init__() + + self.collection = None + + def read(self, document): + if isinstance(document, list): + return lmap(self.read, document) + + q = {'id': document['id']} + try: + return self.collection.find_one(q) + except ValueError: + pass + return None + + def update(self, document): + if isinstance(document, list): + return lmap(self.update, document) + + q = {'id': document['id']} + try: + self.collection.update_one(q, {'$set': document}, upsert=True) + return self.read(document) + except ValueError: + pass + return None + + def compile(self, document): + return document + + def query(self, q): + return self.collection.find(q) + + def delete(self, q): + return self.collection.find_one_and_delete(q) + + def create_id(self, document): + return None + + def create_hash(self, document): + return None diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py index cf8f946d..64870f84 100644 --- a/utils/Sync/sync/album.py +++ b/utils/Sync/sync/album.py @@ -1,5 +1,6 @@ import os from glob import glob +from lxml import etree import lxml.html @@ -8,10 +9,25 @@ from db import db from models.product import create_product from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date from sync.image import sync_image -from utils.fn import lmap, first, map_cases, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns +from utils.fn import lmap, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns, first, map_cases +from utils.hash import file_hash from utils.image import create_image from utils.io import read_yaml_md -from utils.text.transform import url_encode_text, interpolate, interpolate_swift +from utils.text.transform import url_encode_text + + +class ImageCreator: + def __init__(self, cache): + super().__init__() + self.cache = cache if cache else set() + + def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False): + hash = file_hash(file) + cached = self.cached(hash) + return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached + + def cached(self, hash): + return self.cache[hash] if hash in self.cache else None def sync_album(record): @@ -24,7 +40,7 @@ def sync_album(record): q = {'id': record['id']} try: collection = db().events - update_result = collection.update_one(q, {'$set': record}, upsert=True) + collection.update_one(q, {'$set': record}, upsert=True) i = collection.find_one({'id': record['id']}) return i except ValueError: @@ -33,115 +49,150 @@ def sync_album(record): return None -def create_album(doc): - sizes = settings.event_image_sizes - # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' - images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' - url = lambda a, i, s, e: 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format( +def create_post(md, image_path_fn): + html = lxml.html.fromstring(md) + + for img in html.cssselect('img'): + src = img.get('src') + path = image_path_fn(src) + + if path: + img.set('src', path) + img.set('class', settings.album_html_img_class) + img.set('data-file', src) + return etree.tounicode(html) + + +def create_url(a, i, s, e): + return 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format( album=a, id=i, size=s, ext=e ) - create_url = lambda aid, pid: lambda size, ext: url(aid, pid, size, ext.lower()) - image_id = lambda i: url_encode_text( - os.path.splitext(i)[0] - ) - image_path = lambda i: os.path.join(doc['folder'], i) - if 'title' not in doc or not doc['title']: - return None - create_album_id = lambda date, title: '{date}-{title}'.format( - date=date.strftime('%Y'), - title=url_encode_text(title) - ) +def url_creator(aid, pid): + return lambda size, ext: create_url(aid, pid, size, ext.lower()) - images = [] - doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) +class Album: + @staticmethod + def id_from_file(i): + return url_encode_text( + os.path.splitext(i)[0] + ) - if 'album' in doc: - # album_context = + def __init__(self, image_creator): + super().__init__() + self.image_creator = image_creator + self.skip_image_processing = False + self.images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' + + def create_image(self, file, url_fn): + ss = settings.album_image_sizes + od = self.images_dir + sp = self.skip_image_processing + + return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp) + + def create_album(self, doc): + # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' + + image_id = Album.id_from_file + image_path = lambda i: os.path.join(doc['folder'], i) + + if 'title' not in doc or not doc['title']: + return None + + create_album_id = lambda date, title: '{date}-{title}'.format( + date=date.strftime('%Y'), + title=url_encode_text(title) + ) + + doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) + + if 'album' in doc: + doc['album'] = self.create_products(doc, image_path) + + def process_post_image(src): + path = image_path(src) + + if os.path.exists(path): + url = url_creator( + doc['id'], + image_id(src) + ) + + image = self.create_image(path, url) + if image: + images.append(image) + return image['data']['big']['url'] + return None + + images = [] + post_html = create_post(doc['post'], process_post_image) + + doc['post'] = post_html + doc['images'] = images + return doc + + def create_products(self, doc, image_file_fn): + image_id = Album.id_from_file iter_album = iterate_iter_over_fns([ lambda product: { **product, - 'image': create_image( - image_path(product['file']), - sizes, - create_url( + 'image': self.create_image( + image_file_fn(product['file']), + url_creator( doc['id'], image_id(product['file']) - ), - images_dir, - skip_processing=True + ) ) }, - lambda product: { - **product, - 'awards': lmap( - lambda award: { - **award, - 'file': os.path.join(doc['folder'], award['file']), - 'assigned': interpolate(award['assigned'], product, interpolate_swift), - # 'awardFor': product - }, - product['awards'] - ) - } + # lambda product: { + # **product, + # 'awards': lmap( + # lambda award: { + # **award, + # 'file': os.path.join(doc['folder'], award['file']), + # 'assigned': interpolate(award['assigned'], product, interpolate_swift), + # # 'awardFor': product + # }, + # product['awards'] + # ) + # } ]) - doc['album'] = lmap( + return lmap( create_product, iter_album(doc['album']) ) - post_html = lxml.html.fromstring(doc['post']) - - for img in post_html.cssselect('img'): - src = img.get('src') - img_path = image_path(src) - - if os.path.exists(img_path): - image = create_image( - img_path, - sizes, - create_url( - doc['id'], - image_id(src) - ), - images_dir, - skip_processing=True - ) - images.append(image) - - img.set('src', image['data']['big']['url']) - img.set('class', settings.album_html_img_class) - img.set('data-file', src) - - doc['post'] = lxml.html.tostring(post_html).decode('utf-8') - doc['images'] = images - return doc - def get_manifest(filepath): y, m = read_yaml_md(filepath) data = {**y} if y else {} _ = lambda key, d=None: data[key] if key in data else d - date = create_date(_('date'), settings.date_formats_direct) + + date = create_date(str(_('date')), settings.date_formats_direct) if date: data['date'] = date - id = _('id') - if id: - data['id'] = str(id) + album_id = _('id') + if album_id: + data['id'] = str(album_id) post = {'post': m} if m else {} return {**data, **post} if __name__ == '__main__': + image_creator = ImageCreator(cache=None) + alb = Album(image_creator=image_creator) + # alb.skip_image_processing = True + albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' os.chdir(albums_dir) @@ -157,6 +208,7 @@ if __name__ == '__main__': # print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md')) # print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md')) # exit() + # albums = [albums[5]] # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE albums = lmap( @@ -200,13 +252,14 @@ if __name__ == '__main__': # MAP EVENT MANIFEST -> EVENT_OBJECT albums = lmap( - create_album, + alb.create_album, albums ) # FILTER INCORRECT EVENTS albums = list(filter(None, albums)) + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB def sync_images(i): return lmap( @@ -221,10 +274,10 @@ if __name__ == '__main__': ) # SYNC EVENT_OBJECT WITH DB - # albums = lmap( - # sync_album, - # albums - # ) + albums = lmap( + sync_album, + albums + ) lprint_json(albums) print('SYNC ALBUMS DONE') diff --git a/utils/Sync/sync/award.py b/utils/Sync/sync/award.py new file mode 100644 index 00000000..8c590241 --- /dev/null +++ b/utils/Sync/sync/award.py @@ -0,0 +1,196 @@ +import os +from glob import glob +from shutil import copyfile +from tempfile import mkstemp + +import settings +from db import db +from sync.image import sync_image +from utils.fn import lmap, lprint_json, key_mapper, lprint +from utils.hash import hash_file, hash_str +from utils.image import create_image +from utils.image.resize import image_magick_pdf_to_img +from utils.text.transform import url_encode_text, url_encode_file + +prefix = 'award' +dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Awards' +dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' +dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' +url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' +url_base_document = 'https://static.shburg.org/art/uploads/{file}' + + +def collection(): + return db().awards + + +def pdf_to_jpg(pdf): + cwd = os.getcwd() + abspdf = os.path.join(cwd, pdf) + _, temp = mkstemp('.jpg') + image_magick_pdf_to_img(abspdf, temp) + return temp + + +def sync_document(record): + if isinstance(record, list): + return lmap( + sync_document, + record + ) + + q = {'id': record['id']} + try: + collection().update_one(q, {'$set': record}, upsert=True) + return collection().find_one(q) + except ValueError: + pass + + return None + + +def read_document(i, query_fn=None): + q = query_fn(i) if query_fn else {'id': i['id']} + try: + return collection().find_one(q) + except ValueError: + pass + + return None + + +def query_documents(q): + return collection().find(q) + + +def delete_document(q): + return collection().find_one_and_delete(q) + + +def create_preview(pdf, sizes, preview_dir): + temp_preview_path = pdf_to_jpg(pdf) + + id = url_encode_text(pdf) + url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext) + + img = create_image(temp_preview_path, sizes, url, preview_dir) + os.remove(temp_preview_path) + return img + + +def create_document(doc): + sizes = settings.awards_image_sizes + file = doc['file'] + + doc['type'] = prefix + doc['url'] = url_base_document.format(file=url_encode_file(file)) + doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews) + doc['file'] = { + 'name': file, + 'size': os.path.getsize((file)) + } + + return doc + + +def create_id(doc): + return url_encode_text('{type}-{file}'.format( + type=prefix, + file=doc['file'] + )) + + +if __name__ == '__main__': + # ALL DOCUMENTS IDS FOUNDED + scope_documents_ids = [] + + # GET DOCUMENT YAML_MANIFEST FILES + os.chdir(dir_documents) + documents = glob('*.pdf') + + # READ YAML_MANIFEST FILES + documents = lmap( + lambda i: { + 'file': i + }, + documents + ) + + # CREATE DOCUMENT IDENTITY + documents = lmap( + lambda i: {**i, 'id': create_id(i)}, + documents + ) + + # CREATE HASH OF DOCUMENT FILE + documents = lmap( + lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])}, + documents + ) + + # CREATE SCOPE OF CURRENT SESSION + scope_documents_ids = lmap( + lambda i: i['id'], + documents + ) + + # SKIP UNTOUCHED DOCUMENTS + documents = lmap( + lambda i: i[0], + filter( + lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']), + lmap( + lambda document: (document, read_document(document)), + documents + ) + )) + + # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT + documents = lmap( + create_document, + documents + ) + + # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('preview', lambda i: sync_image(i)['_id']), + documents + ) + + # COPY FILE -> STATIC_DIR/URL_FILENAME + out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url'])) + lmap( + lambda doc: copyfile( + doc['file']['name'], + out_path(doc) + ), + documents + ) + + # SYNC DOCUMENT_OBJECT WITH DB + documents = lmap( + sync_document, + documents + ) + + documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = lmap( + delete_document, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) + ) + + # SCOPE + print('SCOPE:') + lprint(scope_documents_ids) + + # DELETE + print('DELETE DOCUMENTS:') + lprint_json(documents_to_remove) + + # DELETE + print('UPDATE DOCUMENTS:') + lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') diff --git a/utils/Sync/sync/collective.py b/utils/Sync/sync/collective.py index 82e6a1ee..585a4935 100644 --- a/utils/Sync/sync/collective.py +++ b/utils/Sync/sync/collective.py @@ -9,6 +9,22 @@ from utils.image import create_image from utils.io import read_yaml_md +def read_teacher_by_id(record): + if isinstance(record, list): + return lmap( + read_teacher_by_id, + record + ) + + q = {'id': record} + try: + return db().collective.find_one(q) + except ValueError: + pass + + return None + + def sync_collective(record): if isinstance(record, list): return lmap( diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index 4d81121d..3427a4fc 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -6,12 +6,20 @@ from tempfile import mkstemp import settings from db import db from sync.image import sync_image -from utils.fn import combine, lmap, lprint, key_mapper +from utils.fn import combine, lmap, lprint_json, key_mapper, lprint +from utils.hash import hash_file, hash_str from utils.image import create_image from utils.image.resize import image_magick_pdf_to_img from utils.io import read_yaml from utils.text.transform import url_encode_text, url_encode_file +prefix = 'document' +dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents' +dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads' +dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images' +url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}' +url_base_document = 'https://static.shburg.org/art/uploads/{file}' + # Adds specified category to each file in files list unwrap_file_list = lambda files, title: map( lambda profile: {'category': title, **profile}, @@ -19,25 +27,6 @@ unwrap_file_list = lambda files, title: map( ) -def sync_document(record): - if isinstance(record, list): - return lmap( - sync_document, - record - ) - - q = {'url': record['url']} - try: - documents = db().documents - update_result = documents.update_one(q, {'$set': record}, upsert=True) - i = documents.find_one({'url': record['url']}) - return i - except ValueError: - pass - - return None - - def pdf_to_jpg(pdf): cwd = os.getcwd() abspdf = os.path.join(cwd, pdf) @@ -46,12 +35,48 @@ def pdf_to_jpg(pdf): return temp +def sync_document(record): + if isinstance(record, list): + return lmap( + sync_document, + record + ) + + q = {'id': record['id']} + try: + documents = db().documents + update_result = documents.update_one(q, {'$set': record}, upsert=True) + i = documents.find_one({'id': record['id']}) + return i + except ValueError: + pass + + return None + + +def read_document(i, query_fn=None): + q = query_fn(i) if query_fn else {'id': i['id']} + try: + return db().documents.find_one(q) + except ValueError: + pass + + return None + + +def query_documents(q): + return db().documents.find(q) + + +def delete_document(q): + return db().documents.find_one_and_delete(q) + + def create_preview(pdf, sizes, preview_dir): temp_preview_path = pdf_to_jpg(pdf) id = url_encode_text(pdf) - url_base = 'https://static.shburg.org/art/images/document-{id}-{size}{ext}' - url = lambda size, ext: url_base.format(id=id, size=size, ext=ext) + url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext) img = create_image(temp_preview_path, sizes, url, preview_dir) os.remove(temp_preview_path) @@ -60,12 +85,11 @@ def create_preview(pdf, sizes, preview_dir): def create_document(doc): sizes = settings.image_sizes - preview_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images' file = doc['file'] - doc['type'] = 'document' - doc['url'] = 'https://static.shburg.org/art/uploads/' + url_encode_file(file) - doc['preview'] = create_preview(file, sizes=sizes, preview_dir=preview_dir) + doc['type'] = prefix + doc['url'] = url_base_document.format(file=url_encode_file(file)) + doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews) doc['file'] = { 'name': file, 'size': os.path.getsize((file)) @@ -74,6 +98,14 @@ def create_document(doc): return doc +def create_id(doc): + return url_encode_text('{type}-{category}-{file}'.format( + type=prefix, + category=doc['category'], + file=doc['file'] + )) + + def unwrap_manifest(param): if isinstance(param, list): return combine(map( @@ -94,21 +126,50 @@ def unwrap_manifest(param): if __name__ == '__main__': - dir = '/Users/tmshv/Dropbox/Dev/Hud school/Documents' - - os.chdir(dir) + # ALL DOCUMENTS IDS FOUNDED + scope_documents_ids = [] # GET DOCUMENT YAML_MANIFEST FILES - files = glob(dir + '/*.yaml') + os.chdir(dir_documents) + documents = glob(dir_documents + '/*.yaml') # READ YAML_MANIFEST FILES - files_data = lmap( + documents = lmap( read_yaml, - files + documents ) # GET FLAT LIST OF DOCUMENTS - documents = unwrap_manifest(files_data) + documents = unwrap_manifest(documents) + + # CREATE DOCUMENT IDENTITY + documents = lmap( + lambda i: {**i, 'id': create_id(i)}, + documents + ) + + # CREATE HASH OF DOCUMENT FILE + documents = lmap( + lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])}, + documents + ) + + # CREATE HASH OF DOCUMENT FILE + scope_documents_ids = lmap( + lambda i: i['id'], + documents + ) + + # SKIP UNTOUCHED DOCUMENTS + documents = lmap( + lambda i: i[0], + filter( + lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']), + lmap( + lambda document: (document, read_document(document)), + documents + ) + )) # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT documents = lmap( @@ -117,15 +178,13 @@ if __name__ == '__main__': ) # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB - sync_preview = lambda i: sync_image(i)['_id'] documents = lmap( - key_mapper('preview', sync_preview), + key_mapper('preview', lambda i: sync_image(i)['_id']), documents ) # COPY FILE -> STATIC_DIR/URL_FILENAME - uploads = '/Users/tmshv/Dropbox/Dev/Hud school/Static/uploads' - out_path = lambda doc: os.path.join(uploads, os.path.basename(doc['url'])) + out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url'])) lmap( lambda doc: copyfile( doc['file']['name'], @@ -140,18 +199,44 @@ if __name__ == '__main__': documents ) - lprint(documents) - print('SYNC DOCUMENTS DONE') + documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = lmap( + delete_document, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) + ) + + # SCOPE + print('SCOPE:') + lprint(scope_documents_ids) + + # DELETE + print('DELETE DOCUMENTS:') + lprint_json(documents_to_remove) + + # DELETE + print('UPDATE DOCUMENTS:') + lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') # DOCUMENT_OBJECT SAMPLE # { - # "_id": ObjectId("56d08411ace9573958e5e490"), - # "url": "https://static.shburg.org/art/uploads/egryul.pdf", - # "title": "Свидетельство о внесении в «ЕГРЮЛ»", - # "preview": ObjectId("56d07d87ace9573958e5e488"), - # "file": { - # "name": ""ЕГРЮЛ.pdf", - # "size": 100500 + # "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0", + # "_id": { + # "$oid": "56d08411ace9573958e5e497" # }, - # "category": "Основные документы" + # "type": "document", + # "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf", + # "category": "Для поступающих", + # "file": { + # "size": 53088, + # "name": "Договор о пожертвовании.pdf" + # }, + # "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf", + # "preview": { + # "$oid": "56d08007ace9573958e5e48f" + # }, + # "title": "Договор о пожертвовании" # } diff --git a/utils/Sync/sync/event.py b/utils/Sync/sync/event.py index 3b376690..cb51a5fb 100644 --- a/utils/Sync/sync/event.py +++ b/utils/Sync/sync/event.py @@ -70,11 +70,15 @@ def create_event(doc): return doc -def get_md_manifest(md_path): +def get_manifest(md_path): y, m = read_yaml_md(md_path) + y = y if y else {} - y['date'] = datetime.strptime(y['date'], '%d.%m.%Y') - y['id'] = str(y['id']) + if 'date' in y: + y['date'] = datetime.strptime(y['date'], '%d.%m.%Y') + + if 'id' in y: + y['id'] = str(y['id']) return { **y, @@ -126,13 +130,13 @@ if __name__ == '__main__': # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST md_glob = lambda i : first(glob(i + '/*.md')) events = lmap( - lambda i: { - **i, + lambda event: { + **event, **map_cases( - i, + event, [( lambda i: md_glob(i['folder']), - lambda i: get_md_manifest(md_glob(i['folder'])), + lambda i: get_manifest(md_glob(i['folder'])), )], lambda i: {} diff --git a/utils/Sync/sync/news.py b/utils/Sync/sync/news.py new file mode 100644 index 00000000..a5e7cfbb --- /dev/null +++ b/utils/Sync/sync/news.py @@ -0,0 +1,97 @@ +import os +from glob import glob + +from db import db +from sync import create_date +from utils.fn import lmap, lprint_json +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + + +def sync_news(record): + if isinstance(record, list): + return lmap( + sync_news, + record + ) + + q = {'id': record['id']} + try: + collection = db().timeline + update_result = collection.update_one(q, {'$set': record}, upsert=True) + i = collection.find_one({'id': record['id']}) + return i + except ValueError: + pass + + return None + + +def create_news(profile): + formats = [ + '%d.%m.%Y', + '%d.%m.%Y %H:%M', + '%d.%m.%Y %H:%M:%S' + ] + + if 'title' not in profile: + return None + + if 'date' in profile: + profile['date'] = create_date(profile['date'], formats) + + if 'until' in profile: + profile['until'] = create_date(profile['until'], formats) + + if 'id' not in profile: + id = lambda date, title: '{date}-{title}'.format( + date=date.strftime('%Y-%m-%d'), + title=url_encode_text(title) + ) + profile['id'] = id(profile['date'], profile['title']) + + return profile + + +if __name__ == '__main__': + news_dir = '/Users/tmshv/Dropbox/Dev/Hud school/News' + os.chdir(news_dir) + + # GET NEWS FILES + file_glob = os.path.join(news_dir, '*.md') + news = glob(file_glob) + + # READ MANIFEST FILES -> NEWS_MANIFEST + news = lmap( + lambda i: (os.path.basename(i),) + read_yaml_md(i), + news + ) + news = lmap( + lambda i: { + **i[1], + 'file': i[0], + 'message': i[2] + }, + news + ) + + # NEWS_MANIFEST -> NEWS_OBJECT + news = lmap( + create_news, + news + ) + + # FILTER BROKEN NEWS_OBJECT + news = list(filter( + None, + news + )) + + # SYNC EVENT_OBJECT WITH DB + events = lmap( + sync_news, + news + ) + + lprint_json(news) + print('SYNC NEWS DONE') diff --git a/utils/Sync/sync/schedule.py b/utils/Sync/sync/schedule.py new file mode 100644 index 00000000..af136e56 --- /dev/null +++ b/utils/Sync/sync/schedule.py @@ -0,0 +1,137 @@ +import os +from glob import glob + +from db import db +from sync import Sync, untouched +from utils.fn import lprint, lprint_json, lmap +from utils.hash import hash_file +from utils.io import read_yaml + +dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules' + + +class SyncSchedule(Sync): + def __init__(self): + super().__init__() + self.collection = db().schedules + + # def sync_collective(self, record): + # if isinstance(record, list): + # return lmap( + # self.sync_collective, + # record + # ) + # + # q = {'id': record['id']} + # try: + # collection = db().collective + # update_result = collection.update_one(q, {'$set': record}, upsert=True) + # i = collection.find_one({'id': record['id']}) + # return i + # except ValueError: + # pass + # + # return None + # + # def compile(self, profile): + # profile['picture'] = create_image( + # profile['picture'], + # sizes, + # lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), + # dir_local_images + # ) + # return profile + # + # def read_document(self, i, query_fn=None): + # q = query_fn(i) if query_fn else {'id': i['id']} + # try: + # return self.collection().find_one(q) + # except ValueError: + # pass + # + # return None + # + # def query_documents(self, q): + # return self.collection().find(q) + # + # def delete_document(self, q): + # return self.collection().find_one_and_delete(q) + + def create_id(self, document): + id = document['id'] if 'id' in document and document['id'] else None + if not id: + id = 'schedule-{period}-{semester}'.format(**document) + document['id'] = id + return document + + def create_hash(self, document): + return { + **document, + 'hash': hash_file(document['file']) + } + + +if __name__ == '__main__': + # ALL DOCUMENTS IDS FOUNDED + scope_documents_ids = [] + + # INIT + os.chdir(dir_documents) + sync = SyncSchedule() + + # COLLECT YAML/MD/JPG/PDF MANIFEST FILES + documents = glob('*.yaml') + + # READ MANIFEST + documents = lmap( + lambda i: { + 'file': i, + **read_yaml(i) + }, + documents + ) + + # # CREATE DOCUMENT IDENTITY + documents = lmap(sync.create_id, documents) + + # CREATE DOCUMENT HASH + documents = lmap(sync.create_hash, documents) + + # # CREATE SCOPE OF CURRENT SESSION + scope_documents_ids = lmap( + lambda i: i['id'], + documents + ) + + # # SKIP UNTOUCHED DOCUMENTS + documents = untouched(documents, sync) + + # DO HEAVY PROCESS WITH DOCUMENTS + + # SYNC TEACHER_PROFILE + documents = lmap( + sync.update, + documents + ) + + documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) + ) + + # SCOPE + print('SCOPE:') + lprint(scope_documents_ids) + + # # DELETE + print('DELETE DOCUMENTS:') + lprint_json(documents_to_remove) + + # DELETE + print('UPDATE DOCUMENTS:') + lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') diff --git a/utils/Sync/sync/teacher.py b/utils/Sync/sync/teacher.py new file mode 100644 index 00000000..e84e3456 --- /dev/null +++ b/utils/Sync/sync/teacher.py @@ -0,0 +1,193 @@ +import os +from glob import glob + +import settings +from db import db +from sync.image import sync_image +from utils.fn import lmap, lprint, key_mapper, lprint_json, ext +from utils.hash import hash_str, hash_file +from utils.image import create_image +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + +dir_teachers = '/Users/tmshv/Dropbox/Dev/Hud school/Collective' +url_base = settings.teacher_image_url_base +dir_local_images = settings.teacher_image_output +sizes = settings.teacher_image_sizes + +class Teacher: + def collection(self): + return db().collective + + def read_teacher_by_id(self, record): + if isinstance(record, list): + return lmap( + self.read_teacher_by_id, + record + ) + + q = {'id': record} + try: + return db().collective.find_one(q) + except ValueError: + pass + + return None + + def sync_collective(self, record): + if isinstance(record, list): + return lmap( + self.sync_collective, + record + ) + + q = {'id': record['id']} + try: + collection = db().collective + update_result = collection.update_one(q, {'$set': record}, upsert=True) + i = collection.find_one({'id': record['id']}) + return i + except ValueError: + pass + + return None + + def compile(self, profile): + profile['picture'] = create_image( + profile['picture'], + sizes, + lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), + dir_local_images + ) + return profile + + def read_document(self, i, query_fn=None): + q = query_fn(i) if query_fn else {'id': i['id']} + try: + return self.collection().find_one(q) + except ValueError: + pass + + return None + + def query_documents(self, q): + return self.collection().find(q) + + def delete_document(self, q): + return self.collection().find_one_and_delete(q) + + def create_id(self, document): + if 'id' in document and document['id']: + return document['id'] + + document['id'] = url_encode_text(document['name']) + + +if __name__ == '__main__': + # ALL DOCUMENTS IDS FOUNDED + scope_documents_ids = [] + + os.chdir(dir_teachers) + sync = Teacher() + + # GET TEACHER YAML/MD MANIFEST FILES + documents = glob('*.md') + + # # READ TEACHER YAML/MD MANIFEST + documents = lmap( + lambda i: (i,) + read_yaml_md(i), + documents + ) + + # COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY + documents = lmap( + lambda i: { + **i[1], + 'file': i[0], + 'biography': i[2] + }, + documents + ) + + # DEFINE PROFILE PICTURE + documents = lmap( + lambda i: { + **i, + 'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']), + }, + documents + ) + + # CREATE DOCUMENT IDENTITY + documents = lmap( + lambda i: {**i, 'id': sync.create_id(i)}, + documents + ) + + # CREATE HASH OF DOCUMENT FILES + documents = lmap( + lambda i: { + **i, 'hash': hash_str( + hash_file(i['file']) + hash_file(i['picture']) + ) + }, + documents + ) + + # CREATE SCOPE OF CURRENT SESSION + scope_documents_ids = lmap( + lambda i: i['id'], + documents + ) + + # SKIP UNTOUCHED DOCUMENTS + documents = lmap( + lambda i: i[0], + filter( + lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), + lmap( + lambda document: (document, sync.read_document(document)), + documents + ) + )) + + # DO HEAVY PROCESS WITH DOCUMENTS + documents = lmap( + sync.compile, + documents + ) + + # # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB + sync_picture = lambda i: None if i is None else sync_image(i)['_id'] + documents = lmap( + key_mapper('picture', sync_picture), + documents + ) + + # # SYNC TEACHER_PROFILE + documents = lmap( + sync.sync_collective, + documents + ) + + documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}}) + documents_to_remove = lmap( + sync.delete_document, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) + ) + + # SCOPE + print('SCOPE:') + lprint(scope_documents_ids) + + # DELETE + print('DELETE DOCUMENTS:') + lprint_json(documents_to_remove) + + # DELETE + print('UPDATE DOCUMENTS:') + lprint_json(documents) + print('[SYNC DOCUMENTS DONE]') diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py index 4273db5b..e73e0a9e 100644 --- a/utils/Sync/utils/fn.py +++ b/utils/Sync/utils/fn.py @@ -30,6 +30,10 @@ lprint_json = lambda s: lprint(lmap( first = lambda i: i[0] if len(i) > 0 else None +def ext(ext): + return lambda i: os.path.splitext(i)[0] + ext + + def map_item_key(item, key, map_fn): """ @@ -42,7 +46,12 @@ def map_item_key(item, key, map_fn): return item -key_mapper = lambda key, fn: lambda i: map_item_key(i, key, fn) +def kmap(key): + return lambda i: i[key] if key in i else None + + +def key_mapper(key, fn): + return lambda i: map_item_key(i, key, fn) def map_cases(param, cases, default_fn=None): diff --git a/utils/Sync/utils/hash.py b/utils/Sync/utils/hash.py index 1c673817..f192e463 100644 --- a/utils/Sync/utils/hash.py +++ b/utils/Sync/utils/hash.py @@ -1,9 +1,30 @@ import hashlib +import json + +from utils.fn import map_cases -def file_hash(file): +def hash_file(file): with open(file, 'rb') as f: data = f.read() - sha = hashlib.sha256(data) - f.close() - return sha.hexdigest() \ No newline at end of file + return hash_data(data) + + +def hash_data(data): + sha = hashlib.sha256(data) + return sha.hexdigest() + + +def hash_str(data): + to_json = lambda i: json.dumps(i, sort_keys=True) + data = map_cases(data, [ + (lambda i: isinstance(i, dict), to_json), + (lambda i: isinstance(i, tuple), to_json), + (lambda i: isinstance(i, list), to_json), + (lambda i: isinstance(i, int), int), + (lambda i: isinstance(i, float), float), + ]) + if isinstance(data, dict): + data = json.dumps(data, sort_keys=True) + + return hash_data(str(data).encode()) diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py index 7f736da3..6e5244c4 100644 --- a/utils/Sync/utils/image/__init__.py +++ b/utils/Sync/utils/image/__init__.py @@ -11,7 +11,6 @@ def read_image(src): try: i = Image.open(src) return i - # return orient(i) except Exception as e: print(src, e) return None @@ -27,8 +26,10 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): :param skip_processing: True/False to skip thumbs generation :return: """ - image = read_image(file) - if not image: + # image = read_image(file) + # if not image: + # return None + if not os.path.exists(file): return None result = {} @@ -36,20 +37,13 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): for size in sizes: size_name, width, height = size - image_width, image_height = image.size - if image_height > image_width: - width, height = height, width - image_url = url_fn(size_name, ext) image_filename = os.path.basename(image_url) local_image_path = os.path.join(output_dir, image_filename) if not skip_processing: - if size_name == 'original': - thumb = optimize(file, local_image_path, quality=90) - else: - thumb = thumbnail(file, local_image_path, (width, height)) - width, height = thumb.size + image = process_image(file, local_image_path, size) + width, height = image.size result[size_name] = { 'url': image_url, @@ -63,3 +57,19 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): 'hash': file_hash(file), 'data': result } + + +def process_image(input_file, output_file, size): + size_name, width, height = size + + image = read_image(input_file) + image_width, image_height = image.size + if image_height > image_width: + width, height = height, width + + if size_name == 'original': + thumb = optimize(input_file, output_file, quality=90) + else: + thumb = thumbnail(input_file, output_file, (width, height)) + + return thumb diff --git a/utils/Sync/utils/io.py b/utils/Sync/utils/io.py index 7e02007a..a0316729 100644 --- a/utils/Sync/utils/io.py +++ b/utils/Sync/utils/io.py @@ -26,11 +26,29 @@ def read_yaml_md(file): data = read_file(file) yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data) + if not yaml_data and not md_data: + return (read_yaml(file), None) + + try: + y = yaml.load(yaml_data) + except: + y = None + + m = markdown(md_data) if md_data else None + return y, m + + +def read_md_yaml(file): + data = read_file(file) + yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data) + + if not yaml_data and not md_data: + return (None, read_md(file)) + try: y = yaml.load(yaml_data) except: y = None m = markdown(md_data) if md_data else None - return y, m diff --git a/utils/Sync/utils/text/split.py b/utils/Sync/utils/text/split.py index 02e4a69a..b4f567b8 100644 --- a/utils/Sync/utils/text/split.py +++ b/utils/Sync/utils/text/split.py @@ -1,4 +1,4 @@ -from utils.fn import lmap, until +from utils.fn import until def split(data, separator=' '): diff --git a/utils/Sync/utils/text/transform.py b/utils/Sync/utils/text/transform.py index d672b95c..871a9610 100644 --- a/utils/Sync/utils/text/transform.py +++ b/utils/Sync/utils/text/transform.py @@ -4,6 +4,9 @@ from functools import reduce import settings from utils.fn import iterate_over_fns +remove_pattr = re.compile('[«»,!&]') +reduce_dash_pattr = re.compile('[-]+') + def translit(text): """ @@ -35,16 +38,52 @@ def interpolate(text, storage, key_fn=None): return storage[key] if key in storage else text -interpolate_swift = lambda i: re.search("([\d\w-]+)", i).group(1) +def interpolate_swift(i): + return re.search("([\d\w-]+)", i).group(1) -remove_pattr = re.compile('[«»,!&]') -remove_punctuation = lambda i: remove_pattr.sub('', i) -space_to_dash = lambda i: i.replace(' ', '-') -dot_to_dash = lambda i: i.replace('.', '-') -remove_underscores = lambda i: i.replace('_', '') -text_lower = lambda i: i.lower() +def remove_punctuation(i): + return remove_pattr.sub('', i) + + +def reduce_dashes(i): + return reduce_dash_pattr.sub('-', i) + + +def space_to_dash(i): + return i.replace(' ', '-') + + +def short_dash(i): + """ + minus -> dash + tire -> dash + :param i: + :return: + """ + return i.replace('–', '-').replace('—', '-') + + +def dot_to_dash(i): + return i.replace('.', '-') + + +def remove_underscores(i): + return i.replace('_', '') + + +def text_lower(i): + return i.lower() + url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower]) -url_encode_text = iterate_over_fns( - [translit, space_to_dash, dot_to_dash, remove_underscores, remove_punctuation, text_lower]) +url_encode_text = iterate_over_fns([ + translit, + space_to_dash, + dot_to_dash, + short_dash, + remove_underscores, + remove_punctuation, + reduce_dashes, + text_lower +])