diff --git a/utils/Sync/README.md b/utils/Sync/README.md index 635b37a8..8efe3484 100644 --- a/utils/Sync/README.md +++ b/utils/Sync/README.md @@ -9,6 +9,7 @@ Sync Hudozka ### Environment variables **YANDEX_DISK_ACCESS_TOKEN** + For access to files using Yandex.Disk API [How to obtain token for testing](https://tech.yandex.ru/oauth/doc/dg/tasks/get-oauth-token-docpage/) @@ -23,7 +24,8 @@ Authorize url. Replace with your app id https://oauth.yandex.ru/authorize?response_type=token&client_id=<идентификатор приложения> ``` -**SYNC_LOCAL_PATH** +#### SYNC_LOCAL_PATH + Path to scan files from Example: @@ -31,7 +33,8 @@ Example: SYNC_LOCAL_PATH=/Users/tmshv/Yandex.Disk ``` -**SYNC_STATIC** +#### SYNC_STATIC + Path to local folder for compiled files (images, uploads) Example: @@ -39,7 +42,8 @@ Example: SYNC_STATIC=~/Desktop/Hudozka Static ``` -**SYNC_PROVIDER** +#### SYNC_PROVIDER + Type of file provider - *fs* for local access @@ -50,10 +54,18 @@ Example: SYNC_PROVIDER=fs ``` -**MONGO_URI** -URI for access to MongoDB -Example: mongodb://localhost:27017/hudozka +#### MONGO_URI + +URI for MongoDB access + +Example: + ```bash + mongodb://localhost:27017/hudozka + ``` + +#### SYNC_ENV + +Environment like *NODE_ENV* -**SYNC_ENV** - *production* default value - *development* don't update/delete documents from DB diff --git a/utils/Sync/db.py b/utils/Sync/db.py index 3c0ad0c7..bfe91db3 100644 --- a/utils/Sync/db.py +++ b/utils/Sync/db.py @@ -6,3 +6,7 @@ import settings def db(): client = MongoClient(settings.database_uri) return client.hudozka + + +def collection(name): + return db()[name] diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index 9e7c7c48..a77ffb29 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -20,14 +20,9 @@ def env(param, default=None): return default -root_dir_gallery = './Gallery' -root_dir_schedule = './Schedules' - -default_description_version = 1 - database_uri = env('MONGO_URI') -teachers_names = { +person_uri = { 'Н.В.Андреева': 'nv-andreeva', 'И.Н.Втюрина': 'in-vturina', 'О.Д.Гоголева': 'od-gogoleva', @@ -94,11 +89,14 @@ date_formats_reverse = [ '%Y.%m.%d' ] +origin = 'Yandex.Disk' + collection_documents = 'documents' collection_schedules = 'schedules' collection_events = 'events' -collection_news = 'news' +collection_news = 'timeline' collection_collective = 'collective' +collection_albums = 'albums' sync_provider_type = env('SYNC_PROVIDER', 'fs') @@ -113,6 +111,7 @@ dir_schedules = f('Hudozka/Site/Schedules') dir_events = f('Hudozka/Site/Events') dir_news = f('Hudozka/Site/News') dir_collective = f('Hudozka/Site/Collective') +dir_gallery = f('Hudozka/Site/Gallery') dir_static = env('SYNC_STATIC', '~/Hudozka Static') dir_static_uploads = absolute(dir_static + '/uploads') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 62dece00..263da35e 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,5 +1,6 @@ import settings -from db import db +from db import db, collection +from sync.album import sync_albums from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider from sync.document import sync_documents @@ -7,6 +8,7 @@ from sync.person import sync_persons from sync.post import sync_posts from sync.schedule import sync_schedules from utils.fn import lprint_json +from utils.image import thumbnail def get_provider(provider_type, root): @@ -19,10 +21,8 @@ def get_provider(provider_type, root): def main(provider, fn, collection_name): if provider: - collection = db()[collection_name] - print('SYNC: %s' % (collection_name.upper())) - u, d = fn(provider, collection, update=settings.do_update, delete=settings.do_update) + u, d = fn(provider, collection(collection_name), update=settings.do_update, delete=settings.do_update) print('DELETE %s: %s' % (collection_name.upper(), 'NO' if not settings.do_delete else str(len(d)))) lprint_json(d) @@ -41,3 +41,4 @@ if __name__ == '__main__': main(io(settings.dir_events), sync_posts, settings.collection_events) main(io(settings.dir_news), sync_posts, settings.collection_news) main(io(settings.dir_collective), sync_persons, settings.collection_collective) + main(io(settings.dir_gallery), sync_albums, settings.collection_albums) diff --git a/utils/Sync/sync/album.py b/utils/Sync/sync/album.py index 183d326b..a924632c 100644 --- a/utils/Sync/sync/album.py +++ b/utils/Sync/sync/album.py @@ -1,182 +1,33 @@ import os -from glob import glob - -import lxml.html -from lxml import etree import settings -from db import db -from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \ - untouched, synced_images_ids -from sync.models import create_product -from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \ - key_mapper, lprint -from utils.hash import hash_file, hash_str +from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \ + synced_images_ids +from sync.core.album import SyncAlbum +from sync.data import list_images +from utils.fn import lmap, lmapfn, key_mapper from utils.image import create_image from utils.io import read_yaml_md -from utils.text.transform import url_encode_text class ImageCreator: - def __init__(self, cache): + def __init__(self, provider, cache): super().__init__() + self.provider = provider self.cache = cache if cache else set() def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False): - hash = hash_file(file) - cached = self.cached(hash) - return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached + filehash = self.provider.hash(file) + cached = self.cached(filehash) + return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached def cached(self, hash): return self.cache[hash] if hash in self.cache else None -def create_post(md, image_path_fn): - html = lxml.html.fromstring(md) - - for img in html.cssselect('img'): - src = img.get('src') - path = image_path_fn(src) - - if path: - img.set('src', path) - img.set('class', settings.album_html_img_class) - img.set('data-file', src) - return etree.tounicode(html) - - -def create_url(a, i, s, e): - return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( - album=a, - id=i, - size=s, - ext=e - ) - - -def url_creator(aid, pid): - return lambda size, ext: create_url(aid, pid, size, ext.lower()) - - -class SyncAlbum(Sync): - @staticmethod - def id_from_file(i): - return url_encode_text( - os.path.splitext(i)[0] - ) - - def __init__(self, image_creator, images_dir): - super().__init__() - self.collection = db().albums - self.image_creator = image_creator - self.skip_image_processing = False - self.images_dir = images_dir - - def create_id(self, document): - if 'id' in document: - return document - - y = str(document['date'].year) - document['id'] = url_encode_text('-'.join([y, document['title']])) - return document - - def create_hash(self, document): - hash_images = lmap( - hash_file, - lmap( - lambda i: os.path.join(document['folder'], i), - document['images'] - ) - ) - document['hash'] = hash_str( - combine([hash_str(document)] + hash_images) - ) - - return document - - def create_image(self, file, url_fn): - ss = settings.album_image_sizes - od = self.images_dir - sp = self.skip_image_processing - - return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp) - - def create_album(self, doc): - # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' - - image_id = SyncAlbum.id_from_file - image_path = lambda i: os.path.join(doc['folder'], i) - - if 'title' not in doc or not doc['title']: - return None - - create_album_id = lambda date, title: '{date}-{title}'.format( - date=date.strftime('%Y'), - title=url_encode_text(title) - ) - - doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) - - if 'album' in doc: - doc['album'] = self.create_products(doc, image_path) - - def process_post_image(src): - path = image_path(src) - - if os.path.exists(path): - url = url_creator( - doc['id'], - image_id(src) - ) - - image = self.create_image(path, url) - if image: - images.append(image) - return image['data']['big']['url'] - return None - - images = [] - post_html = create_post(doc['post'], process_post_image) - - doc['post'] = post_html - doc['images'] = images - return doc - - def create_products(self, doc, image_file_fn): - image_id = SyncAlbum.id_from_file - iter_album = iterate_iter_over_fns([ - lambda product: { - **product, - 'image': self.create_image( - image_file_fn(product['file']), - url_creator( - doc['id'], - image_id(product['file']) - ) - ) - }, - # lambda product: { - # **product, - # 'awards': lmap( - # lambda award: { - # **award, - # 'file': os.path.join(doc['folder'], award['file']), - # 'assigned': interpolate(award['assigned'], product, interpolate_swift), - # # 'awardFor': product - # }, - # product['awards'] - # ) - # } - ]) - - return lmap( - create_product, - iter_album(doc['album']) - ) - - -def get_manifest(filepath): - y, m = read_yaml_md(filepath) +def get_manifest(provider, filepath): + data = provider.read(filepath).read().decode('utf-8') + y, m = read_yaml_md(data) data = {**y} if y else {} _ = lambda key, d=None: data[key] if key in data else d @@ -193,76 +44,60 @@ def get_manifest(filepath): return {**data, **post} -def main(dir_documents, sync): - os.chdir(dir_documents) - - # GET ALBUMS FOLDERS - documents = lmap( - lambda i: i.path, - filter( - lambda i: i.is_dir(), - os.scandir('.') - ) +def album_from_folder(sync, path): + # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE + date, title = create_date_and_title_from_folder_name( + os.path.basename(path) ) - # documents = [documents[3]] - # documents = documents[:15] - - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - documents = lmap( - lambda i: (i,) + create_date_and_title_from_folder_name( - os.path.basename(i) - ), - documents + images = lmap( + lambda i: os.path.relpath(i, path), + list_images(sync.provider, path) ) # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT - glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG']) - documents = lmap( - lambda i: { - 'folder': i[0], - 'date': i[1], - 'title': i[2], - 'post': create_post_from_image_list(lmap( - lambda path: os.path.relpath(path, i[0]), - glob_in(i[0]) - )) - }, - documents - ) + document = { + 'folder': path, + 'date': date, + 'title': title, + 'post': create_post_from_image_list(images), + 'images': images + } # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - md_glob = lambda i: first(glob(i + '/*.md')) - documents = lmap( - lambda album: { - **album, - **map_cases( - album, - [( - lambda album: md_glob(album['folder']), - lambda album: get_manifest(md_glob(album['folder'])), - )], - lambda i: {} - ) - }, - documents - ) - documents = lmap( - lambda i: { - **i, - 'images': lmap( - lambda path: os.path.relpath(path, i['folder']), - list_images(i['folder']) - ) - }, - documents - ) + manifest = sync.provider.type_filter(path, '.md') + manifest = manifest[0] if len(manifest) else None + manifest = get_manifest(sync.provider, manifest) if manifest else {} + manifest = manifest if manifest else {} + + document = { + **document, + **manifest + } # CREATE DOCUMENT IDENTITY - documents = lmap(sync.create_id, documents) + document = sync.create_id(document) # CREATE HASH OF DOCUMENT FILE - documents = lmap(sync.create_hash, documents) + document = sync.create_hash(document) + + return document + + +def main(sync, do_update, do_delete): + # GET ALBUMS FOLDERS + documents = lmap( + lambda i: i, + filter( + lambda i: sync.provider.is_dir(i), + sync.provider.scan('.') + ) + ) + + documents = lmap( + lambda folder: album_from_folder(sync, folder), + documents + ) # CREATE SCOPE OF CURRENT SESSION scope_documents_ids = lmapfn(documents)( @@ -274,11 +109,11 @@ def main(dir_documents, sync): # MAP EVENT MANIFEST -> EVENT_OBJECT documents = lmap( - alb.create_album, + sync.create_album, documents ) - # FILTER INCORRECT EVENTS + # # FILTER INCORRECT EVENTS documents = list(filter(None, documents)) # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB @@ -288,63 +123,42 @@ def main(dir_documents, sync): ) # SYNC OBJECT WITH DB - documents = lmap( - sync.update, - documents - ) + if do_update: + documents = lmap( + sync.update, + documents + ) documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}}) - documents_to_remove = lmap( - sync.delete, - map( - lambda i: {'_id': i['_id']}, - documents_to_remove + + if do_delete: + documents_to_remove = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_remove + ) ) + + return documents, documents_to_remove + + +def sync_albums(provider, collection, update=True, delete=True): + """ + :param collection: + :param provider: + :param update: + :param delete: + :return: + """ + + return main( + SyncAlbum( + collection, + provider, + image_creator=ImageCreator(provider, cache=None), + images_dir=settings.dir_static_images + ), + do_update=update, + do_delete=delete ) - - # SCOPE - print('SCOPE:') - lprint(scope_documents_ids) - - # DELETE - print('DELETE DOCUMENTS:') - lprint_json(lmap( - lambda i: i['id'], - documents_to_remove - )) - - # DELETE - print('UPDATE DOCUMENTS:') - lprint_json(lmap( - lambda i: i['id'], - documents - )) - # lprint_json(documents) - print('[SYNC DOCUMENTS DONE]') - - -if __name__ == '__main__': - image_creator = ImageCreator(cache=None) - alb = SyncAlbum(image_creator=image_creator, images_dir=settings.dir_static_images) - # alb.skip_image_processing = True - - # dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery' - # dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Gallery' - dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/Gallery' - main(dir_documents=dir_docs, sync=alb) - - # dir_docs += '/2013 Валяние Зверушки' - # dir_docs += '/Валяние Зверушки.md' - - # dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/News/2015 Выставка 70-летия победы.md' - - # print(frontmatter.load(dir_docs).content) - # d = read_file(dir_docs) - # d = d.replace('\u0D0A', '\n') - # print(frontmatter.loads(d).metadata) - # print(parse_yaml_front_matter(d)) - # - # print(len(re.split('^---', d, 2, re.MULTILINE))) - # print(re.split('^---', d, 2)[1]) - # - # print(read_yaml_md(dir_docs)) diff --git a/utils/Sync/sync/core/album.py b/utils/Sync/sync/core/album.py new file mode 100644 index 00000000..d68a571b --- /dev/null +++ b/utils/Sync/sync/core/album.py @@ -0,0 +1,145 @@ +import os +from lxml import etree + +import lxml.html +import settings +from sync.core import Sync +from sync.models.product import create_product +from utils.fn import lmap, iterate_iter_over_fns, combine +from utils.hash import hash_file, hash_str +from utils.text.transform import url_encode_text + + +def create_post(md, image_path_fn): + html = lxml.html.fromstring(md) + + for img in html.cssselect('img'): + src = img.get('src') + path = image_path_fn(src) + + if path: + img.set('src', path) + img.set('class', settings.album_html_img_class) + img.set('data-file', src) + return etree.tounicode(html) + + +def create_url(a, i, s, e): + return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format( + album=a, + id=i, + size=s, + ext=e + ) + + +def url_creator(aid, pid): + return lambda size, ext: create_url(aid, pid, size, ext.lower()) + + +class SyncAlbum(Sync): + @staticmethod + def id_from_file(i): + return url_encode_text( + os.path.splitext(i)[0] + ) + + def __init__(self, collection, provider, image_creator, images_dir): + super().__init__() + self.collection = collection + self.provider = provider + self.image_creator = image_creator + self.skip_image_processing = False + self.images_dir = images_dir + + def create_id(self, document): + if 'id' in document: + return document + + y = str(document['date'].year) + document['id'] = url_encode_text('-'.join([y, document['title']])) + return document + + def create_hash(self, document): + files = sorted(document['images']) + hashes = lmap( + self.provider.hash, + lmap( + lambda i: os.path.join(document['folder'], i), + files + ) + ) + + document['hash'] = hash_str( + combine([hash_str(document)] + hashes) + ) + + return document + + def create_image(self, file, url_fn): + ss = settings.album_image_sizes + od = self.images_dir + sp = self.skip_image_processing + + return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp) + + def create_album(self, doc): + # url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}' + + image_id = SyncAlbum.id_from_file + image_path = lambda i: os.path.join(doc['folder'], i) + + if 'title' not in doc or not doc['title']: + return None + + create_album_id = lambda date, title: '{date}-{title}'.format( + date=date.strftime('%Y'), + title=url_encode_text(title) + ) + + doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title']) + + if 'album' in doc: + doc['album'] = self.create_products(doc, image_path) + + def process_post_image(src): + path = image_path(src) + + if self.provider.exists(path): + url = url_creator( + doc['id'], + image_id(src) + ) + + image = self.create_image(path, url) + if image: + images.append(image) + return image['data']['big']['url'] + return None + + images = [] + post_html = create_post(doc['post'], process_post_image) + + doc['post'] = post_html + doc['images'] = images + return doc + + def create_products(self, doc, image_file_fn): + image_id = SyncAlbum.id_from_file + iter_album = iterate_iter_over_fns([ + lambda product: { + **product, + 'image': self.create_image( + image_file_fn(product['file']), + url_creator( + doc['id'], + image_id(product['file']) + ) + ) + } + ]) + + return lmap( + create_product, + iter_album(doc['album']) + ) diff --git a/utils/Sync/sync/core/person.py b/utils/Sync/sync/core/person.py index b78b8af7..cf3e2728 100644 --- a/utils/Sync/sync/core/person.py +++ b/utils/Sync/sync/core/person.py @@ -1,6 +1,6 @@ import settings from sync.core import Sync -from utils.hash import hash_str, hash_file +from utils.hash import hash_str from utils.image import create_image from utils.text.transform import url_encode_text @@ -16,8 +16,9 @@ class SyncPerson(Sync): self.dir_local_images = dir_local_images def compile(self, profile): + image_path = self.provider.get_abs(profile['picture']) profile['picture'] = create_image( - profile['picture'], + image_path, sizes, lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext), self.dir_local_images diff --git a/utils/Sync/sync/core/post.py b/utils/Sync/sync/core/post.py index 2b0cc8c9..e440451f 100644 --- a/utils/Sync/sync/core/post.py +++ b/utils/Sync/sync/core/post.py @@ -12,7 +12,7 @@ from utils.text.transform import url_encode_text, url_encode_file class SyncPost(Sync): - def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes): + def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin): super().__init__() self.collection = collection self.provider = provider @@ -20,6 +20,7 @@ class SyncPost(Sync): self.type = document_type self.image_url_base = image_url_base self.dir_local_images = dir_local_images + self.origin = origin def create_id(self, document): new_id = url_encode_text(document['title']) @@ -77,12 +78,14 @@ class SyncPost(Sync): document['post'] = lxml.html.tostring(post_html).decode('utf-8') document['images'] = images + document['origin'] = self.origin return document def create_remove_query(self, query): if self.type: query = { **query, - 'type': self.type + 'type': self.type, + 'orogin': self.origin } return query diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index e6022a53..19ef6f8b 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -30,6 +30,9 @@ class Provider: def size(self, path): return 0 + def exists(self, path): + return True + def get_abs(self, path): a = os.path.join(self.root, path) return os.path.normpath(a) diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py index 0f3ed756..5e8e74cd 100644 --- a/utils/Sync/sync/data/fs.py +++ b/utils/Sync/sync/data/fs.py @@ -41,6 +41,10 @@ class FSProvider(Provider): file = self.get_abs(path) return os.path.getsize(file) + def exists(self, path): + file = self.get_abs(path) + return os.path.exists(file) + def is_dir(self, path): return os.path.isdir(self.get_abs(path)) diff --git a/utils/Sync/sync/document.py b/utils/Sync/sync/document.py index c1161ac3..985179db 100644 --- a/utils/Sync/sync/document.py +++ b/utils/Sync/sync/document.py @@ -1,5 +1,4 @@ import os -from glob import glob import settings from sync import synced_image_id, untouched @@ -7,7 +6,6 @@ from sync.core.document import SyncDocument from sync.data import scan_subdirs from sync.models.document import Document from utils.fn import combine, lmap, key_mapper -from utils.io import read_yaml def sync_documents(provider, collection, update=True, delete=True): @@ -177,19 +175,3 @@ def unwrap_manifest(param): return None return param - - -def documents_from_yaml(dirpath): - # GET DOCUMENT YAML_MANIFEST FILES - documents = glob(dirpath + '/*.yaml') - - # READ YAML_MANIFEST FILES - documents = lmap( - read_yaml, - documents - ) - - # GET FLAT LIST OF DOCUMENTS - documents = unwrap_manifest(documents) - - return documents diff --git a/utils/Sync/sync/models/product.py b/utils/Sync/sync/models/product.py index c7abf5ad..48760d36 100644 --- a/utils/Sync/sync/models/product.py +++ b/utils/Sync/sync/models/product.py @@ -1,5 +1,7 @@ +import settings +from db import collection +from sync.core.person import SyncPerson from sync.image import sync_image -from sync.teacher import Teacher from utils.fn import key_mapper from utils.text.transform import interpolate_swift @@ -18,10 +20,13 @@ def resolve_value(param, store): unpop_image = key_mapper('image', lambda i: sync_image(i)['_id']) - def resolve_teacher(i): - sync = Teacher() - return resolve_value(i, lambda i: sync.read({'id': i})) + sync = SyncPerson( + collection(settings.collection_collective), + None, + '' + ) + return resolve_value(i, lambda teacher_id: sync.read({'id': teacher_id})) def create_product(doc): @@ -33,7 +38,7 @@ def create_product(doc): unpop_image(product) product['teacher'] = resolve_teacher(doc['teacher']) - if(product['teacher']): + if (product['teacher']): unpop_teacher(product) return product diff --git a/utils/Sync/sync/person.py b/utils/Sync/sync/person.py index d0d2dc83..f4a91b81 100644 --- a/utils/Sync/sync/person.py +++ b/utils/Sync/sync/person.py @@ -33,6 +33,10 @@ def sync_persons(provider, collection, update=True, delete=True): ) +def make_person(provider, file): + pass + + def main(sync, update_documents=False, delete_documents=False): # GET TEACHER YAML/MD MANIFEST FILES documents = sync.provider.type_filter('', '.md') @@ -75,17 +79,7 @@ def main(sync, update_documents=False, delete_documents=False): ) # SKIP UNTOUCHED DOCUMENTS - documents = untouched(documents, sync) - - # documents = lmap( - # lambda i: i[0], - # filter( - # lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']), - # lmap( - # lambda document: (document, sync.open(document)), - # documents - # ) - # )) + # documents = untouched(documents, sync) # DO HEAVY PROCESS WITH DOCUMENTS documents = lmap( diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 348adef8..055ef419 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -36,10 +36,10 @@ def create_document_from_folder(sync, path, file_time_formats): ) document = { + 'images': images, + 'post': post, **document, **params, - 'images': images, - 'post': post } document = sync.create_id(document) @@ -135,20 +135,20 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True): # SKIP UNTOUCHED DOCUMENTS documents = untouched(documents, sync) - # # MAP EVENT MANIFEST -> EVENT_OBJECT - # documents = lmap( - # sync.create, - # documents - # ) + # MAP EVENT MANIFEST -> EVENT_OBJECT + documents = lmap( + sync.create, + documents + ) # FILTER INCORRECT EVENTS documents = list(filter(None, documents)) - # # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB - # documents = lmap( - # key_mapper('images', synced_images_ids), - # documents - # ) + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('images', synced_images_ids), + documents + ) # SYNC EVENT_OBJECT WITH DB if update_documents: @@ -179,10 +179,11 @@ def sync_posts(provider, collection, update=True, delete=True): SyncPost( collection, provider, - None, + 'post', image_url_base, settings.dir_static_images, - settings.event_image_sizes + settings.event_image_sizes, + origin=settings.origin ), file_time_formats=['%Y.%m.%d'], update_documents=update, diff --git a/utils/Sync/sync/schedule.py b/utils/Sync/sync/schedule.py index 9f3ec973..05737efb 100644 --- a/utils/Sync/sync/schedule.py +++ b/utils/Sync/sync/schedule.py @@ -1,17 +1,17 @@ -from yaml import load as yaml from toml import loads as toml_str +from yaml import load as yaml from sync import untouched from sync.core.schedule import SyncSchedule -from sync.data import scan_subdirs, get_data +from sync.data import get_data from utils.fn import lmap, constant -from utils.io import read_yaml dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules' def sync_schedules(provider, collection, update=True, delete=True): """ + :param collection: :param provider: :param update: :param delete: diff --git a/utils/Sync/utils/image/resize.py b/utils/Sync/utils/image/resize.py index 759440a9..9d5c001b 100644 --- a/utils/Sync/utils/image/resize.py +++ b/utils/Sync/utils/image/resize.py @@ -10,13 +10,13 @@ import utils.image def thumbnail(src, dest, size, ext=None): - # image_magick_resize(src, dest, size, quality=85) - # return utils.image.read_image(dest) + image_magick_resize(src, dest, size, quality=85) + return utils.image.read_image(dest) - image = utils.image.read_image(src) - image.thumbnail(size) - image.save(override_extension(dest, ext)) - return image + # image = utils.image.read_image(src) + # image.thumbnail(size) + # image.save(override_extension(dest, ext)) + # return image def optimize(src, dest, quality=85, ext=None): @@ -69,9 +69,9 @@ def orient(image): return image -def image_magick_resize(input_file, output_file, size, quality=85): +def image_magick_resize(ifile, ofile, size, quality=85): s = '%dx%d' % size - call(['convert', input_file, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), output_file]) + call(['/usr/local/bin/convert', ifile, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), ofile]) # def image_magick_resize(input_file, output_file, size, quality=85): diff --git a/utils/Sync/utils/io.py b/utils/Sync/utils/io.py index 6288f874..5b70372b 100644 --- a/utils/Sync/utils/io.py +++ b/utils/Sync/utils/io.py @@ -1,54 +1,14 @@ import re - -import frontmatter import yaml -from markdown import markdown -from utils.text.split import split, split_with +from kazimir import kazimir_to_html -# FM_BOUNDARY = re.compile(r'^-{3,}$', re.MULTILINE) FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE) -def read_file(file): - with open(file, 'rb') as f: - data = f.read().decode('utf-8') - return data - - -def read_yaml(file): - with open(file, 'rb') as f: - data = f.read().decode('utf-8') - return yaml.load(data) - - -def read_md(file): - with open(file, 'rb') as f: - data = f.read().decode('utf-8') - return markdown(data) - def read_yaml_md(data): yaml_front_matter, content = parse_yaml_front_matter(data) - return yaml_front_matter, markdown(content) - - # data = frontmatter.load(file) - # return data.metadata, markdown(data.content) - - -def read_md_yaml(file): - data = read_file(file) - yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data) - - if not yaml_data and not md_data: - return (None, read_md(file)) - - try: - y = yaml.load(yaml_data) - except: - y = None - - m = markdown(md_data) if md_data else None - return y, m + return yaml_front_matter, kazimir_to_html(content) def parse_yaml_front_matter(text, **defaults): @@ -75,4 +35,4 @@ def parse_yaml_front_matter(text, **defaults): if isinstance(fm, dict): metadata.update(fm) - return metadata, content.strip() \ No newline at end of file + return metadata, content.strip() diff --git a/utils/Sync/utils/validation.py b/utils/Sync/utils/validation.py index ac1d1889..547eefd5 100644 --- a/utils/Sync/utils/validation.py +++ b/utils/Sync/utils/validation.py @@ -9,10 +9,3 @@ __check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings. def check_description_filename(filename): return __check_description_filename_exp.match(filename) - - -def suggest_description_filename(filename): - bn = os.path.basename(filename) - bn = os.path.splitext(bn)[0] - v = settings.default_description_version - return '%s [%s].md' % (bn, v)