diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py index a77ffb29..73ceac37 100644 --- a/utils/Sync/settings.py +++ b/utils/Sync/settings.py @@ -97,6 +97,7 @@ collection_events = 'events' collection_news = 'timeline' collection_collective = 'collective' collection_albums = 'albums' +collection_pages = 'pages' sync_provider_type = env('SYNC_PROVIDER', 'fs') @@ -112,6 +113,7 @@ dir_events = f('Hudozka/Site/Events') dir_news = f('Hudozka/Site/News') dir_collective = f('Hudozka/Site/Collective') dir_gallery = f('Hudozka/Site/Gallery') +dir_pages = f('Hudozka/Site/Pages') dir_static = env('SYNC_STATIC', '~/Hudozka Static') dir_static_uploads = absolute(dir_static + '/uploads') diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 263da35e..533a4359 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -4,6 +4,7 @@ from sync.album import sync_albums from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider from sync.document import sync_documents +from sync.page import sync_pages from sync.person import sync_persons from sync.post import sync_posts from sync.schedule import sync_schedules @@ -42,3 +43,4 @@ if __name__ == '__main__': main(io(settings.dir_news), sync_posts, settings.collection_news) main(io(settings.dir_collective), sync_persons, settings.collection_collective) main(io(settings.dir_gallery), sync_albums, settings.collection_albums) + main(io(settings.dir_pages), sync_pages, settings.collection_pages) diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py index d2a37c33..879a2b4b 100644 --- a/utils/Sync/sync/__init__.py +++ b/utils/Sync/sync/__init__.py @@ -93,6 +93,16 @@ def images_from_html(md): return images +def title_from_html(md): + if not md: + return None + + doc = lxml.html.fromstring(md) + ts = doc.cssselect('h1') + if len(ts): + return ts[0].text + + create_post_from_image_list = lambda images: markdown( '\n'.join(map( lambda i: '![]({img})'.format(img=i), diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py index 483d75f5..14e73d43 100644 --- a/utils/Sync/sync/core/__init__.py +++ b/utils/Sync/sync/core/__init__.py @@ -49,3 +49,6 @@ class Sync: def create_hash(self, document): return None + + def create_remove_query(self, query): + return query diff --git a/utils/Sync/sync/page.py b/utils/Sync/sync/page.py new file mode 100644 index 00000000..3c0217a8 --- /dev/null +++ b/utils/Sync/sync/page.py @@ -0,0 +1,148 @@ +import os + +import settings +from sync.core import Sync +from sync.data import list_images +from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ + synced_images_ids, untouched, title_from_html +from sync.core.post import SyncPost +from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant +from utils.hash import hash_str +from utils.io import read_yaml_md +from utils.text.transform import url_encode_text + + +def create_document_from_folder(provider, path): + document = { + 'file': os.path.basename(path), + 'folder': os.path.dirname(path), + 'title': os.path.basename(path), + } + + manifest = get_manifest(provider, path) + document = { + **document, + **manifest, + } + + if 'url' not in document: + return None + + document = create_id(document) + document = create_hash(provider, document) + return document + + +def get_manifest(provider, path): + data = provider.read(path).read().decode('utf-8') + y, m = read_yaml_md(data) + y = y if y else {} + + if 'date' in y: + y['date'] = create_date(y['date'], settings.date_formats) + + if 'until' in y: + y['until'] = create_date(y['until'], settings.date_formats) + + if 'id' in y: + y['id'] = str(y['id']) + + return { + **y, + 'data': m, + 'title': title_from_html(m), + 'images': images_from_html(m) + } + + +def create_id(document): + new_id = url_encode_text(document['url']) + return { + **document, + 'id': document['id'] if 'id' in document else new_id + } + + +def create_hash(provider, document): + files = sorted([document['file']] + document['images']) + hashes = lmap( + provider.hash, + lmap( + lambda x: os.path.join(document['folder'], x), + files + ) + ) + + return { + **document, + 'hash':hash_str(''.join(hashes)) + } + + +def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True): + sync = Sync() + sync.collection = collection + + documents = list(filter( + lambda x: provider.is_dir(x), + provider.scan('.') + )) + + documents = lmap( + lambda x: provider.type_filter(x, '.md'), + documents + ) + + documents = list(filter( + lambda x: len(x), + documents + )) + + documents = lmap( + lambda x: create_document_from_folder(provider, x[0]), + documents + ) + + documents = list(filter(None, documents)) + + # CREATE SCOPE OF CURRENT SESSION + documents_ids = lmapfn(documents)( + lambda i: i['id'] + ) + + # SKIP UNTOUCHED DOCUMENTS + if skip_unchanged: + documents = untouched(documents, sync) + + # # MAP EVENT MANIFEST -> EVENT_OBJECT + # documents = lmap( + # sync.create, + # documents + # ) + + # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB + documents = lmap( + key_mapper('images', synced_images_ids), + documents + ) + + # SYNC EVENT_OBJECT WITH DB + if update: + documents = lmap( + sync.update, + documents + ) + + documents_to_delete = [] + if delete: + remove_query = sync.create_remove_query({'id': {'$nin': documents_ids}}) + documents_to_delete = sync.query(remove_query) + documents_to_delete = lmap( + sync.delete, + map( + lambda i: {'_id': i['_id']}, + documents_to_delete + ) + ) + + return documents, documents_to_delete diff --git a/utils/Sync/utils/text/transform.py b/utils/Sync/utils/text/transform.py index dc56c219..8aaaed59 100644 --- a/utils/Sync/utils/text/transform.py +++ b/utils/Sync/utils/text/transform.py @@ -5,7 +5,10 @@ from unidecode import unidecode from utils.fn import iterate_over_fns remove_pattr = re.compile('[«»<>,!&"\']') -reduce_dash_pattr = re.compile('[-]+') +reduce_dash_patterns = [ + (re.compile('[-]+'), '-'), + (re.compile('^-'), '') +] def translit(text): @@ -32,7 +35,10 @@ def remove_punctuation(i): def reduce_dashes(i): - return reduce_dash_pattr.sub('-', i) + value = i + for reg_exp, replace in reduce_dash_patterns: + value = reg_exp.sub(replace, value) + return value def space_to_dash(i): @@ -53,6 +59,10 @@ def dot_to_dash(i): return i.replace('.', '-') +def slash_to_dash(i): + return i.replace('/', '-') + + def remove_underscores(i): return i.replace('_', '') @@ -76,6 +86,7 @@ url_encode_text = iterate_over_fns([ space_to_dash, dot_to_dash, short_dash, + slash_to_dash, remove_underscores, remove_punctuation, reduce_dashes,