[sync] + pages support

utils/Sync/settings.py

@@ -97,6 +97,7 @@ collection_events = 'events'
 collection_news = 'timeline'
 collection_collective = 'collective'
 collection_albums = 'albums'
+collection_pages = 'pages'
 
 sync_provider_type = env('SYNC_PROVIDER', 'fs')
 
@@ -112,6 +113,7 @@ dir_events = f('Hudozka/Site/Events')
 dir_news = f('Hudozka/Site/News')
 dir_collective = f('Hudozka/Site/Collective')
 dir_gallery = f('Hudozka/Site/Gallery')
+dir_pages = f('Hudozka/Site/Pages')
 
 dir_static = env('SYNC_STATIC', '~/Hudozka Static')
 dir_static_uploads = absolute(dir_static + '/uploads')

utils/Sync/sync.py

@@ -4,6 +4,7 @@ from sync.album import sync_albums
 from sync.data.fs import FSProvider
 from sync.data.yandexdisk import YDProvider
 from sync.document import sync_documents
+from sync.page import sync_pages
 from sync.person import sync_persons
 from sync.post import sync_posts
 from sync.schedule import sync_schedules
@@ -42,3 +43,4 @@ if __name__ == '__main__':
     main(io(settings.dir_news), sync_posts, settings.collection_news)
     main(io(settings.dir_collective), sync_persons, settings.collection_collective)
     main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
+    main(io(settings.dir_pages), sync_pages, settings.collection_pages)

utils/Sync/sync/__init__.py

@@ -93,6 +93,16 @@ def images_from_html(md):
     return images
 
 
+def title_from_html(md):
+    if not md:
+        return None
+
+    doc = lxml.html.fromstring(md)
+    ts = doc.cssselect('h1')
+    if len(ts):
+        return ts[0].text
+
+
 create_post_from_image_list = lambda images: markdown(
     '\n'.join(map(
         lambda i: '![]({img})'.format(img=i),

utils/Sync/sync/core/__init__.py

@@ -49,3 +49,6 @@ class Sync:
 
     def create_hash(self, document):
         return None
+
+    def create_remove_query(self, query):
+        return query

utils/Sync/sync/page.py

@@ -0,0 +1,148 @@
+import os
+
+import settings
+from sync.core import Sync
+from sync.data import list_images
+from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
+    synced_images_ids, untouched, title_from_html
+from sync.core.post import SyncPost
+from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
+from utils.hash import hash_str
+from utils.io import read_yaml_md
+from utils.text.transform import url_encode_text
+
+
+def create_document_from_folder(provider, path):
+    document = {
+        'file': os.path.basename(path),
+        'folder': os.path.dirname(path),
+        'title': os.path.basename(path),
+    }
+
+    manifest = get_manifest(provider, path)
+    document = {
+        **document,
+        **manifest,
+    }
+
+    if 'url' not in document:
+        return None
+
+    document = create_id(document)
+    document = create_hash(provider, document)
+    return document
+
+
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    y, m = read_yaml_md(data)
+    y = y if y else {}
+
+    if 'date' in y:
+        y['date'] = create_date(y['date'], settings.date_formats)
+
+    if 'until' in y:
+        y['until'] = create_date(y['until'], settings.date_formats)
+
+    if 'id' in y:
+        y['id'] = str(y['id'])
+
+    return {
+        **y,
+        'data': m,
+        'title': title_from_html(m),
+        'images': images_from_html(m)
+    }
+
+
+def create_id(document):
+    new_id = url_encode_text(document['url'])
+    return {
+        **document,
+        'id': document['id'] if 'id' in document else new_id
+    }
+
+
+def create_hash(provider, document):
+    files = sorted([document['file']] + document['images'])
+    hashes = lmap(
+        provider.hash,
+        lmap(
+            lambda x: os.path.join(document['folder'], x),
+            files
+        )
+    )
+
+    return {
+        **document,
+        'hash':hash_str(''.join(hashes))
+    }
+
+
+def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True):
+    sync = Sync()
+    sync.collection = collection
+
+    documents = list(filter(
+        lambda x: provider.is_dir(x),
+        provider.scan('.')
+    ))
+
+    documents = lmap(
+        lambda x: provider.type_filter(x, '.md'),
+        documents
+    )
+
+    documents = list(filter(
+        lambda x: len(x),
+        documents
+    ))
+
+    documents = lmap(
+        lambda x: create_document_from_folder(provider, x[0]),
+        documents
+    )
+
+    documents = list(filter(None, documents))
+
+    # CREATE SCOPE OF CURRENT SESSION
+    documents_ids = lmapfn(documents)(
+        lambda i: i['id']
+    )
+
+    # SKIP UNTOUCHED DOCUMENTS
+    if skip_unchanged:
+        documents = untouched(documents, sync)
+
+    # # MAP EVENT MANIFEST -> EVENT_OBJECT
+    # documents = lmap(
+    #     sync.create,
+    #     documents
+    # )
+
+    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
+    documents = lmap(
+        key_mapper('images', synced_images_ids),
+        documents
+    )
+
+    # SYNC EVENT_OBJECT WITH DB
+    if update:
+        documents = lmap(
+            sync.update,
+            documents
+        )
+
+    documents_to_delete = []
+    if delete:
+        remove_query = sync.create_remove_query({'id': {'$nin': documents_ids}})
+        documents_to_delete = sync.query(remove_query)
+        documents_to_delete = lmap(
+            sync.delete,
+            map(
+                lambda i: {'_id': i['_id']},
+                documents_to_delete
+            )
+        )
+
+    return documents, documents_to_delete

utils/Sync/utils/text/transform.py

@@ -5,7 +5,10 @@ from unidecode import unidecode
 from utils.fn import iterate_over_fns
 
 remove_pattr = re.compile('[«»<>,!&"\']')
-reduce_dash_pattr = re.compile('[-]+')
+reduce_dash_patterns = [
+    (re.compile('[-]+'), '-'),
+    (re.compile('^-'), '')
+]
 
 
 def translit(text):
@@ -32,7 +35,10 @@ def remove_punctuation(i):
 
 
 def reduce_dashes(i):
-    return reduce_dash_pattr.sub('-', i)
+    value = i
+    for reg_exp, replace in reduce_dash_patterns:
+        value = reg_exp.sub(replace, value)
+    return value
 
 
 def space_to_dash(i):
@@ -53,6 +59,10 @@ def dot_to_dash(i):
     return i.replace('.', '-')
 
 
+def slash_to_dash(i):
+    return i.replace('/', '-')
+
+
 def remove_underscores(i):
     return i.replace('_', '')
 
@@ -76,6 +86,7 @@ url_encode_text = iterate_over_fns([
     space_to_dash,
     dot_to_dash,
     short_dash,
+    slash_to_dash,
     remove_underscores,
     remove_punctuation,
     reduce_dashes,