[Sync] + new Article sync

utils/Sync/sync.py

@@ -1,6 +1,7 @@
 import settings
 from db import collection
 from sync.core import Sync
+from sync.core.SyncArticle import SyncArticle
 from sync.core.SyncDocument import SyncDocument
 from sync.core.SyncPage import SyncPage
 from sync.core.SyncPerson import SyncPerson
@@ -57,17 +58,22 @@ async def run(run_interval=0):
         collection=c(settings.collection_collective),
     )
 
+    articles = SyncArticle(
+        provider=io(settings.dir_articles),
+        collection=c(settings.collection_articles),
+    )
+
     while True:
         # main(io(settings.dir_documents), sync_documents, settings.collection_documents)
         # main(io(settings.dir_awards), sync_documents, settings.collection_awards)
         # main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
-        # main(io(settings.dir_articles), sync_posts, settings.collection_articles)
         # main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
 
         await asyncio.wait([
             main(documents),
             main(pages),
             main(persons),
+            main(articles),
         ])
 
         if run_interval == 0:

utils/Sync/sync/core/SyncArticle.py

@@ -0,0 +1,58 @@
+import os
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import Provider
+from sync.models.Article import Article
+from sync.models.Person import Person
+
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Article')
+
+
+class SyncArticle(Sync):
+    async def run(self):
+        logger.info('Checking for update')
+
+        documents = await Article.scan(self.provider)
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Item(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        documents = await untouched(documents)
+        logger.info('Changed {} Items(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Items to update')
+
+            for document in documents:
+                logger.info('Updating Item {}'.format(document.id))
+
+                await document.build(
+                    sizes=settings.image_sizes,
+                )
+                await document.save()
+
+                logger.info('Updated Item {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Items to delete')
+            else:
+                for document in documents_delete:
+                    logger.info('Deleting Item {}'.format(document['id']))
+
+                    q = {'_id': document['_id']}
+                    self.delete(q)
+
+        # await self.clean()

utils/Sync/sync/core/post.py

@@ -1,92 +0,0 @@
-import os
-from tempfile import mkstemp
-
-import lxml.html
-
-import kazimir
-from sync.core import Sync
-from utils.fn import last_good, lmap, combine
-from utils.hash import hash_str, hash_file
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class SyncPost(Sync):
-    def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.sizes = sizes
-        self.type = document_type
-        self.image_url_base = image_url_base
-        self.dir_local_images = dir_local_images
-        self.origin = origin
-
-    def create_id(self, document):
-        new_id = url_encode_text(document['title'])
-        document['id'] = document['id'] if 'id' in document else new_id
-        return document
-
-    def create_hash(self, document):
-        files = sorted(document['images'])
-        hashes = lmap(
-            self.provider.hash,
-            lmap(
-                lambda i: os.path.join(document['folder'], i),
-                files
-            )
-        )
-
-        document['hash'] = hash_str(
-            combine([hash_str(document)] + hashes)
-        )
-
-        return document
-
-    def create(self, document):
-        if 'date' not in document or not document['date']:
-            return None
-
-        if 'title' not in document or not document['title']:
-            return None
-
-        if self.type:
-            document['type'] = self.type
-
-        post_html = lxml.html.fromstring(document['post'])
-        images = []
-        for img in post_html.cssselect('img'):
-            src = img.get('src')
-            relative_image_path = os.path.join(document['folder'], src)
-
-            img_path = self.provider.get_local(relative_image_path)
-            if os.path.exists(img_path):
-                img_id = url_encode_text(os.path.splitext(src)[0])
-                url_fn = lambda size, ext: self.image_url_base.format(
-                    id=document['id'],
-                    img=img_id,
-                    size=size,
-                    ext=ext.lower()
-                )
-
-                image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
-                if image:
-                    images.append(image)
-                    img.set('src', image['data']['big']['url'])
-                else:
-                    print('fail: ', document['folder'], src)
-
-        document['post'] = kazimir.html_from_tree(post_html)
-        document['images'] = images
-        document['origin'] = self.origin
-        return document
-
-    def create_remove_query(self, query):
-        if self.type:
-            query = {
-                **query,
-                'type': self.type,
-                'origin': self.origin
-            }
-        return query

utils/Sync/sync/models/Article.py

@@ -0,0 +1,192 @@
+import logging
+
+import os
+
+import lxml.html
+
+import kazimir
+import settings
+from db import collection
+from kazimir import kazimir_to_html
+from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date
+from sync.data import Provider, list_images
+from sync.models import Model
+from sync.models.Image import Image
+
+from utils.fn import swap_ext, first, map_cases, constant, combine
+from utils.hash import hash_str
+from utils.io import read_yaml_md, parse_yaml_front_matter
+from utils.text.transform import url_encode_text
+
+logger = logging.getLogger(settings.name + '.Person')
+store = collection(settings.collection_articles)
+
+
+def find(item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Article(Model):
+    @staticmethod
+    async def scan(provider):
+        documents = [i for i in provider.scan('.') if provider.is_dir(i)]
+        documents = [Article.read(provider, i) for i in documents]
+        return documents
+
+    @staticmethod
+    def read(provider, path):
+        file_time_formats = ['%Y.%m.%d']
+
+        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
+        doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
+        document = {
+            'folder': path,
+            'date': doc_date,
+            'title': doc_title,
+        }
+
+        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
+        images = list_images(provider, path)
+        images = [os.path.relpath(i, path) for i in images]
+        document['images'] = images
+        document['post'] = create_post_from_image_list(images)
+
+        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
+        manifest_file = md_from_folder(provider, path)
+        if manifest_file:
+            manifest = get_manifest(provider, manifest_file)
+        else:
+            manifest = {}
+
+        return Article(
+            provider,
+            path,
+            params={
+                **document,
+                **manifest,
+            },
+        )
+
+    def __init__(self, provider, file, params=None):
+        # self.__id_template: str = '{category}-{file}'
+        # self.__url_template: str = settings.document_url_template
+        # self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.params = params if params else {}
+        self.origin = settings.origin
+        self.version = '2'
+        self.post = None
+        self.images = None
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.__set_id()
+        self.__set_hash()
+
+    def validate(self):
+        document = self.params
+        if 'date' not in document or not document['date']:
+            return None
+
+        if 'title' not in document or not document['title']:
+            return None
+
+    async def build(self, **kwargs):
+        sizes = kwargs['sizes']
+        post, images = await create_post(self.provider, self.id, self.params, sizes)
+        self.post = post
+        self.images = images
+
+        post, images = await create_post(self.provider, self.id, self.params, sizes)
+        self.post = kazimir.html_from_tree(post)
+        self.images = images
+
+    def bake(self):
+        return {
+            **self.params,
+            'images': [i.id for i in self.images],
+            'post': self.post,
+            'id': self.id,
+            'hash': self.hash,
+            'file': self.file,
+            'origin': self.origin,
+            'version': self.version,
+        }
+
+    def __set_id(self):
+        if 'id' in self.params:
+            self.id = self.params['id']
+        else:
+            self.id = url_encode_text(self.params['title'])
+
+    def __set_hash(self):
+        f = self.params['folder']
+        images = sorted(self.params['images'])
+        images = [os.path.join(f, i) for i in images]
+        images = [self.provider.hash(i) for i in images]
+
+        self.hash = hash_str(
+            combine([hash_str(self.params)] + images)
+        )
+
+    def __str__(self):
+        return '<Article hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def md_from_folder(provider, i):
+    return first(provider.type_filter(i, '.md'))
+
+
+def get_manifest(provider, path):
+    data = provider.read(path).read().decode('utf-8')
+    manifest = parse_yaml_front_matter(data)
+
+    if 'date' in manifest:
+        manifest['date'] = create_date(manifest['date'], settings.date_formats)
+
+    if 'until' in manifest:
+        manifest['until'] = create_date(manifest['until'], settings.date_formats)
+
+    if 'id' in manifest:
+        manifest['id'] = str(manifest['id'])
+
+    html = kazimir_to_html(manifest['content'])
+    return {
+        **manifest,
+        'post': html,
+        'images': images_from_html(html)  # Add images for hashing
+    }
+
+
+async def create_post(provider, doc_id, document, sizes):
+    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
+
+    post_html = lxml.html.fromstring(document['post'])
+    images = []
+    for img in post_html.cssselect('img'):
+        src = img.get('src')
+        relative_image_path = os.path.join(document['folder'], src)
+
+        img_path = provider.get_local(relative_image_path)
+        if os.path.exists(img_path):
+            img_id = url_encode_text(os.path.splitext(src)[0])
+            url_fn = lambda file, size, ext: image_url_base.format(
+                id=doc_id,
+                img=img_id,
+                size=size,
+                ext=ext.lower()
+            )
+
+            image = await Image.new(provider, img_path, sizes, url_fn)
+            if image:
+                url = image.get_size('big')['url']
+                images.append(image)
+                img.set('src', url)
+            else:
+                logger.error('Fail to get Image', document['folder'], src)
+    return post_html, images

utils/Sync/sync/post.py

@@ -1,192 +0,0 @@
-import os
-
-import settings
-from kazimir import kazimir_to_html
-from sync.data import list_images
-from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
-    synced_images_ids, untouched
-from sync.core.post import SyncPost
-from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
-from utils.io import read_yaml_md, parse_yaml_front_matter
-
-
-def create_document_from_folder(sync, path, file_time_formats):
-    # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
-    doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
-    document = {
-        'folder': path,
-        'date': doc_date,
-        'title': doc_title,
-    }
-
-    # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
-    images = lmap(
-        lambda img: os.path.relpath(img, path),
-        list_images(sync.provider, path)
-    )
-    post = create_post_from_image_list(images)
-
-    # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
-    params = map_cases(
-        document,
-        [(
-            lambda doc: md_from_folder(sync.provider, path),
-            lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)),
-        )],
-        constant({})
-    )
-
-    document = {
-        'images': images,
-        'post': post,
-        **document,
-        **params,
-    }
-
-    document = sync.create_id(document)
-    document = sync.create_hash(document)
-    return document
-
-
-def get_manifest(provider, path):
-    data = provider.read(path).read().decode('utf-8')
-    manifest = parse_yaml_front_matter(data)
-
-    if 'date' in manifest:
-        manifest['date'] = create_date(manifest['date'], settings.date_formats)
-
-    if 'until' in manifest:
-        manifest['until'] = create_date(manifest['until'], settings.date_formats)
-
-    if 'id' in manifest:
-        manifest['id'] = str(manifest['id'])
-
-    html = kazimir_to_html(manifest['content'])
-    return {
-        **manifest,
-        'post': html,
-        'images': images_from_html(html)  # Add images for hashing
-    }
-
-
-def md_from_folder(provider, i):
-    return first(provider.type_filter(i, '.md'))
-
-
-def get_folder_documents(sync, file_time_formats):
-    # GET EVENT FOLDERS
-    documents = lmap(
-        lambda i: i,
-        filter(
-            lambda i: sync.provider.is_dir(i),
-            sync.provider.scan('.')
-        )
-    )
-
-    return lmap(
-        lambda doc: create_document_from_folder(sync, doc, file_time_formats),
-        documents
-    )
-
-
-# GET EVENT FILES
-def get_file_documents(sync, file_time_formats):
-    documents = sync.provider.type_filter('.', '.md')
-    documents = lmap(
-        lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats),
-        documents
-    )
-
-    documents = lmap(
-        lambda i: {
-            'file': i[0],
-            'date': i[1],
-            'title': i[2],
-        },
-        documents
-    )
-
-    # OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST
-    documents = lmap(
-        lambda i: {
-            **i,
-            **get_manifest(sync.provider, i['file'])
-        },
-        documents
-    )
-
-    # CREATE DOCUMENT IDENTITY
-    documents = lmap(sync.create_id, documents)
-
-    # CREATE HASH OF DOCUMENT FILE
-    documents = lmap(sync.create_hash, documents)
-
-    return documents
-
-
-def main(sync, file_time_formats, update_documents=True, delete_documents=True):
-    documents = get_folder_documents(sync, file_time_formats)
-    documents += get_file_documents(sync, file_time_formats)
-
-    # CREATE SCOPE OF CURRENT SESSION
-    scope_documents_ids = lmapfn(documents)(
-        lambda i: i['id']
-    )
-
-    # SKIP UNTOUCHED DOCUMENTS
-    # documents = untouched(documents, sync)
-
-    # MAP EVENT MANIFEST -> EVENT_OBJECT
-    documents = lmap(
-        sync.create,
-        documents
-    )
-
-    # FILTER INCORRECT EVENTS
-    documents = list(filter(None, documents))
-
-    # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
-    documents = lmap(
-        key_mapper('images', synced_images_ids),
-        documents
-    )
-
-    # SYNC EVENT_OBJECT WITH DB
-    if update_documents:
-        documents = lmap(
-            sync.update,
-            documents
-        )
-
-    documents_to_delete = []
-    if delete_documents:
-        remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
-        documents_to_delete = sync.query(remove_query)
-        documents_to_delete = lmap(
-            sync.delete,
-            map(
-                lambda i: {'_id': i['_id']},
-                documents_to_delete
-            )
-        )
-
-    return documents, documents_to_delete
-
-
-def sync_posts(provider, collection, update=True, delete=True):
-    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
-
-    return main(
-        SyncPost(
-            collection,
-            provider,
-            'post',
-            image_url_base,
-            settings.dir_static_images,
-            settings.event_image_sizes,
-            origin=settings.origin
-        ),
-        file_time_formats=['%Y.%m.%d'],
-        update_documents=update,
-        delete_documents=delete
-    )