hudozka/utils/Sync/sync/models/Article.py

import logging

import os

import lxml.html

import kazimir
import settings
from db import collection
from kazimir import kazimir_to_html
from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date
from sync.data import Provider, list_images
from sync.models import Model
from sync.models.Image import Image

from utils.fn import swap_ext, first, map_cases, constant, combine
from utils.hash import hash_str
from utils.io import read_yaml_md, parse_yaml_front_matter
from utils.text.transform import url_encode_text

logger = logging.getLogger(settings.name + '.Article')
store = collection(settings.collection_articles)


class Article(Model):
    @staticmethod
    async def find(query):
        return store.find(query)

    @staticmethod
    async def delete(query):
        return store.find_one_and_delete(query)

    @staticmethod
    async def scan(provider):
        documents = [i for i in provider.scan('.') if provider.is_dir(i)]
        documents = [Article.read(provider, i) for i in documents]
        return documents

    @staticmethod
    def read(provider, path):
        file_time_formats = ['%Y.%m.%d']

        # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
        doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
        document = {
            'folder': path,
            'date': doc_date,
            'title': doc_title,
        }

        # CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
        images = list_images(provider, path)
        images = [os.path.relpath(i, path) for i in images]
        document['images'] = images
        document['post'] = create_post_from_image_list(images)

        # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
        manifest_file = md_from_folder(provider, path)
        if manifest_file:
            manifest = get_manifest(provider, manifest_file)
        else:
            manifest = {}

        return Article(
            provider,
            path,
            params={
                **document,
                **manifest,
            },
        )

    def __init__(self, provider, file, params=None):
        # self.__id_template: str = '{category}-{file}'
        # self.__url_template: str = settings.document_url_template
        # self.__url_preview_template: str = settings.document_url_preview_template

        self.origin = settings.origin
        self.version = '2'
        self.post = None
        self.images = None
        super().__init__(provider, store, file, params=params)

    def init(self):
        self.__set_id()
        self.__set_hash()

    def validate(self):
        document = self.params
        if 'date' not in document or not document['date']:
            return None

        if 'title' not in document or not document['title']:
            return None

    async def build(self, **kwargs):
        sizes = kwargs['sizes']
        post, images = await create_post(self.provider, self.id, self.params, sizes)
        self.post = post
        self.images = images

        post, images = await create_post(self.provider, self.id, self.params, sizes)
        self.post = kazimir.html_from_tree(post)
        self.images = images

        if 'preview' in self.params:
            preview = self.get_param('preview')
            preview = self._get_relpath(preview)
            preview = await Image.new(self.provider, preview, sizes)
            self.preview = preview
        elif len(self.images):
            self.preview = self.images[0]
        else:
            self.preview = 'https://art.shlisselburg.org/entrance.jpg'

    def bake(self):
        return {
            **self.params,
            'images': [i.id for i in self.images],
            'post': self.post,
            'id': self.id,
            'hash': self.hash,
            'file': self.file,
            'origin': self.origin,
            'version': self.version,
            'preview': self.preview if isinstance(self.preview, str) else self.preview.id,
        }

    def __set_id(self):
        if 'id' in self.params:
            self.id = self.params['id']
        else:
            self.id = url_encode_text(self.params['title'])

    def __set_hash(self):
        f = self.params['folder']
        images = sorted(self.params['images'])
        images = [os.path.join(f, i) for i in images]
        images = [self.provider.hash(i) for i in images]

        self.hash = hash_str(
            combine([hash_str(self.params)] + images)
        )

    def __str__(self):
        return '<Article file={} id={}>'.format(self.file, self.id)


def md_from_folder(provider, i):
    return first(provider.type_filter(i, '.md'))


def get_manifest(provider, path):
    data = provider.read(path).read().decode('utf-8')
    manifest = parse_yaml_front_matter(data)

    if 'date' in manifest:
        manifest['date'] = create_date(manifest['date'], settings.date_formats)

    if 'until' in manifest:
        manifest['until'] = create_date(manifest['until'], settings.date_formats)

    if 'id' in manifest:
        manifest['id'] = str(manifest['id'])

    html = kazimir_to_html(manifest['content'])
    return {
        **manifest,
        'post': html,
        'images': images_from_html(html)  # Add images for hashing
    }


async def create_post(provider, doc_id, document, sizes):
    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'

    post_html = lxml.html.fromstring(document['post'])
    images = []
    for img in post_html.cssselect('img'):
        src = img.get('src')
        relative_image_path = os.path.join(document['folder'], src)

        img_path = provider.get_local(relative_image_path)
        if os.path.exists(img_path):
            img_id = url_encode_text(os.path.splitext(src)[0])
            url_fn = lambda file, size, ext: image_url_base.format(
                id=doc_id,
                img=img_id,
                size=size,
                ext=ext.lower()
            )

            image = await Image.new(provider, img_path, sizes, url_fn)
            if image:
                url = image.get_size('big')['url']
                images.append(image)
                img.set('src', url)
            else:
                logger.error('Fail to get Image', document['folder'], src)
    return post_html, images