hudozka/utils/Sync/sync/document.py

import os
from glob import glob
from shutil import copyfile
from tempfile import mkstemp

import settings
from db import db
from sync.image import sync_image
from utils.fn import combine, lmap, lprint_json, key_mapper, lprint
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
from utils.io import read_yaml
from utils.text.transform import url_encode_text, url_encode_file

prefix = 'document'
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
url_base_document = 'https://static.shburg.org/art/uploads/{file}'

# Adds specified category to each file in files list
unwrap_file_list = lambda files, title: map(
    lambda profile: {'category': title, **profile},
    files
)


def pdf_to_jpg(pdf):
    cwd = os.getcwd()
    abspdf = os.path.join(cwd, pdf)
    _, temp = mkstemp('.jpg')
    image_magick_pdf_to_img(abspdf, temp)
    return temp


def sync_document(record):
    if isinstance(record, list):
        return lmap(
            sync_document,
            record
        )

    q = {'id': record['id']}
    try:
        documents = db().documents
        update_result = documents.update_one(q, {'$set': record}, upsert=True)
        i = documents.find_one({'id': record['id']})
        return i
    except ValueError:
        pass

    return None


def read_document(i, query_fn=None):
    q = query_fn(i) if query_fn else {'id': i['id']}
    try:
        return db().documents.find_one(q)
    except ValueError:
        pass

    return None


def query_documents(q):
    return db().documents.find(q)


def delete_document(q):
    return db().documents.find_one_and_delete(q)


def create_preview(pdf, sizes, preview_dir):
    temp_preview_path = pdf_to_jpg(pdf)

    id = url_encode_text(pdf)
    url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)

    img = create_image(temp_preview_path, sizes, url, preview_dir)
    os.remove(temp_preview_path)
    return img


def create_document(doc):
    sizes = settings.image_sizes
    file = doc['file']

    doc['type'] = prefix
    doc['url'] = url_base_document.format(file=url_encode_file(file))
    doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
    doc['file'] = {
        'name': file,
        'size': os.path.getsize((file))
    }

    return doc


def create_id(doc):
    return url_encode_text('{type}-{category}-{file}'.format(
        type=prefix,
        category=doc['category'],
        file=doc['file']
    ))


def unwrap_manifest(param):
    if isinstance(param, list):
        return combine(map(
            unwrap_manifest,
            param
        ))

    if 'files' in param:
        return lmap(
            unwrap_manifest,
            unwrap_file_list(param['files'], param['title'])
        )

    if 'file' not in param:
        return None

    return param


if __name__ == '__main__':
    # ALL DOCUMENTS IDS FOUNDED
    scope_documents_ids = []

    # GET DOCUMENT YAML_MANIFEST FILES
    os.chdir(dir_documents)
    documents = glob(dir_documents + '/*.yaml')

    # READ YAML_MANIFEST FILES
    documents = lmap(
        read_yaml,
        documents
    )

    # GET FLAT LIST OF DOCUMENTS
    documents = unwrap_manifest(documents)

    # CREATE DOCUMENT IDENTITY
    documents = lmap(
        lambda i: {**i, 'id': create_id(i)},
        documents
    )

    # CREATE HASH OF DOCUMENT FILE
    documents = lmap(
        lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
        documents
    )

    # CREATE HASH OF DOCUMENT FILE
    scope_documents_ids = lmap(
        lambda i: i['id'],
        documents
    )

    # SKIP UNTOUCHED DOCUMENTS
    documents = lmap(
        lambda i: i[0],
        filter(
            lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
            lmap(
                lambda document: (document, read_document(document)),
                documents
            )
        ))

    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
    documents = lmap(
        create_document,
        documents
    )

    # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
    documents = lmap(
        key_mapper('preview', lambda i: sync_image(i)['_id']),
        documents
    )

    # COPY FILE -> STATIC_DIR/URL_FILENAME
    out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
    lmap(
        lambda doc: copyfile(
            doc['file']['name'],
            out_path(doc)
        ),
        documents
    )

    # SYNC DOCUMENT_OBJECT WITH DB
    documents = lmap(
        sync_document,
        documents
    )

    documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
    documents_to_remove = lmap(
        delete_document,
        map(
            lambda i: {'_id': i['_id']},
            documents_to_remove
        )
    )

    # SCOPE
    print('SCOPE:')
    lprint(scope_documents_ids)

    # DELETE
    print('DELETE DOCUMENTS:')
    lprint_json(documents_to_remove)

    # DELETE
    print('UPDATE DOCUMENTS:')
    lprint_json(documents)
    print('[SYNC DOCUMENTS DONE]')

    # DOCUMENT_OBJECT SAMPLE
    # {
    #     "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
    #     "_id": {
    #         "$oid": "56d08411ace9573958e5e497"
    #     },
    #     "type": "document",
    #     "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
    #     "category": "Для поступающих",
    #     "file": {
    #         "size": 53088,
    #         "name": "Договор о пожертвовании.pdf"
    #     },
    #     "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
    #     "preview": {
    #         "$oid": "56d08007ace9573958e5e48f"
    #     },
    #     "title": "Договор о пожертвовании"
    # }