hudozka/utils/Sync/sync/document.py

import os
from glob import glob

import settings
from db import db
from sync import synced_image_id, untouched
from sync.core.document import SyncDocument
from sync.data.fs import FSProvider
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn, kmap
from utils.io import read_yaml


def sync_documents(update=True, delete=True):
    provider = FSProvider(settings.dir_documents)

    static_path = static_path_fn(settings.dir_static_uploads)

    u, d = main(
        SyncDocument(
            db()[settings.collection_documents],
            provider,
            sizes=settings.image_sizes,
            dir_static_previews=settings.dir_static_images,
            url_base_preview=settings.url_base_preview,
            url_base_document=settings.url_base_document,
        ),
        static_path,
        update_documents=update,
        delete_documents=delete
    )

    print('DELETE DOCUMENTS: %s' % ('NO' if not delete else str(len(d))))
    lprint_json(d)

    print('UPDATE DOCUMENTS: %s' % ('NO' if not update else str(len(u))))
    lprint_json(u)


def main(sync, static_path, update_documents, delete_documents):
    documents = get_documents(sync)
    documents_id = lmap(
        lambda i: i['id'],
        documents
    )

    # CHECKING
    file_names = set(lmap(
        lambda i: i['file'],
        documents
    ))
    if len(file_names) != len(documents):
        raise Exception('File names should be unique')

    # SKIP UNTOUCHED DOCUMENTS
    documents = untouched(documents, sync)

    # COPY FILE -> STATIC_DIR/URL_FILENAME
    for i in documents:
        sync.provider.copy(
            i['file'],
            static_path(i)
        )

    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
    documents = lmap(
        sync.create,
        documents
    )

    if update_documents:
        # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
        documents = lmap(
            key_mapper('preview', lambda i: synced_image_id(i)),
            documents
        )

        # SYNC DOCUMENT_OBJECT WITH DB
        documents = lmap(
            sync.update,
            documents
        )

    # DELETING
    documents_delete = sync.query({'id': {'$nin': documents_id}})
    if delete_documents:
        documents_delete = lmap(
            sync.delete,
            map(
                lambda i: {'_id': i['_id']},
                documents_delete
            )
        )

    return documents, documents_delete


def get_documents(sync):
    # documents = documents_from_yaml()
    documents = documents_from_subdirs(sync.provider)

    # SETUP TITLE BASED ON FILE NAME
    documents = lmapfn(documents)(
        lambda i: {
            **i,
            'title': os.path.splitext(os.path.basename(i['file']))[0]
        }
    )

    # CHOICE BETTER TITLE
    documents = lmapfn(documents)(
        lambda i: {
            **i,
            'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
        }
    )

    documents = lmap(sync.create_id, documents)
    documents = lmap(sync.create_hash, documents)
    documents = lmap(sync.create_url, documents)

    return documents


def static_path_fn(dir):
    return lambda document: os.path.join(dir, os.path.basename(document['url']))


def categorize_files_list(files, title):
    """
    Adds specified category to each file in files list
    :param files:
    :param title:
    :return:
    """
    return map(
        lambda profile: {
            **profile,
            'category': title
        },
        files
    )


def unwrap_manifest(param):
    if isinstance(param, list):
        return combine(map(
            unwrap_manifest,
            param
        ))

    if 'files' in param:
        return lmap(
            unwrap_manifest,
            categorize_files_list(param['files'], param['title'])
        )

    if 'file' not in param:
        return None

    return param


def documents_from_yaml(dirpath):
    # GET DOCUMENT YAML_MANIFEST FILES
    documents = glob(dirpath + '/*.yaml')

    # READ YAML_MANIFEST FILES
    documents = lmap(
        read_yaml,
        documents
    )

    # GET FLAT LIST OF DOCUMENTS
    documents = unwrap_manifest(documents)

    return documents


def documents_from_subdirs(provider):
    documents = lmap(
        lambda i: i.path,
        filter(
            lambda i: i.is_dir(),
            provider.scan('.')
        )
    )

    documents = lmap(
        lambda folder: lmap(
            lambda path: {
                'file': provider.get_rel(path),
                'category': os.path.basename(folder)
            },
            provider.glob(folder + '/*.pdf')
        ),
        documents
    )

    documents = combine(documents)
    return documents


def get_pdf_title(provider, file):
    from PyPDF2 import PdfFileReader
    from PyPDF2.generic import TextStringObject
    from PyPDF2.generic import IndirectObject

    pdf = PdfFileReader(provider.open(file, 'rb'))
    info = pdf.getDocumentInfo()

    if info:
        if type(info.title) == TextStringObject:
            return str(info.title)

        if type(info.title_raw) == IndirectObject:
            o = pdf.getObject(info.title_raw)
            return str(o)
    return None


def until_none(ls):
    better = None
    for i in ls:
        if (i is not None) and i != '':
            better = i
    return better


if __name__ == '__main__':
    sync_documents()
    # sync_documents(delete=False, update=False)

    # DOCUMENT_OBJECT SAMPLE
    # {
    #     "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
    #     "_id": {
    #         "$oid": "56d08411ace9573958e5e497"
    #     },
    #     "type": "document",
    #     "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
    #     "category": "Для поступающих",
    #     "file": {
    #         "size": 53088,
    #         "name": "Договор о пожертвовании.pdf"
    #     },
    #     "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
    #     "preview": {
    #         "$oid": "56d08007ace9573958e5e48f"
    #     },
    #     "title": "Договор о пожертвовании"
    # }