hudozka/utils/Sync/sync/document.py

import os

import requests

import settings
from sync import synced_image_id, untouched
from sync.core.document import SyncDocument
from sync.data import scan_subdirs
from sync.models.document import Document
from utils.fn import combine, lmap, key_mapper


def sync_documents(provider, collection, update=True, delete=True):
    """
    DOCUMENT_OBJECT SAMPLE
    {
        "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
        "_id": {
            "$oid": "56d08411ace9573958e5e497"
        },
        "type": "document",
        "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
        "category": "Для поступающих",
        "file": {
            "size": 53088,
            "name": "Договор о пожертвовании.pdf"
        },
        "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
        "preview": {
            "$oid": "56d08007ace9573958e5e48f"
        },
        "title": "Договор о пожертвовании"
    }

    :param provider:
    :param update:
    :param delete:
    :return:
    """

    static_path = static_path_fn(settings.dir_static_uploads)

    return main(
        SyncDocument(
            collection,
            provider,
            sizes=settings.image_sizes,
            dir_static_previews=settings.dir_static_images,
            url_base_preview=settings.url_base_preview,
            url_base_document=settings.url_base_document,
        ),
        static_path,
        update_documents=update,
        delete_documents=delete
    )


def upload(provider, document):
    f = document['file']
    p = provider.get_local(f)
    n = os.path.basename(document['url'])

    files = {'file': open(p, 'rb')}
    # values = {'DB': 'photcat', 'OUT': 'csv', 'SHORT': 'short'}

    r = requests.put(
        url='https://static.shlisselburg.org/upload/art/uploads/{}'.format(n),
        files=files,
        auth=('uploader', '1207690962638465')
    )
    if r.status_code == 200:
        print('Document {} uploaded'.format(document['url']))
    else:
        print('Failed to upload document {}'.format(document['url']))
        print('https://static.shlisselburg.org/upload/art/uploads/{}'.format(n))
        print(r.request.headers)
        print(r.headers)
        print(r.text)
        print()


def main(sync, static_path, update_documents, delete_documents):
    documents = scan_subdirs(sync.provider, '.pdf')
    documents = lmap(
        lambda document: read_document(sync, document),
        documents
    )
    documents_id = lmap(
        lambda document: document['id'],
        documents
    )

    # CHECKING
    file_names = set(lmap(
        lambda document: document['file'],
        documents
    ))
    if len(file_names) != len(documents):
        raise Exception('File names should be unique')

    # SKIP UNTOUCHED DOCUMENTS
    documents = untouched(documents, sync)

    # COPY FILE -> STATIC_DIR/URL_FILENAME
    for i in documents:
        sync.provider.copy(
            i['file'],
            static_path(i)
        )

    for i in documents:
        upload(sync.provider, i)

    # MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
    documents = lmap(
        sync.create,
        documents
    )

    if update_documents:
        # REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
        documents = lmap(
            key_mapper('preview', lambda img: synced_image_id(img)),
            documents
        )

        # SYNC DOCUMENT_OBJECT WITH DB
        documents = lmap(
            sync.update,
            documents
        )

    # DELETING
    documents_delete = sync.query({'id': {'$nin': documents_id}})
    if delete_documents:
        documents_delete = lmap(
            sync.delete,
            map(
                lambda document: {'_id': document['_id']},
                documents_delete
            )
        )

    return documents, documents_delete


def read_document(sync, path):
    name = os.path.basename(path)

    document = {
        'file': path,
        'category': os.path.dirname(path),
        'title': os.path.splitext(name)[0]
    }

    document = sync.create_id(document)
    document = sync.create_url(document)
    document = sync.create_hash(document)

    return document


def create_document(provider, m):
    # dir_static_previews = settings.dir_static_images,
    # url_base_preview = settings.url_base_preview,
    # url_base_document = settings.url_base_document
    return Document(provider, m, {})


def static_path_fn(dir_path):
    return lambda document: os.path.join(dir_path, os.path.basename(document['url']))


def categorize_files_list(files, title):
    """
    Adds specified category to each file in files list
    :param files:
    :param title:
    :return:
    """
    return map(
        lambda profile: {
            **profile,
            'category': title
        },
        files
    )


def unwrap_manifest(param):
    if isinstance(param, list):
        return combine(map(
            unwrap_manifest,
            param
        ))

    if 'files' in param:
        return lmap(
            unwrap_manifest,
            categorize_files_list(param['files'], param['title'])
        )

    if 'file' not in param:
        return None

    return param