hudozka/modules/Fetch/fetch.py

#!/usr/local/bin/python3

import json
import os
import re
import string
from datetime import datetime
from functools import reduce
from os import makedirs

import json5
import sys

import requests

from utils import until, copy
from vk import vk_init, vk_wall_get_by_id, vk_photos_get_albums, vk_photos_get

config = json5.load(open('config.json5', 'r'))
vk_init(config['vk']['appId'], config['vk']['appSecret'])

pattern = re.compile(r'https?://vk\.com/wall([-\d]+)_(\d+)')
ms = []

size_priority = ['w', 'z', 'y']


# get_album_item = until(size_priority)


def get_item_by_size(size, item):
    for s in item:
        if s['type'] == size:
            return s

    return None


def get_album_item(item):
    for s in size_priority:
        img = get_item_by_size(s, item)
        if img:
            return img
    return None


def is_valid_url(i: str):
    return pattern.match(i) is not None


def add_middleware(var: str, fn):
    ms.append((var, fn))


def apply_middleware(data: str, mw: tuple):
    key, fn = mw
    value = fn(data)
    form = {}
    form[mw[0]] = ''
    data.format(**form)
    return data


def apply_middlewares(output_dir, template, data):
    result = template
    for mw in ms:
        key, fn = mw
        form = {key: fn(output_dir, data)}
        result = string.Template(result)
        result = result.safe_substitute(**form)

    return result


def attach_album(output_dir, data):
    """
    {
        'size': 13,
         'title': 'Каллиграфия 27.11.2016',
         'owner_id': -132442859,
         'updated': 1480340404,
         'description': '',
         'id': '238711857',
         'thumb': {'user_id': 100, 'access_key': '428f3dad13bdd5d730', 'owner_id': -132442859,
                   'date': 1480340404,
                   'photo_807': 'https://pp.vk.me/c837134/v837134463/11056/ZaoO1h5Idaw.jpg',
                   'id': 456239050,
                   'width': 1024, 'photo_75': 'https://pp.vk.me/c837134/v837134463/11053/aHOALfgr0VU.jpg',
                   'photo_130': 'https://pp.vk.me/c837134/v837134463/11054/czcuc7o-bN4.jpg', 'text': '',
                   'photo_604': 'https://pp.vk.me/c837134/v837134463/11055/P7yUE6WitSA.jpg', 'height': 683,
                   'photo_1280': 'https://pp.vk.me/c837134/v837134463/11057/eLG5Akgx3fk.jpg',
                   'album_id': 238711857
        },
        'created': 1480340311
    }
    :param data:
    :return:
    """
    oid = data['owner_id']
    aid = data['id']

    album = vk_photos_get(oid, aid)['response']['items']

    files = []
    for file in album:
        img = get_album_item(file['sizes'])
        if img:
            files.append(img['src'])
            # files.append(file['sizes'])

    download_files(files, output_dir)
    return '\n'.join(list(
        map(
            lambda i: '![](%s)' % i,
            map(
                lambda f: os.path.basename(f),
                files
            )
        )
    ))


def get_date(ts: int):
    date = datetime.fromtimestamp(ts)
    return date.strftime("%Y.%m.%d")


def get_folder_name(post):
    date_formatted = get_date(post['date'])

    return '{date} vk-{owner_id}-{id}'.format(
        date=date_formatted,
        owner_id=post['owner_id'],
        id=post['id'],
    )


def download_files(files, output):
    for url in files:
        file_out = os.path.join(output, os.path.basename(url))
        req = requests.get(url)
        data = req.content
        f = open(file_out, 'wb')
        f.write(data)
        f.close()


attachment_types = {
    'album': attach_album
}


def get_post_attachments(output_dir, data):
    attachments = []
    for a in data['attachments']:
        a_type = a['type']
        if a_type in attachment_types:
            fn = attachment_types[a_type]

            attachment = fn(output_dir, a[a_type])
            attachments.append(attachment)

    return '\n'.join(attachments)


if __name__ == '__main__':
    add_middleware('id', lambda _, data: 'vk-{owner_id}-{id}'.format(
        owner_id=data['owner_id'],
        id=data['id'],
    ))

    add_middleware('date', lambda _, data: get_date(data['date']))
    add_middleware('text', lambda _, data: data['text'])
    add_middleware('attachments', get_post_attachments)

    article_file = open('template.md', 'r', encoding='utf-8')
    article = article_file.read()
    article_file.close()

    urls = sys.argv[1:]
    for url in urls:
        if not is_valid_url(url):
            raise Exception('Cannot perform URL %s' % url)

        vid = '%s_%s' % pattern.findall(url)[0]
        res = vk_wall_get_by_id(vid)
        if 'response' in res:
            post = res['response'][0]

            if 'copy_history' in post:
                copy(post['copy_history'][0], post)

            output_dir = os.path.join(config['outputDir'], get_folder_name(post))
            makedirs(output_dir, exist_ok=True)

            article = apply_middlewares(output_dir, article, post)

            article_file = open(os.path.join(output_dir, 'vk.md'), 'w', encoding='utf-8')
            article_file.write(article)
            article_file.close()

            # print(json.dumps(post, indent=4, ensure_ascii=False))