ya trash

modules/Fetch/fetch.py

@@ -0,0 +1,202 @@
+#!/usr/local/bin/python3
+
+import json
+import os
+import re
+import string
+from datetime import datetime
+from functools import reduce
+from os import makedirs
+
+import json5
+import sys
+
+import requests
+
+from utils import until, copy
+from vk import vk_init, vk_wall_get_by_id, vk_photos_get_albums, vk_photos_get
+
+config = json5.load(open('config.json5', 'r'))
+vk_init(config['vk']['appId'], config['vk']['appSecret'])
+
+pattern = re.compile(r'https?://vk\.com/wall([-\d]+)_(\d+)')
+ms = []
+
+size_priority = ['w', 'z', 'y']
+
+
+# get_album_item = until(size_priority)
+
+
+def get_item_by_size(size, item):
+    for s in item:
+        if s['type'] == size:
+            return s
+
+    return None
+
+
+def get_album_item(item):
+    for s in size_priority:
+        img = get_item_by_size(s, item)
+        if img:
+            return img
+    return None
+
+
+def is_valid_url(i: str):
+    return pattern.match(i) is not None
+
+
+def add_middleware(var: str, fn):
+    ms.append((var, fn))
+
+
+def apply_middleware(data: str, mw: tuple):
+    key, fn = mw
+    value = fn(data)
+    form = {}
+    form[mw[0]] = ''
+    data.format(**form)
+    return data
+
+
+def apply_middlewares(output_dir, template, data):
+    result = template
+    for mw in ms:
+        key, fn = mw
+        form = {key: fn(output_dir, data)}
+        result = string.Template(result)
+        result = result.safe_substitute(**form)
+
+    return result
+
+
+def attach_album(output_dir, data):
+    """
+    {
+        'size': 13,
+         'title': 'Каллиграфия 27.11.2016',
+         'owner_id': -132442859,
+         'updated': 1480340404,
+         'description': '',
+         'id': '238711857',
+         'thumb': {'user_id': 100, 'access_key': '428f3dad13bdd5d730', 'owner_id': -132442859,
+                   'date': 1480340404,
+                   'photo_807': 'https://pp.vk.me/c837134/v837134463/11056/ZaoO1h5Idaw.jpg',
+                   'id': 456239050,
+                   'width': 1024, 'photo_75': 'https://pp.vk.me/c837134/v837134463/11053/aHOALfgr0VU.jpg',
+                   'photo_130': 'https://pp.vk.me/c837134/v837134463/11054/czcuc7o-bN4.jpg', 'text': '',
+                   'photo_604': 'https://pp.vk.me/c837134/v837134463/11055/P7yUE6WitSA.jpg', 'height': 683,
+                   'photo_1280': 'https://pp.vk.me/c837134/v837134463/11057/eLG5Akgx3fk.jpg',
+                   'album_id': 238711857
+        },
+        'created': 1480340311
+    }
+    :param data:
+    :return:
+    """
+    oid = data['owner_id']
+    aid = data['id']
+
+    album = vk_photos_get(oid, aid)['response']['items']
+
+    files = []
+    for file in album:
+        img = get_album_item(file['sizes'])
+        if img:
+            files.append(img['src'])
+            # files.append(file['sizes'])
+
+    download_files(files, output_dir)
+    return '\n'.join(list(
+        map(
+            lambda i: '![](%s)' % i,
+            map(
+                lambda f: os.path.basename(f),
+                files
+            )
+        )
+    ))
+
+
+def get_date(ts: int):
+    date = datetime.fromtimestamp(ts)
+    return date.strftime("%Y.%m.%d")
+
+
+def get_folder_name(post):
+    date_formatted = get_date(post['date'])
+
+    return '{date} vk-{owner_id}-{id}'.format(
+        date=date_formatted,
+        owner_id=post['owner_id'],
+        id=post['id'],
+    )
+
+
+def download_files(files, output):
+    for url in files:
+        file_out = os.path.join(output, os.path.basename(url))
+        req = requests.get(url)
+        data = req.content
+        f = open(file_out, 'wb')
+        f.write(data)
+        f.close()
+
+
+attachment_types = {
+    'album': attach_album
+}
+
+
+def get_post_attachments(output_dir, data):
+    attachments = []
+    for a in data['attachments']:
+        a_type = a['type']
+        if a_type in attachment_types:
+            fn = attachment_types[a_type]
+
+            attachment = fn(output_dir, a[a_type])
+            attachments.append(attachment)
+
+    return '\n'.join(attachments)
+
+
+if __name__ == '__main__':
+    add_middleware('id', lambda _, data: 'vk-{owner_id}-{id}'.format(
+        owner_id=data['owner_id'],
+        id=data['id'],
+    ))
+
+    add_middleware('date', lambda _, data: get_date(data['date']))
+    add_middleware('text', lambda _, data: data['text'])
+    add_middleware('attachments', get_post_attachments)
+
+    article_file = open('template.md', 'r', encoding='utf-8')
+    article = article_file.read()
+    article_file.close()
+
+    urls = sys.argv[1:]
+    for url in urls:
+        if not is_valid_url(url):
+            raise Exception('Cannot perform URL %s' % url)
+
+        vid = '%s_%s' % pattern.findall(url)[0]
+        res = vk_wall_get_by_id(vid)
+        if 'response' in res:
+            post = res['response'][0]
+
+            if 'copy_history' in post:
+                copy(post['copy_history'][0], post)
+
+            output_dir = os.path.join(config['outputDir'], get_folder_name(post))
+            makedirs(output_dir, exist_ok=True)
+
+            article = apply_middlewares(output_dir, article, post)
+
+            article_file = open(os.path.join(output_dir, 'vk.md'), 'w', encoding='utf-8')
+            article_file.write(article)
+            article_file.close()
+
+            # print(json.dumps(post, indent=4, ensure_ascii=False))