From 84745ead6ddc7de33e74fa771f152c87b537fa41 Mon Sep 17 00:00:00 2001
From: Roman Timashev <tmshv@yandex-team.ru>
Date: Sun, 17 Sep 2017 13:40:09 +0300
Subject: [PATCH] [Sync] refactoring, removed old code, cleanup

---
 utils/Sync/sync/__init__.py           | 11 ----
 utils/Sync/sync/core/SyncDocument.py  |  5 +-
 utils/Sync/sync/core/__init__.py      |  8 ---
 utils/Sync/sync/data/__init__.py      | 20 ++----
 utils/Sync/sync/data/fs.py            |  4 +-
 utils/Sync/sync/data/yandexdisk.py    |  3 +-
 utils/Sync/sync/image.py              | 20 ------
 utils/Sync/sync/models/Album.py       |  2 +-
 utils/Sync/sync/models/Document.py    |  5 +-
 utils/Sync/sync/models/Page.py        | 22 ++-----
 utils/Sync/sync/models/product.py     | 16 ++---
 utils/Sync/test/sync/test_document.py |  2 -
 utils/Sync/utils/fn.py                | 70 +++-----------------
 utils/Sync/utils/hash.py              |  7 +-
 utils/Sync/utils/image/__init__.py    |  7 +-
 utils/Sync/utils/image/resize.py      | 93 ++++-----------------------
 utils/Sync/utils/text/transform.py    | 14 ++--
 17 files changed, 61 insertions(+), 248 deletions(-)
 delete mode 100644 utils/Sync/sync/image.py

diff --git a/utils/Sync/sync/__init__.py b/utils/Sync/sync/__init__.py
index 2cb7c6c1..a8ea13a2 100644
--- a/utils/Sync/sync/__init__.py
+++ b/utils/Sync/sync/__init__.py
@@ -1,16 +1,13 @@
 import os
 import re
 from datetime import datetime
-from glob import glob
 from lxml import etree
 
 import lxml.html
 from markdown import markdown
 
 import settings
-from sync.image import sync_image
 from sync.models import Model
-from utils.fn import lmap, combine
 
 
 def create_date(date_str, date_formats=None):
@@ -66,14 +63,6 @@ async def untouched(items: [Model]) -> [Model]:
     # return filtered_documents
 
 
-def synced_images_ids(images):
-    return [i['_id'] for i in sync_image(images)]
-
-
-def synced_image_id(image):
-    return sync_image(image)['_id']
-
-
 def create_date_and_title_from_folder_name(folder_name, date_formats=None):
     m = folder_name_pattern.findall(folder_name)
     if not m:
diff --git a/utils/Sync/sync/core/SyncDocument.py b/utils/Sync/sync/core/SyncDocument.py
index 704b493a..8806130b 100644
--- a/utils/Sync/sync/core/SyncDocument.py
+++ b/utils/Sync/sync/core/SyncDocument.py
@@ -6,11 +6,10 @@ import logging
 import settings
 from sync import untouched
 from sync.core import Sync
-from sync.data import scan_subdirs, Provider
+from sync.data import Provider
 from sync.models.Document import Document
 
-from sync.models import Model
-from utils.fn import last_good, lmap
+from utils.fn import last_good
 from utils.hash import hash_str
 from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
diff --git a/utils/Sync/sync/core/__init__.py b/utils/Sync/sync/core/__init__.py
index 2df6a9d6..dd44d4d8 100644
--- a/utils/Sync/sync/core/__init__.py
+++ b/utils/Sync/sync/core/__init__.py
@@ -1,6 +1,4 @@
 from sync.data import Provider
-from sync.models import Model
-from utils.fn import lmap
 
 
 def get_id(item):
@@ -60,11 +58,5 @@ class Sync:
     def delete(self, q):
         return self.collection.find_one_and_delete(q)
 
-    def create_id(self, document):
-        return None
-
-    def create_hash(self, document):
-        return None
-
     def create_remove_query(self, query):
         return query
diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py
index e1ede610..07d71e6b 100644
--- a/utils/Sync/sync/data/__init__.py
+++ b/utils/Sync/sync/data/__init__.py
@@ -1,10 +1,7 @@
 import os
 from io import BytesIO
 
-import requests
-
-import settings
-from utils.fn import combine, lmap
+from utils.fn import combine
 
 
 class Provider:
@@ -61,16 +58,11 @@ def scan_subdirs(provider, ext, path='.'):
     :return: list of files relative to provider.root
     """
     items = provider.scan(path)
-    items = list(filter(
+    items = filter(
         lambda i: provider.is_dir(i),
         items
-    ))
-
-    items = lmap(
-        lambda folder: provider.type_filter(folder, ext),
-        items
     )
-
+    items = [provider.type_filter(folder, ext) for folder in items]
     items = combine(items)
     return items
 
@@ -82,7 +74,5 @@ def get_data(provider, path, transform):
 
 def list_images(provider, path):
     image_types = ['.jpg', '.JPG', '.png', '.PNG']
-    return combine(lmap(
-        lambda ext: provider.type_filter(path, ext),
-        image_types
-    ))
+    images = [provider.type_filter(path, ext) for ext in image_types]
+    return combine(images)
diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py
index d203ce29..2674110f 100644
--- a/utils/Sync/sync/data/fs.py
+++ b/utils/Sync/sync/data/fs.py
@@ -4,7 +4,6 @@ from glob import glob
 from shutil import copyfile
 
 from sync.data import Provider
-from utils.fn import lmap
 from utils.hash import hash_file
 
 
@@ -18,7 +17,8 @@ class FSProvider(Provider):
         return [self.get_rel(path) for path in paths]
 
     def glob(self, pattr):
-        return lmap(self.get_rel, glob(self.get_abs(pattr)))
+        items = glob(self.get_abs(pattr))
+        return [self.get_rel(i) for i in items]
 
     def type_filter(self, path, ext):
         return list(filter(
diff --git a/utils/Sync/sync/data/yandexdisk.py b/utils/Sync/sync/data/yandexdisk.py
index ea8be6c4..f19678a4 100644
--- a/utils/Sync/sync/data/yandexdisk.py
+++ b/utils/Sync/sync/data/yandexdisk.py
@@ -8,7 +8,6 @@ from tempfile import mkstemp
 import requests
 
 from sync.data import Provider
-from utils.fn import lmap
 from yandexdisk import YDClient
 
 
@@ -34,7 +33,7 @@ class YDProvider(Provider):
             lambda fp: re.sub('disk:', '', fp),
             files
         )
-        return lmap(self.get_rel, files)
+        return [self.get_rel(i) for i in files]
 
     def glob(self, pattr):
         return []
diff --git a/utils/Sync/sync/image.py b/utils/Sync/sync/image.py
deleted file mode 100644
index c4fdfec0..00000000
--- a/utils/Sync/sync/image.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from db import db
-from utils.fn import lmap
-
-
-def sync_image(record):
-    if isinstance(record, list):
-        return lmap(
-            sync_image,
-            record
-        )
-
-    q = {'hash': record['hash']}
-    try:
-        db().images.update_one(q, {'$set': record}, upsert=True)
-        i = db().images.find_one({'hash': record['hash']})
-        return i
-    except ValueError:
-        pass
-
-    return None
diff --git a/utils/Sync/sync/models/Album.py b/utils/Sync/sync/models/Album.py
index 3c8fa145..0dd35cb2 100644
--- a/utils/Sync/sync/models/Album.py
+++ b/utils/Sync/sync/models/Album.py
@@ -14,7 +14,7 @@ from sync.data import Provider, list_images
 from sync.models import Model
 from sync.models.Image import Image
 
-from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns
+from utils.fn import swap_ext, first, map_cases, constant, combine
 from utils.hash import hash_str
 from utils.io import read_yaml_md, parse_yaml_front_matter
 from utils.text.transform import url_encode_text
diff --git a/utils/Sync/sync/models/Document.py b/utils/Sync/sync/models/Document.py
index d426f2d9..64d162d0 100644
--- a/utils/Sync/sync/models/Document.py
+++ b/utils/Sync/sync/models/Document.py
@@ -1,16 +1,13 @@
 import os
 from tempfile import mkstemp
 
-import requests
-
 import settings
 from db import collection
 from sync.data import scan_subdirs, request
 from sync.models import Model
 from sync.models.Image import Image
-from utils.fn import last_good, lmap
+from utils.fn import last_good
 from utils.hash import hash_str
-from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
 from utils.text.transform import url_encode_text, url_encode_file
 
diff --git a/utils/Sync/sync/models/Page.py b/utils/Sync/sync/models/Page.py
index db365d72..473bea59 100644
--- a/utils/Sync/sync/models/Page.py
+++ b/utils/Sync/sync/models/Page.py
@@ -34,13 +34,13 @@ class Page(Model):
         documents = [i for i in provider.scan('.') if provider.is_dir(i)]
         documents = [provider.type_filter(i, '.md') for i in documents]
         documents = [i[0] for i in documents if len(i)]
-        documents = [Page.read_path(provider, i) for i in documents]
+        documents = [Page.read(provider, i) for i in documents]
         documents = [i for i in documents if i]
 
         return documents
 
     @staticmethod
-    def read_path(provider, path):
+    def read(provider, path):
         params = {
             'file': os.path.basename(path),
             'folder': os.path.dirname(path),
@@ -89,17 +89,6 @@ class Page(Model):
 
         return not (self.hash == i['hash'])
 
-    async def save(self):
-        document = self.bake()
-
-        query = {'id': document['id']}
-        try:
-            self.store.update_one(query, {'$set': document}, upsert=True)
-            return self
-        except ValueError:
-            pass
-        return None
-
     async def setup_images(self, sizes, url_factory):
         folder = self.params['folder']
         images = []
@@ -142,15 +131,16 @@ class Page(Model):
 
     def __set_id(self):
         if 'id' in self.params:
-            self.id = self.params['id']
+            self.id = self.get_param('id')
         else:
-            self.id = url_encode_text(self.params['url'])
+            self.id = url_encode_text(self.get_param('url'))
 
     def __set_url(self):
         self.url = self.params['url']
 
     def __set_hash(self):
-        images = [os.path.join(self.params['folder'], i) for i in self.params['images']]
+        f = self.get_param('folder')
+        images = [os.path.join(f, i) for i in self.get_param('images')]
         files = sorted([self.file] + images)
         hashes = [self.provider.hash(i) for i in files]
 
diff --git a/utils/Sync/sync/models/product.py b/utils/Sync/sync/models/product.py
index 48760d36..42534242 100644
--- a/utils/Sync/sync/models/product.py
+++ b/utils/Sync/sync/models/product.py
@@ -1,8 +1,6 @@
 import settings
 from db import collection
 from sync.core.person import SyncPerson
-from sync.image import sync_image
-from utils.fn import key_mapper
 from utils.text.transform import interpolate_swift
 
 
@@ -17,9 +15,6 @@ def resolve_value(param, store):
     return store(i)
 
 
-unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
-
-
 def resolve_teacher(i):
     sync = SyncPerson(
         collection(settings.collection_collective),
@@ -32,13 +27,14 @@ def resolve_teacher(i):
 def create_product(doc):
     # author, image, title, awards, teacher, date
 
-    unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
+    # unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
+    # unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
 
     product = {**doc}
-    unpop_image(product)
+    # unpop_image(product)
 
-    product['teacher'] = resolve_teacher(doc['teacher'])
-    if (product['teacher']):
-        unpop_teacher(product)
+    # product['teacher'] = resolve_teacher(doc['teacher'])
+    # if (product['teacher']):
+        # unpop_teacher(product)
 
     return product
diff --git a/utils/Sync/test/sync/test_document.py b/utils/Sync/test/sync/test_document.py
index 9c13dc39..4db32b89 100644
--- a/utils/Sync/test/sync/test_document.py
+++ b/utils/Sync/test/sync/test_document.py
@@ -3,8 +3,6 @@ import unittest
 import settings
 from sync.data import scan_subdirs
 from sync.data.fs import FSProvider
-from sync.models.Document import Document
-from utils.fn import lmap
 
 
 class MyTestCase(unittest.TestCase):
diff --git a/utils/Sync/utils/fn.py b/utils/Sync/utils/fn.py
index c1fd0ca6..a927a76e 100644
--- a/utils/Sync/utils/fn.py
+++ b/utils/Sync/utils/fn.py
@@ -1,9 +1,5 @@
-import json
 import os
 from functools import reduce
-from glob import glob
-
-import bson.json_util
 
 
 def constant(value):
@@ -20,32 +16,13 @@ def combine(items):
     )
 
 
-def to_json(i):
-    return json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
+def compose(*fns):
+    return lambda i: reduce(
+        lambda out, fn: fn(out),
+        fns, i
+    )
 
 
-iterate_over_fns = lambda fns: lambda i: reduce(
-    lambda out, fn: fn(out),
-    fns, i
-)
-
-iterate_iter_over_fns = lambda fns: lambda i: lmap(
-    iterate_over_fns(fns),
-    i
-)
-
-
-def lmap(fn, ls):
-    return list(map(fn, ls))
-
-
-def lmapfn(ls):
-    return lambda fn: list(map(fn, ls))
-
-
-lprint = lambda i: lmap(print, i)
-lprint_json = lambda i: lprint(lmap(to_json, i))
-
 first = lambda i: i[0] if len(i) > 0 else None
 
 
@@ -53,32 +30,11 @@ def swap_ext(extension: str):
     return lambda i: os.path.splitext(i)[0] + extension
 
 
-def map_item_key(item, key, map_fn):
-    """
-
-    :param item:
-    :param key:
-    :param map_fn:
-    :return:
-    """
-    if key in item:
-        item[key] = map_fn(item[key])
-    return item
-
-
-def kmap(key):
-    return lambda i: i[key] if key in i else None
-
-
-def key_mapper(key, fn):
-    return lambda i: map_item_key(i, key, fn)
-
-
-def map_cases(param, cases, default_fn=None):
-    for c, fn in cases:
-        if c(param):
+def map_cases(param, cases):
+    for predicate, fn in cases:
+        if predicate(param):
             return fn(param)
-    return param if not default_fn else default_fn(param)
+    return param
 
 
 def until(condition_fn, iter_fn, items, default_value=None):
@@ -89,14 +45,6 @@ def until(condition_fn, iter_fn, items, default_value=None):
     return default_value
 
 
-# glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
-# glob_in('./images')
-dir_globber = lambda mask_list: lambda dir: combine(map(
-    lambda mask: glob(os.path.join(dir, mask)),
-    mask_list
-))
-
-
 def last_good(ls):
     better = None
     for i in ls:
diff --git a/utils/Sync/utils/hash.py b/utils/Sync/utils/hash.py
index ffef5494..010f3cce 100644
--- a/utils/Sync/utils/hash.py
+++ b/utils/Sync/utils/hash.py
@@ -1,6 +1,9 @@
 import hashlib
+import json
 
-from utils.fn import map_cases, to_json
+import bson.json_util
+
+from utils.fn import map_cases
 
 
 def hash_file(file):
@@ -15,6 +18,8 @@ def hash_data(data):
 
 
 def hash_str(data):
+    to_json = lambda i: json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
+
     data = map_cases(data, [
         (lambda i: isinstance(i, dict), to_json),
         (lambda i: isinstance(i, tuple), to_json),
diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py
index 6aac3f27..6f0c6137 100644
--- a/utils/Sync/utils/image/__init__.py
+++ b/utils/Sync/utils/image/__init__.py
@@ -6,9 +6,7 @@ from typing import Optional
 from PIL import Image
 
 import settings
-from utils.hash import hash_file
-# from utils.image import resize
-from utils.image.resize import optimize, thumbnail, orient
+from utils.image.resize import optimize, thumbnail
 
 logger = logging.getLogger(settings.name)
 
@@ -50,9 +48,6 @@ def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[di
                 continue
 
             width, height = image.size
-            # upload_url = upload(local_image_path, '/images/{}'.format(image_filename))
-            # if not upload_url:
-            #     print('Image {} not uploaded'.format(file))
 
         result[size_name] = {
             'url': image_url,
diff --git a/utils/Sync/utils/image/resize.py b/utils/Sync/utils/image/resize.py
index bfaed8ad..817dbcb3 100644
--- a/utils/Sync/utils/image/resize.py
+++ b/utils/Sync/utils/image/resize.py
@@ -1,94 +1,30 @@
-import os
-
 from subprocess import call
 
-import smartcrop
-from PIL import Image
-from smartcrop import SmartCrop
-
 import utils.image
 
 
-def thumbnail(src, dest, size, ext=None):
+def thumbnail(src, dest, size):
     image_magick_resize(src, dest, size, quality=85)
     return utils.image.read_image(dest)
 
-    # image = utils.image.read_image(src)
-    # image.thumbnail(size)
-    # image.save(override_extension(dest, ext))
-    # return image
 
-
-def optimize(src, dest, quality=85, ext=None):
+def optimize(src, dest, quality=85):
     image = utils.image.read_image(src)
     if image:
-        image.save(override_extension(dest, ext), quality=quality)
+        image.save(dest, quality=quality)
     return image
 
 
-def override_extension(dest, ext=None):
-    if ext is not None:
-        dest = os.path.splitext(dest)[0] + ext
-
-    return dest
-
-
-def crop(src, dest, size):
-    width, height = size
-    image = utils.image.read_image(src)
-    crop_options = smartcrop.DEFAULTS
-    crop_options['width'] = width
-    crop_options['height'] = height
-
-    crop_instance = SmartCrop()
-    result = crop_instance.crop(image.copy(), crop_options)
-
-    box = (result['topCrop']['x'],
-           result['topCrop']['y'],
-           result['topCrop']['x'] + result['topCrop']['width'],
-           result['topCrop']['y'] + result['topCrop']['height'])
-
-    i = utils.image.read_image(src).crop(box)
-    i.thumbnail(size, Image.ANTIALIAS)
-    i.save(dest)
-
-
-def orient(image):
-    orientation = 0x0112
-    rotations = {
-        3: Image.ROTATE_180,
-        6: Image.ROTATE_270,
-        8: Image.ROTATE_90
-    }
-
-    if hasattr(image, '_getexif'):
-        exif = image._getexif()
-        if exif is not None:
-            o = exif[orientation]
-            if o in rotations:
-                return image.transpose(rotations[o])
-    return image
-
-
-def image_magick_resize(ifile, ofile, size, quality=85):
+def image_magick_resize(input_file, output_file, size, quality=85):
     s = '%dx%d' % size
-    call(['/usr/local/bin/convert', ifile, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), ofile])
-
-
-# def image_magick_resize(input_file, output_file, size, quality=85):
-#     from wand.image import Image
-#
-#     w, h = size
-#     s = '{w}x{h}>'.format(w=w, h=h)
-#
-#     with Image(filename=input_file) as img:
-#         # print(img.size)
-#         # print(list(img.metadata.items()))
-#         with img.clone() as i:
-#             i.transform(resize=s)
-#             i.auto_orient()
-#             i.strip()
-#             i.save(filename=output_file)
+    call([
+        '/usr/local/bin/convert', input_file,
+        '-strip',
+        '-auto-orient',
+        '-resize', s,
+        '-quality', str(quality),
+        output_file,
+    ])
 
 
 def image_magick_pdf_to_img(input_file, output_file):
@@ -97,7 +33,6 @@ def image_magick_pdf_to_img(input_file, output_file):
         '-q',
         '-sDEVICE=jpeg',
         '-dLastPage=1',
-        '-o',
-        output_file,
-        input_file
+        '-o', output_file,
+        input_file,
     ])
diff --git a/utils/Sync/utils/text/transform.py b/utils/Sync/utils/text/transform.py
index e37eb5ac..4380a687 100644
--- a/utils/Sync/utils/text/transform.py
+++ b/utils/Sync/utils/text/transform.py
@@ -2,7 +2,7 @@ import re
 
 from unidecode import unidecode
 
-from utils.fn import iterate_over_fns
+from utils.fn import compose
 
 remove_pattr = re.compile('[«»<>#,!&"\']')
 reduce_dash_patterns = [
@@ -71,17 +71,17 @@ def text_lower(i):
     return i.lower()
 
 
-url_encode_file = iterate_over_fns([
+url_encode_file = compose(
     translit,
     space_to_dash,
     short_dash,
     remove_underscores,
     remove_punctuation,
     reduce_dashes,
-    text_lower
-])
+    text_lower,
+)
 
-url_encode_text = iterate_over_fns([
+url_encode_text = compose(
     translit,
     space_to_dash,
     dot_to_dash,
@@ -90,5 +90,5 @@ url_encode_text = iterate_over_fns([
     remove_underscores,
     remove_punctuation,
     reduce_dashes,
-    text_lower
-])
+    text_lower,
+)