tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

[Sync] refactoring, removed old code, cleanup

Browse Source
This commit is contained in:
Roman Timashev committed 2017-09-17 13:40:09 +03:00
1 parent 32843f4d09
commit 84745ead6d
17 files changed
+61 -248
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/Sync/sync/__init__.py
-11
View File
@@ -1,16 +1,13 @@
import os
import re
from datetime import datetime
from glob import glob
from lxml import etree
import lxml.html
from markdown import markdown
import settings
from sync.image import sync_image
from sync.models import Model
from utils.fn import lmap, combine
def create_date(date_str, date_formats=None):
@@ -66,14 +63,6 @@ async def untouched(items: [Model]) -> [Model]:
# return filtered_documents
def synced_images_ids(images):
return [i['_id'] for i in sync_image(images)]
def synced_image_id(image):
return sync_image(image)['_id']
def create_date_and_title_from_folder_name(folder_name, date_formats=None):
m = folder_name_pattern.findall(folder_name)
if not m:
utils/Sync/sync/core/SyncDocument.py
+2 -3
View File
@@ -6,11 +6,10 @@ import logging
import settings
from sync import untouched
from sync.core import Sync
from sync.data import scan_subdirs, Provider
from sync.data import Provider
from sync.models.Document import Document
from sync.models import Model
from utils.fn import last_good, lmap
from utils.fn import last_good
from utils.hash import hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
utils/Sync/sync/core/__init__.py
-8
View File
@@ -1,6 +1,4 @@
from sync.data import Provider
from sync.models import Model
from utils.fn import lmap
def get_id(item):
@@ -60,11 +58,5 @@ class Sync:
def delete(self, q):
return self.collection.find_one_and_delete(q)
def create_id(self, document):
return None
def create_hash(self, document):
return None
def create_remove_query(self, query):
return query
utils/Sync/sync/data/__init__.py
+5 -15
View File
@@ -1,10 +1,7 @@
import os
from io import BytesIO
import requests
import settings
from utils.fn import combine, lmap
from utils.fn import combine
class Provider:
@@ -61,16 +58,11 @@ def scan_subdirs(provider, ext, path='.'):
:return: list of files relative to provider.root
"""
items = provider.scan(path)
items = list(filter(
items = filter(
lambda i: provider.is_dir(i),
items
))
items = lmap(
lambda folder: provider.type_filter(folder, ext),
items
)
items = [provider.type_filter(folder, ext) for folder in items]
items = combine(items)
return items
@@ -82,7 +74,5 @@ def get_data(provider, path, transform):
def list_images(provider, path):
image_types = ['.jpg', '.JPG', '.png', '.PNG']
return combine(lmap(
lambda ext: provider.type_filter(path, ext),
image_types
))
images = [provider.type_filter(path, ext) for ext in image_types]
return combine(images)
utils/Sync/sync/data/fs.py
+2 -2
View File
@@ -4,7 +4,6 @@ from glob import glob
from shutil import copyfile
from sync.data import Provider
from utils.fn import lmap
from utils.hash import hash_file
@@ -18,7 +17,8 @@ class FSProvider(Provider):
return [self.get_rel(path) for path in paths]
def glob(self, pattr):
return lmap(self.get_rel, glob(self.get_abs(pattr)))
items = glob(self.get_abs(pattr))
return [self.get_rel(i) for i in items]
def type_filter(self, path, ext):
return list(filter(
utils/Sync/sync/data/yandexdisk.py
+1 -2
View File
@@ -8,7 +8,6 @@ from tempfile import mkstemp
import requests
from sync.data import Provider
from utils.fn import lmap
from yandexdisk import YDClient
@@ -34,7 +33,7 @@ class YDProvider(Provider):
lambda fp: re.sub('disk:', '', fp),
files
)
return lmap(self.get_rel, files)
return [self.get_rel(i) for i in files]
def glob(self, pattr):
return []
utils/Sync/sync/image.py
-20
View File
@@ -1,20 +0,0 @@
from db import db
from utils.fn import lmap
def sync_image(record):
if isinstance(record, list):
return lmap(
sync_image,
record
)
q = {'hash': record['hash']}
try:
db().images.update_one(q, {'$set': record}, upsert=True)
i = db().images.find_one({'hash': record['hash']})
return i
except ValueError:
pass
return None
utils/Sync/sync/models/Album.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sync.data import Provider, list_images
from sync.models import Model
from sync.models.Image import Image
from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns
from utils.fn import swap_ext, first, map_cases, constant, combine
from utils.hash import hash_str
from utils.io import read_yaml_md, parse_yaml_front_matter
from utils.text.transform import url_encode_text
utils/Sync/sync/models/Document.py
+1 -4
View File
@@ -1,16 +1,13 @@
import os
from tempfile import mkstemp
import requests
import settings
from db import collection
from sync.data import scan_subdirs, request
from sync.models import Model
from sync.models.Image import Image
from utils.fn import last_good, lmap
from utils.fn import last_good
from utils.hash import hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
from utils.text.transform import url_encode_text, url_encode_file
utils/Sync/sync/models/Page.py
+6 -16
View File
@@ -34,13 +34,13 @@ class Page(Model):
documents = [i for i in provider.scan('.') if provider.is_dir(i)]
documents = [provider.type_filter(i, '.md') for i in documents]
documents = [i[0] for i in documents if len(i)]
documents = [Page.read_path(provider, i) for i in documents]
documents = [Page.read(provider, i) for i in documents]
documents = [i for i in documents if i]
return documents
@staticmethod
def read_path(provider, path):
def read(provider, path):
params = {
'file': os.path.basename(path),
'folder': os.path.dirname(path),
@@ -89,17 +89,6 @@ class Page(Model):
return not (self.hash == i['hash'])
async def save(self):
document = self.bake()
query = {'id': document['id']}
try:
self.store.update_one(query, {'$set': document}, upsert=True)
return self
except ValueError:
pass
return None
async def setup_images(self, sizes, url_factory):
folder = self.params['folder']
images = []
@@ -142,15 +131,16 @@ class Page(Model):
def __set_id(self):
if 'id' in self.params:
self.id = self.params['id']
self.id = self.get_param('id')
else:
self.id = url_encode_text(self.params['url'])
self.id = url_encode_text(self.get_param('url'))
def __set_url(self):
self.url = self.params['url']
def __set_hash(self):
images = [os.path.join(self.params['folder'], i) for i in self.params['images']]
f = self.get_param('folder')
images = [os.path.join(f, i) for i in self.get_param('images')]
files = sorted([self.file] + images)
hashes = [self.provider.hash(i) for i in files]
utils/Sync/sync/models/product.py
+6 -10
View File
@@ -1,8 +1,6 @@
import settings
from db import collection
from sync.core.person import SyncPerson
from sync.image import sync_image
from utils.fn import key_mapper
from utils.text.transform import interpolate_swift
@@ -17,9 +15,6 @@ def resolve_value(param, store):
return store(i)
unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
def resolve_teacher(i):
sync = SyncPerson(
collection(settings.collection_collective),
@@ -32,13 +27,14 @@ def resolve_teacher(i):
def create_product(doc):
# author, image, title, awards, teacher, date
unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
# unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
# unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
product = {**doc}
unpop_image(product)
# unpop_image(product)
product['teacher'] = resolve_teacher(doc['teacher'])
if (product['teacher']):
unpop_teacher(product)
# product['teacher'] = resolve_teacher(doc['teacher'])
# if (product['teacher']):
# unpop_teacher(product)
return product
utils/Sync/test/sync/test_document.py
-2
View File
@@ -3,8 +3,6 @@ import unittest
import settings
from sync.data import scan_subdirs
from sync.data.fs import FSProvider
from sync.models.Document import Document
from utils.fn import lmap
class MyTestCase(unittest.TestCase):
utils/Sync/utils/fn.py
+9 -61
View File
@@ -1,9 +1,5 @@
import json
import os
from functools import reduce
from glob import glob
import bson.json_util
def constant(value):
@@ -20,32 +16,13 @@ def combine(items):
)
def to_json(i):
return json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
def compose(*fns):
return lambda i: reduce(
lambda out, fn: fn(out),
fns, i
)
iterate_over_fns = lambda fns: lambda i: reduce(
lambda out, fn: fn(out),
fns, i
)
iterate_iter_over_fns = lambda fns: lambda i: lmap(
iterate_over_fns(fns),
i
)
def lmap(fn, ls):
return list(map(fn, ls))
def lmapfn(ls):
return lambda fn: list(map(fn, ls))
lprint = lambda i: lmap(print, i)
lprint_json = lambda i: lprint(lmap(to_json, i))
first = lambda i: i[0] if len(i) > 0 else None
@@ -53,32 +30,11 @@ def swap_ext(extension: str):
return lambda i: os.path.splitext(i)[0] + extension
def map_item_key(item, key, map_fn):
"""
:param item:
:param key:
:param map_fn:
:return:
"""
if key in item:
item[key] = map_fn(item[key])
return item
def kmap(key):
return lambda i: i[key] if key in i else None
def key_mapper(key, fn):
return lambda i: map_item_key(i, key, fn)
def map_cases(param, cases, default_fn=None):
for c, fn in cases:
if c(param):
def map_cases(param, cases):
for predicate, fn in cases:
if predicate(param):
return fn(param)
return param if not default_fn else default_fn(param)
return param
def until(condition_fn, iter_fn, items, default_value=None):
@@ -89,14 +45,6 @@ def until(condition_fn, iter_fn, items, default_value=None):
return default_value
# glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
# glob_in('./images')
dir_globber = lambda mask_list: lambda dir: combine(map(
lambda mask: glob(os.path.join(dir, mask)),
mask_list
))
def last_good(ls):
better = None
for i in ls:
utils/Sync/utils/hash.py
+6 -1
View File
@@ -1,6 +1,9 @@
import hashlib
import json
from utils.fn import map_cases, to_json
import bson.json_util
from utils.fn import map_cases
def hash_file(file):
@@ -15,6 +18,8 @@ def hash_data(data):
def hash_str(data):
to_json = lambda i: json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
data = map_cases(data, [
(lambda i: isinstance(i, dict), to_json),
(lambda i: isinstance(i, tuple), to_json),
utils/Sync/utils/image/__init__.py
+1 -6
View File
@@ -6,9 +6,7 @@ from typing import Optional
from PIL import Image
import settings
from utils.hash import hash_file
# from utils.image import resize
from utils.image.resize import optimize, thumbnail, orient
from utils.image.resize import optimize, thumbnail
logger = logging.getLogger(settings.name)
@@ -50,9 +48,6 @@ def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[di
continue
width, height = image.size
# upload_url = upload(local_image_path, '/images/{}'.format(image_filename))
# if not upload_url:
# print('Image {} not uploaded'.format(file))
result[size_name] = {
'url': image_url,
utils/Sync/utils/image/resize.py
+14 -79
View File
@@ -1,94 +1,30 @@
import os
from subprocess import call
import smartcrop
from PIL import Image
from smartcrop import SmartCrop
import utils.image
def thumbnail(src, dest, size, ext=None):
def thumbnail(src, dest, size):
image_magick_resize(src, dest, size, quality=85)
return utils.image.read_image(dest)
# image = utils.image.read_image(src)
# image.thumbnail(size)
# image.save(override_extension(dest, ext))
# return image
def optimize(src, dest, quality=85, ext=None):
def optimize(src, dest, quality=85):
image = utils.image.read_image(src)
if image:
image.save(override_extension(dest, ext), quality=quality)
image.save(dest, quality=quality)
return image
def override_extension(dest, ext=None):
if ext is not None:
dest = os.path.splitext(dest)[0] + ext
return dest
def crop(src, dest, size):
width, height = size
image = utils.image.read_image(src)
crop_options = smartcrop.DEFAULTS
crop_options['width'] = width
crop_options['height'] = height
crop_instance = SmartCrop()
result = crop_instance.crop(image.copy(), crop_options)
box = (result['topCrop']['x'],
result['topCrop']['y'],
result['topCrop']['x'] + result['topCrop']['width'],
result['topCrop']['y'] + result['topCrop']['height'])
i = utils.image.read_image(src).crop(box)
i.thumbnail(size, Image.ANTIALIAS)
i.save(dest)
def orient(image):
orientation = 0x0112
rotations = {
3: Image.ROTATE_180,
6: Image.ROTATE_270,
8: Image.ROTATE_90
}
if hasattr(image, '_getexif'):
exif = image._getexif()
if exif is not None:
o = exif[orientation]
if o in rotations:
return image.transpose(rotations[o])
return image
def image_magick_resize(ifile, ofile, size, quality=85):
def image_magick_resize(input_file, output_file, size, quality=85):
s = '%dx%d' % size
call(['/usr/local/bin/convert', ifile, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), ofile])
# def image_magick_resize(input_file, output_file, size, quality=85):
# from wand.image import Image
#
# w, h = size
# s = '{w}x{h}>'.format(w=w, h=h)
#
# with Image(filename=input_file) as img:
# # print(img.size)
# # print(list(img.metadata.items()))
# with img.clone() as i:
# i.transform(resize=s)
# i.auto_orient()
# i.strip()
# i.save(filename=output_file)
call([
'/usr/local/bin/convert', input_file,
'-strip',
'-auto-orient',
'-resize', s,
'-quality', str(quality),
output_file,
])
def image_magick_pdf_to_img(input_file, output_file):
@@ -97,7 +33,6 @@ def image_magick_pdf_to_img(input_file, output_file):
'-q',
'-sDEVICE=jpeg',
'-dLastPage=1',
'-o',
output_file,
input_file
'-o', output_file,
input_file,
])
utils/Sync/utils/text/transform.py
+7 -7
View File
@@ -2,7 +2,7 @@ import re
from unidecode import unidecode
from utils.fn import iterate_over_fns
from utils.fn import compose
remove_pattr = re.compile('[«»<>#,!&"\']')
reduce_dash_patterns = [
@@ -71,17 +71,17 @@ def text_lower(i):
return i.lower()
url_encode_file = iterate_over_fns([
url_encode_file = compose(
translit,
space_to_dash,
short_dash,
remove_underscores,
remove_punctuation,
reduce_dashes,
text_lower
])
text_lower,
)
url_encode_text = iterate_over_fns([
url_encode_text = compose(
translit,
space_to_dash,
dot_to_dash,
@@ -90,5 +90,5 @@ url_encode_text = iterate_over_fns([
remove_underscores,
remove_punctuation,
reduce_dashes,
text_lower
])
text_lower,
)