Sync util big update

This commit is contained in:
tmshv committed 2016-02-27 16:19:08 +03:00
1 parent 1e80bae401
commit 40321327d6
30 files changed
+1001 -159

No files matched your search

+68
View File
@@ -0,0 +1,68 @@
import json
import os
from functools import reduce
from glob import glob
from bson import json_util
combine = lambda i: reduce(
lambda s, i: s + i,
i
)
iterate_over_fns = lambda fns: lambda i: reduce(
lambda out, fn: fn(out),
fns, i
)
iterate_iter_over_fns = lambda fns: lambda i: lmap(
iterate_over_fns(fns),
i
)
lmap = lambda fn, ls: list(map(fn, ls))
lprint = lambda i: lmap(print, i)
lprint_json = lambda s: lprint(lmap(
lambda i: json.dumps(i, ensure_ascii=False, indent=4, default=json_util.default),
s
))
first = lambda i: i[0] if len(i) > 0 else None
def map_item_key(item, key, map_fn):
"""
:param item:
:param key:
:param map_fn:
:return:
"""
item[key] = map_fn(item[key])
return item
key_mapper = lambda key, fn: lambda i: map_item_key(i, key, fn)
def map_cases(param, cases, default_fn=None):
for c, fn in cases:
if c(param):
return fn(param)
return param if not default_fn else default_fn(param)
def until(condition_fn, iter_fn, items, default_value=None):
for i in items:
result = iter_fn(i)
if condition_fn(result):
return result
return default_value
# glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
# glob_in('./images')
dir_globber = lambda mask_list: lambda dir: combine(map(
lambda mask: glob(os.path.join(dir, mask)),
mask_list
))
+9
View File
@@ -0,0 +1,9 @@
import hashlib
def file_hash(file):
with open(file, 'rb') as f:
data = f.read()
sha = hashlib.sha256(data)
f.close()
return sha.hexdigest()
+55
View File
@@ -0,0 +1,55 @@
import os
from PIL import Image
from utils.hash import file_hash
from utils.image import resize
from utils.image.resize import optimize, thumbnail
def read_image(src):
try:
return Image.open(src)
except FileNotFoundError:
return None
def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
"""
:param file: Image path to process
:param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
:param url_fn: function (size_name, ext) for creating image url
:param output_dir: path to local folder for thumbs storing
:param skip_processing: True/False to skip thumbs generation
:return:
"""
image = read_image(file)
if not image:
return None
result = {}
_, ext = os.path.splitext(file)
for size in sizes:
size_name, width, height = size
image_url = url_fn(size_name, ext)
image_filename = os.path.basename(image_url)
d = os.path.join(output_dir, image_filename)
if not skip_processing:
thumb = optimize(image, d, quality=90) if size_name == 'original' else thumbnail(image, d, (width, height))
width, height = thumb.size
result[size_name] = {
'url': image_url,
'size': size_name,
'width': width,
'height': height
}
return {
'file': os.path.basename(file),
'hash': file_hash(file),
'data': result
}
+31
View File
@@ -0,0 +1,31 @@
import os
from subprocess import call
def thumbnail(image, dest, size, ext=None):
image.thumbnail(size)
image.save(override_extension(dest, ext))
return image
def optimize(image, dest, quality=85, ext=None):
image.save(override_extension(dest, ext), quality=quality)
return image
def override_extension(dest, ext=None):
if ext is not None:
dest = os.path.splitext(dest)[0] + ext
return dest
def image_magick_resize(input_file, output_file, size, quality=85):
s = '%dx%d' % size
call(['convert', input_file, '-resize', s, '-quality', str(quality), output_file])
def image_magick_pdf_to_img(input_file, output_file):
i = "{image}[0]".format(image=input_file)
call(['convert', i, output_file])
+36
View File
@@ -0,0 +1,36 @@
import yaml
from markdown import markdown
from utils.text.split import split, split_with
def read_file(file):
with open(file, 'rb') as f:
data = f.read().decode('utf-8')
return data
def read_yaml(file):
with open(file, 'rb') as f:
data = f.read().decode('utf-8')
return yaml.load(data)
def read_md(file):
with open(file, 'rb') as f:
data = f.read().decode('utf-8')
return markdown(data)
def read_yaml_md(file):
data = read_file(file)
yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data)
try:
y = yaml.load(yaml_data)
except:
y = None
m = markdown(md_data) if md_data else None
return y, m
+29
View File
@@ -0,0 +1,29 @@
import re
def get_following_tags(element, stop_tag_pattr):
stop_exp = re.compile(stop_tag_pattr)
following = element.xpath('./following-sibling::*')
tags = []
i = 0
while True:
try:
tag = following[i]
if stop_exp.match(tag.tag):
break
tags.append(tag)
except IndexError as tag:
break
i += 1
return tags
def get_text_following_by_tag(element, tag, contains, strip=True):
e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains))
if len(e) > 0:
if strip:
return e[0].strip()
else:
return e[0]
else:
return None
+18
View File
@@ -0,0 +1,18 @@
from utils.fn import lmap, until
def split(data, separator=' '):
try:
s = data.index(separator)
return data[:s], data[s:]
except ValueError:
return None, None
def split_with(separators):
return lambda data: until(
lambda i: i[0] and i[1],
lambda s: split(data, s),
separators,
(None, None)
)
+50
View File
@@ -0,0 +1,50 @@
import re
from functools import reduce
import settings
from utils.fn import iterate_over_fns
def translit(text):
"""
RUSSIAN TEXT => ENGLISH TEXT
based on table like:
(
['а', 'б', 'в'],
['a', 'b', 'v']
)
Leave if character is not founded
"""
table = settings.translit_table
ins, outs = table
out = lambda i: outs[ins.index(i)]
replace = lambda i: out(i) if i in ins else i
return reduce(
lambda t, i: t + replace(i),
list(text),
''
)
def interpolate(text, storage, key_fn=None):
if not key_fn:
key_fn = lambda i: i
key = key_fn(text)
return storage[key] if key in storage else text
interpolate_swift = lambda i: re.search("([\d\w-]+)", i).group(1)
remove_pattr = re.compile('[«»,!&]')
remove_punctuation = lambda i: remove_pattr.sub('', i)
space_to_dash = lambda i: i.replace(' ', '-')
dot_to_dash = lambda i: i.replace('.', '-')
remove_underscores = lambda i: i.replace('_', '')
text_lower = lambda i: i.lower()
url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower])
url_encode_text = iterate_over_fns(
[translit, space_to_dash, dot_to_dash, remove_underscores, remove_punctuation, text_lower])
+18
View File
@@ -0,0 +1,18 @@
import os
import re
import settings
__author__ = 'Roman Timashev'
__check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings.description_available_chars)
def check_description_filename(filename):
return __check_description_filename_exp.match(filename)
def suggest_description_filename(filename):
bn = os.path.basename(filename)
bn = os.path.splitext(bn)[0]
v = settings.default_description_version
return '%s [%s].md' % (bn, v)