Sync util big update
This commit is contained in:
1 parent
1e80bae401
commit
40321327d6
30 files changed
+1001
-159
No files matched your search
@@ -0,0 +1,68 @@
|
||||
import json
|
||||
import os
|
||||
from functools import reduce
|
||||
from glob import glob
|
||||
|
||||
from bson import json_util
|
||||
|
||||
combine = lambda i: reduce(
|
||||
lambda s, i: s + i,
|
||||
i
|
||||
)
|
||||
|
||||
iterate_over_fns = lambda fns: lambda i: reduce(
|
||||
lambda out, fn: fn(out),
|
||||
fns, i
|
||||
)
|
||||
|
||||
iterate_iter_over_fns = lambda fns: lambda i: lmap(
|
||||
iterate_over_fns(fns),
|
||||
i
|
||||
)
|
||||
|
||||
lmap = lambda fn, ls: list(map(fn, ls))
|
||||
lprint = lambda i: lmap(print, i)
|
||||
lprint_json = lambda s: lprint(lmap(
|
||||
lambda i: json.dumps(i, ensure_ascii=False, indent=4, default=json_util.default),
|
||||
s
|
||||
))
|
||||
|
||||
first = lambda i: i[0] if len(i) > 0 else None
|
||||
|
||||
|
||||
def map_item_key(item, key, map_fn):
|
||||
"""
|
||||
|
||||
:param item:
|
||||
:param key:
|
||||
:param map_fn:
|
||||
:return:
|
||||
"""
|
||||
item[key] = map_fn(item[key])
|
||||
return item
|
||||
|
||||
|
||||
key_mapper = lambda key, fn: lambda i: map_item_key(i, key, fn)
|
||||
|
||||
|
||||
def map_cases(param, cases, default_fn=None):
|
||||
for c, fn in cases:
|
||||
if c(param):
|
||||
return fn(param)
|
||||
return param if not default_fn else default_fn(param)
|
||||
|
||||
|
||||
def until(condition_fn, iter_fn, items, default_value=None):
|
||||
for i in items:
|
||||
result = iter_fn(i)
|
||||
if condition_fn(result):
|
||||
return result
|
||||
return default_value
|
||||
|
||||
|
||||
# glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
|
||||
# glob_in('./images')
|
||||
dir_globber = lambda mask_list: lambda dir: combine(map(
|
||||
lambda mask: glob(os.path.join(dir, mask)),
|
||||
mask_list
|
||||
))
|
||||
@@ -0,0 +1,9 @@
|
||||
import hashlib
|
||||
|
||||
|
||||
def file_hash(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read()
|
||||
sha = hashlib.sha256(data)
|
||||
f.close()
|
||||
return sha.hexdigest()
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from utils.hash import file_hash
|
||||
from utils.image import resize
|
||||
from utils.image.resize import optimize, thumbnail
|
||||
|
||||
|
||||
def read_image(src):
|
||||
try:
|
||||
return Image.open(src)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
|
||||
"""
|
||||
|
||||
:param file: Image path to process
|
||||
:param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
|
||||
:param url_fn: function (size_name, ext) for creating image url
|
||||
:param output_dir: path to local folder for thumbs storing
|
||||
:param skip_processing: True/False to skip thumbs generation
|
||||
:return:
|
||||
"""
|
||||
image = read_image(file)
|
||||
if not image:
|
||||
return None
|
||||
|
||||
result = {}
|
||||
_, ext = os.path.splitext(file)
|
||||
for size in sizes:
|
||||
size_name, width, height = size
|
||||
|
||||
image_url = url_fn(size_name, ext)
|
||||
image_filename = os.path.basename(image_url)
|
||||
d = os.path.join(output_dir, image_filename)
|
||||
|
||||
if not skip_processing:
|
||||
thumb = optimize(image, d, quality=90) if size_name == 'original' else thumbnail(image, d, (width, height))
|
||||
width, height = thumb.size
|
||||
|
||||
result[size_name] = {
|
||||
'url': image_url,
|
||||
'size': size_name,
|
||||
'width': width,
|
||||
'height': height
|
||||
}
|
||||
|
||||
return {
|
||||
'file': os.path.basename(file),
|
||||
'hash': file_hash(file),
|
||||
'data': result
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import os
|
||||
|
||||
from subprocess import call
|
||||
|
||||
|
||||
def thumbnail(image, dest, size, ext=None):
|
||||
image.thumbnail(size)
|
||||
image.save(override_extension(dest, ext))
|
||||
return image
|
||||
|
||||
|
||||
def optimize(image, dest, quality=85, ext=None):
|
||||
image.save(override_extension(dest, ext), quality=quality)
|
||||
return image
|
||||
|
||||
|
||||
def override_extension(dest, ext=None):
|
||||
if ext is not None:
|
||||
dest = os.path.splitext(dest)[0] + ext
|
||||
|
||||
return dest
|
||||
|
||||
|
||||
def image_magick_resize(input_file, output_file, size, quality=85):
|
||||
s = '%dx%d' % size
|
||||
call(['convert', input_file, '-resize', s, '-quality', str(quality), output_file])
|
||||
|
||||
|
||||
def image_magick_pdf_to_img(input_file, output_file):
|
||||
i = "{image}[0]".format(image=input_file)
|
||||
call(['convert', i, output_file])
|
||||
@@ -0,0 +1,36 @@
|
||||
import yaml
|
||||
from markdown import markdown
|
||||
|
||||
from utils.text.split import split, split_with
|
||||
|
||||
|
||||
def read_file(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
return data
|
||||
|
||||
|
||||
def read_yaml(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
return yaml.load(data)
|
||||
|
||||
|
||||
def read_md(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
return markdown(data)
|
||||
|
||||
|
||||
def read_yaml_md(file):
|
||||
data = read_file(file)
|
||||
yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data)
|
||||
|
||||
try:
|
||||
y = yaml.load(yaml_data)
|
||||
except:
|
||||
y = None
|
||||
|
||||
m = markdown(md_data) if md_data else None
|
||||
|
||||
return y, m
|
||||
@@ -0,0 +1,29 @@
|
||||
import re
|
||||
|
||||
|
||||
def get_following_tags(element, stop_tag_pattr):
|
||||
stop_exp = re.compile(stop_tag_pattr)
|
||||
following = element.xpath('./following-sibling::*')
|
||||
tags = []
|
||||
i = 0
|
||||
while True:
|
||||
try:
|
||||
tag = following[i]
|
||||
if stop_exp.match(tag.tag):
|
||||
break
|
||||
tags.append(tag)
|
||||
except IndexError as tag:
|
||||
break
|
||||
i += 1
|
||||
return tags
|
||||
|
||||
|
||||
def get_text_following_by_tag(element, tag, contains, strip=True):
|
||||
e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains))
|
||||
if len(e) > 0:
|
||||
if strip:
|
||||
return e[0].strip()
|
||||
else:
|
||||
return e[0]
|
||||
else:
|
||||
return None
|
||||
@@ -0,0 +1,18 @@
|
||||
from utils.fn import lmap, until
|
||||
|
||||
|
||||
def split(data, separator=' '):
|
||||
try:
|
||||
s = data.index(separator)
|
||||
return data[:s], data[s:]
|
||||
except ValueError:
|
||||
return None, None
|
||||
|
||||
|
||||
def split_with(separators):
|
||||
return lambda data: until(
|
||||
lambda i: i[0] and i[1],
|
||||
lambda s: split(data, s),
|
||||
separators,
|
||||
(None, None)
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
import re
|
||||
from functools import reduce
|
||||
|
||||
import settings
|
||||
from utils.fn import iterate_over_fns
|
||||
|
||||
|
||||
def translit(text):
|
||||
"""
|
||||
RUSSIAN TEXT => ENGLISH TEXT
|
||||
based on table like:
|
||||
(
|
||||
['а', 'б', 'в'],
|
||||
['a', 'b', 'v']
|
||||
)
|
||||
Leave if character is not founded
|
||||
"""
|
||||
table = settings.translit_table
|
||||
ins, outs = table
|
||||
out = lambda i: outs[ins.index(i)]
|
||||
replace = lambda i: out(i) if i in ins else i
|
||||
|
||||
return reduce(
|
||||
lambda t, i: t + replace(i),
|
||||
list(text),
|
||||
''
|
||||
)
|
||||
|
||||
|
||||
def interpolate(text, storage, key_fn=None):
|
||||
if not key_fn:
|
||||
key_fn = lambda i: i
|
||||
|
||||
key = key_fn(text)
|
||||
return storage[key] if key in storage else text
|
||||
|
||||
|
||||
interpolate_swift = lambda i: re.search("([\d\w-]+)", i).group(1)
|
||||
|
||||
remove_pattr = re.compile('[«»,!&]')
|
||||
remove_punctuation = lambda i: remove_pattr.sub('', i)
|
||||
|
||||
space_to_dash = lambda i: i.replace(' ', '-')
|
||||
dot_to_dash = lambda i: i.replace('.', '-')
|
||||
remove_underscores = lambda i: i.replace('_', '')
|
||||
text_lower = lambda i: i.lower()
|
||||
|
||||
url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower])
|
||||
url_encode_text = iterate_over_fns(
|
||||
[translit, space_to_dash, dot_to_dash, remove_underscores, remove_punctuation, text_lower])
|
||||
@@ -0,0 +1,18 @@
|
||||
import os
|
||||
import re
|
||||
import settings
|
||||
|
||||
__author__ = 'Roman Timashev'
|
||||
|
||||
__check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings.description_available_chars)
|
||||
|
||||
|
||||
def check_description_filename(filename):
|
||||
return __check_description_filename_exp.match(filename)
|
||||
|
||||
|
||||
def suggest_description_filename(filename):
|
||||
bn = os.path.basename(filename)
|
||||
bn = os.path.splitext(bn)[0]
|
||||
v = settings.default_description_version
|
||||
return '%s [%s].md' % (bn, v)
|
||||
Reference in new issue
Block a user