sync moved to modules

modules/sync/utils/fn.py

@@ -0,0 +1,45 @@
+import os
+from functools import reduce
+
+
+def constant(value):
+    return lambda _: value
+
+
+def combine(items):
+    if len(items) == 0:
+        return []
+
+    return reduce(
+        lambda total, i: total + i,
+        items
+    )
+
+
+def compose(*fns):
+    return lambda i: reduce(
+        lambda out, fn: fn(out),
+        fns, i
+    )
+
+
+first = lambda i: i[0] if len(i) > 0 else None
+
+
+def swap_ext(extension: str):
+    return lambda i: os.path.splitext(i)[0] + extension
+
+
+def map_cases(param, cases):
+    for predicate, fn in cases:
+        if predicate(param):
+            return fn(param)
+    return param
+
+
+def until(condition_fn, iter_fn, items, default_value=None):
+    for i in items:
+        result = iter_fn(i)
+        if condition_fn(result):
+            return result
+    return default_value

modules/sync/utils/hash.py

@@ -0,0 +1,38 @@
+import hashlib
+import json
+
+import bson.json_util
+
+from utils.fn import map_cases
+
+
+def hash_file(file):
+    with open(file, 'rb') as f:
+        data = f.read()
+        return hash_data(data)
+
+
+def hash_data(data):
+    sha = hashlib.sha256(data)
+    return sha.hexdigest()
+
+
+def hash_str(data):
+    to_json = lambda i: json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
+
+    data = map_cases(data, [
+        (lambda i: isinstance(i, dict), to_json),
+        (lambda i: isinstance(i, tuple), to_json),
+        (lambda i: isinstance(i, list), to_json),
+        (lambda i: isinstance(i, int), int),
+        (lambda i: isinstance(i, float), float),
+    ])
+    if isinstance(data, dict):
+        data = to_json(data)
+
+    return hash_data(str(data).encode())
+
+
+def md5(data: str) -> str:
+    i = hashlib.md5(data.encode('utf-8'))
+    return i.hexdigest()

modules/sync/utils/image.py

@@ -0,0 +1,35 @@
+import asyncio
+from subprocess import call
+
+
+async def image_magick_resize(input_file, output_file, size, quality=85):
+    interlace = 'Plane'
+    s = '%dx%d' % size
+
+    cmd = [
+        '/usr/local/bin/convert',
+        '"%s"' % input_file,
+
+        '-strip',
+        '-interlace', interlace,
+        '-auto-orient',
+        '-resize', s,
+        '-quality', str(quality),
+
+        '"%s"' % output_file,
+    ]
+
+    cmd = ' '.join(cmd)
+    process = await asyncio.create_subprocess_shell(cmd)
+    await process.wait()
+
+
+async def image_magick_pdf_to_img(input_file, output_file):
+    call([
+        '/usr/local/bin/gs',
+        '-q',
+        '-sDEVICE=jpeg',
+        '-dLastPage=1',
+        '-o', output_file,
+        input_file,
+    ])

modules/sync/utils/io.py

@@ -0,0 +1,47 @@
+import re
+import yaml
+
+from kazimir import markdown_to_html
+
+FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
+
+
+def read_yaml_md(data):
+    yaml_front_matter, content = split_yaml_front_matter(data)
+    return yaml_front_matter, markdown_to_html(content)
+
+
+def parse_yaml_front_matter(data: str):
+    fm, content = split_yaml_front_matter(data)
+    fm = fm if fm else {}
+    return {
+        **fm,
+        'content': content,
+    }
+
+
+def split_yaml_front_matter(text, **defaults):
+    """
+    Parse text with YAML frontmatter, return metadata and content.
+    Pass in optional metadata defaults as keyword args.
+
+    If frontmatter is not found, returns an empty metadata dictionary
+    and original text content.
+    """
+
+    # metadata starts with defaults
+    metadata = defaults.copy()
+
+    # split on the first two triple-dashes
+    try:
+        _, fm, content = FM_BOUNDARY.split(text, 2)
+    except ValueError:
+        # if we can't split, bail
+        return metadata, text
+
+    # parse yaml, now that we have frontmatter
+    fm = yaml.safe_load(fm)
+    if isinstance(fm, dict):
+        metadata.update(fm)
+
+    return metadata, content.strip()

modules/sync/utils/lxml_utils.py

@@ -0,0 +1,29 @@
+import re
+
+
+def get_following_tags(element, stop_tag_pattr):
+    stop_exp = re.compile(stop_tag_pattr)
+    following = element.xpath('./following-sibling::*')
+    tags = []
+    i = 0
+    while True:
+        try:
+            tag = following[i]
+            if stop_exp.match(tag.tag):
+                break
+            tags.append(tag)
+        except IndexError as tag:
+            break
+        i += 1
+    return tags
+
+
+def get_text_following_by_tag(element, tag, contains, strip=True):
+    e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains))
+    if len(e) > 0:
+        if strip:
+            return e[0].strip()
+        else:
+            return e[0]
+    else:
+        return None

modules/sync/utils/text/file.py

@@ -0,0 +1,29 @@
+import os
+
+import math
+
+
+def default_ext(ext: str):
+    return ext.upper()
+
+
+def get_ext(filename):
+    ext_s = ['.png', '.pdf', '.jpg', '.jpeg', '.gif', '.doc']
+    ext_d = ['ПНГ', 'ПДФ', 'ЖПГ', 'ЖПГ', 'ГИФ', 'ДОК']
+
+    _, ext = os.path.splitext(filename)
+
+    if ext in ext_s:
+        i = ext_s.index(ext)
+        return ext_d[i]
+    else:
+        default_ext(ext)
+
+
+def get_size(number_of_bytes, precision=1):
+    units = ['байт', 'КБ', 'МБ', 'ГБ', 'ТБ', 'ПБ']
+
+    number = math.floor(math.log(number_of_bytes) / math.log(1024))
+    size = (number_of_bytes / math.pow(1024, math.floor(number)))
+    size = round(size, precision)
+    return f'{size} {units[number]}'

modules/sync/utils/text/split.py

@@ -0,0 +1,18 @@
+from utils.fn import until
+
+
+def split(data, separator=' '):
+    try:
+        s = data.index(separator)
+        return data[:s], data[s:]
+    except ValueError:
+        return None, None
+
+
+def split_with(separators):
+    return lambda data: until(
+        lambda i: i[0] and i[1],
+        lambda s: split(data, s),
+        separators,
+        (None, None)
+    )

modules/sync/utils/text/transform.py

@@ -0,0 +1,94 @@
+import re
+
+from unidecode import unidecode
+
+from utils.fn import compose
+
+remove_pattr = re.compile('[«»<>#,!&"\']')
+reduce_dash_patterns = [
+    (re.compile('[-]+'), '-'),
+    (re.compile('^-'), '')
+]
+
+
+def translit(text):
+    """
+    RUSSIAN TEXT => ENGLISH TEXT
+    """
+    return unidecode(text)
+
+
+def interpolate(text, storage, key_fn=None):
+    if not key_fn:
+        key_fn = lambda i: i
+
+    key = key_fn(text)
+    return storage[key] if key in storage else text
+
+
+def interpolate_swift(i):
+    return re.search("([\d\w-]+)", i).group(1)
+
+
+def remove_punctuation(i):
+    return remove_pattr.sub('', i)
+
+
+def reduce_dashes(i):
+    value = i
+    for reg_exp, replace in reduce_dash_patterns:
+        value = reg_exp.sub(replace, value)
+    return value
+
+
+def space_to_dash(i):
+    return i.replace(' ', '-')
+
+
+def short_dash(i):
+    """
+    minus -> dash
+    tire -> dash
+    :param i:
+    :return:
+    """
+    return i.replace('–', '-').replace('—', '-')
+
+
+def dot_to_dash(i):
+    return i.replace('.', '-')
+
+
+def slash_to_dash(i):
+    return i.replace('/', '-')
+
+
+def remove_underscores(i):
+    return i.replace('_', '')
+
+
+def text_lower(i):
+    return i.lower()
+
+
+url_encode_file = compose(
+    translit,
+    space_to_dash,
+    short_dash,
+    remove_underscores,
+    remove_punctuation,
+    reduce_dashes,
+    text_lower,
+)
+
+url_encode_text = compose(
+    translit,
+    space_to_dash,
+    dot_to_dash,
+    short_dash,
+    slash_to_dash,
+    remove_underscores,
+    remove_punctuation,
+    reduce_dashes,
+    text_lower,
+)

modules/sync/utils/validation.py

@@ -0,0 +1,11 @@
+import os
+import re
+import settings
+
+__author__ = 'Roman Timashev'
+
+__check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings.description_available_chars)
+
+
+def check_description_filename(filename):
+    return __check_description_filename_exp.match(filename)