From 0693e7c758fb0031d803f96b6b49aa35eff4d389 Mon Sep 17 00:00:00 2001
From: Roman Timashev <roman@tmshv.ru>
Date: Sat, 9 Jan 2016 17:59:42 +0300
Subject: [PATCH] + sync utils

---
 utils/Sync/Album.py          |  22 ++++
 utils/Sync/db-test.py        |  13 ++
 utils/Sync/img.py            |  25 ++++
 utils/Sync/lxml_utils.py     |  29 +++++
 utils/Sync/read_gallery_1.py | 112 ++++++++++++++++++
 utils/Sync/settings.py       |  53 +++++++++
 utils/Sync/sync.py           | 223 +++++++++++++++++++++++++++++++++++
 utils/Sync/sync_gallery.py   | 210 +++++++++++++++++++++++++++++++++
 utils/Sync/sync_schedules.py |  31 +++++
 utils/Sync/validation.py     |  18 +++
 10 files changed, 736 insertions(+)
 create mode 100644 utils/Sync/Album.py
 create mode 100644 utils/Sync/db-test.py
 create mode 100644 utils/Sync/img.py
 create mode 100644 utils/Sync/lxml_utils.py
 create mode 100644 utils/Sync/read_gallery_1.py
 create mode 100644 utils/Sync/settings.py
 create mode 100644 utils/Sync/sync.py
 create mode 100644 utils/Sync/sync_gallery.py
 create mode 100644 utils/Sync/sync_schedules.py
 create mode 100644 utils/Sync/validation.py

diff --git a/utils/Sync/Album.py b/utils/Sync/Album.py
new file mode 100644
index 00000000..59cec1f2
--- /dev/null
+++ b/utils/Sync/Album.py
@@ -0,0 +1,22 @@
+__author__ = 'Roman Timashev'
+
+
+class Album:
+    def __init__(self, type, title, uri, course, course_uri, date, teacher, comment):
+        super().__init__()
+        self.products = []
+        self.type = type
+        self.title = title
+        self.uri = uri
+        self.course = course
+        self.course_uri = course_uri
+        self.date = date
+        self.teacher = teacher
+        self.comment = comment
+
+    def add_product(self, product):
+        self.products.append(product)
+
+    def __repr__(self, *args, **kwargs):
+        return '[%s by %s (%d items)]' % (self.title, self.teacher, len(self.products))
+
diff --git a/utils/Sync/db-test.py b/utils/Sync/db-test.py
new file mode 100644
index 00000000..2c8a1a95
--- /dev/null
+++ b/utils/Sync/db-test.py
@@ -0,0 +1,13 @@
+from pymongo import MongoClient
+import settings
+
+__author__ = 'Roman Timashev'
+
+
+client = MongoClient(settings.database_uri)
+p = client.hudozka.posts
+
+print(list(p.find()))
+
+# print(dir(p))
+# print(p.insert({'a': 1}))
\ No newline at end of file
diff --git a/utils/Sync/img.py b/utils/Sync/img.py
new file mode 100644
index 00000000..e4a1fb9d
--- /dev/null
+++ b/utils/Sync/img.py
@@ -0,0 +1,25 @@
+import os
+
+__author__ = 'Roman Timashev'
+
+from PIL import Image
+
+
+def read_image(src):
+    return Image.open(src)
+
+
+def minify(image, dest, size, ext=None):
+    image.thumbnail(size)
+    image.save(override_extension(dest, ext))
+
+
+def optimize(image, dest, quality=85, ext=None):
+    image.save(override_extension(dest, ext), quality=quality)
+
+
+def override_extension(dest, ext=None):
+    if ext is not None:
+        dest = os.path.splitext(dest)[0] + ext
+
+    return dest
\ No newline at end of file
diff --git a/utils/Sync/lxml_utils.py b/utils/Sync/lxml_utils.py
new file mode 100644
index 00000000..4972ced0
--- /dev/null
+++ b/utils/Sync/lxml_utils.py
@@ -0,0 +1,29 @@
+import re
+
+
+def get_following_tags(element, stop_tag_pattr):
+    stop_exp = re.compile(stop_tag_pattr)
+    following = element.xpath('./following-sibling::*')
+    tags = []
+    i = 0
+    while True:
+        try:
+            tag = following[i]
+            if stop_exp.match(tag.tag):
+                break
+            tags.append(tag)
+        except IndexError as tag:
+            break
+        i += 1
+    return tags
+
+
+def get_text_following_by_tag(element, tag, contains, strip=True):
+    e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains))
+    if len(e) > 0:
+        if strip:
+            return e[0].strip()
+        else:
+            return e[0]
+    else:
+        return None
\ No newline at end of file
diff --git a/utils/Sync/read_gallery_1.py b/utils/Sync/read_gallery_1.py
new file mode 100644
index 00000000..a010c4f1
--- /dev/null
+++ b/utils/Sync/read_gallery_1.py
@@ -0,0 +1,112 @@
+from datetime import datetime
+import re
+import lxml.html
+from markdown import markdown
+import time
+from Album import Album
+import lxml_utils
+import settings
+
+__author__ = 'Roman Timashev'
+
+artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER+settings.CYR_UPPER))
+# artist_name_exp = re.compile('(.+)(\s\(\d+ лет\))?$')
+artist_age_exp = re.compile('.+\((\d+) лет\)$')
+
+class GalleryReader:
+    def __init__(self, path):
+        super().__init__()
+        self.html = None
+        self.description_path = path
+
+    def read(self):
+        with open(self.description_path, 'r', encoding='utf-8') as dfile:
+            data = dfile.read()
+            md = markdown(data)
+            self.html = lxml.html.fromstring(md)
+
+            # ALBUM
+            try:
+                album_title = self.html.xpath('.//h1/a/text()')[0]
+                album_uri = self.html.xpath('.//h1/a/@href')[0]
+            except:
+                raise Exception('Cannot read album title/uri')
+
+            # COURSE
+            try:
+                course_title = self.html.xpath('.//p/a/text()')[0]
+                course_uri = self.html.xpath('.//p/a/@href')[0]
+            except:
+                raise Exception('Cannot read course title/uri')
+
+            # TEACHER
+            teacher = lxml_utils.get_text_following_by_tag(self.html, 'em', "Преподаватель")
+            teacher = re.sub(r'\s', '', teacher)
+            if teacher not in settings.teachers_names:
+                raise Exception('Teacher %s is not found' % teacher)
+            teacher_name = settings.teachers_names[teacher]
+
+            # DATE
+            date = self.get_date()
+            if date is None:
+                raise Exception('Date field is empty')
+
+            # PRODUCT
+            content = self.get_drawings()
+
+            a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date, teacher=teacher_name, comment=None)
+            for p in content:
+                a.add_product(p)
+            return a
+
+    def get_drawings(self):
+        drawings = []
+        h3_list = self.html.xpath('.//h3')
+        for h3 in h3_list:
+            artist_name_age = h3.xpath('./a')
+            if len(artist_name_age) == 0:
+                raise Exception('Artist name should be a link')
+            artist_name_age = artist_name_age[0].text
+
+            # ARTIST NAME [required]
+            if artist_name_age is None:
+                raise Exception('Artist name section is empty')
+
+            exp = artist_name_exp.search(artist_name_age)
+            if exp is None:
+                raise Exception('Artist name is not specified')
+            artist_name = exp.group(1).strip()
+
+            # ARTIST AGE [optional]
+            exp = artist_age_exp.search(artist_name_age)
+            if exp is not None and len(exp.groups()) > 0:
+                artist_age = int(exp.group(1))
+            else:
+                artist_age = None
+
+            image_filename = h3.xpath('./a/@href')[0]
+
+            # PRODUCT TITLE [optional]
+            drawing_data = lxml_utils.get_following_tags(h3, 'h3')
+            if len(drawing_data) == 0:
+                product_title = None
+                # raise Exception('Product title not specified')
+            else:
+                product_title = drawing_data[0].text[1:-1]
+
+            drawings.append((artist_name, artist_age, image_filename, product_title))
+
+        return drawings
+
+    def get_date(self):
+        string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата")
+        if string is None:
+            return None
+
+        for f in settings.date_formats:
+            try:
+                date = time.strptime(string, f)
+                return datetime.fromtimestamp(time.mktime(date))
+            except ValueError:
+                continue
+        return None
diff --git a/utils/Sync/settings.py b/utils/Sync/settings.py
new file mode 100644
index 00000000..f95b4f98
--- /dev/null
+++ b/utils/Sync/settings.py
@@ -0,0 +1,53 @@
+__author__ = 'Roman Timashev'
+
+root_dir_gallery = './Gallery'
+root_dir_schedule = './Schedules'
+
+default_description_version = 1
+
+database_uri = 'mongodb://localhost/hudozka'
+
+CYR_LOWER = 'йцукенгшщзхъфывапролджэячсмитьбю'
+CYR_UPPER = 'ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮ'
+EN_LOWER = 'qwertyuiopasdfghjklzxcvbnm'
+EN_UPPER = 'QWERTYUIOPASDFGHJKLZXCVBNM'
+DIGITS = '0123456789'
+description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-'
+
+translit_table = (
+    'абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ',
+    'abvgdeejzijklmnoprstufhzcss_y_euaABVGDEEJZIJKLMNOPRSTUFHZCSS_Y_EUA'
+)
+
+teachers_names = {
+    'Н.В.Андреева': 'nv-andreeva',
+    'И.Н.Втюрина': 'in-vturina',
+    'О.Д.Гоголева': 'od-gogoleva',
+    'В.А.Саржин': 'va-sarzhin',
+    'Р.К.Тимашев': 'rk-timashev',
+    'А.С.Тимашева': 'as-timasheva',
+    # '': '',
+}
+
+image_processing = True
+
+image_save_dir = './static_images'
+
+image_base_url = 'https://static.shburg.org/art/image/'
+image_ext = '.jpg'
+image_sizes = [
+    ('original', None, None),
+    ('big', 1500, 667),
+    ('medium', 400, 300),
+    ('small', 250, 175),
+    ('little', 100, 100),
+    ('preview', 50, 50),
+]
+
+date_formats = [
+    '%Y',
+    '%m.%Y',
+    '%d.%m.%Y',
+    '%d.%m.%Y %H:%M',
+    '%d.%m.%Y %H:%M:%S'
+]
diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py
new file mode 100644
index 00000000..7cecd4cf
--- /dev/null
+++ b/utils/Sync/sync.py
@@ -0,0 +1,223 @@
+from glob import glob
+import os
+import markdown as md
+import lxml.html as html
+from lxml.etree import tostring
+import re
+import pymongo
+import requests
+import img
+import lxml_utils
+
+__author__ = 'tmshv'
+
+# db = pymongo.MongoClient('mongodb://localhost:27017')['stickers']
+
+# images_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Static/images')
+catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Hudozka/Gallery')
+catalog_files = os.listdir(catalog_dir)
+
+
+def lesson_info(page):
+    lesson = list(filter(lambda h2: not h2.xpath('./a'), page.xpath('//h2')))[0]
+    content = lxml_utils.get_following_tags(lesson, 'h2')
+    for p in lxml_utils.get_following_tags(lesson, 'h2'):
+        print(p, tostring(p, encoding='utf-8').decode())
+
+    # print(content[0].xpath('./em/text()')[0].strip())
+    # print(content[0].xpath('./text()')[0].strip())
+
+    # print(content[1].xpath('./text()'))
+    # return content
+
+    date_str = '12.10.2013'
+
+    #todo: make as `date`
+    date = date_str
+
+    return {
+        'name': 'Collage',
+        'course': 'painting',
+        'technology': ['pastel', 'huel', 'glue'],
+        'date': date,
+    }
+
+
+def drawings_info(page):
+    out = []
+    pics = list(filter(lambda h2: h2.xpath('./a'), page.xpath('//h2')))
+    for p in pics:
+        content = lxml_utils.get_following_tags(p, 'h2')[0]
+        author_info = content.xpath('./text()')[0].split('\n')
+        out.append({
+            'name': p.xpath('./a/text()')[0],
+            'image': p.xpath('./a/@href')[0],
+            'author': author_info[0],
+            'author_age': int(author_info[1].split(' ')[0])
+        })
+    return out
+
+
+def generate_image_filename(src, course, year, theme, author, size, i,ext):
+    if ext is None:
+        ext = os.path.splitext(src)[1]
+
+    if i == 0:
+        filename = '{course}-{year}-{theme}-{author}-{size}.{ext}'
+    else:
+        filename = '{course}-{year}-{theme}-{author}-{index}-{size}.{ext}'\
+
+    filename = filename.format(
+        course=course,
+        year=year,
+        theme=theme,
+        author=author,
+        size=size,
+        index=i,
+        ext=ext
+    )
+    return filename
+
+
+for cf in catalog_files:
+    os.chdir(catalog_dir)
+    if os.path.isdir(cf):
+        print('======================================================================================')
+        print(cf)
+
+        product_dir = os.path.join(catalog_dir, cf)
+        os.chdir(product_dir)
+
+        def matching_files(param):
+            if isinstance(param, list):
+                return list(map(matching_files, param))
+            else:
+                return glob(param)
+
+
+        product_file = matching_files('*.md')[0]
+
+        def from_file(path):
+            f = open(path, 'r')
+            data = f.read()
+            f.close()
+            return data
+
+        markdown_text = from_file(product_file)
+        html_text = md.markdown(markdown_text)
+        page = html.fromstring(html_text)
+        print(html_text)
+
+        # name
+        name = page.xpath('//h1/text()')[0]
+
+        # lesson
+        # filtering paragraph with no link in h2 title
+        lesson = lesson_info(page)
+        print(lesson)
+
+        drawings = drawings_info(page)
+        for d in drawings:
+            print(d)
+
+        products = list(map(
+            lambda drawing: {
+                'image': drawing['image'],
+                'name': drawing['name'],
+                'author': drawing['author'],
+                'authorAge': drawing['author_age'],
+                'course': lesson['course'],
+                'theme': lesson['name'],
+                'rating': 10,
+                'date': lesson['date']
+            },
+            drawings
+        ))
+
+        for p in products:
+            print(p)
+
+            # # price
+            # def str_price_to_int(text):
+            # items = text.split('\n')
+            # out = list(map(lambda p: int(re.sub('[^\d]+', '', p)), items))
+            # return out
+            #
+            #
+            # price_text = page.xpath('//h2[contains(text(), "Цена")]/following-sibling::p/text()')[0]
+            # product_price = str_price_to_int(price_text)
+            #
+            # # colors
+            # def str_colors_to_code_list(text):
+            #     cl = re.split('\s+', text)
+            #     return ttcolor.color_by_code(cl)
+            #
+            # color_text = page.xpath('//h2[contains(text(), "Цвета")]/following-sibling::p/text()')[0]
+            # product_colors = str_colors_to_code_list(color_text)
+            #
+            # # text
+            # def format_text(text):
+            #     try:
+            #         res = requests.post("http://mdash.ru/api.v1.php", {
+            #             'text': text
+            #         })
+            #         return res.json()['result']
+            #
+            #     except Exception as e:
+            #
+            #         return text
+            #
+            # product_text = page.xpath('//h2[contains(text(), "Текст")]/following-sibling::p/text()')[0]
+            # # product_text = format_text(product_text)
+            #
+            #
+            # # qualities
+            # def get_quality(li):
+            #     q = {
+            #         'image': li.xpath('./img/@src')[0],
+            #         'text': li.xpath('text()')[0].strip()
+            #     }
+            #     return q
+            #
+            #
+            # def qualities_list(elements):
+            #     return list(map(get_quality, elements))
+            #
+            #
+            # qualities_elements = page.xpath('//h2[contains(text(), "Свойства")]/following-sibling::ul/li')
+            # qualities = qualities_list(qualities_elements)
+            #
+            #
+
+            product_images = [p['image']]
+
+            image_sizes = [
+                (50, 50, 'small'),
+                (300, 300, 'preview'),
+                (800, 800, 'medium'),
+                (2000, 2000, 'big'),
+                (0, 0, 'original')
+            ]
+
+            images = []
+            i = 0
+            for image in product_images:
+                pi = {'source': image}
+
+                for size in image_sizes:
+                    size_name = size[2]
+                    image_filename = generate_image_filename(image, size_name, product_sku, i, 'png')
+                    d = os.path.join(images_dir, image_filename)
+                    pi[size_name] = 'http://static.touchtwin.ru/images/{img}'.format(img=image_filename)
+                    if size_name == 'original':
+                        img.optimize(image, d, quality=90)
+                    else:
+                        img.minify(image, d, size[:2])
+
+                images.append(pi)
+                i += 1
+
+            p['images'] = images
+            del p['image']
+
+            # db['products'].update({'uri': product_uri}, p, upsert=True)
\ No newline at end of file
diff --git a/utils/Sync/sync_gallery.py b/utils/Sync/sync_gallery.py
new file mode 100644
index 00000000..35b944a6
--- /dev/null
+++ b/utils/Sync/sync_gallery.py
@@ -0,0 +1,210 @@
+from glob import glob
+import os
+import re
+from pymongo import MongoClient
+import img
+from read_gallery_1 import GalleryReader
+
+import settings
+import validation
+
+__author__ = 'Roman Timashev'
+
+description_version_exp = re.compile('\[(\d+)\]\.md$')
+
+dir_gallery = settings.root_dir_gallery
+catalog_dir = os.path.expanduser(dir_gallery)
+catalog_files = os.listdir(catalog_dir)
+os.makedirs(settings.image_save_dir, exist_ok=True)
+
+
+def get_description_version(filename):
+    v = description_version_exp.search(filename)
+    return v.group(1)
+
+
+def get_description_reader(version):
+    if version == '1':
+        return GalleryReader
+
+    return None
+
+
+def process_image(name, filename, sizes):
+    ext = settings.image_ext
+
+    thumbs = {}
+
+    for size, width, height in sizes:
+        source_image = img.read_image(filename)
+        sw, sh = source_image.size
+        if sw < sh:
+            width, height = height, width
+
+        out_image_filename = '{name}-{size}{ext}'.format(name=name, size=size, ext=ext)
+        out_image_local_path = os.path.join(settings.image_save_dir, out_image_filename)
+        out_image_url = os.path.join(settings.image_base_url, out_image_filename)
+
+        if width and height:
+            tw = width
+            th = height
+        else:
+            tw = sw
+            th = sh
+
+        thumbs[size] = {
+            'url': out_image_url,
+            'size': size,
+            'width': sw,
+            'height': sh
+        }
+
+        # print('generate %s for %s' % (size, filename), out_image_url)
+
+        if settings.image_processing:
+            if width and height:
+                img.minify(source_image, out_image_local_path, (width, height))
+            else:
+                img.optimize(source_image, out_image_local_path)
+
+    return thumbs
+
+
+def process_product(album, product):
+    tr = {ord(a):ord(b) for a, b in zip(*settings.translit_table)}
+
+    author_name, author_age, filename, product_title = product
+    author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr)
+    image_id = '%s-%s-%s-%s' % (
+        album.date.year,
+        re.sub(r'[/]', '', album.uri),
+        author_name_encoded,
+        os.path.splitext(filename)[0]
+    )
+
+    image = process_image(image_id, filename, settings.image_sizes)
+    return (author_name, author_age, image, product_title)
+
+
+def get_db():
+    client = MongoClient(settings.database_uri)
+    return client.hudozka
+
+
+def clear_db():
+    db = get_db()
+    db.products.drop()
+    db.albums.drop()
+
+
+def sync_product(product, type):
+    db = get_db()
+    author_name, author_age, image, product_title = product
+
+    product_record = db['products'].insert({
+        'author': author_name,
+        'authorAge': author_age,
+        'title': product_title,
+        'type': type,
+        'content': image
+    })
+
+    print('inserted', product_record)
+
+    return product_record
+
+def sync_album(album):
+    print('syncing %s' % album.title)
+    db = get_db()
+
+    product_ids = list(map(
+        lambda product: sync_product(product, album.type),
+        album.products
+    ))
+
+    album_title = album.title
+    album_uri = album.uri
+    album_teacher = album.teacher
+    album_date = album.date
+    album_comment = album.comment
+    album_course = album.course
+    album_course_uri = album.course_uri
+
+    db['albums'].insert({
+        'title': album_title,
+        'uri': album_uri,
+        'date': album_date,
+        'description': album_comment,
+        'course': album_course,
+        'course_uri': album_course_uri,
+        'teacher': album_teacher,
+        'content': product_ids
+    })
+
+
+def read_gallery_item(dir_name, sync=True):
+    """
+    Read directory with .md and images and return an instance of Album.
+    None if reads fails.
+
+    :param dir_name:
+    :param sync:
+    :return:
+    """
+
+    set_path = os.path.join(dir_gallery, i)
+    os.chdir(set_path)
+
+    md = glob('*.md')
+    if len(md) > 0:
+        description_filename = md[0]
+        description_path = os.path.join(set_path, description_filename)
+
+        #if validation.check_description_filename(description_filename):
+        if True:
+            #v = get_description_version(description_filename)
+            v = '1'
+            DescriptionReader = get_description_reader(v)
+            if DescriptionReader is not None:
+                r = DescriptionReader(description_path)
+                album = None
+                try:
+                    album = r.read()
+
+                except Exception as e:
+                    print('Error with reading description file %s' % (description_path))
+                    print(e)
+                    return
+
+                try:
+                    album.products = list(map(
+                        lambda p : process_product(album, p),
+                        album.products
+                    ))
+                except Exception as e:
+                    print('Error with image processing %s' % (description_path))
+                    print(e)
+
+                if sync:
+                    try:
+                        sync_album(album)
+                    except Exception as e:
+                        print('Cannot sync %s' % (description_path))
+                        print(e)
+
+            else:
+                print('No reader for description file %s' % (description_path))
+
+        else:
+            new_name = validation.suggest_description_filename(description_filename)
+            os.rename(description_filename, new_name)
+            read_gallery_item(dir_name)
+
+
+if __name__ == '__main__':
+    # clear_db()
+    for i in catalog_files:
+        read_gallery_item(i, sync=False)
+        print()
+
+    exit()
diff --git a/utils/Sync/sync_schedules.py b/utils/Sync/sync_schedules.py
new file mode 100644
index 00000000..fc7125c6
--- /dev/null
+++ b/utils/Sync/sync_schedules.py
@@ -0,0 +1,31 @@
+from glob import glob
+import json
+import os
+import re
+from pymongo import MongoClient
+import settings
+
+__author__ = 'tmshv'
+
+
+db = MongoClient(settings.database_uri).hudozka
+db.schedules.drop()
+
+
+for schedule_path in glob(settings.root_dir_schedule + '/*.json'):
+    with open(schedule_path, 'r') as schedule_file:
+        name = os.path.basename(schedule_path)
+
+        period = re.search('(\d{4}-\d{4})', name).group(1)
+        semester = re.search('([a-zA-Z]+)\.json$', name).group(1)
+
+        schedule = json.loads(schedule_file.read())
+        # print(name, period, semester, schedule)
+
+        db.schedules.insert({
+            'period': period,
+            'semester': semester,
+            'schedule': schedule
+        })
+
+print('schedule sync complete')
\ No newline at end of file
diff --git a/utils/Sync/validation.py b/utils/Sync/validation.py
new file mode 100644
index 00000000..ac1d1889
--- /dev/null
+++ b/utils/Sync/validation.py
@@ -0,0 +1,18 @@
+import os
+import re
+import settings
+
+__author__ = 'Roman Timashev'
+
+__check_description_filename_exp = re.compile('^[%s]+(\[\d+\])\.md$' % settings.description_available_chars)
+
+
+def check_description_filename(filename):
+    return __check_description_filename_exp.match(filename)
+
+
+def suggest_description_filename(filename):
+    bn = os.path.basename(filename)
+    bn = os.path.splitext(bn)[0]
+    v = settings.default_description_version
+    return '%s [%s].md' % (bn, v)