hudozka/utils/Sync/sync.py

import os
from glob import glob

import lxml.html as html
import markdown as md
from lxml.etree import tostring

import lxml_utils
from image import resize

__author__ = 'tmshv'

# db = pymongo.MongoClient('mongodb://localhost:27017')['stickers']

# images_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Static/images')
catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Hudozka/Gallery')
catalog_files = os.listdir(catalog_dir)


def lesson_info(page):
    lesson = list(filter(lambda h2: not h2.xpath('./a'), page.xpath('//h2')))[0]
    content = lxml_utils.get_following_tags(lesson, 'h2')
    for p in lxml_utils.get_following_tags(lesson, 'h2'):
        print(p, tostring(p, encoding='utf-8').decode())

    # print(content[0].xpath('./em/text()')[0].strip())
    # print(content[0].xpath('./text()')[0].strip())

    # print(content[1].xpath('./text()'))
    # return content

    date_str = '12.10.2013'

    #todo: make as `date`
    date = date_str

    return {
        'name': 'Collage',
        'course': 'painting',
        'technology': ['pastel', 'huel', 'glue'],
        'date': date,
    }


def drawings_info(page):
    out = []
    pics = list(filter(lambda h2: h2.xpath('./a'), page.xpath('//h2')))
    for p in pics:
        content = lxml_utils.get_following_tags(p, 'h2')[0]
        author_info = content.xpath('./text()')[0].split('\n')
        out.append({
            'name': p.xpath('./a/text()')[0],
            'image': p.xpath('./a/@href')[0],
            'author': author_info[0],
            'author_age': int(author_info[1].split(' ')[0])
        })
    return out


def generate_image_filename(src, course, year, theme, author, size, i,ext):
    if ext is None:
        ext = os.path.splitext(src)[1]

    if i == 0:
        filename = '{course}-{year}-{theme}-{author}-{size}.{ext}'
    else:
        filename = '{course}-{year}-{theme}-{author}-{index}-{size}.{ext}'\

    filename = filename.format(
        course=course,
        year=year,
        theme=theme,
        author=author,
        size=size,
        index=i,
        ext=ext
    )
    return filename


for cf in catalog_files:
    os.chdir(catalog_dir)
    if os.path.isdir(cf):
        print('======================================================================================')
        print(cf)

        product_dir = os.path.join(catalog_dir, cf)
        os.chdir(product_dir)

        def matching_files(param):
            if isinstance(param, list):
                return list(map(matching_files, param))
            else:
                return glob(param)


        product_file = matching_files('*.md')[0]

        def from_file(path):
            f = open(path, 'r')
            data = f.read()
            f.close()
            return data

        markdown_text = from_file(product_file)
        html_text = md.markdown(markdown_text)
        page = html.fromstring(html_text)
        print(html_text)

        # name
        name = page.xpath('//h1/text()')[0]

        # lesson
        # filtering paragraph with no link in h2 title
        lesson = lesson_info(page)
        print(lesson)

        drawings = drawings_info(page)
        for d in drawings:
            print(d)

        products = list(map(
            lambda drawing: {
                'image': drawing['image'],
                'name': drawing['name'],
                'author': drawing['author'],
                'authorAge': drawing['author_age'],
                'course': lesson['course'],
                'theme': lesson['name'],
                'rating': 10,
                'date': lesson['date']
            },
            drawings
        ))

        for p in products:
            print(p)

            # # price
            # def str_price_to_int(text):
            # items = text.split('\n')
            # out = list(map(lambda p: int(re.sub('[^\d]+', '', p)), items))
            # return out
            #
            #
            # price_text = page.xpath('//h2[contains(text(), "Цена")]/following-sibling::p/text()')[0]
            # product_price = str_price_to_int(price_text)
            #
            # # colors
            # def str_colors_to_code_list(text):
            #     cl = re.split('\s+', text)
            #     return ttcolor.color_by_code(cl)
            #
            # color_text = page.xpath('//h2[contains(text(), "Цвета")]/following-sibling::p/text()')[0]
            # product_colors = str_colors_to_code_list(color_text)
            #
            # # text
            # def format_text(text):
            #     try:
            #         res = requests.post("http://mdash.ru/api.v1.php", {
            #             'text': text
            #         })
            #         return res.json()['result']
            #
            #     except Exception as e:
            #
            #         return text
            #
            # product_text = page.xpath('//h2[contains(text(), "Текст")]/following-sibling::p/text()')[0]
            # # product_text = format_text(product_text)
            #
            #
            # # qualities
            # def get_quality(li):
            #     q = {
            #         'image': li.xpath('./img/@src')[0],
            #         'text': li.xpath('text()')[0].strip()
            #     }
            #     return q
            #
            #
            # def qualities_list(elements):
            #     return list(map(get_quality, elements))
            #
            #
            # qualities_elements = page.xpath('//h2[contains(text(), "Свойства")]/following-sibling::ul/li')
            # qualities = qualities_list(qualities_elements)
            #
            #

            product_images = [p['image']]

            image_sizes = [
                (50, 50, 'small'),
                (300, 300, 'preview'),
                (800, 800, 'medium'),
                (2000, 2000, 'big'),
                (0, 0, 'original')
            ]

            images = []
            i = 0
            for image in product_images:
                pi = {'source': image}

                for size in image_sizes:
                    size_name = size[2]
                    image_filename = generate_image_filename(image, size_name, product_sku, i, 'png')
                    d = os.path.join(images_dir, image_filename)
                    pi[size_name] = 'http://static.touchtwin.ru/images/{img}'.format(img=image_filename)
                    if size_name == 'original':
                        resize.optimize(image, d, quality=90)
                    else:
                        resize.minify(image, d, size[:2])

                images.append(pi)
                i += 1

            p['images'] = images
            del p['image']

            # db['products'].update({'uri': product_uri}, p, upsert=True)