Sync util big update
This commit is contained in:
tmshv committed
1 parent
1e80bae401
commit
40321327d6
30 files changed
+1001
-159
No files matched your search
@@ -1,14 +0,0 @@
|
||||
__author__ = 'Roman Timashev'
|
||||
|
||||
|
||||
class Product:
|
||||
def __init__(self, author_name, author_age, image, product_title):
|
||||
super().__init__()
|
||||
self.author_name = author_name
|
||||
self.author_age = author_age
|
||||
self.image = image
|
||||
self.product_title = product_title
|
||||
self.image_original_path = None
|
||||
|
||||
def to_tuple(self):
|
||||
return self.author_name, self.author_age, self.image, self.product_title, self.image_original_path
|
||||
@@ -1,13 +0,0 @@
|
||||
from pymongo import MongoClient
|
||||
import settings
|
||||
|
||||
__author__ = 'Roman Timashev'
|
||||
|
||||
|
||||
client = MongoClient(settings.database_uri)
|
||||
p = client.hudozka.posts
|
||||
|
||||
print(list(p.find()))
|
||||
|
||||
# print(dir(p))
|
||||
# print(p.insert({'a': 1}))
|
||||
@@ -0,0 +1,8 @@
|
||||
from pymongo import MongoClient
|
||||
|
||||
import settings
|
||||
|
||||
|
||||
def db():
|
||||
client = MongoClient(settings.database_uri)
|
||||
return client.hudozka
|
||||
@@ -1,8 +0,0 @@
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def read_image(src):
|
||||
try:
|
||||
return Image.open(src)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
@@ -1,35 +0,0 @@
|
||||
import os
|
||||
|
||||
from image.image import read_image
|
||||
from image.resize import optimize, thumbnail
|
||||
|
||||
|
||||
def image_processor(images_dir, public_base_url, sizes, name_generator):
|
||||
def process(file):
|
||||
image = read_image(file)
|
||||
if not image:
|
||||
return None
|
||||
|
||||
basename = os.path.basename(file)
|
||||
uri, ext = os.path.splitext(basename)
|
||||
|
||||
image_dict = {}
|
||||
for size in sizes:
|
||||
size_name, width, height = size
|
||||
|
||||
image_filename = name_generator(uri, size_name, ext)
|
||||
d = os.path.join(images_dir, image_filename)
|
||||
image_url = os.path.join(public_base_url, image_filename)
|
||||
|
||||
do_optimize = size_name == 'original'
|
||||
thumb = optimize(image, d, quality=90) if do_optimize else thumbnail(image, d, (width, height))
|
||||
width, height = thumb.size
|
||||
image_dict[size_name] = {
|
||||
'url': image_url,
|
||||
'size': size_name,
|
||||
'width': width,
|
||||
'height': height
|
||||
}
|
||||
|
||||
return image_dict
|
||||
return process
|
||||
File renamed without changes.
@@ -0,0 +1,8 @@
|
||||
# JSON
|
||||
# {
|
||||
# "assigned": "\\(author)",
|
||||
# "file": "1 diploma.jpg"
|
||||
# }
|
||||
def create_award(doc):
|
||||
return {**doc}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from sync.image import sync_image
|
||||
from utils.fn import key_mapper
|
||||
|
||||
|
||||
unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
|
||||
|
||||
|
||||
def create_product(doc):
|
||||
# author, image, title, awards, teacher, date
|
||||
|
||||
product = {**doc}
|
||||
unpop_image(product)
|
||||
|
||||
return product
|
||||
@@ -6,11 +6,11 @@ from datetime import datetime
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
|
||||
import lxml_utils
|
||||
import settings
|
||||
from Album import Album
|
||||
from Product import Product
|
||||
from image import resize
|
||||
from models.Album import Album
|
||||
from models.product import Product
|
||||
from utils import lxml_utils
|
||||
from utils.image import resize
|
||||
|
||||
__author__ = 'Roman Timashev'
|
||||
|
||||
@@ -128,9 +128,11 @@ class GalleryReader:
|
||||
return Product(author_name, author_age, image, product_title)
|
||||
|
||||
def create_image(self, name, filename, sizes, ext, base_url):
|
||||
print('DEPRECATED: use utils.image.__init__.create_image')
|
||||
|
||||
thumbs = {}
|
||||
for size, width, height in sizes:
|
||||
source_image = resize.read_image(filename)
|
||||
source_image = read_image(filename)
|
||||
sw, sh = source_image.size
|
||||
if sw < sh:
|
||||
width, height = height, width
|
||||
@@ -150,6 +152,8 @@ class GalleryReader:
|
||||
return thumbs
|
||||
|
||||
def get_date(self):
|
||||
print('DEPRECATED: use sync.__init__.create_date')
|
||||
|
||||
string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата")
|
||||
if string is None:
|
||||
return None
|
||||
|
||||
+22
-3
@@ -14,9 +14,10 @@ EN_UPPER = 'QWERTYUIOPASDFGHJKLZXCVBNM'
|
||||
DIGITS = '0123456789'
|
||||
description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-'
|
||||
|
||||
|
||||
translit_table = (
|
||||
'абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ',
|
||||
'abvgdeejzijklmnoprstufhzcss_y_euaABVGDEEJZIJKLMNOPRSTUFHZCSS_Y_EUA'
|
||||
['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'],
|
||||
['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', '_']
|
||||
)
|
||||
|
||||
teachers_names = {
|
||||
@@ -34,7 +35,9 @@ image_save_dir = './static_images'
|
||||
|
||||
image_base_url = 'https://static.shburg.org/art/image/'
|
||||
collective_image_base_url = 'https://static.shburg.org/art/collective/'
|
||||
collective_image_output = '/Users/tmshv/Dropbox/Dev/Hud School/Static/collective'
|
||||
collective_image_output = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
|
||||
album_html_img_class = 'hudozka-product'
|
||||
|
||||
image_ext = '.jpg'
|
||||
image_sizes = [
|
||||
@@ -46,6 +49,10 @@ image_sizes = [
|
||||
('preview', 50, 50),
|
||||
]
|
||||
|
||||
event_image_sizes = [
|
||||
('big', 1200, 800)
|
||||
]
|
||||
|
||||
date_formats = [
|
||||
'%Y',
|
||||
'%m.%Y',
|
||||
@@ -53,3 +60,15 @@ date_formats = [
|
||||
'%d.%m.%Y %H:%M',
|
||||
'%d.%m.%Y %H:%M:%S'
|
||||
]
|
||||
|
||||
date_formats_direct = [
|
||||
'%Y',
|
||||
'%m.%Y',
|
||||
'%d.%m.%Y',
|
||||
]
|
||||
|
||||
date_formats_reverse = [
|
||||
'%Y',
|
||||
'%Y.%m',
|
||||
'%Y.%m.%d'
|
||||
]
|
||||
@@ -0,0 +1,59 @@
|
||||
import re
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from markdown import markdown
|
||||
|
||||
|
||||
def create_date(date_str, date_formats=None):
|
||||
"""
|
||||
2016
|
||||
2016.10
|
||||
2016.10.10
|
||||
-> datetime
|
||||
|
||||
:param date_str:
|
||||
:return:
|
||||
"""
|
||||
if not date_formats:
|
||||
date_formats = [
|
||||
'%Y',
|
||||
'%Y.%m',
|
||||
'%Y.%m.%d'
|
||||
]
|
||||
|
||||
for format in date_formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, format)
|
||||
except:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
# 2016 На крыльях бабочек
|
||||
# 2016.10 На крыльях бабочек
|
||||
# 2016.10.10 На крыльях бабочек
|
||||
folder_name_pattern = re.compile('([\d.]+)(.*)')
|
||||
|
||||
|
||||
def create_date_and_title_from_folder_name(folder_name, date_formats=None):
|
||||
m = folder_name_pattern.findall(folder_name)
|
||||
if not m:
|
||||
return None, None
|
||||
|
||||
date_str, title = m[0]
|
||||
title = title.strip()
|
||||
date = create_date(date_str, date_formats)
|
||||
|
||||
if not date:
|
||||
return None, title
|
||||
|
||||
return date, title
|
||||
|
||||
|
||||
create_post_from_image_list = lambda images: markdown(
|
||||
'\n'.join(map(
|
||||
lambda i: ''.format(img=i),
|
||||
images
|
||||
))
|
||||
)
|
||||
@@ -0,0 +1,230 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from models.product import create_product
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, first, map_cases, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text, interpolate, interpolate_swift
|
||||
|
||||
|
||||
def sync_album(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_album,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().events
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_album(doc):
|
||||
sizes = settings.event_image_sizes
|
||||
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
|
||||
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
url = lambda a, i, s, e: 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
|
||||
album=a,
|
||||
id=i,
|
||||
size=s,
|
||||
ext=e
|
||||
)
|
||||
create_url = lambda aid, pid: lambda size, ext: url(aid, pid, size, ext.lower())
|
||||
image_id = lambda i: url_encode_text(
|
||||
os.path.splitext(i)[0]
|
||||
)
|
||||
image_path = lambda i: os.path.join(doc['folder'], i)
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
return None
|
||||
|
||||
create_album_id = lambda date, title: '{date}-{title}'.format(
|
||||
date=date.strftime('%Y'),
|
||||
title=url_encode_text(title)
|
||||
)
|
||||
|
||||
images = []
|
||||
|
||||
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
|
||||
|
||||
if 'album' in doc:
|
||||
# album_context =
|
||||
iter_album = iterate_iter_over_fns([
|
||||
lambda product: {
|
||||
**product,
|
||||
'image': create_image(
|
||||
image_path(product['file']),
|
||||
sizes,
|
||||
create_url(
|
||||
doc['id'],
|
||||
image_id(product['file'])
|
||||
),
|
||||
images_dir,
|
||||
skip_processing=True
|
||||
)
|
||||
},
|
||||
lambda product: {
|
||||
**product,
|
||||
'awards': lmap(
|
||||
lambda award: {
|
||||
**award,
|
||||
'file': os.path.join(doc['folder'], award['file']),
|
||||
'assigned': interpolate(award['assigned'], product, interpolate_swift),
|
||||
# 'awardFor': product
|
||||
},
|
||||
product['awards']
|
||||
)
|
||||
}
|
||||
])
|
||||
|
||||
doc['album'] = lmap(
|
||||
create_product,
|
||||
iter_album(doc['album'])
|
||||
)
|
||||
|
||||
post_html = lxml.html.fromstring(doc['post'])
|
||||
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
img_path = image_path(src)
|
||||
|
||||
if os.path.exists(img_path):
|
||||
image = create_image(
|
||||
img_path,
|
||||
sizes,
|
||||
create_url(
|
||||
doc['id'],
|
||||
image_id(src)
|
||||
),
|
||||
images_dir,
|
||||
skip_processing=True
|
||||
)
|
||||
images.append(image)
|
||||
|
||||
img.set('src', image['data']['big']['url'])
|
||||
img.set('class', settings.album_html_img_class)
|
||||
img.set('data-file', src)
|
||||
|
||||
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
|
||||
doc['images'] = images
|
||||
return doc
|
||||
|
||||
|
||||
def get_manifest(filepath):
|
||||
y, m = read_yaml_md(filepath)
|
||||
|
||||
data = {**y} if y else {}
|
||||
_ = lambda key, d=None: data[key] if key in data else d
|
||||
date = create_date(_('date'), settings.date_formats_direct)
|
||||
if date:
|
||||
data['date'] = date
|
||||
|
||||
id = _('id')
|
||||
if id:
|
||||
data['id'] = str(id)
|
||||
|
||||
post = {'post': m} if m else {}
|
||||
return {**data, **post}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
|
||||
os.chdir(albums_dir)
|
||||
|
||||
# GET ALBUMS FOLDERS
|
||||
albums = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
# print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md'))
|
||||
# print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md'))
|
||||
# exit()
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
albums = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i)
|
||||
),
|
||||
albums
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
|
||||
albums = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
'post': create_post_from_image_list(lmap(
|
||||
lambda path: os.path.relpath(path, i[0]),
|
||||
glob_in(i[0])
|
||||
))
|
||||
},
|
||||
albums
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
md_glob = lambda i: first(glob(i + '/*.md'))
|
||||
albums = lmap(
|
||||
lambda album: {
|
||||
**album,
|
||||
**map_cases(
|
||||
album,
|
||||
[(
|
||||
lambda album: md_glob(album['folder']),
|
||||
lambda album: get_manifest(md_glob(album['folder'])),
|
||||
)],
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
albums
|
||||
)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
albums = lmap(
|
||||
create_album,
|
||||
albums
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
albums = list(filter(None, albums))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
def sync_images(i):
|
||||
return lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(i)
|
||||
)
|
||||
|
||||
|
||||
albums = lmap(
|
||||
key_mapper('images', sync_images),
|
||||
albums
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
# albums = lmap(
|
||||
# sync_album,
|
||||
# albums
|
||||
# )
|
||||
|
||||
lprint_json(albums)
|
||||
print('SYNC ALBUMS DONE')
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
|
||||
|
||||
def sync_collective(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_collective,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().collective
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_profile(profile):
|
||||
sizes = settings.image_sizes
|
||||
images_dir = settings.collective_image_output
|
||||
|
||||
url_base = 'https://static.shburg.org/art/image/teacher-{id}-{size}{ext}'
|
||||
url_fn = lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext)
|
||||
|
||||
profile['picture'] = create_image(profile['picture'], sizes, url_fn, images_dir)
|
||||
return profile
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
|
||||
os.chdir(dir)
|
||||
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
files = glob('*.md')
|
||||
|
||||
# READ TEACHER YAML/MD MANIFEST
|
||||
collective = lmap(
|
||||
read_yaml_md,
|
||||
files
|
||||
)
|
||||
|
||||
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
|
||||
collective = lmap(
|
||||
lambda i: {'biography': i[1], **i[0]},
|
||||
collective
|
||||
)
|
||||
|
||||
# MAP TEACHER_MANIFEST -> TEACHER_PROFILE
|
||||
collective = lmap(
|
||||
create_profile,
|
||||
collective
|
||||
)
|
||||
|
||||
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
collective = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
collective
|
||||
)
|
||||
|
||||
lprint(collective)
|
||||
print('SYNC COLLECTIVE DONE')
|
||||
@@ -0,0 +1,139 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
from tempfile import mkstemp
|
||||
|
||||
from shutil import copyfile
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.io import read_yaml
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
# Adds specified category to each file in files list
|
||||
unwrap_file_list = lambda files, title: map(
|
||||
lambda profile: {'category': title, **profile},
|
||||
files
|
||||
)
|
||||
|
||||
|
||||
def sync_document(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_document,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'url': record['url']}
|
||||
try:
|
||||
documents = db().documents
|
||||
update_result = documents.update_one(q, {'$set': record}, upsert=True)
|
||||
i = documents.find_one({'url': record['url']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def pdf_to_jpg(pdf):
|
||||
cwd = os.getcwd()
|
||||
abspdf = os.path.join(cwd, pdf)
|
||||
_, temp = mkstemp('.jpg')
|
||||
image_magick_pdf_to_img(abspdf, temp)
|
||||
return temp
|
||||
|
||||
|
||||
def create_preview(pdf, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(pdf)
|
||||
|
||||
id = url_encode_text(pdf)
|
||||
url_base = 'https://static.shburg.org/art/image/document-{id}-{size}{ext}'
|
||||
url = lambda size, ext: url_base.format(id=id, size=size, ext=ext)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
|
||||
|
||||
def create_document(doc):
|
||||
sizes = settings.image_sizes
|
||||
preview_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
|
||||
doc['url'] = 'https://static.shburg.org/art/uploads/' + url_encode_file(doc['file'])
|
||||
doc['preview'] = create_preview(doc['file'], sizes=sizes, preview_dir=preview_dir)
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def unwrap_manifest(param):
|
||||
if isinstance(param, list):
|
||||
return combine(map(
|
||||
unwrap_manifest,
|
||||
param
|
||||
))
|
||||
|
||||
if 'files' in param:
|
||||
return lmap(
|
||||
unwrap_manifest,
|
||||
unwrap_file_list(param['files'], param['title'])
|
||||
)
|
||||
|
||||
if 'file' not in param:
|
||||
return None
|
||||
|
||||
return param
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
|
||||
|
||||
os.chdir(dir)
|
||||
|
||||
# GET DOCUMENT YAML_MANIFEST FILES
|
||||
files = glob(dir + '/*.yaml')
|
||||
|
||||
# READ YAML_MANIFEST FILES
|
||||
files_data = lmap(
|
||||
read_yaml,
|
||||
files
|
||||
)
|
||||
|
||||
# GET FLAT LIST OF DOCUMENTS
|
||||
documents = unwrap_manifest(files_data)
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
create_document,
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_preview = lambda i: sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('preview', sync_preview),
|
||||
documents
|
||||
)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
out_path = lambda doc: os.path.join('/Users/tmshv/Dropbox/Dev/Hud school/Static/uploads', os.path.basename(doc['url']))
|
||||
lmap(
|
||||
lambda doc : copyfile(
|
||||
doc['file'],
|
||||
out_path(doc)
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync_document,
|
||||
documents
|
||||
)
|
||||
|
||||
lprint(documents)
|
||||
print('SYNC DOCUMENTS DONE')
|
||||
@@ -0,0 +1,169 @@
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint, first, map_cases, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
def sync_event(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_event,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().events
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_event(doc):
|
||||
if 'date' not in doc or not doc['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
return None
|
||||
|
||||
doc['id'] = doc['id'] if 'id' in doc else url_encode_text(doc['title'])
|
||||
|
||||
url_base = 'https://static.shburg.org/art/image/event-{id}-{img}-{size}{ext}'
|
||||
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
sizes = settings.event_image_sizes
|
||||
|
||||
post_html = lxml.html.fromstring(doc['post'])
|
||||
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
img_path = os.path.join(doc['folder'], src)
|
||||
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda size, ext: url_base.format(id=doc['id'], img=img_id, size=size, ext=ext.lower())
|
||||
|
||||
image = create_image(img_path, sizes, url_fn, images_dir)
|
||||
images.append(image)
|
||||
img.set('src', image['data']['big']['url'])
|
||||
|
||||
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
|
||||
doc['images'] = images
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def get_md_manifest(md_path):
|
||||
y, m = read_yaml_md(md_path)
|
||||
|
||||
y['date'] = datetime.strptime(y['date'], '%d.%m.%Y')
|
||||
y['id'] = str(y['id'])
|
||||
|
||||
return {
|
||||
**y,
|
||||
'post': m
|
||||
}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
|
||||
|
||||
os.chdir(dir)
|
||||
|
||||
# GET EVENT FOLDERS
|
||||
events = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
events = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i), ['%Y.%m.%d']
|
||||
),
|
||||
events
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
|
||||
glob_in = lambda dir: combine(map(
|
||||
lambda type: glob(os.path.join(dir, type)),
|
||||
image_types
|
||||
))
|
||||
events = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
'post': create_post_from_image_list(lmap(
|
||||
lambda path: os.path.relpath(path, i[0]),
|
||||
glob_in(i[0])
|
||||
))
|
||||
},
|
||||
events
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
md_glob = lambda i : first(glob(i + '/*.md'))
|
||||
events = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
**map_cases(
|
||||
i,
|
||||
[(
|
||||
lambda i: md_glob(i['folder']),
|
||||
lambda i: get_md_manifest(md_glob(i['folder'])),
|
||||
)],
|
||||
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
events
|
||||
)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
events = lmap(
|
||||
create_event,
|
||||
events
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
events = list(filter(None, events))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
sync_images = lambda i: lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(i)
|
||||
)
|
||||
events = lmap(
|
||||
key_mapper('images', sync_images),
|
||||
events
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
events = lmap(
|
||||
sync_event,
|
||||
events
|
||||
)
|
||||
|
||||
lprint(events)
|
||||
print('SYNC EVENTS DONE')
|
||||
@@ -0,0 +1,20 @@
|
||||
from db import db
|
||||
from utils.fn import lmap
|
||||
|
||||
|
||||
def sync_image(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_image,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'hash': record['hash']}
|
||||
try:
|
||||
db().images.update_one(q, {'$set': record}, upsert=True)
|
||||
i = db().images.find_one({'hash': record['hash']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
@@ -4,7 +4,7 @@ from glob import glob
|
||||
from pymongo import MongoClient
|
||||
|
||||
import settings
|
||||
from image import resize
|
||||
from utils.image import resize
|
||||
|
||||
|
||||
def generate_image_filename(image, size_name, ext):
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import yaml
|
||||
from markdown import markdown
|
||||
from pymongo import MongoClient
|
||||
|
||||
import settings
|
||||
from image.image_processor import image_processor
|
||||
from text.split import split
|
||||
|
||||
|
||||
def get_db():
|
||||
client = MongoClient(settings.database_uri)
|
||||
return client.hudozka
|
||||
|
||||
|
||||
def sync(collective):
|
||||
clear_db()
|
||||
db = get_db()
|
||||
for a in collective:
|
||||
db.collective.insert(a)
|
||||
|
||||
|
||||
def clear_db():
|
||||
db = get_db()
|
||||
db.collective.drop()
|
||||
|
||||
|
||||
def process_profile(md_path):
|
||||
sizes = settings.image_sizes
|
||||
images_dir = settings.collective_image_output
|
||||
public_base_url = settings.collective_image_base_url
|
||||
|
||||
generate_image_filename = lambda image, size_name, ext: '{image}-{size}{ext}'.format(
|
||||
image=image,
|
||||
size=size_name,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
ip = image_processor(images_dir, public_base_url, sizes, generate_image_filename)
|
||||
|
||||
with open(md_path, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
yaml_data, md_data = split(data)
|
||||
|
||||
profile = yaml.load(yaml_data)
|
||||
text = markdown(md_data) if md_data else None
|
||||
|
||||
img = os.path.join(
|
||||
os.path.dirname(md_path),
|
||||
profile['picture']
|
||||
)
|
||||
profile['picture'] = ip(img)
|
||||
profile['biography'] = text
|
||||
|
||||
return profile
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
|
||||
files = glob(dir + '/*.md')
|
||||
|
||||
collective = list(map(
|
||||
process_profile,
|
||||
files
|
||||
))
|
||||
|
||||
list(map(print, collective))
|
||||
sync(collective)
|
||||
@@ -6,8 +6,8 @@ from glob import glob
|
||||
from pymongo import MongoClient
|
||||
|
||||
import settings
|
||||
from image import resize as img_util
|
||||
from read_gallery_1 import GalleryReader
|
||||
from utils.image import resize as img_util
|
||||
|
||||
__author__ = 'Roman Timashev'
|
||||
|
||||
|
||||
@@ -5,15 +5,15 @@ import lxml.html as html
|
||||
import markdown as md
|
||||
from lxml.etree import tostring
|
||||
|
||||
import lxml_utils
|
||||
from image import resize
|
||||
from utils import lxml_utils
|
||||
from utils.image import resize
|
||||
|
||||
__author__ = 'tmshv'
|
||||
|
||||
# db = pymongo.MongoClient('mongodb://localhost:27017')['stickers']
|
||||
|
||||
# images_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Static/images')
|
||||
catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Hudozka/Gallery')
|
||||
catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud School/Gallery')
|
||||
catalog_files = os.listdir(catalog_dir)
|
||||
|
||||
|
||||
@@ -79,6 +79,8 @@ def generate_image_filename(src, course, year, theme, author, size, i,ext):
|
||||
|
||||
|
||||
for cf in catalog_files:
|
||||
break
|
||||
|
||||
os.chdir(catalog_dir)
|
||||
if os.path.isdir(cf):
|
||||
print('======================================================================================')
|
||||
@@ -1,6 +0,0 @@
|
||||
def split(data, separator='\n\n'):
|
||||
try:
|
||||
s = data.index(separator)
|
||||
return data[:s], data[s:]
|
||||
except ValueError:
|
||||
return data, None
|
||||
@@ -0,0 +1,68 @@
|
||||
import json
|
||||
import os
|
||||
from functools import reduce
|
||||
from glob import glob
|
||||
|
||||
from bson import json_util
|
||||
|
||||
combine = lambda i: reduce(
|
||||
lambda s, i: s + i,
|
||||
i
|
||||
)
|
||||
|
||||
iterate_over_fns = lambda fns: lambda i: reduce(
|
||||
lambda out, fn: fn(out),
|
||||
fns, i
|
||||
)
|
||||
|
||||
iterate_iter_over_fns = lambda fns: lambda i: lmap(
|
||||
iterate_over_fns(fns),
|
||||
i
|
||||
)
|
||||
|
||||
lmap = lambda fn, ls: list(map(fn, ls))
|
||||
lprint = lambda i: lmap(print, i)
|
||||
lprint_json = lambda s: lprint(lmap(
|
||||
lambda i: json.dumps(i, ensure_ascii=False, indent=4, default=json_util.default),
|
||||
s
|
||||
))
|
||||
|
||||
first = lambda i: i[0] if len(i) > 0 else None
|
||||
|
||||
|
||||
def map_item_key(item, key, map_fn):
|
||||
"""
|
||||
|
||||
:param item:
|
||||
:param key:
|
||||
:param map_fn:
|
||||
:return:
|
||||
"""
|
||||
item[key] = map_fn(item[key])
|
||||
return item
|
||||
|
||||
|
||||
key_mapper = lambda key, fn: lambda i: map_item_key(i, key, fn)
|
||||
|
||||
|
||||
def map_cases(param, cases, default_fn=None):
|
||||
for c, fn in cases:
|
||||
if c(param):
|
||||
return fn(param)
|
||||
return param if not default_fn else default_fn(param)
|
||||
|
||||
|
||||
def until(condition_fn, iter_fn, items, default_value=None):
|
||||
for i in items:
|
||||
result = iter_fn(i)
|
||||
if condition_fn(result):
|
||||
return result
|
||||
return default_value
|
||||
|
||||
|
||||
# glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
|
||||
# glob_in('./images')
|
||||
dir_globber = lambda mask_list: lambda dir: combine(map(
|
||||
lambda mask: glob(os.path.join(dir, mask)),
|
||||
mask_list
|
||||
))
|
||||
@@ -0,0 +1,9 @@
|
||||
import hashlib
|
||||
|
||||
|
||||
def file_hash(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read()
|
||||
sha = hashlib.sha256(data)
|
||||
f.close()
|
||||
return sha.hexdigest()
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from utils.hash import file_hash
|
||||
from utils.image import resize
|
||||
from utils.image.resize import optimize, thumbnail
|
||||
|
||||
|
||||
def read_image(src):
|
||||
try:
|
||||
return Image.open(src)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
|
||||
"""
|
||||
|
||||
:param file: Image path to process
|
||||
:param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
|
||||
:param url_fn: function (size_name, ext) for creating image url
|
||||
:param output_dir: path to local folder for thumbs storing
|
||||
:param skip_processing: True/False to skip thumbs generation
|
||||
:return:
|
||||
"""
|
||||
image = read_image(file)
|
||||
if not image:
|
||||
return None
|
||||
|
||||
result = {}
|
||||
_, ext = os.path.splitext(file)
|
||||
for size in sizes:
|
||||
size_name, width, height = size
|
||||
|
||||
image_url = url_fn(size_name, ext)
|
||||
image_filename = os.path.basename(image_url)
|
||||
d = os.path.join(output_dir, image_filename)
|
||||
|
||||
if not skip_processing:
|
||||
thumb = optimize(image, d, quality=90) if size_name == 'original' else thumbnail(image, d, (width, height))
|
||||
width, height = thumb.size
|
||||
|
||||
result[size_name] = {
|
||||
'url': image_url,
|
||||
'size': size_name,
|
||||
'width': width,
|
||||
'height': height
|
||||
}
|
||||
|
||||
return {
|
||||
'file': os.path.basename(file),
|
||||
'hash': file_hash(file),
|
||||
'data': result
|
||||
}
|
||||
@@ -24,3 +24,8 @@ def override_extension(dest, ext=None):
|
||||
def image_magick_resize(input_file, output_file, size, quality=85):
|
||||
s = '%dx%d' % size
|
||||
call(['convert', input_file, '-resize', s, '-quality', str(quality), output_file])
|
||||
|
||||
|
||||
def image_magick_pdf_to_img(input_file, output_file):
|
||||
i = "{image}[0]".format(image=input_file)
|
||||
call(['convert', i, output_file])
|
||||
@@ -0,0 +1,36 @@
|
||||
import yaml
|
||||
from markdown import markdown
|
||||
|
||||
from utils.text.split import split, split_with
|
||||
|
||||
|
||||
def read_file(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
return data
|
||||
|
||||
|
||||
def read_yaml(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
return yaml.load(data)
|
||||
|
||||
|
||||
def read_md(file):
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read().decode('utf-8')
|
||||
return markdown(data)
|
||||
|
||||
|
||||
def read_yaml_md(file):
|
||||
data = read_file(file)
|
||||
yaml_data, md_data = split_with(['\n\n', '\r\n\r\n'])(data)
|
||||
|
||||
try:
|
||||
y = yaml.load(yaml_data)
|
||||
except:
|
||||
y = None
|
||||
|
||||
m = markdown(md_data) if md_data else None
|
||||
|
||||
return y, m
|
||||
File renamed without changes.
@@ -0,0 +1,18 @@
|
||||
from utils.fn import lmap, until
|
||||
|
||||
|
||||
def split(data, separator=' '):
|
||||
try:
|
||||
s = data.index(separator)
|
||||
return data[:s], data[s:]
|
||||
except ValueError:
|
||||
return None, None
|
||||
|
||||
|
||||
def split_with(separators):
|
||||
return lambda data: until(
|
||||
lambda i: i[0] and i[1],
|
||||
lambda s: split(data, s),
|
||||
separators,
|
||||
(None, None)
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
import re
|
||||
from functools import reduce
|
||||
|
||||
import settings
|
||||
from utils.fn import iterate_over_fns
|
||||
|
||||
|
||||
def translit(text):
|
||||
"""
|
||||
RUSSIAN TEXT => ENGLISH TEXT
|
||||
based on table like:
|
||||
(
|
||||
['а', 'б', 'в'],
|
||||
['a', 'b', 'v']
|
||||
)
|
||||
Leave if character is not founded
|
||||
"""
|
||||
table = settings.translit_table
|
||||
ins, outs = table
|
||||
out = lambda i: outs[ins.index(i)]
|
||||
replace = lambda i: out(i) if i in ins else i
|
||||
|
||||
return reduce(
|
||||
lambda t, i: t + replace(i),
|
||||
list(text),
|
||||
''
|
||||
)
|
||||
|
||||
|
||||
def interpolate(text, storage, key_fn=None):
|
||||
if not key_fn:
|
||||
key_fn = lambda i: i
|
||||
|
||||
key = key_fn(text)
|
||||
return storage[key] if key in storage else text
|
||||
|
||||
|
||||
interpolate_swift = lambda i: re.search("([\d\w-]+)", i).group(1)
|
||||
|
||||
remove_pattr = re.compile('[«»,!&]')
|
||||
remove_punctuation = lambda i: remove_pattr.sub('', i)
|
||||
|
||||
space_to_dash = lambda i: i.replace(' ', '-')
|
||||
dot_to_dash = lambda i: i.replace('.', '-')
|
||||
remove_underscores = lambda i: i.replace('_', '')
|
||||
text_lower = lambda i: i.lower()
|
||||
|
||||
url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower])
|
||||
url_encode_text = iterate_over_fns(
|
||||
[translit, space_to_dash, dot_to_dash, remove_underscores, remove_punctuation, text_lower])
|
||||
File renamed without changes.
Reference in new issue
Block a user