tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

sync update: caching, refactoring, other

Browse Source
This commit is contained in:
tmshv committed 2016-03-06 02:24:24 +03:00
1 parent e68cfd7b2c
commit fe409c04a4
23 files changed
+758 -1588
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/Sync/crop_files.py
-22
View File
@@ -1,22 +0,0 @@
import os
from glob import glob
from utils.fn import lprint, lmap
from utils.image.resize import crop
size = (400, 400)
# files = glob('/Users/tmshv/Dropbox/Dev/Hud School/Static/images/product*medium.jpg')
# files = files[:3]
files = ['product-2015-maketirovanie-doma-peterburga-img0298-medium.jpg','product-2015-zhivopis-img0308-medium.jpg','product-2015-maketirovanie-img0240-medium.jpg','product-2015-podvodnyj-mir-stankovaya-kompoziziya-img0346-medium.jpg','product-2015-risunok-img0037-medium.jpg','product-2015-skazki-mg1037-medium.jpg','product-2015-stankovaya-kompoziziya-img0350-medium.jpg','product-2015-zvety-stankovaya-kompoziziya-img0348-medium.jpg','product-2015-gravyura-mg1014-medium.jpg','product-2015-guash-mg1024-medium.jpg','product-2015-shlisselburg-1-medium.jpg','product-2015-dpk-portret-v-tehnike-applikaziya-img0356-medium.jpg','product-70-img3869-medium.jpg','product-2015-domiki-20150406-1613-medium.jpg','product-2015-skulptura-vov-20150406-1591-medium.jpg','product-2015-history-competition-2-medium.jpg','product-2015-rozhdestvenskaya-skazka-2-medium.jpg','product-2016-zimnie-zabavy-1-medium.jpg','product-2016-na-krylyah-babochek-1-medium.jpg','product-2016-narodnye-motivy-1-medium.jpg','product-around-the-world-1-medium.jpg',]
src = lambda i: '/Users/tmshv/Dropbox/Dev/Hud School/Static/images/' + os.path.basename(i)
dest = lambda i: '/Users/tmshv/Desktop/smartcrop_sandbox/' + os.path.basename(i)
for i in files:
try:
crop(src(i), dest(i), size)
except:
print(i)
utils/Sync/models/product.py
+3 -2
View File
@@ -1,5 +1,5 @@
from sync.teacher import read_teacher_by_id
from sync.image import sync_image
from sync.teacher import Teacher
from utils.fn import key_mapper
from utils.text.transform import interpolate_swift
@@ -20,7 +20,8 @@ unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
def resolve_teacher(i):
return resolve_value(i, read_teacher_by_id)
sync = Teacher()
return resolve_value(i, lambda i: sync.read({'id': i}))
def create_product(doc):
utils/Sync/read_gallery_1.py
-167
View File
@@ -1,167 +0,0 @@
import os
import re
import time
from datetime import datetime
import lxml.html
from markdown import markdown
import settings
from models.Album import Album
from models.product import Product
from utils import lxml_utils
from utils.image import resize
__author__ = 'Roman Timashev'
artist_name_exp = re.compile('([ %s]+)(\s\(\d+ лет\))?$' % (settings.CYR_LOWER + settings.CYR_UPPER))
# artist_name_exp = re.compile('(.+)(\s\(\d+ лет\))?$')
artist_age_exp = re.compile('.+\((\d+) лет\)$')
class GalleryReader:
def __init__(self, dir, path):
super().__init__()
self.html = None
self.description_path = path
self.dir_path = dir
self.image_sizes = settings.image_sizes
self.image_ext = settings.image_ext
self.image_base_url = settings.image_base_url
def read(self):
with open(self.description_path, 'r', encoding='utf-8') as dfile:
data = dfile.read()
md = markdown(data)
self.html = lxml.html.fromstring(md)
# ALBUM
try:
album_title = self.html.xpath('.//h1/a/text()')[0]
album_uri = self.html.xpath('.//h1/a/@href')[0]
except:
raise Exception('Cannot read album title/uri')
# COURSE
try:
course_title = self.html.xpath('.//p/a/text()')[0]
course_uri = self.html.xpath('.//p/a/@href')[0]
except:
raise Exception('Cannot read course title/uri')
# TEACHER
teacher = lxml_utils.get_text_following_by_tag(self.html, 'em', "Преподаватель")
teacher = re.sub(r'\s', '', teacher)
if teacher not in settings.teachers_names:
raise Exception('Teacher %s is not found' % teacher)
teacher_name = settings.teachers_names[teacher]
# DATE
date = self.get_date()
if date is None:
raise Exception('Date field is empty')
# PRODUCT
content = self.get_drawings()
a = Album(type='1', title=album_title, uri=album_uri, course=course_title, course_uri=course_uri, date=date,
teacher=teacher_name, comment=None)
for p in content:
author_name, author_age, filename, product_title = p
product = self.create_product(a, author_name, author_age, filename, product_title)
product.image_original_path = os.path.join(self.dir_path, filename)
a.add_product(product)
return a
def get_drawings(self):
drawings = []
h3_list = self.html.xpath('.//h3')
for h3 in h3_list:
artist_name_age = h3.xpath('./a')
if len(artist_name_age) == 0:
raise Exception('Artist name should be a link')
artist_name_age = artist_name_age[0].text
# ARTIST NAME [required]
if artist_name_age is None:
raise Exception('Artist name section is empty')
exp = artist_name_exp.search(artist_name_age)
if exp is None:
raise Exception('Artist name is not specified')
artist_name = exp.group(1).strip()
# ARTIST AGE [optional]
exp = artist_age_exp.search(artist_name_age)
if exp is not None and len(exp.groups()) > 0:
artist_age = int(exp.group(1))
else:
artist_age = None
image_filename = h3.xpath('./a/@href')[0]
# PRODUCT TITLE [optional]
drawing_data = lxml_utils.get_following_tags(h3, 'h3')
if len(drawing_data) == 0:
product_title = None
# raise Exception('Product title not specified')
else:
product_title = drawing_data[0].text[1:-1]
drawings.append((artist_name, artist_age, image_filename, product_title))
return drawings
def create_product(self, album, author_name, author_age, filename, product_title):
tr = {ord(a): ord(b) for a, b in zip(*settings.translit_table)}
author_name_encoded = re.sub(r'[\s()]', '', author_name).lower().translate(tr)
image_id = '%s-%s-%s-%s' % (
album.date.year,
re.sub(r'[/]', '', album.uri),
author_name_encoded,
os.path.splitext(filename)[0]
)
image = self.create_image(image_id, filename, self.image_sizes, self.image_ext, self.image_base_url)
return Product(author_name, author_age, image, product_title)
def create_image(self, name, filename, sizes, ext, base_url):
print('DEPRECATED: use utils.image.__init__.create_image')
thumbs = {}
for size, width, height in sizes:
source_image = read_image(filename)
sw, sh = source_image.size
if sw < sh:
width, height = height, width
width = width if width else sw
height = height if height else sh
out_filename = '{name}-{w}x{h}{ext}'.format(name=name, w=width, h=height, ext=ext)
out_url = os.path.join(base_url, out_filename)
thumbs[size] = {
'url': out_url,
'size': size,
'width': width,
'height': height
}
return thumbs
def get_date(self):
print('DEPRECATED: use sync.__init__.create_date')
string = lxml_utils.get_text_following_by_tag(self.html, 'em', "Дата")
if string is None:
return None
for f in settings.date_formats:
try:
date = time.strptime(string, f)
return datetime.fromtimestamp(time.mktime(date))
except ValueError:
continue
return None
utils/Sync/settings.py
+1
View File
@@ -34,6 +34,7 @@ image_processing = True
image_save_dir = './static_images'
image_base_url = 'https://static.shburg.org/art/images/'
image_output = '/Users/tmshv/Desktop/Hudozka Static/images'
album_html_img_class = 'hudozka-product'
utils/Sync/sync/__init__.py
+36 -3
View File
@@ -1,10 +1,21 @@
import os
import re
from datetime import datetime
from glob import glob
import lxml.html
from markdown import markdown
from db import db
from utils.fn import lmap, key_mapper
from sync.image import sync_image
from utils.fn import lmap, combine
def list_images(dir):
image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
return combine(map(
lambda type: glob(os.path.join(dir, type)),
image_types
))
def create_date(date_str, date_formats=None):
@@ -50,13 +61,25 @@ def untouched(documents, store):
))
def synced_images_ids(images):
ids = lmap(
lambda image: image['_id'],
sync_image(images)
)
return ids
def synced_image_id(image):
return sync_image(image)['_id']
def create_date_and_title_from_folder_name(folder_name, date_formats=None):
m = folder_name_pattern.findall(folder_name)
if not m:
return None, None
date_str, title = m[0]
title = title.strip()
title = os.path.splitext(title.strip())[0]
date = create_date(date_str, date_formats)
if not date:
@@ -65,6 +88,16 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
return date, title
def images_from_html(md):
post_html = lxml.html.fromstring(md)
images = []
for img in post_html.cssselect('img'):
src = img.get('src')
images.append(src)
return images
create_post_from_image_list = lambda images: markdown(
'\n'.join(map(
lambda i: '![]({img})'.format(img=i),
utils/Sync/sync/album.py
+118 -69
View File
@@ -1,16 +1,17 @@
import os
from glob import glob
from lxml import etree
import lxml.html
from lxml import etree
import settings
from db import db
from models.product import create_product
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date
from sync.image import sync_image
from utils.fn import lmap, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns, first, map_cases
from utils.hash import file_hash
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \
untouched, synced_images_ids
from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \
key_mapper, lprint
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text
@@ -22,7 +23,7 @@ class ImageCreator:
self.cache = cache if cache else set()
def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
hash = file_hash(file)
hash = hash_file(file)
cached = self.cached(hash)
return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
@@ -30,25 +31,6 @@ class ImageCreator:
return self.cache[hash] if hash in self.cache else None
def sync_album(record):
if isinstance(record, list):
return lmap(
sync_album,
record
)
q = {'id': record['id']}
try:
collection = db().events
collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
def create_post(md, image_path_fn):
html = lxml.html.fromstring(md)
@@ -64,7 +46,7 @@ def create_post(md, image_path_fn):
def create_url(a, i, s, e):
return 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
album=a,
id=i,
size=s,
@@ -76,18 +58,41 @@ def url_creator(aid, pid):
return lambda size, ext: create_url(aid, pid, size, ext.lower())
class Album:
class SyncAlbum(Sync):
@staticmethod
def id_from_file(i):
return url_encode_text(
os.path.splitext(i)[0]
)
def __init__(self, image_creator):
def __init__(self, image_creator, images_dir):
super().__init__()
self.collection = db().albums
self.image_creator = image_creator
self.skip_image_processing = False
self.images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
self.images_dir = images_dir
def create_id(self, document):
if 'id' in document:
return document
y = str(document['date'].year)
document['id'] = url_encode_text('-'.join([y, document['title']]))
return document
def create_hash(self, document):
hash_images = lmap(
hash_file,
lmap(
lambda i: os.path.join(document['folder'], i),
document['images']
)
)
document['hash'] = hash_str(
combine([hash_str(document)] + hash_images)
)
return document
def create_image(self, file, url_fn):
ss = settings.album_image_sizes
@@ -99,7 +104,7 @@ class Album:
def create_album(self, doc):
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
image_id = Album.id_from_file
image_id = SyncAlbum.id_from_file
image_path = lambda i: os.path.join(doc['folder'], i)
if 'title' not in doc or not doc['title']:
@@ -138,7 +143,7 @@ class Album:
return doc
def create_products(self, doc, image_file_fn):
image_id = Album.id_from_file
image_id = SyncAlbum.id_from_file
iter_album = iterate_iter_over_fns([
lambda product: {
**product,
@@ -188,16 +193,11 @@ def get_manifest(filepath):
return {**data, **post}
if __name__ == '__main__':
image_creator = ImageCreator(cache=None)
alb = Album(image_creator=image_creator)
# alb.skip_image_processing = True
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
os.chdir(albums_dir)
def main(dir_documents, sync):
os.chdir(dir_documents)
# GET ALBUMS FOLDERS
albums = lmap(
documents = lmap(
lambda i: i.path,
filter(
lambda i: i.is_dir(),
@@ -205,22 +205,20 @@ if __name__ == '__main__':
)
)
# print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md'))
# print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md'))
# exit()
# albums = [albums[5]]
# documents = [documents[3]]
# documents = documents[:15]
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
albums = lmap(
documents = lmap(
lambda i: (i,) + create_date_and_title_from_folder_name(
os.path.basename(i)
),
albums
documents
)
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
albums = lmap(
documents = lmap(
lambda i: {
'folder': i[0],
'date': i[1],
@@ -230,12 +228,12 @@ if __name__ == '__main__':
glob_in(i[0])
))
},
albums
documents
)
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
md_glob = lambda i: first(glob(i + '/*.md'))
albums = lmap(
documents = lmap(
lambda album: {
**album,
**map_cases(
@@ -247,37 +245,88 @@ if __name__ == '__main__':
lambda i: {}
)
},
albums
documents
)
documents = lmap(
lambda i: {
**i,
'images': lmap(
lambda path: os.path.relpath(path, i['folder']),
list_images(i['folder'])
)
},
documents
)
# CREATE DOCUMENT IDENTITY
documents = lmap(sync.create_id, documents)
# CREATE HASH OF DOCUMENT FILE
documents = lmap(sync.create_hash, documents)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmapfn(documents)(
lambda i: i['id']
)
# SKIP UNTOUCHED DOCUMENTS
documents = untouched(documents, sync)
# MAP EVENT MANIFEST -> EVENT_OBJECT
albums = lmap(
documents = lmap(
alb.create_album,
albums
documents
)
# FILTER INCORRECT EVENTS
albums = list(filter(None, albums))
documents = list(filter(None, documents))
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
def sync_images(i):
return lmap(
lambda image: image['_id'],
sync_image(i)
documents = lmap(
key_mapper('images', synced_images_ids),
documents
)
# SYNC OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
)
albums = lmap(
key_mapper('images', sync_images),
albums
)
# SYNC EVENT_OBJECT WITH DB
albums = lmap(
sync_album,
albums
)
# SCOPE
print('SCOPE:')
lprint(scope_documents_ids)
lprint_json(albums)
print('SYNC ALBUMS DONE')
# DELETE
print('DELETE DOCUMENTS:')
lprint_json(lmap(
lambda i: i['id'],
documents_to_remove
))
# DELETE
print('UPDATE DOCUMENTS:')
lprint_json(lmap(
lambda i: i['id'],
documents
))
# lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
image_creator = ImageCreator(cache=None)
alb = SyncAlbum(image_creator=image_creator, images_dir=settings.image_output)
alb.skip_image_processing = True
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
main(dir_documents=albums_dir, sync=alb)
utils/Sync/sync/award.py
-196
View File
@@ -1,196 +0,0 @@
import os
from glob import glob
from shutil import copyfile
from tempfile import mkstemp
import settings
from db import db
from sync.image import sync_image
from utils.fn import lmap, lprint_json, key_mapper, lprint
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
from utils.text.transform import url_encode_text, url_encode_file
prefix = 'award'
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Awards'
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
url_base_document = 'https://static.shburg.org/art/uploads/{file}'
def collection():
return db().awards
def pdf_to_jpg(pdf):
cwd = os.getcwd()
abspdf = os.path.join(cwd, pdf)
_, temp = mkstemp('.jpg')
image_magick_pdf_to_img(abspdf, temp)
return temp
def sync_document(record):
if isinstance(record, list):
return lmap(
sync_document,
record
)
q = {'id': record['id']}
try:
collection().update_one(q, {'$set': record}, upsert=True)
return collection().find_one(q)
except ValueError:
pass
return None
def read_document(i, query_fn=None):
q = query_fn(i) if query_fn else {'id': i['id']}
try:
return collection().find_one(q)
except ValueError:
pass
return None
def query_documents(q):
return collection().find(q)
def delete_document(q):
return collection().find_one_and_delete(q)
def create_preview(pdf, sizes, preview_dir):
temp_preview_path = pdf_to_jpg(pdf)
id = url_encode_text(pdf)
url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
img = create_image(temp_preview_path, sizes, url, preview_dir)
os.remove(temp_preview_path)
return img
def create_document(doc):
sizes = settings.awards_image_sizes
file = doc['file']
doc['type'] = prefix
doc['url'] = url_base_document.format(file=url_encode_file(file))
doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
doc['file'] = {
'name': file,
'size': os.path.getsize((file))
}
return doc
def create_id(doc):
return url_encode_text('{type}-{file}'.format(
type=prefix,
file=doc['file']
))
if __name__ == '__main__':
# ALL DOCUMENTS IDS FOUNDED
scope_documents_ids = []
# GET DOCUMENT YAML_MANIFEST FILES
os.chdir(dir_documents)
documents = glob('*.pdf')
# READ YAML_MANIFEST FILES
documents = lmap(
lambda i: {
'file': i
},
documents
)
# CREATE DOCUMENT IDENTITY
documents = lmap(
lambda i: {**i, 'id': create_id(i)},
documents
)
# CREATE HASH OF DOCUMENT FILE
documents = lmap(
lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
documents
)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmap(
lambda i: i['id'],
documents
)
# SKIP UNTOUCHED DOCUMENTS
documents = lmap(
lambda i: i[0],
filter(
lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, read_document(document)),
documents
)
))
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
documents = lmap(
create_document,
documents
)
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('preview', lambda i: sync_image(i)['_id']),
documents
)
# COPY FILE -> STATIC_DIR/URL_FILENAME
out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
lmap(
lambda doc: copyfile(
doc['file']['name'],
out_path(doc)
),
documents
)
# SYNC DOCUMENT_OBJECT WITH DB
documents = lmap(
sync_document,
documents
)
documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
delete_document,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
)
)
# SCOPE
print('SCOPE:')
lprint(scope_documents_ids)
# DELETE
print('DELETE DOCUMENTS:')
lprint_json(documents_to_remove)
# DELETE
print('UPDATE DOCUMENTS:')
lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
utils/Sync/sync/collective.py
-91
View File
@@ -1,91 +0,0 @@
import os
from glob import glob
import settings
from db import db
from sync.image import sync_image
from utils.fn import lmap, lprint, key_mapper
from utils.image import create_image
from utils.io import read_yaml_md
def read_teacher_by_id(record):
if isinstance(record, list):
return lmap(
read_teacher_by_id,
record
)
q = {'id': record}
try:
return db().collective.find_one(q)
except ValueError:
pass
return None
def sync_collective(record):
if isinstance(record, list):
return lmap(
sync_collective,
record
)
q = {'id': record['id']}
try:
collection = db().collective
update_result = collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
def create_profile(profile):
sizes = settings.image_sizes
images_dir = settings.collective_image_output
url_base = 'https://static.shburg.org/art/image/teacher-{id}-{size}{ext}'
url_fn = lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext)
profile['picture'] = create_image(profile['picture'], sizes, url_fn, images_dir)
return profile
if __name__ == '__main__':
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
os.chdir(dir)
# GET TEACHER YAML/MD MANIFEST FILES
files = glob('*.md')
# READ TEACHER YAML/MD MANIFEST
collective = lmap(
read_yaml_md,
files
)
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
collective = lmap(
lambda i: {'biography': i[1], **i[0]},
collective
)
# MAP TEACHER_MANIFEST -> TEACHER_PROFILE
collective = lmap(
create_profile,
collective
)
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
collective = lmap(
key_mapper('picture', sync_picture),
collective
)
lprint(collective)
print('SYNC COLLECTIVE DONE')
utils/Sync/sync/document.py
+154 -128
View File
@@ -5,26 +5,67 @@ from tempfile import mkstemp
import settings
from db import db
from sync.image import sync_image
from utils.fn import combine, lmap, lprint_json, key_mapper, lprint
from sync import Sync, untouched, synced_image_id
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
from utils.io import read_yaml
from utils.text.transform import url_encode_text, url_encode_file
prefix = 'document'
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
url_base_document = 'https://static.shburg.org/art/uploads/{file}'
# Adds specified category to each file in files list
unwrap_file_list = lambda files, title: map(
lambda profile: {'category': title, **profile},
files
)
class SyncDocument(Sync):
def __init__(self, collection_name, document_type, sizes):
super().__init__()
self.collection = db()[collection_name]
self.document_type = document_type
self.sizes = sizes
self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
self.url_base_document = 'https://static.shburg.org/art/uploads/{file}'
def create(self, document):
file = document['file']
filename = os.path.basename(document['file'])
document['type'] = self.document_type
document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
document['file'] = {
'name': filename,
'size': os.path.getsize((file))
}
return document
def create_id(self, document):
bn = os.path.basename(document['file'])
document['id'] = url_encode_text('{type}-{category}-{file}'.format(
type=self.document_type,
category=document['category'],
file=bn
))
return document
def create_url(self, document):
filename = os.path.basename(document['file'])
document['url'] = self.url_base_document.format(file=url_encode_file(filename))
return document
def create_hash(self, document):
document['hash'] = hash_str(
hash_str(document) + hash_file(document['file'])
)
return document
def create_preview(self, pdf, sizes, preview_dir):
temp_preview_path = pdf_to_jpg(pdf)
id = url_encode_text(pdf)
url = lambda size, ext: self.url_base_preview.format(id=id, size=size, ext=ext)
img = create_image(temp_preview_path, sizes, url, preview_dir)
os.remove(temp_preview_path)
return img
def pdf_to_jpg(pdf):
@@ -35,75 +76,20 @@ def pdf_to_jpg(pdf):
return temp
def sync_document(record):
if isinstance(record, list):
return lmap(
sync_document,
record
)
q = {'id': record['id']}
try:
documents = db().documents
update_result = documents.update_one(q, {'$set': record}, upsert=True)
i = documents.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
def read_document(i, query_fn=None):
q = query_fn(i) if query_fn else {'id': i['id']}
try:
return db().documents.find_one(q)
except ValueError:
pass
return None
def query_documents(q):
return db().documents.find(q)
def delete_document(q):
return db().documents.find_one_and_delete(q)
def create_preview(pdf, sizes, preview_dir):
temp_preview_path = pdf_to_jpg(pdf)
id = url_encode_text(pdf)
url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
img = create_image(temp_preview_path, sizes, url, preview_dir)
os.remove(temp_preview_path)
return img
def create_document(doc):
sizes = settings.image_sizes
file = doc['file']
doc['type'] = prefix
doc['url'] = url_base_document.format(file=url_encode_file(file))
doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
doc['file'] = {
'name': file,
'size': os.path.getsize((file))
}
return doc
def create_id(doc):
return url_encode_text('{type}-{category}-{file}'.format(
type=prefix,
category=doc['category'],
file=doc['file']
))
def categorize_files_list(files, title):
"""
Adds specified category to each file in files list
:param files:
:param title:
:return:
"""
return map(
lambda profile: {
**profile,
'category': title
},
files
)
def unwrap_manifest(param):
@@ -116,7 +102,7 @@ def unwrap_manifest(param):
if 'files' in param:
return lmap(
unwrap_manifest,
unwrap_file_list(param['files'], param['title'])
categorize_files_list(param['files'], param['title'])
)
if 'file' not in param:
@@ -125,12 +111,9 @@ def unwrap_manifest(param):
return param
if __name__ == '__main__':
# ALL DOCUMENTS IDS FOUNDED
scope_documents_ids = []
def documents_from_yaml():
# GET DOCUMENT YAML_MANIFEST FILES
os.chdir(dir_documents)
documents = glob(dir_documents + '/*.yaml')
# READ YAML_MANIFEST FILES
@@ -142,85 +125,128 @@ if __name__ == '__main__':
# GET FLAT LIST OF DOCUMENTS
documents = unwrap_manifest(documents)
return documents
def documents_from_subdirs():
documents = lmap(
lambda i: i.path,
filter(
lambda i: i.is_dir(),
os.scandir('.')
)
)
documents = lmap(
lambda folder: lmap(
lambda path: {
'file': path,
'title': os.path.splitext(os.path.basename(os.path.relpath(path, folder)))[0],
'category': os.path.basename(folder)
},
glob(folder + '/*.pdf')
),
documents
)
documents = combine(documents)
return documents
def main(dir_documents, dir_static_uploads, sync):
os.chdir(dir_documents)
# documents = documents_from_yaml()
documents = documents_from_subdirs()
# CREATE DOCUMENT IDENTITY
documents = lmap(
lambda i: {**i, 'id': create_id(i)},
documents
)
documents = lmap(sync.create_id, documents)
# CREATE HASH OF DOCUMENT FILE
documents = lmap(
lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
documents
)
documents = lmap(sync.create_hash, documents)
# CREATE HASH OF DOCUMENT FILE
# CREATE SCOPE
scope_documents_ids = lmap(
lambda i: i['id'],
documents
)
file_names = set(lmap(
lambda i: i['file'],
documents
))
if len(file_names) != len(documents):
raise Exception('File names should be unique')
# SKIP UNTOUCHED DOCUMENTS
documents = lmap(
lambda i: i[0],
filter(
lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, read_document(document)),
documents
)
))
documents = untouched(documents, sync)
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
documents = lmap(
create_document,
documents
)
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('preview', lambda i: sync_image(i)['_id']),
documents
)
# URL
documents = lmap(sync.create_url, documents)
# COPY FILE -> STATIC_DIR/URL_FILENAME
out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
lmap(
lambda doc: copyfile(
doc['file']['name'],
doc['file'],
out_path(doc)
),
documents
)
# SYNC DOCUMENT_OBJECT WITH DB
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
documents = lmap(
sync_document,
sync.create,
documents
)
documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('preview', lambda i: synced_image_id(i)),
documents
)
# SYNC DOCUMENT_OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
delete_document,
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
)
)
# SCOPE
print('SCOPE:')
print('SCOPE:', len(scope_documents_ids))
lprint(scope_documents_ids)
# DELETE
print('DELETE DOCUMENTS:')
print('DELETE DOCUMENTS:', len(documents_to_remove))
lprint_json(documents_to_remove)
# DELETE
print('UPDATE DOCUMENTS:')
print('UPDATE DOCUMENTS:', len(documents))
lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
main(
dir_documents,
dir_static_uploads,
SyncDocument(
'documents',
'document',
sizes=settings.image_sizes
)
)
# DOCUMENT_OBJECT SAMPLE
# {
# "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
utils/Sync/sync/event.py
-174
View File
@@ -1,174 +0,0 @@
import os
from datetime import datetime
from glob import glob
import lxml.html
import settings
from db import db
from sync import create_date_and_title_from_folder_name, create_post_from_image_list
from sync.image import sync_image
from utils.fn import combine, lmap, lprint, first, map_cases, key_mapper
from utils.image import create_image
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text
def sync_event(record):
if isinstance(record, list):
return lmap(
sync_event,
record
)
q = {'id': record['id']}
try:
collection = db().events
update_result = collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
def create_event(doc):
if 'date' not in doc or not doc['date']:
return None
if 'title' not in doc or not doc['title']:
return None
doc['id'] = doc['id'] if 'id' in doc else url_encode_text(doc['title'])
url_base = 'https://static.shburg.org/art/images/event-{id}-{img}-{size}{ext}'
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
sizes = settings.event_image_sizes
post_html = lxml.html.fromstring(doc['post'])
images = []
for img in post_html.cssselect('img'):
src = img.get('src')
img_path = os.path.join(doc['folder'], src)
if os.path.exists(img_path):
img_id = url_encode_text(os.path.splitext(src)[0])
url_fn = lambda size, ext: url_base.format(id=doc['id'], img=img_id, size=size, ext=ext.lower())
image = create_image(img_path, sizes, url_fn, images_dir)
if image:
images.append(image)
img.set('src', image['data']['big']['url'])
else:
print('fail: ', doc['folder'], src)
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
doc['images'] = images
return doc
def get_manifest(md_path):
y, m = read_yaml_md(md_path)
y = y if y else {}
if 'date' in y:
y['date'] = datetime.strptime(y['date'], '%d.%m.%Y')
if 'id' in y:
y['id'] = str(y['id'])
return {
**y,
'post': m
}
if __name__ == '__main__':
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
os.chdir(dir)
# GET EVENT FOLDERS
events = lmap(
lambda i: i.path,
filter(
lambda i: i.is_dir(),
os.scandir('.')
)
)
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
events = lmap(
lambda i: (i,) + create_date_and_title_from_folder_name(
os.path.basename(i), ['%Y.%m.%d']
),
events
)
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
glob_in = lambda dir: combine(map(
lambda type: glob(os.path.join(dir, type)),
image_types
))
events = lmap(
lambda i: {
'folder': i[0],
'date': i[1],
'title': i[2],
'post': create_post_from_image_list(lmap(
lambda path: os.path.relpath(path, i[0]),
glob_in(i[0])
))
},
events
)
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
md_glob = lambda i : first(glob(i + '/*.md'))
events = lmap(
lambda event: {
**event,
**map_cases(
event,
[(
lambda i: md_glob(i['folder']),
lambda i: get_manifest(md_glob(i['folder'])),
)],
lambda i: {}
)
},
events
)
# MAP EVENT MANIFEST -> EVENT_OBJECT
events = lmap(
create_event,
events
)
# FILTER INCORRECT EVENTS
events = list(filter(None, events))
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
sync_images = lambda i: lmap(
lambda image: image['_id'],
sync_image(i)
)
events = lmap(
key_mapper('images', sync_images),
events
)
# SYNC EVENT_OBJECT WITH DB
events = lmap(
sync_event,
events
)
lprint(events)
print('SYNC EVENTS DONE')
utils/Sync/sync/image.py
+40
View File
@@ -2,6 +2,46 @@ from db import db
from utils.fn import lmap
# class SyncImage(Sync):
# def __init__(self, sizes, save_path, create_url_fn):
# super().__init__()
# self.sizes = sizes
# self.save_path = save_path
# self.create_url_fn = create_url_fn
# self.collection = db().images
#
# def create_id(self, document):
# id = document['id'] if 'id' in document and document['id'] else None
# if not id:
# id = 'schedule-{period}-{semester}'.format(**document)
# document['id'] = id
# return document
#
# def read(self, document):
# return super().read(document)
#
# def create_hash(self, document):
# return {
# **document,
# 'hash': hash_file(document)
# }
#
# def create(self, document):
# image = create_image(document, self.sizes, self.create_url_fn, self.save_path)
#
# return {
# 'file': document,
# 'hash': self.create_hash(document),
# 'data': image['data']
# }
#
# def find_or_create(self, document):
# i = self.read(document)
# if not i:
# return self.create(document)
# return i
def sync_image(record):
if isinstance(record, list):
return lmap(
utils/Sync/sync/news.py
-97
View File
@@ -1,97 +0,0 @@
import os
from glob import glob
from db import db
from sync import create_date
from utils.fn import lmap, lprint_json
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text
def sync_news(record):
if isinstance(record, list):
return lmap(
sync_news,
record
)
q = {'id': record['id']}
try:
collection = db().timeline
update_result = collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
def create_news(profile):
formats = [
'%d.%m.%Y',
'%d.%m.%Y %H:%M',
'%d.%m.%Y %H:%M:%S'
]
if 'title' not in profile:
return None
if 'date' in profile:
profile['date'] = create_date(profile['date'], formats)
if 'until' in profile:
profile['until'] = create_date(profile['until'], formats)
if 'id' not in profile:
id = lambda date, title: '{date}-{title}'.format(
date=date.strftime('%Y-%m-%d'),
title=url_encode_text(title)
)
profile['id'] = id(profile['date'], profile['title'])
return profile
if __name__ == '__main__':
news_dir = '/Users/tmshv/Dropbox/Dev/Hud school/News'
os.chdir(news_dir)
# GET NEWS FILES
file_glob = os.path.join(news_dir, '*.md')
news = glob(file_glob)
# READ MANIFEST FILES -> NEWS_MANIFEST
news = lmap(
lambda i: (os.path.basename(i),) + read_yaml_md(i),
news
)
news = lmap(
lambda i: {
**i[1],
'file': i[0],
'message': i[2]
},
news
)
# NEWS_MANIFEST -> NEWS_OBJECT
news = lmap(
create_news,
news
)
# FILTER BROKEN NEWS_OBJECT
news = list(filter(
None,
news
))
# SYNC EVENT_OBJECT WITH DB
events = lmap(
sync_news,
news
)
lprint_json(news)
print('SYNC NEWS DONE')
utils/Sync/sync/post.py
+312
View File
@@ -0,0 +1,312 @@
import os
from datetime import datetime
from glob import glob
import lxml.html
import settings
from db import db
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \
images_from_html, list_images, synced_images_ids
from sync.image import sync_image
from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text
class SyncPost(Sync):
def __init__(self, collection_name, document_type, image_url_base, dir_local_images, sizes):
super().__init__()
self.collection = db()[collection_name]
self.sizes = sizes
self.type = document_type
self.image_url_base = image_url_base
self.dir_local_images = dir_local_images
def create_id(self, document):
document['id'] = document['id'] if 'id' in document else url_encode_text(document['title'])
return document
def create_hash(self, document):
hash_images = lmap(
hash_file,
lmap(
lambda i: os.path.join(document['folder'], i),
document['images']
)
)
document['hash'] = hash_str(
combine([hash_str(document)] + hash_images)
)
return document
def create(self, document):
if 'date' not in document or not document['date']:
return None
if 'title' not in document or not document['title']:
return None
if self.type:
document['type'] = self.type
post_html = lxml.html.fromstring(document['post'])
images = []
for img in post_html.cssselect('img'):
src = img.get('src')
img_path = os.path.abspath(
os.path.join(document['folder'], src)
)
if os.path.exists(img_path):
img_id = url_encode_text(os.path.splitext(src)[0])
url_fn = lambda size, ext: self.image_url_base.format(id=document['id'], img=img_id, size=size,
ext=ext.lower())
image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
if image:
images.append(image)
img.set('src', image['data']['big']['url'])
else:
print('fail: ', document['folder'], src)
document['post'] = lxml.html.tostring(post_html).decode('utf-8')
document['images'] = images
return document
def get_manifest(path):
y, m = read_yaml_md(path)
y = y if y else {}
if 'date' in y:
y['date'] = create_date(y['date'], settings.date_formats)
if 'until' in y:
y['until'] = create_date(y['until'], settings.date_formats)
if 'id' in y:
y['id'] = str(y['id'])
return {
**y,
'post': m,
'images': images_from_html(m)
}
def md_from_folder(i):
return first(glob(i + '/*.md'))
def get_folder_documents(sync, file_time_formats):
# GET EVENT FOLDERS
documents = lmap(
lambda i: i.path,
filter(
lambda i: i.is_dir(),
os.scandir('.')
)
)
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
documents = lmap(
lambda i: (i,) + create_date_and_title_from_folder_name(
os.path.basename(i), file_time_formats
),
documents
)
documents = lmap(
lambda i: {
'folder': i[0],
'date': i[1],
'title': i[2],
},
documents
)
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
documents = lmap(
lambda i: {
**i,
'images': lmap(
lambda path: os.path.relpath(path, i['folder']),
list_images(i['folder'])
)
},
documents
)
documents = lmap(
lambda i: {
**i,
'post': create_post_from_image_list(i['images'])
},
documents
)
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
documents = lmap(
lambda i: {
**i,
**map_cases(
i,
[(
lambda i: md_from_folder(i['folder']),
lambda i: get_manifest(md_from_folder(i['folder'])),
)],
lambda i: {}
)
},
documents
)
# CREATE DOCUMENT IDENTITY
documents = lmap(sync.create_id, documents)
# CREATE HASH OF DOCUMENT FILE
documents = lmap(sync.create_hash, documents)
return documents
# GET EVENT FILES
def get_file_documents(sync, file_time_formats):
documents = glob('*.md')
documents = lmap(
lambda i: (i,) + create_date_and_title_from_folder_name(
os.path.basename(i), file_time_formats
),
documents
)
documents = lmap(
lambda i: {
'file': i[0],
'date': i[1],
'title': i[2],
},
documents
)
# OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST
documents = lmap(
lambda i: {
**i,
**get_manifest(i['file'])
},
documents
)
# CREATE DOCUMENT IDENTITY
documents = lmap(sync.create_id, documents)
# CREATE HASH OF DOCUMENT FILE
documents = lmap(sync.create_hash, documents)
return documents
def main(dir_documents, sync, file_time_formats):
# INIT
os.chdir(dir_documents)
documents = get_folder_documents(sync, file_time_formats)
documents += get_file_documents(sync, file_time_formats)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmapfn(documents)(
lambda i: i['id']
)
# SKIP UNTOUCHED DOCUMENTS
documents = lmap(
lambda i: i[0],
filter(
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, sync.read(document)),
documents
)
))
# MAP EVENT MANIFEST -> EVENT_OBJECT
documents = lmap(
sync.create,
documents
)
# FILTER INCORRECT EVENTS
documents = list(filter(None, documents))
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('images', synced_images_ids),
documents
)
# SYNC EVENT_OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
)
)
# SCOPE
print('SCOPE:')
lprint(scope_documents_ids)
# DELETE
print('DELETE DOCUMENTS:')
lprint_json(lmap(
lambda i: i['id'],
documents_to_remove
))
# DELETE
print('UPDATE DOCUMENTS:')
lprint_json(lmap(
lambda i: i['id'],
documents
))
lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
main(
'/Users/tmshv/Dropbox/Dev/Hud school/Events',
SyncPost(
'events',
None,
image_url_base,
settings.image_output,
settings.event_image_sizes
),
file_time_formats=['%Y.%m.%d']
)
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
main(
'/Users/tmshv/Dropbox/Dev/Hud school/News',
SyncPost(
'timeline',
'post',
image_url_base,
settings.image_output,
settings.event_image_sizes
),
file_time_formats=settings.date_formats_reverse
)
utils/Sync/sync/schedule.py
-42
View File
@@ -15,48 +15,6 @@ class SyncSchedule(Sync):
super().__init__()
self.collection = db().schedules
# def sync_collective(self, record):
# if isinstance(record, list):
# return lmap(
# self.sync_collective,
# record
# )
#
# q = {'id': record['id']}
# try:
# collection = db().collective
# update_result = collection.update_one(q, {'$set': record}, upsert=True)
# i = collection.find_one({'id': record['id']})
# return i
# except ValueError:
# pass
#
# return None
#
# def compile(self, profile):
# profile['picture'] = create_image(
# profile['picture'],
# sizes,
# lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
# dir_local_images
# )
# return profile
#
# def read_document(self, i, query_fn=None):
# q = query_fn(i) if query_fn else {'id': i['id']}
# try:
# return self.collection().find_one(q)
# except ValueError:
# pass
#
# return None
#
# def query_documents(self, q):
# return self.collection().find(q)
#
# def delete_document(self, q):
# return self.collection().find_one_and_delete(q)
def create_id(self, document):
id = document['id'] if 'id' in document and document['id'] else None
if not id:
utils/Sync/sync/teacher.py
+28 -76
View File
@@ -3,8 +3,9 @@ from glob import glob
import settings
from db import db
from sync import Sync
from sync.image import sync_image
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
from utils.hash import hash_str, hash_file
from utils.image import create_image
from utils.io import read_yaml_md
@@ -15,42 +16,11 @@ url_base = settings.teacher_image_url_base
dir_local_images = settings.teacher_image_output
sizes = settings.teacher_image_sizes
class Teacher:
def collection(self):
return db().collective
def read_teacher_by_id(self, record):
if isinstance(record, list):
return lmap(
self.read_teacher_by_id,
record
)
q = {'id': record}
try:
return db().collective.find_one(q)
except ValueError:
pass
return None
def sync_collective(self, record):
if isinstance(record, list):
return lmap(
self.sync_collective,
record
)
q = {'id': record['id']}
try:
collection = db().collective
update_result = collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
class Teacher(Sync):
def __init__(self):
super().__init__()
self.collection = db().collective
def compile(self, profile):
profile['picture'] = create_image(
@@ -61,39 +31,27 @@ class Teacher:
)
return profile
def read_document(self, i, query_fn=None):
q = query_fn(i) if query_fn else {'id': i['id']}
try:
return self.collection().find_one(q)
except ValueError:
pass
return None
def query_documents(self, q):
return self.collection().find(q)
def delete_document(self, q):
return self.collection().find_one_and_delete(q)
def create_id(self, document):
if 'id' in document and document['id']:
return document['id']
return document
document['id'] = url_encode_text(document['name'])
return document
def create_hash(self, document):
document['hash'] = hash_str(
hash_file(document['file']) + hash_file(document['picture'])
)
return document
if __name__ == '__main__':
# ALL DOCUMENTS IDS FOUNDED
scope_documents_ids = []
def main(dir_teachers, sync):
os.chdir(dir_teachers)
sync = Teacher()
# GET TEACHER YAML/MD MANIFEST FILES
documents = glob('*.md')
# # READ TEACHER YAML/MD MANIFEST
# READ TEACHER YAML/MD MANIFEST
documents = lmap(
lambda i: (i,) + read_yaml_md(i),
documents
@@ -119,20 +77,10 @@ if __name__ == '__main__':
)
# CREATE DOCUMENT IDENTITY
documents = lmap(
lambda i: {**i, 'id': sync.create_id(i)},
documents
)
documents = lmapfn(documents)(sync.create_id)
# CREATE HASH OF DOCUMENT FILES
documents = lmap(
lambda i: {
**i, 'hash': hash_str(
hash_file(i['file']) + hash_file(i['picture'])
)
},
documents
)
documents = lmapfn(documents)(sync.create_hash)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmap(
@@ -146,7 +94,7 @@ if __name__ == '__main__':
filter(
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, sync.read_document(document)),
lambda document: (document, sync.read(document)),
documents
)
))
@@ -157,22 +105,22 @@ if __name__ == '__main__':
documents
)
# # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
documents = lmap(
key_mapper('picture', sync_picture),
documents
)
# # SYNC TEACHER_PROFILE
# SYNC TEACHER_PROFILE
documents = lmap(
sync.sync_collective,
sync.update,
documents
)
documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}})
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete_document,
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
@@ -191,3 +139,7 @@ if __name__ == '__main__':
print('UPDATE DOCUMENTS:')
lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
main(dir_teachers, Teacher())
utils/Sync/sync_awards.py
-90
View File
@@ -1,90 +0,0 @@
import os
from glob import glob
from pymongo import MongoClient
import settings
from utils.image import resize
def generate_image_filename(image, size_name, ext):
"""
Example: 2015-history-competition-1-small.jpg
:param image:
:param size_name:
:param ext:
:return:
"""
return '{image}-{size}{ext}'.format(
image=image,
size=size_name,
ext=ext
)
def process_file(file):
sizes = settings.image_sizes
image = resize.read_image(file)
images_dir = '/Users/tmshv/Desktop/Hudozka Awards'
public_base_url = 'https://static.shlisselburg.org/art/awards/'
basename = os.path.basename(file)
uri, ext = os.path.splitext(basename)
award = {
'uri': uri,
'thumbs': {}
}
for size in sizes:
size_name, width, height = size
image_filename = generate_image_filename(uri, size_name, ext)
d = os.path.join(images_dir, image_filename)
image_url = os.path.join(public_base_url, image_filename)
thumb = resize.optimize(image, d, quality=90) if size_name == 'original' else resize.thumbnail(image, d, (width, height))
width, height = thumb.size
award['thumbs'][size_name] = {
'url': image_url,
'size': size_name,
'width': width,
'height': height
}
return award
def get_db():
client = MongoClient(settings.database_uri)
return client.hudozka
def sync(awards):
clear_db()
db = get_db()
for a in awards:
db.awards.insert(a)
def clear_db():
db = get_db()
db.awards.drop()
if __name__ == '__main__':
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Awards'
files = glob(dir + '/*')
awards = list(map(
process_file,
files
# map(
# lambda f : os.path.join(dir, f),
# files
# )
))
print(awards)
sync(awards)
utils/Sync/sync_gallery.py
-154
View File
@@ -1,154 +0,0 @@
import os
import re
from functools import reduce
from glob import glob
from pymongo import MongoClient
import settings
from read_gallery_1 import GalleryReader
from utils.image import resize as img_util
__author__ = 'Roman Timashev'
description_version_exp = re.compile('\[(\d+)\]\.md$')
dir_gallery = settings.root_dir_gallery
catalog_dir = os.path.expanduser(dir_gallery)
catalog_files = os.listdir(catalog_dir)
os.makedirs(settings.image_save_dir, exist_ok=True)
def get_description_version(filename):
v = description_version_exp.search(filename)
return v.group(1)
def get_description_reader(version):
if version == '1':
return GalleryReader
return None
def process_product_image(product, save_dir):
for k in product.image.keys():
thumb = product.image[k]
filename = os.path.basename(thumb['url'])
outpath = os.path.join(save_dir, filename)
img_util.image_magick_resize(
product.image_original_path,
outpath,
(thumb['width'], thumb['height'])
)
def get_db():
client = MongoClient(settings.database_uri)
return client.hudozka
def clear_db():
db = get_db()
db.products.drop()
db.albums.drop()
def sync_product(product, type):
db = get_db()
author_name, author_age, image, product_title, original_path = product.to_tuple()
return db['products'].insert(dict(
author=author_name,
authorAge=author_age,
title=product_title,
type=type,
content=image
))
def sync_album(album):
db = get_db()
product_ids = list(map(
lambda product: sync_product(product, album.type),
album.products
))
return db['albums'].insert(dict(
title=album.title,
uri=album.uri,
date=album.date,
description=album.comment,
course=album.course,
course_uri=album.course_uri,
teacher=album.teacher,
content=product_ids
))
def read_gallery_item(dir_name):
"""
Read directory with .md and images and return an instance of Album.
None if reads fails.
:param dir_name:
:return:
"""
set_path = os.path.join(dir_gallery, dir_name)
os.chdir(set_path)
md = glob('*.md')
if len(md) > 0:
description_filename = md[0]
description_path = os.path.join(set_path, description_filename)
v = '1'
DescriptionReader = get_description_reader(v)
if DescriptionReader is not None:
r = DescriptionReader(set_path, description_path)
try:
return r.read()
except Exception as e:
print('Error with reading description file %s' % (description_path))
print(e)
return None
if __name__ == '__main__':
print('--> READING GALLERY CATALOGS')
albums = list(map(read_gallery_item, catalog_files))
print('--> GALLERY TO SYNC')
albums = list(filter(lambda a: a is not None, albums))
for a in albums:
print(a)
img = False
save_dir = settings.image_save_dir
print('--> PROCESSING IMAGES to %s (%s)' % (save_dir, img))
if img:
products = reduce(lambda p, a: p + a.products, albums, [])
print('Products image to process %s' % len(products))
for p in products:
print('Process %s' % p.image_original_path)
process_product_image(p, save_dir)
sync = True
print('--> SYNCING ALBUMS (%s)' % sync)
if sync:
print('--> DROP DB')
clear_db()
for a in albums:
try:
a_id = sync_album(a)
print('SYNC DONE (%s: %s)' % a_id, a.title)
except Exception as e:
print('Cannot sync %s' % (a))
print(e)
exit()
utils/Sync/sync_old.py
-224
View File
@@ -1,224 +0,0 @@
import os
from glob import glob
import lxml.html as html
import markdown as md
from lxml.etree import tostring
from utils import lxml_utils
from utils.image import resize
__author__ = 'tmshv'
# db = pymongo.MongoClient('mongodb://localhost:27017')['stickers']
# images_dir = os.path.expanduser('~/Dropbox/Dev/Hud school/Static/images')
catalog_dir = os.path.expanduser('~/Dropbox/Dev/Hud School/Gallery')
catalog_files = os.listdir(catalog_dir)
def lesson_info(page):
lesson = list(filter(lambda h2: not h2.xpath('./a'), page.xpath('//h2')))[0]
content = lxml_utils.get_following_tags(lesson, 'h2')
for p in lxml_utils.get_following_tags(lesson, 'h2'):
print(p, tostring(p, encoding='utf-8').decode())
# print(content[0].xpath('./em/text()')[0].strip())
# print(content[0].xpath('./text()')[0].strip())
# print(content[1].xpath('./text()'))
# return content
date_str = '12.10.2013'
#todo: make as `date`
date = date_str
return {
'name': 'Collage',
'course': 'painting',
'technology': ['pastel', 'huel', 'glue'],
'date': date,
}
def drawings_info(page):
out = []
pics = list(filter(lambda h2: h2.xpath('./a'), page.xpath('//h2')))
for p in pics:
content = lxml_utils.get_following_tags(p, 'h2')[0]
author_info = content.xpath('./text()')[0].split('\n')
out.append({
'name': p.xpath('./a/text()')[0],
'image': p.xpath('./a/@href')[0],
'author': author_info[0],
'author_age': int(author_info[1].split(' ')[0])
})
return out
def generate_image_filename(src, course, year, theme, author, size, i,ext):
if ext is None:
ext = os.path.splitext(src)[1]
if i == 0:
filename = '{course}-{year}-{theme}-{author}-{size}.{ext}'
else:
filename = '{course}-{year}-{theme}-{author}-{index}-{size}.{ext}'\
filename = filename.format(
course=course,
year=year,
theme=theme,
author=author,
size=size,
index=i,
ext=ext
)
return filename
for cf in catalog_files:
break
os.chdir(catalog_dir)
if os.path.isdir(cf):
print('======================================================================================')
print(cf)
product_dir = os.path.join(catalog_dir, cf)
os.chdir(product_dir)
def matching_files(param):
if isinstance(param, list):
return list(map(matching_files, param))
else:
return glob(param)
product_file = matching_files('*.md')[0]
def from_file(path):
f = open(path, 'r')
data = f.read()
f.close()
return data
markdown_text = from_file(product_file)
html_text = md.markdown(markdown_text)
page = html.fromstring(html_text)
print(html_text)
# name
name = page.xpath('//h1/text()')[0]
# lesson
# filtering paragraph with no link in h2 title
lesson = lesson_info(page)
print(lesson)
drawings = drawings_info(page)
for d in drawings:
print(d)
products = list(map(
lambda drawing: {
'image': drawing['image'],
'name': drawing['name'],
'author': drawing['author'],
'authorAge': drawing['author_age'],
'course': lesson['course'],
'theme': lesson['name'],
'rating': 10,
'date': lesson['date']
},
drawings
))
for p in products:
print(p)
# # price
# def str_price_to_int(text):
# items = text.split('\n')
# out = list(map(lambda p: int(re.sub('[^\d]+', '', p)), items))
# return out
#
#
# price_text = page.xpath('//h2[contains(text(), "Цена")]/following-sibling::p/text()')[0]
# product_price = str_price_to_int(price_text)
#
# # colors
# def str_colors_to_code_list(text):
# cl = re.split('\s+', text)
# return ttcolor.color_by_code(cl)
#
# color_text = page.xpath('//h2[contains(text(), "Цвета")]/following-sibling::p/text()')[0]
# product_colors = str_colors_to_code_list(color_text)
#
# # text
# def format_text(text):
# try:
# res = requests.post("http://mdash.ru/api.v1.php", {
# 'text': text
# })
# return res.json()['result']
#
# except Exception as e:
#
# return text
#
# product_text = page.xpath('//h2[contains(text(), "Текст")]/following-sibling::p/text()')[0]
# # product_text = format_text(product_text)
#
#
# # qualities
# def get_quality(li):
# q = {
# 'image': li.xpath('./img/@src')[0],
# 'text': li.xpath('text()')[0].strip()
# }
# return q
#
#
# def qualities_list(elements):
# return list(map(get_quality, elements))
#
#
# qualities_elements = page.xpath('//h2[contains(text(), "Свойства")]/following-sibling::ul/li')
# qualities = qualities_list(qualities_elements)
#
#
product_images = [p['image']]
image_sizes = [
(50, 50, 'small'),
(300, 300, 'preview'),
(800, 800, 'medium'),
(2000, 2000, 'big'),
(0, 0, 'original')
]
images = []
i = 0
for image in product_images:
pi = {'source': image}
for size in image_sizes:
size_name = size[2]
image_filename = generate_image_filename(image, size_name, product_sku, i, 'png')
d = os.path.join(images_dir, image_filename)
pi[size_name] = 'http://static.touchtwin.ru/images/{img}'.format(img=image_filename)
if size_name == 'original':
resize.optimize(image, d, quality=90)
else:
resize.minify(image, d, size[:2])
images.append(pi)
i += 1
p['images'] = images
del p['image']
# db['products'].update({'uri': product_uri}, p, upsert=True)
utils/Sync/sync_schedules.py
-38
View File
@@ -1,38 +0,0 @@
from glob import glob
import json
import os
import re
from pymongo import MongoClient
import settings
__author__ = 'tmshv'
def process_file(schedule_path):
with open(schedule_path, 'rb') as schedule_file:
data = schedule_file.read().decode('utf-8')
try:
return json.loads(data)
except json.decoder.JSONDecodeError:
print(schedule_path)
if __name__ == '__main__':
db = MongoClient(settings.database_uri).hudozka
db.schedules.drop()
# dir_schedule = settings.root_dir_schedule
dir_schedule = '/Users/tmshv/Dropbox/Dev/Hud School/Schedules/'
files = glob(dir_schedule + '*.json')
ss = list(map(process_file, files))
list(map(print, ss))
list(map(
db.schedules.insert,
filter(
lambda i : i is not None,
ss
)
))
print('schedule sync complete')
utils/Sync/utils/fn.py
+21 -10
View File
@@ -3,12 +3,19 @@ import os
from functools import reduce
from glob import glob
from bson import json_util
import bson.json_util
def combine(items):
return reduce(
lambda total, i: total + i,
items
)
def to_json(i):
return json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
combine = lambda i: reduce(
lambda s, i: s + i,
i
)
iterate_over_fns = lambda fns: lambda i: reduce(
lambda out, fn: fn(out),
@@ -20,12 +27,16 @@ iterate_iter_over_fns = lambda fns: lambda i: lmap(
i
)
lmap = lambda fn, ls: list(map(fn, ls))
def lmap(fn, ls):
return list(map(fn, ls))
def lmapfn(ls):
return lambda fn: list(map(fn, ls))
lprint = lambda i: lmap(print, i)
lprint_json = lambda s: lprint(lmap(
lambda i: json.dumps(i, ensure_ascii=False, indent=4, default=json_util.default),
s
))
lprint_json = lambda i: lprint(lmap(to_json, i))
first = lambda i: i[0] if len(i) > 0 else None
utils/Sync/utils/hash.py
+2 -3
View File
@@ -1,7 +1,7 @@
import hashlib
import json
from utils.fn import map_cases
from utils.fn import map_cases, to_json
def hash_file(file):
@@ -16,7 +16,6 @@ def hash_data(data):
def hash_str(data):
to_json = lambda i: json.dumps(i, sort_keys=True)
data = map_cases(data, [
(lambda i: isinstance(i, dict), to_json),
(lambda i: isinstance(i, tuple), to_json),
@@ -25,6 +24,6 @@ def hash_str(data):
(lambda i: isinstance(i, float), float),
])
if isinstance(data, dict):
data = json.dumps(data, sort_keys=True)
data = to_json(data)
return hash_data(str(data).encode())
utils/Sync/utils/image/__init__.py
+2 -2
View File
@@ -2,7 +2,7 @@ import os
from PIL import Image
from utils.hash import file_hash
from utils.hash import hash_file
from utils.image import resize
from utils.image.resize import optimize, thumbnail, orient
@@ -54,7 +54,7 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False):
return {
'file': os.path.basename(file),
'hash': file_hash(file),
'hash': hash_file(file),
'data': result
}
utils/Sync/utils/image/resize.py
+41
View File
@@ -2,7 +2,9 @@ import os
from subprocess import call
import smartcrop
from PIL import Image
from smartcrop import SmartCrop
import utils.image
@@ -30,6 +32,29 @@ def override_extension(dest, ext=None):
return dest
def crop(src, dest, size):
width, height = size
image = utils.image.read_image(src)
crop_options = smartcrop.DEFAULTS
crop_options['width'] = width
crop_options['height'] = height
crop = SmartCrop()
result = crop.crop(image.copy(), crop_options)
box = (result['topCrop']['x'],
result['topCrop']['y'],
result['topCrop']['x'] + result['topCrop']['width'],
result['topCrop']['y'] + result['topCrop']['height'])
i = utils.image\
.read_image(src)\
.crop(box)
i.thumbnail(size, Image.ANTIALIAS)
i.save(dest)
def orient(image):
orientation = 0x0112
rotations = {
@@ -52,6 +77,22 @@ def image_magick_resize(input_file, output_file, size, quality=85):
call(['convert', input_file, '-strip', '-auto-orient', '-resize', s, '-quality', str(quality), output_file])
# def image_magick_resize(input_file, output_file, size, quality=85):
# from wand.image import Image
#
# w, h = size
# s = '{w}x{h}>'.format(w=w, h=h)
#
# with Image(filename=input_file) as img:
# # print(img.size)
# # print(list(img.metadata.items()))
# with img.clone() as i:
# i.transform(resize=s)
# i.auto_orient()
# i.strip()
# i.save(filename=output_file)
def image_magick_pdf_to_img(input_file, output_file):
i = "{image}[0]".format(image=input_file)
call(['convert', i, output_file])