FUCKING BIG SYNC UPDATE: caching, events, albums, documents, awards, news, schedules, teachers, other

This commit is contained in:
tmshv committed 2016-03-03 01:36:50 +03:00
1 parent 359cc430f5
commit f07c39c317
20 files changed
+1190 -166

No files matched your search

+132 -79
View File
@@ -1,5 +1,6 @@
import os
from glob import glob
from lxml import etree
import lxml.html
@@ -8,10 +9,25 @@ from db import db
from models.product import create_product
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date
from sync.image import sync_image
from utils.fn import lmap, first, map_cases, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns
from utils.fn import lmap, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns, first, map_cases
from utils.hash import file_hash
from utils.image import create_image
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text, interpolate, interpolate_swift
from utils.text.transform import url_encode_text
class ImageCreator:
def __init__(self, cache):
super().__init__()
self.cache = cache if cache else set()
def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
hash = file_hash(file)
cached = self.cached(hash)
return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
def cached(self, hash):
return self.cache[hash] if hash in self.cache else None
def sync_album(record):
@@ -24,7 +40,7 @@ def sync_album(record):
q = {'id': record['id']}
try:
collection = db().events
update_result = collection.update_one(q, {'$set': record}, upsert=True)
collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
@@ -33,115 +49,150 @@ def sync_album(record):
return None
def create_album(doc):
sizes = settings.event_image_sizes
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
url = lambda a, i, s, e: 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
def create_post(md, image_path_fn):
html = lxml.html.fromstring(md)
for img in html.cssselect('img'):
src = img.get('src')
path = image_path_fn(src)
if path:
img.set('src', path)
img.set('class', settings.album_html_img_class)
img.set('data-file', src)
return etree.tounicode(html)
def create_url(a, i, s, e):
return 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
album=a,
id=i,
size=s,
ext=e
)
create_url = lambda aid, pid: lambda size, ext: url(aid, pid, size, ext.lower())
image_id = lambda i: url_encode_text(
os.path.splitext(i)[0]
)
image_path = lambda i: os.path.join(doc['folder'], i)
if 'title' not in doc or not doc['title']:
return None
create_album_id = lambda date, title: '{date}-{title}'.format(
date=date.strftime('%Y'),
title=url_encode_text(title)
)
def url_creator(aid, pid):
return lambda size, ext: create_url(aid, pid, size, ext.lower())
images = []
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
class Album:
@staticmethod
def id_from_file(i):
return url_encode_text(
os.path.splitext(i)[0]
)
if 'album' in doc:
# album_context =
def __init__(self, image_creator):
super().__init__()
self.image_creator = image_creator
self.skip_image_processing = False
self.images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
def create_image(self, file, url_fn):
ss = settings.album_image_sizes
od = self.images_dir
sp = self.skip_image_processing
return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
def create_album(self, doc):
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
image_id = Album.id_from_file
image_path = lambda i: os.path.join(doc['folder'], i)
if 'title' not in doc or not doc['title']:
return None
create_album_id = lambda date, title: '{date}-{title}'.format(
date=date.strftime('%Y'),
title=url_encode_text(title)
)
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
if 'album' in doc:
doc['album'] = self.create_products(doc, image_path)
def process_post_image(src):
path = image_path(src)
if os.path.exists(path):
url = url_creator(
doc['id'],
image_id(src)
)
image = self.create_image(path, url)
if image:
images.append(image)
return image['data']['big']['url']
return None
images = []
post_html = create_post(doc['post'], process_post_image)
doc['post'] = post_html
doc['images'] = images
return doc
def create_products(self, doc, image_file_fn):
image_id = Album.id_from_file
iter_album = iterate_iter_over_fns([
lambda product: {
**product,
'image': create_image(
image_path(product['file']),
sizes,
create_url(
'image': self.create_image(
image_file_fn(product['file']),
url_creator(
doc['id'],
image_id(product['file'])
),
images_dir,
skip_processing=True
)
)
},
lambda product: {
**product,
'awards': lmap(
lambda award: {
**award,
'file': os.path.join(doc['folder'], award['file']),
'assigned': interpolate(award['assigned'], product, interpolate_swift),
# 'awardFor': product
},
product['awards']
)
}
# lambda product: {
# **product,
# 'awards': lmap(
# lambda award: {
# **award,
# 'file': os.path.join(doc['folder'], award['file']),
# 'assigned': interpolate(award['assigned'], product, interpolate_swift),
# # 'awardFor': product
# },
# product['awards']
# )
# }
])
doc['album'] = lmap(
return lmap(
create_product,
iter_album(doc['album'])
)
post_html = lxml.html.fromstring(doc['post'])
for img in post_html.cssselect('img'):
src = img.get('src')
img_path = image_path(src)
if os.path.exists(img_path):
image = create_image(
img_path,
sizes,
create_url(
doc['id'],
image_id(src)
),
images_dir,
skip_processing=True
)
images.append(image)
img.set('src', image['data']['big']['url'])
img.set('class', settings.album_html_img_class)
img.set('data-file', src)
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
doc['images'] = images
return doc
def get_manifest(filepath):
y, m = read_yaml_md(filepath)
data = {**y} if y else {}
_ = lambda key, d=None: data[key] if key in data else d
date = create_date(_('date'), settings.date_formats_direct)
date = create_date(str(_('date')), settings.date_formats_direct)
if date:
data['date'] = date
id = _('id')
if id:
data['id'] = str(id)
album_id = _('id')
if album_id:
data['id'] = str(album_id)
post = {'post': m} if m else {}
return {**data, **post}
if __name__ == '__main__':
image_creator = ImageCreator(cache=None)
alb = Album(image_creator=image_creator)
# alb.skip_image_processing = True
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
os.chdir(albums_dir)
@@ -157,6 +208,7 @@ if __name__ == '__main__':
# print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md'))
# print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md'))
# exit()
# albums = [albums[5]]
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
albums = lmap(
@@ -200,13 +252,14 @@ if __name__ == '__main__':
# MAP EVENT MANIFEST -> EVENT_OBJECT
albums = lmap(
create_album,
alb.create_album,
albums
)
# FILTER INCORRECT EVENTS
albums = list(filter(None, albums))
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
def sync_images(i):
return lmap(
@@ -221,10 +274,10 @@ if __name__ == '__main__':
)
# SYNC EVENT_OBJECT WITH DB
# albums = lmap(
# sync_album,
# albums
# )
albums = lmap(
sync_album,
albums
)
lprint_json(albums)
print('SYNC ALBUMS DONE')