tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

[sync] updated: + albums, kazimir rendering, other changes

Browse Source
This commit is contained in:
tmshv committed 2016-11-11 02:34:19 +03:00
1 parent 2ea3c1f322
commit dc8363ee84
18 files changed
+331 -410
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/Sync/sync/album.py
+93 -279
View File
@@ -1,182 +1,33 @@
import os
from glob import glob
import lxml.html
from lxml import etree
import settings
from db import db
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \
untouched, synced_images_ids
from sync.models import create_product
from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \
key_mapper, lprint
from utils.hash import hash_file, hash_str
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, untouched, \
synced_images_ids
from sync.core.album import SyncAlbum
from sync.data import list_images
from utils.fn import lmap, lmapfn, key_mapper
from utils.image import create_image
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text
class ImageCreator:
def __init__(self, cache):
def __init__(self, provider, cache):
super().__init__()
self.provider = provider
self.cache = cache if cache else set()
def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
hash = hash_file(file)
cached = self.cached(hash)
return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
filehash = self.provider.hash(file)
cached = self.cached(filehash)
return create_image(self.provider.get_abs(file), sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
def cached(self, hash):
return self.cache[hash] if hash in self.cache else None
def create_post(md, image_path_fn):
html = lxml.html.fromstring(md)
for img in html.cssselect('img'):
src = img.get('src')
path = image_path_fn(src)
if path:
img.set('src', path)
img.set('class', settings.album_html_img_class)
img.set('data-file', src)
return etree.tounicode(html)
def create_url(a, i, s, e):
return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
album=a,
id=i,
size=s,
ext=e
)
def url_creator(aid, pid):
return lambda size, ext: create_url(aid, pid, size, ext.lower())
class SyncAlbum(Sync):
@staticmethod
def id_from_file(i):
return url_encode_text(
os.path.splitext(i)[0]
)
def __init__(self, image_creator, images_dir):
super().__init__()
self.collection = db().albums
self.image_creator = image_creator
self.skip_image_processing = False
self.images_dir = images_dir
def create_id(self, document):
if 'id' in document:
return document
y = str(document['date'].year)
document['id'] = url_encode_text('-'.join([y, document['title']]))
return document
def create_hash(self, document):
hash_images = lmap(
hash_file,
lmap(
lambda i: os.path.join(document['folder'], i),
document['images']
)
)
document['hash'] = hash_str(
combine([hash_str(document)] + hash_images)
)
return document
def create_image(self, file, url_fn):
ss = settings.album_image_sizes
od = self.images_dir
sp = self.skip_image_processing
return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
def create_album(self, doc):
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
image_id = SyncAlbum.id_from_file
image_path = lambda i: os.path.join(doc['folder'], i)
if 'title' not in doc or not doc['title']:
return None
create_album_id = lambda date, title: '{date}-{title}'.format(
date=date.strftime('%Y'),
title=url_encode_text(title)
)
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
if 'album' in doc:
doc['album'] = self.create_products(doc, image_path)
def process_post_image(src):
path = image_path(src)
if os.path.exists(path):
url = url_creator(
doc['id'],
image_id(src)
)
image = self.create_image(path, url)
if image:
images.append(image)
return image['data']['big']['url']
return None
images = []
post_html = create_post(doc['post'], process_post_image)
doc['post'] = post_html
doc['images'] = images
return doc
def create_products(self, doc, image_file_fn):
image_id = SyncAlbum.id_from_file
iter_album = iterate_iter_over_fns([
lambda product: {
**product,
'image': self.create_image(
image_file_fn(product['file']),
url_creator(
doc['id'],
image_id(product['file'])
)
)
},
# lambda product: {
# **product,
# 'awards': lmap(
# lambda award: {
# **award,
# 'file': os.path.join(doc['folder'], award['file']),
# 'assigned': interpolate(award['assigned'], product, interpolate_swift),
# # 'awardFor': product
# },
# product['awards']
# )
# }
])
return lmap(
create_product,
iter_album(doc['album'])
)
def get_manifest(filepath):
y, m = read_yaml_md(filepath)
def get_manifest(provider, filepath):
data = provider.read(filepath).read().decode('utf-8')
y, m = read_yaml_md(data)
data = {**y} if y else {}
_ = lambda key, d=None: data[key] if key in data else d
@@ -193,76 +44,60 @@ def get_manifest(filepath):
return {**data, **post}
def main(dir_documents, sync):
os.chdir(dir_documents)
# GET ALBUMS FOLDERS
documents = lmap(
lambda i: i.path,
filter(
lambda i: i.is_dir(),
os.scandir('.')
)
def album_from_folder(sync, path):
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
date, title = create_date_and_title_from_folder_name(
os.path.basename(path)
)
# documents = [documents[3]]
# documents = documents[:15]
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
documents = lmap(
lambda i: (i,) + create_date_and_title_from_folder_name(
os.path.basename(i)
),
documents
images = lmap(
lambda i: os.path.relpath(i, path),
list_images(sync.provider, path)
)
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
documents = lmap(
lambda i: {
'folder': i[0],
'date': i[1],
'title': i[2],
'post': create_post_from_image_list(lmap(
lambda path: os.path.relpath(path, i[0]),
glob_in(i[0])
))
},
documents
)
document = {
'folder': path,
'date': date,
'title': title,
'post': create_post_from_image_list(images),
'images': images
}
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
md_glob = lambda i: first(glob(i + '/*.md'))
documents = lmap(
lambda album: {
**album,
**map_cases(
album,
[(
lambda album: md_glob(album['folder']),
lambda album: get_manifest(md_glob(album['folder'])),
)],
lambda i: {}
)
},
documents
)
documents = lmap(
lambda i: {
**i,
'images': lmap(
lambda path: os.path.relpath(path, i['folder']),
list_images(i['folder'])
)
},
documents
)
manifest = sync.provider.type_filter(path, '.md')
manifest = manifest[0] if len(manifest) else None
manifest = get_manifest(sync.provider, manifest) if manifest else {}
manifest = manifest if manifest else {}
document = {
**document,
**manifest
}
# CREATE DOCUMENT IDENTITY
documents = lmap(sync.create_id, documents)
document = sync.create_id(document)
# CREATE HASH OF DOCUMENT FILE
documents = lmap(sync.create_hash, documents)
document = sync.create_hash(document)
return document
def main(sync, do_update, do_delete):
# GET ALBUMS FOLDERS
documents = lmap(
lambda i: i,
filter(
lambda i: sync.provider.is_dir(i),
sync.provider.scan('.')
)
)
documents = lmap(
lambda folder: album_from_folder(sync, folder),
documents
)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmapfn(documents)(
@@ -274,11 +109,11 @@ def main(dir_documents, sync):
# MAP EVENT MANIFEST -> EVENT_OBJECT
documents = lmap(
alb.create_album,
sync.create_album,
documents
)
# FILTER INCORRECT EVENTS
# # FILTER INCORRECT EVENTS
documents = list(filter(None, documents))
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
@@ -288,63 +123,42 @@ def main(dir_documents, sync):
)
# SYNC OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
if do_update:
documents = lmap(
sync.update,
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
if do_delete:
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
)
)
return documents, documents_to_remove
def sync_albums(provider, collection, update=True, delete=True):
"""
:param collection:
:param provider:
:param update:
:param delete:
:return:
"""
return main(
SyncAlbum(
collection,
provider,
image_creator=ImageCreator(provider, cache=None),
images_dir=settings.dir_static_images
),
do_update=update,
do_delete=delete
)
# SCOPE
print('SCOPE:')
lprint(scope_documents_ids)
# DELETE
print('DELETE DOCUMENTS:')
lprint_json(lmap(
lambda i: i['id'],
documents_to_remove
))
# DELETE
print('UPDATE DOCUMENTS:')
lprint_json(lmap(
lambda i: i['id'],
documents
))
# lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
image_creator = ImageCreator(cache=None)
alb = SyncAlbum(image_creator=image_creator, images_dir=settings.dir_static_images)
# alb.skip_image_processing = True
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
# dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Gallery'
dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/Gallery'
main(dir_documents=dir_docs, sync=alb)
# dir_docs += '/2013 Валяние Зверушки'
# dir_docs += '/Валяние Зверушки.md'
# dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/News/2015 Выставка 70-летия победы.md'
# print(frontmatter.load(dir_docs).content)
# d = read_file(dir_docs)
# d = d.replace('\u0D0A', '\n')
# print(frontmatter.loads(d).metadata)
# print(parse_yaml_front_matter(d))
#
# print(len(re.split('^---', d, 2, re.MULTILINE)))
# print(re.split('^---', d, 2)[1])
#
# print(read_yaml_md(dir_docs))
utils/Sync/sync/core/album.py
+145
View File
@@ -0,0 +1,145 @@
import os
from lxml import etree
import lxml.html
import settings
from sync.core import Sync
from sync.models.product import create_product
from utils.fn import lmap, iterate_iter_over_fns, combine
from utils.hash import hash_file, hash_str
from utils.text.transform import url_encode_text
def create_post(md, image_path_fn):
html = lxml.html.fromstring(md)
for img in html.cssselect('img'):
src = img.get('src')
path = image_path_fn(src)
if path:
img.set('src', path)
img.set('class', settings.album_html_img_class)
img.set('data-file', src)
return etree.tounicode(html)
def create_url(a, i, s, e):
return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
album=a,
id=i,
size=s,
ext=e
)
def url_creator(aid, pid):
return lambda size, ext: create_url(aid, pid, size, ext.lower())
class SyncAlbum(Sync):
@staticmethod
def id_from_file(i):
return url_encode_text(
os.path.splitext(i)[0]
)
def __init__(self, collection, provider, image_creator, images_dir):
super().__init__()
self.collection = collection
self.provider = provider
self.image_creator = image_creator
self.skip_image_processing = False
self.images_dir = images_dir
def create_id(self, document):
if 'id' in document:
return document
y = str(document['date'].year)
document['id'] = url_encode_text('-'.join([y, document['title']]))
return document
def create_hash(self, document):
files = sorted(document['images'])
hashes = lmap(
self.provider.hash,
lmap(
lambda i: os.path.join(document['folder'], i),
files
)
)
document['hash'] = hash_str(
combine([hash_str(document)] + hashes)
)
return document
def create_image(self, file, url_fn):
ss = settings.album_image_sizes
od = self.images_dir
sp = self.skip_image_processing
return self.image_creator.create_image(file, ss, url_fn, od, skip_processing=sp)
def create_album(self, doc):
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
image_id = SyncAlbum.id_from_file
image_path = lambda i: os.path.join(doc['folder'], i)
if 'title' not in doc or not doc['title']:
return None
create_album_id = lambda date, title: '{date}-{title}'.format(
date=date.strftime('%Y'),
title=url_encode_text(title)
)
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
if 'album' in doc:
doc['album'] = self.create_products(doc, image_path)
def process_post_image(src):
path = image_path(src)
if self.provider.exists(path):
url = url_creator(
doc['id'],
image_id(src)
)
image = self.create_image(path, url)
if image:
images.append(image)
return image['data']['big']['url']
return None
images = []
post_html = create_post(doc['post'], process_post_image)
doc['post'] = post_html
doc['images'] = images
return doc
def create_products(self, doc, image_file_fn):
image_id = SyncAlbum.id_from_file
iter_album = iterate_iter_over_fns([
lambda product: {
**product,
'image': self.create_image(
image_file_fn(product['file']),
url_creator(
doc['id'],
image_id(product['file'])
)
)
}
])
return lmap(
create_product,
iter_album(doc['album'])
)
utils/Sync/sync/core/person.py
+3 -2
View File
@@ -1,6 +1,6 @@
import settings
from sync.core import Sync
from utils.hash import hash_str, hash_file
from utils.hash import hash_str
from utils.image import create_image
from utils.text.transform import url_encode_text
@@ -16,8 +16,9 @@ class SyncPerson(Sync):
self.dir_local_images = dir_local_images
def compile(self, profile):
image_path = self.provider.get_abs(profile['picture'])
profile['picture'] = create_image(
profile['picture'],
image_path,
sizes,
lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
self.dir_local_images
utils/Sync/sync/core/post.py
+5 -2
View File
@@ -12,7 +12,7 @@ from utils.text.transform import url_encode_text, url_encode_file
class SyncPost(Sync):
def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes):
def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
super().__init__()
self.collection = collection
self.provider = provider
@@ -20,6 +20,7 @@ class SyncPost(Sync):
self.type = document_type
self.image_url_base = image_url_base
self.dir_local_images = dir_local_images
self.origin = origin
def create_id(self, document):
new_id = url_encode_text(document['title'])
@@ -77,12 +78,14 @@ class SyncPost(Sync):
document['post'] = lxml.html.tostring(post_html).decode('utf-8')
document['images'] = images
document['origin'] = self.origin
return document
def create_remove_query(self, query):
if self.type:
query = {
**query,
'type': self.type
'type': self.type,
'orogin': self.origin
}
return query
utils/Sync/sync/data/__init__.py
+3
View File
@@ -30,6 +30,9 @@ class Provider:
def size(self, path):
return 0
def exists(self, path):
return True
def get_abs(self, path):
a = os.path.join(self.root, path)
return os.path.normpath(a)
utils/Sync/sync/data/fs.py
+4
View File
@@ -41,6 +41,10 @@ class FSProvider(Provider):
file = self.get_abs(path)
return os.path.getsize(file)
def exists(self, path):
file = self.get_abs(path)
return os.path.exists(file)
def is_dir(self, path):
return os.path.isdir(self.get_abs(path))
utils/Sync/sync/document.py
-18
View File
@@ -1,5 +1,4 @@
import os
from glob import glob
import settings
from sync import synced_image_id, untouched
@@ -7,7 +6,6 @@ from sync.core.document import SyncDocument
from sync.data import scan_subdirs
from sync.models.document import Document
from utils.fn import combine, lmap, key_mapper
from utils.io import read_yaml
def sync_documents(provider, collection, update=True, delete=True):
@@ -177,19 +175,3 @@ def unwrap_manifest(param):
return None
return param
def documents_from_yaml(dirpath):
# GET DOCUMENT YAML_MANIFEST FILES
documents = glob(dirpath + '/*.yaml')
# READ YAML_MANIFEST FILES
documents = lmap(
read_yaml,
documents
)
# GET FLAT LIST OF DOCUMENTS
documents = unwrap_manifest(documents)
return documents
utils/Sync/sync/models/product.py
+10 -5
View File
@@ -1,5 +1,7 @@
import settings
from db import collection
from sync.core.person import SyncPerson
from sync.image import sync_image
from sync.teacher import Teacher
from utils.fn import key_mapper
from utils.text.transform import interpolate_swift
@@ -18,10 +20,13 @@ def resolve_value(param, store):
unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
def resolve_teacher(i):
sync = Teacher()
return resolve_value(i, lambda i: sync.read({'id': i}))
sync = SyncPerson(
collection(settings.collection_collective),
None,
''
)
return resolve_value(i, lambda teacher_id: sync.read({'id': teacher_id}))
def create_product(doc):
@@ -33,7 +38,7 @@ def create_product(doc):
unpop_image(product)
product['teacher'] = resolve_teacher(doc['teacher'])
if(product['teacher']):
if (product['teacher']):
unpop_teacher(product)
return product
utils/Sync/sync/person.py
+5 -11
View File
@@ -33,6 +33,10 @@ def sync_persons(provider, collection, update=True, delete=True):
)
def make_person(provider, file):
pass
def main(sync, update_documents=False, delete_documents=False):
# GET TEACHER YAML/MD MANIFEST FILES
documents = sync.provider.type_filter('', '.md')
@@ -75,17 +79,7 @@ def main(sync, update_documents=False, delete_documents=False):
)
# SKIP UNTOUCHED DOCUMENTS
documents = untouched(documents, sync)
# documents = lmap(
# lambda i: i[0],
# filter(
# lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
# lmap(
# lambda document: (document, sync.open(document)),
# documents
# )
# ))
# documents = untouched(documents, sync)
# DO HEAVY PROCESS WITH DOCUMENTS
documents = lmap(
utils/Sync/sync/post.py
+15 -14
View File
@@ -36,10 +36,10 @@ def create_document_from_folder(sync, path, file_time_formats):
)
document = {
'images': images,
'post': post,
**document,
**params,
'images': images,
'post': post
}
document = sync.create_id(document)
@@ -135,20 +135,20 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
# SKIP UNTOUCHED DOCUMENTS
documents = untouched(documents, sync)
# # MAP EVENT MANIFEST -> EVENT_OBJECT
# documents = lmap(
# sync.create,
# documents
# )
# MAP EVENT MANIFEST -> EVENT_OBJECT
documents = lmap(
sync.create,
documents
)
# FILTER INCORRECT EVENTS
documents = list(filter(None, documents))
# # REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
# documents = lmap(
# key_mapper('images', synced_images_ids),
# documents
# )
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('images', synced_images_ids),
documents
)
# SYNC EVENT_OBJECT WITH DB
if update_documents:
@@ -179,10 +179,11 @@ def sync_posts(provider, collection, update=True, delete=True):
SyncPost(
collection,
provider,
None,
'post',
image_url_base,
settings.dir_static_images,
settings.event_image_sizes
settings.event_image_sizes,
origin=settings.origin
),
file_time_formats=['%Y.%m.%d'],
update_documents=update,
utils/Sync/sync/schedule.py
+3 -3
View File
@@ -1,17 +1,17 @@
from yaml import load as yaml
from toml import loads as toml_str
from yaml import load as yaml
from sync import untouched
from sync.core.schedule import SyncSchedule
from sync.data import scan_subdirs, get_data
from sync.data import get_data
from utils.fn import lmap, constant
from utils.io import read_yaml
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules'
def sync_schedules(provider, collection, update=True, delete=True):
"""
:param collection:
:param provider:
:param update:
:param delete: