tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

[sync] person (teachers) updated

Browse Source
This commit is contained in:
tmshv committed 2016-10-15 16:01:25 +03:00
1 parent 54fbe4ef9c
commit 38d3c8d65c
5 files changed
+167 -145
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/Sync/settings.py
+2
View File
@@ -98,6 +98,7 @@ collection_documents = 'documents'
collection_schedules = 'schedules'
collection_events = 'events'
collection_news = 'news'
collection_collective = 'collective'
sync_provider_type = env('SYNC_PROVIDER', 'fs')
@@ -111,6 +112,7 @@ dir_documents = f('Hudozka/Site/Documents')
dir_schedules = f('Hudozka/Site/Schedules')
dir_events = f('Hudozka/Site/Events')
dir_news = f('Hudozka/Site/News')
dir_collective = f('Hudozka/Site/Collective')
dir_static = env('SYNC_STATIC', '~/Hudozka Static')
dir_static_uploads = absolute(dir_static + '/uploads')
utils/Sync/sync.py
+2
View File
@@ -3,6 +3,7 @@ from db import db
from sync.data.fs import FSProvider
from sync.data.yandexdisk import YDProvider
from sync.document import sync_documents
from sync.person import sync_persons
from sync.post import sync_posts
from sync.schedule import sync_schedules
from utils.fn import lprint_json
@@ -39,3 +40,4 @@ if __name__ == '__main__':
main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
main(io(settings.dir_events), sync_posts, settings.collection_events)
main(io(settings.dir_news), sync_posts, settings.collection_news)
main(io(settings.dir_collective), sync_persons, settings.collection_collective)
utils/Sync/sync/core/person.py
+38
View File
@@ -0,0 +1,38 @@
import settings
from sync.core import Sync
from utils.hash import hash_str, hash_file
from utils.image import create_image
from utils.text.transform import url_encode_text
url_base = settings.teacher_image_url_base
sizes = settings.teacher_image_sizes
class SyncPerson(Sync):
def __init__(self, collection, provider, dir_local_images):
super().__init__()
self.collection = collection
self.provider = provider
self.dir_local_images = dir_local_images
def compile(self, profile):
profile['picture'] = create_image(
profile['picture'],
sizes,
lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
self.dir_local_images
)
return profile
def create_id(self, document):
if 'id' in document and document['id']:
return document
document['id'] = url_encode_text(document['name'])
return document
def create_hash(self, document):
document['hash'] = hash_str(
self.provider.hash(document['file']) + self.provider.hash(document['picture'])
)
return document
utils/Sync/sync/person.py
+125
View File
@@ -0,0 +1,125 @@
import os
from glob import glob
import settings
from sync import untouched
from sync.core.person import SyncPerson
from sync.image import sync_image
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
from utils.io import read_yaml_md
url_base = settings.teacher_image_url_base
dir_local_images = settings.teacher_image_output
sizes = settings.teacher_image_sizes
def sync_persons(provider, collection, update=True, delete=True):
"""
:param collection:
:param provider:
:param update:
:param delete:
:return:
"""
return main(
SyncPerson(
collection,
provider,
dir_local_images
),
update_documents=update,
delete_documents=delete
)
def main(sync, update_documents=False, delete_documents=False):
# GET TEACHER YAML/MD MANIFEST FILES
documents = sync.provider.type_filter('', '.md')
# READ TEACHER YAML/MD MANIFEST
documents = lmap(
lambda i: (i,) + get_manifest(sync.provider, i),
documents
)
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
documents = lmap(
lambda i: {
**i[1],
'file': i[0],
'biography': i[2]
},
documents
)
# DEFINE PROFILE PICTURE
documents = lmap(
lambda i: {
**i,
'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
},
documents
)
# CREATE DOCUMENT IDENTITY
documents = lmapfn(documents)(sync.create_id)
# CREATE HASH OF DOCUMENT FILES
documents = lmapfn(documents)(sync.create_hash)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmap(
lambda i: i['id'],
documents
)
# SKIP UNTOUCHED DOCUMENTS
documents = untouched(documents, sync)
# documents = lmap(
# lambda i: i[0],
# filter(
# lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
# lmap(
# lambda document: (document, sync.open(document)),
# documents
# )
# ))
# DO HEAVY PROCESS WITH DOCUMENTS
documents = lmap(
sync.compile,
documents
)
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
documents = lmap(
key_mapper('picture', sync_picture),
documents
)
# SYNC TEACHER_PROFILE
if update_documents:
documents = lmap(
sync.update,
documents
)
documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
if delete_documents:
documents_to_delete = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_delete
)
)
return documents, documents_to_delete
def get_manifest(provider, path):
data = provider.read(path).read().decode('utf-8')
return read_yaml_md(data)
utils/Sync/sync/teacher.py
-145
View File
@@ -1,145 +0,0 @@
import os
from glob import glob
import settings
from db import db
from sync import Sync
from sync.image import sync_image
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
from utils.hash import hash_str, hash_file
from utils.image import create_image
from utils.io import read_yaml_md
from utils.text.transform import url_encode_text
dir_teachers = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
url_base = settings.teacher_image_url_base
dir_local_images = settings.teacher_image_output
sizes = settings.teacher_image_sizes
class Teacher(Sync):
def __init__(self):
super().__init__()
self.collection = db().collective
def compile(self, profile):
profile['picture'] = create_image(
profile['picture'],
sizes,
lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
dir_local_images
)
return profile
def create_id(self, document):
if 'id' in document and document['id']:
return document
document['id'] = url_encode_text(document['name'])
return document
def create_hash(self, document):
document['hash'] = hash_str(
hash_file(document['file']) + hash_file(document['picture'])
)
return document
def main(dir_teachers, sync):
os.chdir(dir_teachers)
# GET TEACHER YAML/MD MANIFEST FILES
documents = glob('*.md')
# READ TEACHER YAML/MD MANIFEST
documents = lmap(
lambda i: (i,) + read_yaml_md(i),
documents
)
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
documents = lmap(
lambda i: {
**i[1],
'file': i[0],
'biography': i[2]
},
documents
)
# DEFINE PROFILE PICTURE
documents = lmap(
lambda i: {
**i,
'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
},
documents
)
# CREATE DOCUMENT IDENTITY
documents = lmapfn(documents)(sync.create_id)
# CREATE HASH OF DOCUMENT FILES
documents = lmapfn(documents)(sync.create_hash)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmap(
lambda i: i['id'],
documents
)
# SKIP UNTOUCHED DOCUMENTS
documents = lmap(
lambda i: i[0],
filter(
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, sync.open(document)),
documents
)
))
# DO HEAVY PROCESS WITH DOCUMENTS
documents = lmap(
sync.compile,
documents
)
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
documents = lmap(
key_mapper('picture', sync_picture),
documents
)
# SYNC TEACHER_PROFILE
documents = lmap(
sync.update,
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
)
)
# SCOPE
print('SCOPE:')
lprint(scope_documents_ids)
# DELETE
print('DELETE DOCUMENTS:')
lprint_json(documents_to_remove)
# DELETE
print('UPDATE DOCUMENTS:')
lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
main(dir_teachers, Teacher())