FUCKING BIG SYNC UPDATE: caching, events, albums, documents, awards, news, schedules, teachers, other
This commit is contained in:
1 parent
359cc430f5
commit
f07c39c317
20 files changed
+1190
-166
No files matched your search
@@ -0,0 +1,193 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext
|
||||
from utils.hash import hash_str, hash_file
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
dir_teachers = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
|
||||
url_base = settings.teacher_image_url_base
|
||||
dir_local_images = settings.teacher_image_output
|
||||
sizes = settings.teacher_image_sizes
|
||||
|
||||
class Teacher:
|
||||
def collection(self):
|
||||
return db().collective
|
||||
|
||||
def read_teacher_by_id(self, record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
self.read_teacher_by_id,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record}
|
||||
try:
|
||||
return db().collective.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def sync_collective(self, record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
self.sync_collective,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().collective
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def compile(self, profile):
|
||||
profile['picture'] = create_image(
|
||||
profile['picture'],
|
||||
sizes,
|
||||
lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
|
||||
dir_local_images
|
||||
)
|
||||
return profile
|
||||
|
||||
def read_document(self, i, query_fn=None):
|
||||
q = query_fn(i) if query_fn else {'id': i['id']}
|
||||
try:
|
||||
return self.collection().find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def query_documents(self, q):
|
||||
return self.collection().find(q)
|
||||
|
||||
def delete_document(self, q):
|
||||
return self.collection().find_one_and_delete(q)
|
||||
|
||||
def create_id(self, document):
|
||||
if 'id' in document and document['id']:
|
||||
return document['id']
|
||||
|
||||
document['id'] = url_encode_text(document['name'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# ALL DOCUMENTS IDS FOUNDED
|
||||
scope_documents_ids = []
|
||||
|
||||
os.chdir(dir_teachers)
|
||||
sync = Teacher()
|
||||
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
documents = glob('*.md')
|
||||
|
||||
# # READ TEACHER YAML/MD MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: (i,) + read_yaml_md(i),
|
||||
documents
|
||||
)
|
||||
|
||||
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i[1],
|
||||
'file': i[0],
|
||||
'biography': i[2]
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# DEFINE PROFILE PICTURE
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'id': sync.create_id(i)},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILES
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i, 'hash': hash_str(
|
||||
hash_file(i['file']) + hash_file(i['picture'])
|
||||
)
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = lmap(
|
||||
lambda i: i[0],
|
||||
filter(
|
||||
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
|
||||
lmap(
|
||||
lambda document: (document, sync.read_document(document)),
|
||||
documents
|
||||
)
|
||||
))
|
||||
|
||||
# DO HEAVY PROCESS WITH DOCUMENTS
|
||||
documents = lmap(
|
||||
sync.compile,
|
||||
documents
|
||||
)
|
||||
|
||||
# # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
documents
|
||||
)
|
||||
|
||||
# # SYNC TEACHER_PROFILE
|
||||
documents = lmap(
|
||||
sync.sync_collective,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
sync.delete_document,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
# SCOPE
|
||||
print('SCOPE:')
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
# DELETE
|
||||
print('DELETE DOCUMENTS:')
|
||||
lprint_json(documents_to_remove)
|
||||
|
||||
# DELETE
|
||||
print('UPDATE DOCUMENTS:')
|
||||
lprint_json(documents)
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
Reference in new issue
Block a user