sync update: caching, refactoring, other

This commit is contained in:
tmshv committed 2016-03-06 02:24:24 +03:00
1 parent e68cfd7b2c
commit fe409c04a4
23 files changed
+758 -1588

No files matched your search

+28 -76
View File
@@ -3,8 +3,9 @@ from glob import glob
import settings
from db import db
from sync import Sync
from sync.image import sync_image
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
from utils.hash import hash_str, hash_file
from utils.image import create_image
from utils.io import read_yaml_md
@@ -15,42 +16,11 @@ url_base = settings.teacher_image_url_base
dir_local_images = settings.teacher_image_output
sizes = settings.teacher_image_sizes
class Teacher:
def collection(self):
return db().collective
def read_teacher_by_id(self, record):
if isinstance(record, list):
return lmap(
self.read_teacher_by_id,
record
)
q = {'id': record}
try:
return db().collective.find_one(q)
except ValueError:
pass
return None
def sync_collective(self, record):
if isinstance(record, list):
return lmap(
self.sync_collective,
record
)
q = {'id': record['id']}
try:
collection = db().collective
update_result = collection.update_one(q, {'$set': record}, upsert=True)
i = collection.find_one({'id': record['id']})
return i
except ValueError:
pass
return None
class Teacher(Sync):
def __init__(self):
super().__init__()
self.collection = db().collective
def compile(self, profile):
profile['picture'] = create_image(
@@ -61,39 +31,27 @@ class Teacher:
)
return profile
def read_document(self, i, query_fn=None):
q = query_fn(i) if query_fn else {'id': i['id']}
try:
return self.collection().find_one(q)
except ValueError:
pass
return None
def query_documents(self, q):
return self.collection().find(q)
def delete_document(self, q):
return self.collection().find_one_and_delete(q)
def create_id(self, document):
if 'id' in document and document['id']:
return document['id']
return document
document['id'] = url_encode_text(document['name'])
return document
def create_hash(self, document):
document['hash'] = hash_str(
hash_file(document['file']) + hash_file(document['picture'])
)
return document
if __name__ == '__main__':
# ALL DOCUMENTS IDS FOUNDED
scope_documents_ids = []
def main(dir_teachers, sync):
os.chdir(dir_teachers)
sync = Teacher()
# GET TEACHER YAML/MD MANIFEST FILES
documents = glob('*.md')
# # READ TEACHER YAML/MD MANIFEST
# READ TEACHER YAML/MD MANIFEST
documents = lmap(
lambda i: (i,) + read_yaml_md(i),
documents
@@ -119,20 +77,10 @@ if __name__ == '__main__':
)
# CREATE DOCUMENT IDENTITY
documents = lmap(
lambda i: {**i, 'id': sync.create_id(i)},
documents
)
documents = lmapfn(documents)(sync.create_id)
# CREATE HASH OF DOCUMENT FILES
documents = lmap(
lambda i: {
**i, 'hash': hash_str(
hash_file(i['file']) + hash_file(i['picture'])
)
},
documents
)
documents = lmapfn(documents)(sync.create_hash)
# CREATE SCOPE OF CURRENT SESSION
scope_documents_ids = lmap(
@@ -146,7 +94,7 @@ if __name__ == '__main__':
filter(
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, sync.read_document(document)),
lambda document: (document, sync.read(document)),
documents
)
))
@@ -157,22 +105,22 @@ if __name__ == '__main__':
documents
)
# # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
documents = lmap(
key_mapper('picture', sync_picture),
documents
)
# # SYNC TEACHER_PROFILE
# SYNC TEACHER_PROFILE
documents = lmap(
sync.sync_collective,
sync.update,
documents
)
documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}})
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete_document,
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
@@ -191,3 +139,7 @@ if __name__ == '__main__':
print('UPDATE DOCUMENTS:')
lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
main(dir_teachers, Teacher())