FUCKING BIG SYNC UPDATE: caching, events, albums, documents, awards, news, schedules, teachers, other
This commit is contained in:
tmshv committed
1 parent
359cc430f5
commit
f07c39c317
20 files changed
+1190
-166
No files matched your search
+132
-47
@@ -6,12 +6,20 @@ from tempfile import mkstemp
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint, key_mapper
|
||||
from utils.fn import combine, lmap, lprint_json, key_mapper, lprint
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.io import read_yaml
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
prefix = 'document'
|
||||
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
|
||||
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
|
||||
dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
|
||||
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
|
||||
url_base_document = 'https://static.shburg.org/art/uploads/{file}'
|
||||
|
||||
# Adds specified category to each file in files list
|
||||
unwrap_file_list = lambda files, title: map(
|
||||
lambda profile: {'category': title, **profile},
|
||||
@@ -19,25 +27,6 @@ unwrap_file_list = lambda files, title: map(
|
||||
)
|
||||
|
||||
|
||||
def sync_document(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_document,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'url': record['url']}
|
||||
try:
|
||||
documents = db().documents
|
||||
update_result = documents.update_one(q, {'$set': record}, upsert=True)
|
||||
i = documents.find_one({'url': record['url']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def pdf_to_jpg(pdf):
|
||||
cwd = os.getcwd()
|
||||
abspdf = os.path.join(cwd, pdf)
|
||||
@@ -46,12 +35,48 @@ def pdf_to_jpg(pdf):
|
||||
return temp
|
||||
|
||||
|
||||
def sync_document(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_document,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
documents = db().documents
|
||||
update_result = documents.update_one(q, {'$set': record}, upsert=True)
|
||||
i = documents.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def read_document(i, query_fn=None):
|
||||
q = query_fn(i) if query_fn else {'id': i['id']}
|
||||
try:
|
||||
return db().documents.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def query_documents(q):
|
||||
return db().documents.find(q)
|
||||
|
||||
|
||||
def delete_document(q):
|
||||
return db().documents.find_one_and_delete(q)
|
||||
|
||||
|
||||
def create_preview(pdf, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(pdf)
|
||||
|
||||
id = url_encode_text(pdf)
|
||||
url_base = 'https://static.shburg.org/art/images/document-{id}-{size}{ext}'
|
||||
url = lambda size, ext: url_base.format(id=id, size=size, ext=ext)
|
||||
url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
@@ -60,12 +85,11 @@ def create_preview(pdf, sizes, preview_dir):
|
||||
|
||||
def create_document(doc):
|
||||
sizes = settings.image_sizes
|
||||
preview_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
file = doc['file']
|
||||
|
||||
doc['type'] = 'document'
|
||||
doc['url'] = 'https://static.shburg.org/art/uploads/' + url_encode_file(file)
|
||||
doc['preview'] = create_preview(file, sizes=sizes, preview_dir=preview_dir)
|
||||
doc['type'] = prefix
|
||||
doc['url'] = url_base_document.format(file=url_encode_file(file))
|
||||
doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
|
||||
doc['file'] = {
|
||||
'name': file,
|
||||
'size': os.path.getsize((file))
|
||||
@@ -74,6 +98,14 @@ def create_document(doc):
|
||||
return doc
|
||||
|
||||
|
||||
def create_id(doc):
|
||||
return url_encode_text('{type}-{category}-{file}'.format(
|
||||
type=prefix,
|
||||
category=doc['category'],
|
||||
file=doc['file']
|
||||
))
|
||||
|
||||
|
||||
def unwrap_manifest(param):
|
||||
if isinstance(param, list):
|
||||
return combine(map(
|
||||
@@ -94,21 +126,50 @@ def unwrap_manifest(param):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
|
||||
|
||||
os.chdir(dir)
|
||||
# ALL DOCUMENTS IDS FOUNDED
|
||||
scope_documents_ids = []
|
||||
|
||||
# GET DOCUMENT YAML_MANIFEST FILES
|
||||
files = glob(dir + '/*.yaml')
|
||||
os.chdir(dir_documents)
|
||||
documents = glob(dir_documents + '/*.yaml')
|
||||
|
||||
# READ YAML_MANIFEST FILES
|
||||
files_data = lmap(
|
||||
documents = lmap(
|
||||
read_yaml,
|
||||
files
|
||||
documents
|
||||
)
|
||||
|
||||
# GET FLAT LIST OF DOCUMENTS
|
||||
documents = unwrap_manifest(files_data)
|
||||
documents = unwrap_manifest(documents)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'id': create_id(i)},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = lmap(
|
||||
lambda i: i[0],
|
||||
filter(
|
||||
lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
|
||||
lmap(
|
||||
lambda document: (document, read_document(document)),
|
||||
documents
|
||||
)
|
||||
))
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
@@ -117,15 +178,13 @@ if __name__ == '__main__':
|
||||
)
|
||||
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_preview = lambda i: sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('preview', sync_preview),
|
||||
key_mapper('preview', lambda i: sync_image(i)['_id']),
|
||||
documents
|
||||
)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
uploads = '/Users/tmshv/Dropbox/Dev/Hud school/Static/uploads'
|
||||
out_path = lambda doc: os.path.join(uploads, os.path.basename(doc['url']))
|
||||
out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
|
||||
lmap(
|
||||
lambda doc: copyfile(
|
||||
doc['file']['name'],
|
||||
@@ -140,18 +199,44 @@ if __name__ == '__main__':
|
||||
documents
|
||||
)
|
||||
|
||||
lprint(documents)
|
||||
print('SYNC DOCUMENTS DONE')
|
||||
documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
delete_document,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
# SCOPE
|
||||
print('SCOPE:')
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
# DELETE
|
||||
print('DELETE DOCUMENTS:')
|
||||
lprint_json(documents_to_remove)
|
||||
|
||||
# DELETE
|
||||
print('UPDATE DOCUMENTS:')
|
||||
lprint_json(documents)
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
|
||||
# DOCUMENT_OBJECT SAMPLE
|
||||
# {
|
||||
# "_id": ObjectId("56d08411ace9573958e5e490"),
|
||||
# "url": "https://static.shburg.org/art/uploads/egryul.pdf",
|
||||
# "title": "Свидетельство о внесении в «ЕГРЮЛ»",
|
||||
# "preview": ObjectId("56d07d87ace9573958e5e488"),
|
||||
# "file": {
|
||||
# "name": ""ЕГРЮЛ.pdf",
|
||||
# "size": 100500
|
||||
# "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
|
||||
# "_id": {
|
||||
# "$oid": "56d08411ace9573958e5e497"
|
||||
# },
|
||||
# "category": "Основные документы"
|
||||
# "type": "document",
|
||||
# "id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
|
||||
# "category": "Для поступающих",
|
||||
# "file": {
|
||||
# "size": 53088,
|
||||
# "name": "Договор о пожертвовании.pdf"
|
||||
# },
|
||||
# "url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
|
||||
# "preview": {
|
||||
# "$oid": "56d08007ace9573958e5e48f"
|
||||
# },
|
||||
# "title": "Договор о пожертвовании"
|
||||
# }
|
||||
Reference in new issue
Block a user