sync update: caching, refactoring, other
This commit is contained in:
tmshv committed
1 parent
e68cfd7b2c
commit
fe409c04a4
23 files changed
+758
-1588
No files matched your search
@@ -1,10 +1,21 @@
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
|
||||
from db import db
|
||||
from utils.fn import lmap, key_mapper
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, combine
|
||||
|
||||
|
||||
def list_images(dir):
|
||||
image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
|
||||
return combine(map(
|
||||
lambda type: glob(os.path.join(dir, type)),
|
||||
image_types
|
||||
))
|
||||
|
||||
|
||||
def create_date(date_str, date_formats=None):
|
||||
@@ -50,13 +61,25 @@ def untouched(documents, store):
|
||||
))
|
||||
|
||||
|
||||
def synced_images_ids(images):
|
||||
ids = lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(images)
|
||||
)
|
||||
return ids
|
||||
|
||||
|
||||
def synced_image_id(image):
|
||||
return sync_image(image)['_id']
|
||||
|
||||
|
||||
def create_date_and_title_from_folder_name(folder_name, date_formats=None):
|
||||
m = folder_name_pattern.findall(folder_name)
|
||||
if not m:
|
||||
return None, None
|
||||
|
||||
date_str, title = m[0]
|
||||
title = title.strip()
|
||||
title = os.path.splitext(title.strip())[0]
|
||||
date = create_date(date_str, date_formats)
|
||||
|
||||
if not date:
|
||||
@@ -65,6 +88,16 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
|
||||
return date, title
|
||||
|
||||
|
||||
def images_from_html(md):
|
||||
post_html = lxml.html.fromstring(md)
|
||||
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
images.append(src)
|
||||
return images
|
||||
|
||||
|
||||
create_post_from_image_list = lambda images: markdown(
|
||||
'\n'.join(map(
|
||||
lambda i: ''.format(img=i),
|
||||
|
||||
+118
-69
@@ -1,16 +1,17 @@
|
||||
import os
|
||||
from glob import glob
|
||||
from lxml import etree
|
||||
|
||||
import lxml.html
|
||||
from lxml import etree
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from models.product import create_product
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns, first, map_cases
|
||||
from utils.hash import file_hash
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, Sync, list_images, \
|
||||
untouched, synced_images_ids
|
||||
from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, first, map_cases, combine, lmapfn, \
|
||||
key_mapper, lprint
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
@@ -22,7 +23,7 @@ class ImageCreator:
|
||||
self.cache = cache if cache else set()
|
||||
|
||||
def create_image(self, file, sizes, url_fn, output_dir, skip_processing=False):
|
||||
hash = file_hash(file)
|
||||
hash = hash_file(file)
|
||||
cached = self.cached(hash)
|
||||
return create_image(file, sizes, url_fn, output_dir, skip_processing=skip_processing) if not cached else cached
|
||||
|
||||
@@ -30,25 +31,6 @@ class ImageCreator:
|
||||
return self.cache[hash] if hash in self.cache else None
|
||||
|
||||
|
||||
def sync_album(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_album,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().events
|
||||
collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_post(md, image_path_fn):
|
||||
html = lxml.html.fromstring(md)
|
||||
|
||||
@@ -64,7 +46,7 @@ def create_post(md, image_path_fn):
|
||||
|
||||
|
||||
def create_url(a, i, s, e):
|
||||
return 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
|
||||
return 'https://static.shburg.org/art/images/product-{album}-{id}-{size}{ext}'.format(
|
||||
album=a,
|
||||
id=i,
|
||||
size=s,
|
||||
@@ -76,18 +58,41 @@ def url_creator(aid, pid):
|
||||
return lambda size, ext: create_url(aid, pid, size, ext.lower())
|
||||
|
||||
|
||||
class Album:
|
||||
class SyncAlbum(Sync):
|
||||
@staticmethod
|
||||
def id_from_file(i):
|
||||
return url_encode_text(
|
||||
os.path.splitext(i)[0]
|
||||
)
|
||||
|
||||
def __init__(self, image_creator):
|
||||
def __init__(self, image_creator, images_dir):
|
||||
super().__init__()
|
||||
self.collection = db().albums
|
||||
self.image_creator = image_creator
|
||||
self.skip_image_processing = False
|
||||
self.images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
self.images_dir = images_dir
|
||||
|
||||
def create_id(self, document):
|
||||
if 'id' in document:
|
||||
return document
|
||||
|
||||
y = str(document['date'].year)
|
||||
document['id'] = url_encode_text('-'.join([y, document['title']]))
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
hash_images = lmap(
|
||||
hash_file,
|
||||
lmap(
|
||||
lambda i: os.path.join(document['folder'], i),
|
||||
document['images']
|
||||
)
|
||||
)
|
||||
document['hash'] = hash_str(
|
||||
combine([hash_str(document)] + hash_images)
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def create_image(self, file, url_fn):
|
||||
ss = settings.album_image_sizes
|
||||
@@ -99,7 +104,7 @@ class Album:
|
||||
def create_album(self, doc):
|
||||
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
|
||||
|
||||
image_id = Album.id_from_file
|
||||
image_id = SyncAlbum.id_from_file
|
||||
image_path = lambda i: os.path.join(doc['folder'], i)
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
@@ -138,7 +143,7 @@ class Album:
|
||||
return doc
|
||||
|
||||
def create_products(self, doc, image_file_fn):
|
||||
image_id = Album.id_from_file
|
||||
image_id = SyncAlbum.id_from_file
|
||||
iter_album = iterate_iter_over_fns([
|
||||
lambda product: {
|
||||
**product,
|
||||
@@ -188,16 +193,11 @@ def get_manifest(filepath):
|
||||
return {**data, **post}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
image_creator = ImageCreator(cache=None)
|
||||
alb = Album(image_creator=image_creator)
|
||||
# alb.skip_image_processing = True
|
||||
|
||||
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
|
||||
os.chdir(albums_dir)
|
||||
def main(dir_documents, sync):
|
||||
os.chdir(dir_documents)
|
||||
|
||||
# GET ALBUMS FOLDERS
|
||||
albums = lmap(
|
||||
documents = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
@@ -205,22 +205,20 @@ if __name__ == '__main__':
|
||||
)
|
||||
)
|
||||
|
||||
# print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md'))
|
||||
# print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md'))
|
||||
# exit()
|
||||
# albums = [albums[5]]
|
||||
# documents = [documents[3]]
|
||||
# documents = documents[:15]
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
albums = lmap(
|
||||
documents = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i)
|
||||
),
|
||||
albums
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
|
||||
albums = lmap(
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
@@ -230,12 +228,12 @@ if __name__ == '__main__':
|
||||
glob_in(i[0])
|
||||
))
|
||||
},
|
||||
albums
|
||||
documents
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
md_glob = lambda i: first(glob(i + '/*.md'))
|
||||
albums = lmap(
|
||||
documents = lmap(
|
||||
lambda album: {
|
||||
**album,
|
||||
**map_cases(
|
||||
@@ -247,37 +245,88 @@ if __name__ == '__main__':
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
albums
|
||||
documents
|
||||
)
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
'images': lmap(
|
||||
lambda path: os.path.relpath(path, i['folder']),
|
||||
list_images(i['folder'])
|
||||
)
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmapfn(documents)(
|
||||
lambda i: i['id']
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
albums = lmap(
|
||||
documents = lmap(
|
||||
alb.create_album,
|
||||
albums
|
||||
documents
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
albums = list(filter(None, albums))
|
||||
|
||||
documents = list(filter(None, documents))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
def sync_images(i):
|
||||
return lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(i)
|
||||
documents = lmap(
|
||||
key_mapper('images', synced_images_ids),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
|
||||
|
||||
albums = lmap(
|
||||
key_mapper('images', sync_images),
|
||||
albums
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
albums = lmap(
|
||||
sync_album,
|
||||
albums
|
||||
)
|
||||
# SCOPE
|
||||
print('SCOPE:')
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
lprint_json(albums)
|
||||
print('SYNC ALBUMS DONE')
|
||||
# DELETE
|
||||
print('DELETE DOCUMENTS:')
|
||||
lprint_json(lmap(
|
||||
lambda i: i['id'],
|
||||
documents_to_remove
|
||||
))
|
||||
|
||||
# DELETE
|
||||
print('UPDATE DOCUMENTS:')
|
||||
lprint_json(lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
))
|
||||
# lprint_json(documents)
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
image_creator = ImageCreator(cache=None)
|
||||
alb = SyncAlbum(image_creator=image_creator, images_dir=settings.image_output)
|
||||
alb.skip_image_processing = True
|
||||
|
||||
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
|
||||
main(dir_documents=albums_dir, sync=alb)
|
||||
@@ -1,196 +0,0 @@
|
||||
import os
|
||||
from glob import glob
|
||||
from shutil import copyfile
|
||||
from tempfile import mkstemp
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint_json, key_mapper, lprint
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
prefix = 'award'
|
||||
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Awards'
|
||||
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
|
||||
dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
|
||||
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
|
||||
url_base_document = 'https://static.shburg.org/art/uploads/{file}'
|
||||
|
||||
|
||||
def collection():
|
||||
return db().awards
|
||||
|
||||
|
||||
def pdf_to_jpg(pdf):
|
||||
cwd = os.getcwd()
|
||||
abspdf = os.path.join(cwd, pdf)
|
||||
_, temp = mkstemp('.jpg')
|
||||
image_magick_pdf_to_img(abspdf, temp)
|
||||
return temp
|
||||
|
||||
|
||||
def sync_document(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_document,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection().update_one(q, {'$set': record}, upsert=True)
|
||||
return collection().find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def read_document(i, query_fn=None):
|
||||
q = query_fn(i) if query_fn else {'id': i['id']}
|
||||
try:
|
||||
return collection().find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def query_documents(q):
|
||||
return collection().find(q)
|
||||
|
||||
|
||||
def delete_document(q):
|
||||
return collection().find_one_and_delete(q)
|
||||
|
||||
|
||||
def create_preview(pdf, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(pdf)
|
||||
|
||||
id = url_encode_text(pdf)
|
||||
url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
|
||||
|
||||
def create_document(doc):
|
||||
sizes = settings.awards_image_sizes
|
||||
file = doc['file']
|
||||
|
||||
doc['type'] = prefix
|
||||
doc['url'] = url_base_document.format(file=url_encode_file(file))
|
||||
doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
|
||||
doc['file'] = {
|
||||
'name': file,
|
||||
'size': os.path.getsize((file))
|
||||
}
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def create_id(doc):
|
||||
return url_encode_text('{type}-{file}'.format(
|
||||
type=prefix,
|
||||
file=doc['file']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# ALL DOCUMENTS IDS FOUNDED
|
||||
scope_documents_ids = []
|
||||
|
||||
# GET DOCUMENT YAML_MANIFEST FILES
|
||||
os.chdir(dir_documents)
|
||||
documents = glob('*.pdf')
|
||||
|
||||
# READ YAML_MANIFEST FILES
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
'file': i
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'id': create_id(i)},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = lmap(
|
||||
lambda i: i[0],
|
||||
filter(
|
||||
lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
|
||||
lmap(
|
||||
lambda document: (document, read_document(document)),
|
||||
documents
|
||||
)
|
||||
))
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
create_document,
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('preview', lambda i: sync_image(i)['_id']),
|
||||
documents
|
||||
)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
|
||||
lmap(
|
||||
lambda doc: copyfile(
|
||||
doc['file']['name'],
|
||||
out_path(doc)
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync_document,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
delete_document,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
# SCOPE
|
||||
print('SCOPE:')
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
# DELETE
|
||||
print('DELETE DOCUMENTS:')
|
||||
lprint_json(documents_to_remove)
|
||||
|
||||
# DELETE
|
||||
print('UPDATE DOCUMENTS:')
|
||||
lprint_json(documents)
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
@@ -1,91 +0,0 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
|
||||
|
||||
def read_teacher_by_id(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
read_teacher_by_id,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record}
|
||||
try:
|
||||
return db().collective.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def sync_collective(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_collective,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().collective
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_profile(profile):
|
||||
sizes = settings.image_sizes
|
||||
images_dir = settings.collective_image_output
|
||||
|
||||
url_base = 'https://static.shburg.org/art/image/teacher-{id}-{size}{ext}'
|
||||
url_fn = lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext)
|
||||
|
||||
profile['picture'] = create_image(profile['picture'], sizes, url_fn, images_dir)
|
||||
return profile
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
|
||||
os.chdir(dir)
|
||||
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
files = glob('*.md')
|
||||
|
||||
# READ TEACHER YAML/MD MANIFEST
|
||||
collective = lmap(
|
||||
read_yaml_md,
|
||||
files
|
||||
)
|
||||
|
||||
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
|
||||
collective = lmap(
|
||||
lambda i: {'biography': i[1], **i[0]},
|
||||
collective
|
||||
)
|
||||
|
||||
# MAP TEACHER_MANIFEST -> TEACHER_PROFILE
|
||||
collective = lmap(
|
||||
create_profile,
|
||||
collective
|
||||
)
|
||||
|
||||
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
collective = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
collective
|
||||
)
|
||||
|
||||
lprint(collective)
|
||||
print('SYNC COLLECTIVE DONE')
|
||||
+154
-128
@@ -5,26 +5,67 @@ from tempfile import mkstemp
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint_json, key_mapper, lprint
|
||||
from sync import Sync, untouched, synced_image_id
|
||||
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.io import read_yaml
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
prefix = 'document'
|
||||
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
|
||||
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
|
||||
dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
|
||||
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
|
||||
url_base_document = 'https://static.shburg.org/art/uploads/{file}'
|
||||
|
||||
# Adds specified category to each file in files list
|
||||
unwrap_file_list = lambda files, title: map(
|
||||
lambda profile: {'category': title, **profile},
|
||||
files
|
||||
)
|
||||
class SyncDocument(Sync):
|
||||
def __init__(self, collection_name, document_type, sizes):
|
||||
super().__init__()
|
||||
self.collection = db()[collection_name]
|
||||
self.document_type = document_type
|
||||
self.sizes = sizes
|
||||
self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
|
||||
self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
|
||||
self.url_base_document = 'https://static.shburg.org/art/uploads/{file}'
|
||||
|
||||
def create(self, document):
|
||||
file = document['file']
|
||||
filename = os.path.basename(document['file'])
|
||||
|
||||
document['type'] = self.document_type
|
||||
document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
|
||||
document['file'] = {
|
||||
'name': filename,
|
||||
'size': os.path.getsize((file))
|
||||
}
|
||||
|
||||
return document
|
||||
|
||||
def create_id(self, document):
|
||||
bn = os.path.basename(document['file'])
|
||||
document['id'] = url_encode_text('{type}-{category}-{file}'.format(
|
||||
type=self.document_type,
|
||||
category=document['category'],
|
||||
file=bn
|
||||
))
|
||||
return document
|
||||
|
||||
def create_url(self, document):
|
||||
filename = os.path.basename(document['file'])
|
||||
document['url'] = self.url_base_document.format(file=url_encode_file(filename))
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
document['hash'] = hash_str(
|
||||
hash_str(document) + hash_file(document['file'])
|
||||
)
|
||||
return document
|
||||
|
||||
def create_preview(self, pdf, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(pdf)
|
||||
|
||||
id = url_encode_text(pdf)
|
||||
url = lambda size, ext: self.url_base_preview.format(id=id, size=size, ext=ext)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
|
||||
|
||||
def pdf_to_jpg(pdf):
|
||||
@@ -35,75 +76,20 @@ def pdf_to_jpg(pdf):
|
||||
return temp
|
||||
|
||||
|
||||
def sync_document(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_document,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
documents = db().documents
|
||||
update_result = documents.update_one(q, {'$set': record}, upsert=True)
|
||||
i = documents.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def read_document(i, query_fn=None):
|
||||
q = query_fn(i) if query_fn else {'id': i['id']}
|
||||
try:
|
||||
return db().documents.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def query_documents(q):
|
||||
return db().documents.find(q)
|
||||
|
||||
|
||||
def delete_document(q):
|
||||
return db().documents.find_one_and_delete(q)
|
||||
|
||||
|
||||
def create_preview(pdf, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(pdf)
|
||||
|
||||
id = url_encode_text(pdf)
|
||||
url = lambda size, ext: url_base_preview.format(id=id, size=size, ext=ext)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
|
||||
|
||||
def create_document(doc):
|
||||
sizes = settings.image_sizes
|
||||
file = doc['file']
|
||||
|
||||
doc['type'] = prefix
|
||||
doc['url'] = url_base_document.format(file=url_encode_file(file))
|
||||
doc['preview'] = create_preview(file, sizes=sizes, preview_dir=dir_static_previews)
|
||||
doc['file'] = {
|
||||
'name': file,
|
||||
'size': os.path.getsize((file))
|
||||
}
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def create_id(doc):
|
||||
return url_encode_text('{type}-{category}-{file}'.format(
|
||||
type=prefix,
|
||||
category=doc['category'],
|
||||
file=doc['file']
|
||||
))
|
||||
def categorize_files_list(files, title):
|
||||
"""
|
||||
Adds specified category to each file in files list
|
||||
:param files:
|
||||
:param title:
|
||||
:return:
|
||||
"""
|
||||
return map(
|
||||
lambda profile: {
|
||||
**profile,
|
||||
'category': title
|
||||
},
|
||||
files
|
||||
)
|
||||
|
||||
|
||||
def unwrap_manifest(param):
|
||||
@@ -116,7 +102,7 @@ def unwrap_manifest(param):
|
||||
if 'files' in param:
|
||||
return lmap(
|
||||
unwrap_manifest,
|
||||
unwrap_file_list(param['files'], param['title'])
|
||||
categorize_files_list(param['files'], param['title'])
|
||||
)
|
||||
|
||||
if 'file' not in param:
|
||||
@@ -125,12 +111,9 @@ def unwrap_manifest(param):
|
||||
return param
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# ALL DOCUMENTS IDS FOUNDED
|
||||
scope_documents_ids = []
|
||||
|
||||
def documents_from_yaml():
|
||||
# GET DOCUMENT YAML_MANIFEST FILES
|
||||
os.chdir(dir_documents)
|
||||
|
||||
documents = glob(dir_documents + '/*.yaml')
|
||||
|
||||
# READ YAML_MANIFEST FILES
|
||||
@@ -142,85 +125,128 @@ if __name__ == '__main__':
|
||||
# GET FLAT LIST OF DOCUMENTS
|
||||
documents = unwrap_manifest(documents)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def documents_from_subdirs():
|
||||
documents = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
documents = lmap(
|
||||
lambda folder: lmap(
|
||||
lambda path: {
|
||||
'file': path,
|
||||
'title': os.path.splitext(os.path.basename(os.path.relpath(path, folder)))[0],
|
||||
'category': os.path.basename(folder)
|
||||
},
|
||||
glob(folder + '/*.pdf')
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
documents = combine(documents)
|
||||
return documents
|
||||
|
||||
|
||||
def main(dir_documents, dir_static_uploads, sync):
|
||||
os.chdir(dir_documents)
|
||||
|
||||
# documents = documents_from_yaml()
|
||||
documents = documents_from_subdirs()
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'id': create_id(i)},
|
||||
documents
|
||||
)
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'hash': hash_str(i) + hash_file(i['file'])},
|
||||
documents
|
||||
)
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
# CREATE SCOPE
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
file_names = set(lmap(
|
||||
lambda i: i['file'],
|
||||
documents
|
||||
))
|
||||
if len(file_names) != len(documents):
|
||||
raise Exception('File names should be unique')
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = lmap(
|
||||
lambda i: i[0],
|
||||
filter(
|
||||
lambda i: i[1] is None or (i[0]['hash'] != i[1]['hash']),
|
||||
lmap(
|
||||
lambda document: (document, read_document(document)),
|
||||
documents
|
||||
)
|
||||
))
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
create_document,
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('preview', lambda i: sync_image(i)['_id']),
|
||||
documents
|
||||
)
|
||||
# URL
|
||||
documents = lmap(sync.create_url, documents)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
out_path = lambda doc: os.path.join(dir_static_uploads, os.path.basename(doc['url']))
|
||||
lmap(
|
||||
lambda doc: copyfile(
|
||||
doc['file']['name'],
|
||||
doc['file'],
|
||||
out_path(doc)
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
sync_document,
|
||||
sync.create,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = query_documents({'id': {'$nin': scope_documents_ids}})
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('preview', lambda i: synced_image_id(i)),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
delete_document,
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
# SCOPE
|
||||
print('SCOPE:')
|
||||
print('SCOPE:', len(scope_documents_ids))
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
# DELETE
|
||||
print('DELETE DOCUMENTS:')
|
||||
print('DELETE DOCUMENTS:', len(documents_to_remove))
|
||||
lprint_json(documents_to_remove)
|
||||
|
||||
# DELETE
|
||||
print('UPDATE DOCUMENTS:')
|
||||
print('UPDATE DOCUMENTS:', len(documents))
|
||||
lprint_json(documents)
|
||||
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
|
||||
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
|
||||
|
||||
main(
|
||||
dir_documents,
|
||||
dir_static_uploads,
|
||||
SyncDocument(
|
||||
'documents',
|
||||
'document',
|
||||
sizes=settings.image_sizes
|
||||
)
|
||||
)
|
||||
|
||||
# DOCUMENT_OBJECT SAMPLE
|
||||
# {
|
||||
# "hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
import os
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint, first, map_cases, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
def sync_event(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_event,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().events
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_event(doc):
|
||||
if 'date' not in doc or not doc['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
return None
|
||||
|
||||
doc['id'] = doc['id'] if 'id' in doc else url_encode_text(doc['title'])
|
||||
|
||||
url_base = 'https://static.shburg.org/art/images/event-{id}-{img}-{size}{ext}'
|
||||
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
sizes = settings.event_image_sizes
|
||||
|
||||
post_html = lxml.html.fromstring(doc['post'])
|
||||
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
img_path = os.path.join(doc['folder'], src)
|
||||
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda size, ext: url_base.format(id=doc['id'], img=img_id, size=size, ext=ext.lower())
|
||||
|
||||
image = create_image(img_path, sizes, url_fn, images_dir)
|
||||
if image:
|
||||
images.append(image)
|
||||
img.set('src', image['data']['big']['url'])
|
||||
else:
|
||||
print('fail: ', doc['folder'], src)
|
||||
|
||||
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
|
||||
doc['images'] = images
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def get_manifest(md_path):
|
||||
y, m = read_yaml_md(md_path)
|
||||
y = y if y else {}
|
||||
|
||||
if 'date' in y:
|
||||
y['date'] = datetime.strptime(y['date'], '%d.%m.%Y')
|
||||
|
||||
if 'id' in y:
|
||||
y['id'] = str(y['id'])
|
||||
|
||||
return {
|
||||
**y,
|
||||
'post': m
|
||||
}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
|
||||
|
||||
os.chdir(dir)
|
||||
|
||||
# GET EVENT FOLDERS
|
||||
events = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
events = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i), ['%Y.%m.%d']
|
||||
),
|
||||
events
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
|
||||
glob_in = lambda dir: combine(map(
|
||||
lambda type: glob(os.path.join(dir, type)),
|
||||
image_types
|
||||
))
|
||||
events = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
'post': create_post_from_image_list(lmap(
|
||||
lambda path: os.path.relpath(path, i[0]),
|
||||
glob_in(i[0])
|
||||
))
|
||||
},
|
||||
events
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
md_glob = lambda i : first(glob(i + '/*.md'))
|
||||
events = lmap(
|
||||
lambda event: {
|
||||
**event,
|
||||
**map_cases(
|
||||
event,
|
||||
[(
|
||||
lambda i: md_glob(i['folder']),
|
||||
lambda i: get_manifest(md_glob(i['folder'])),
|
||||
)],
|
||||
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
events
|
||||
)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
events = lmap(
|
||||
create_event,
|
||||
events
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
events = list(filter(None, events))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
sync_images = lambda i: lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(i)
|
||||
)
|
||||
events = lmap(
|
||||
key_mapper('images', sync_images),
|
||||
events
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
events = lmap(
|
||||
sync_event,
|
||||
events
|
||||
)
|
||||
|
||||
lprint(events)
|
||||
print('SYNC EVENTS DONE')
|
||||
@@ -2,6 +2,46 @@ from db import db
|
||||
from utils.fn import lmap
|
||||
|
||||
|
||||
# class SyncImage(Sync):
|
||||
# def __init__(self, sizes, save_path, create_url_fn):
|
||||
# super().__init__()
|
||||
# self.sizes = sizes
|
||||
# self.save_path = save_path
|
||||
# self.create_url_fn = create_url_fn
|
||||
# self.collection = db().images
|
||||
#
|
||||
# def create_id(self, document):
|
||||
# id = document['id'] if 'id' in document and document['id'] else None
|
||||
# if not id:
|
||||
# id = 'schedule-{period}-{semester}'.format(**document)
|
||||
# document['id'] = id
|
||||
# return document
|
||||
#
|
||||
# def read(self, document):
|
||||
# return super().read(document)
|
||||
#
|
||||
# def create_hash(self, document):
|
||||
# return {
|
||||
# **document,
|
||||
# 'hash': hash_file(document)
|
||||
# }
|
||||
#
|
||||
# def create(self, document):
|
||||
# image = create_image(document, self.sizes, self.create_url_fn, self.save_path)
|
||||
#
|
||||
# return {
|
||||
# 'file': document,
|
||||
# 'hash': self.create_hash(document),
|
||||
# 'data': image['data']
|
||||
# }
|
||||
#
|
||||
# def find_or_create(self, document):
|
||||
# i = self.read(document)
|
||||
# if not i:
|
||||
# return self.create(document)
|
||||
# return i
|
||||
|
||||
|
||||
def sync_image(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
from db import db
|
||||
from sync import create_date
|
||||
from utils.fn import lmap, lprint_json
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
def sync_news(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_news,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().timeline
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_news(profile):
|
||||
formats = [
|
||||
'%d.%m.%Y',
|
||||
'%d.%m.%Y %H:%M',
|
||||
'%d.%m.%Y %H:%M:%S'
|
||||
]
|
||||
|
||||
if 'title' not in profile:
|
||||
return None
|
||||
|
||||
if 'date' in profile:
|
||||
profile['date'] = create_date(profile['date'], formats)
|
||||
|
||||
if 'until' in profile:
|
||||
profile['until'] = create_date(profile['until'], formats)
|
||||
|
||||
if 'id' not in profile:
|
||||
id = lambda date, title: '{date}-{title}'.format(
|
||||
date=date.strftime('%Y-%m-%d'),
|
||||
title=url_encode_text(title)
|
||||
)
|
||||
profile['id'] = id(profile['date'], profile['title'])
|
||||
|
||||
return profile
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
news_dir = '/Users/tmshv/Dropbox/Dev/Hud school/News'
|
||||
os.chdir(news_dir)
|
||||
|
||||
# GET NEWS FILES
|
||||
file_glob = os.path.join(news_dir, '*.md')
|
||||
news = glob(file_glob)
|
||||
|
||||
# READ MANIFEST FILES -> NEWS_MANIFEST
|
||||
news = lmap(
|
||||
lambda i: (os.path.basename(i),) + read_yaml_md(i),
|
||||
news
|
||||
)
|
||||
news = lmap(
|
||||
lambda i: {
|
||||
**i[1],
|
||||
'file': i[0],
|
||||
'message': i[2]
|
||||
},
|
||||
news
|
||||
)
|
||||
|
||||
# NEWS_MANIFEST -> NEWS_OBJECT
|
||||
news = lmap(
|
||||
create_news,
|
||||
news
|
||||
)
|
||||
|
||||
# FILTER BROKEN NEWS_OBJECT
|
||||
news = list(filter(
|
||||
None,
|
||||
news
|
||||
))
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
events = lmap(
|
||||
sync_news,
|
||||
news
|
||||
)
|
||||
|
||||
lprint_json(news)
|
||||
print('SYNC NEWS DONE')
|
||||
@@ -0,0 +1,312 @@
|
||||
import os
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \
|
||||
images_from_html, list_images, synced_images_ids
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn
|
||||
from utils.hash import hash_file, hash_str
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
class SyncPost(Sync):
|
||||
def __init__(self, collection_name, document_type, image_url_base, dir_local_images, sizes):
|
||||
super().__init__()
|
||||
self.collection = db()[collection_name]
|
||||
self.sizes = sizes
|
||||
self.type = document_type
|
||||
self.image_url_base = image_url_base
|
||||
self.dir_local_images = dir_local_images
|
||||
|
||||
def create_id(self, document):
|
||||
document['id'] = document['id'] if 'id' in document else url_encode_text(document['title'])
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
hash_images = lmap(
|
||||
hash_file,
|
||||
lmap(
|
||||
lambda i: os.path.join(document['folder'], i),
|
||||
document['images']
|
||||
)
|
||||
)
|
||||
|
||||
document['hash'] = hash_str(
|
||||
combine([hash_str(document)] + hash_images)
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def create(self, document):
|
||||
if 'date' not in document or not document['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in document or not document['title']:
|
||||
return None
|
||||
|
||||
if self.type:
|
||||
document['type'] = self.type
|
||||
|
||||
post_html = lxml.html.fromstring(document['post'])
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
img_path = os.path.abspath(
|
||||
os.path.join(document['folder'], src)
|
||||
)
|
||||
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda size, ext: self.image_url_base.format(id=document['id'], img=img_id, size=size,
|
||||
ext=ext.lower())
|
||||
|
||||
image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
|
||||
if image:
|
||||
images.append(image)
|
||||
img.set('src', image['data']['big']['url'])
|
||||
else:
|
||||
print('fail: ', document['folder'], src)
|
||||
|
||||
document['post'] = lxml.html.tostring(post_html).decode('utf-8')
|
||||
document['images'] = images
|
||||
return document
|
||||
|
||||
|
||||
def get_manifest(path):
|
||||
y, m = read_yaml_md(path)
|
||||
y = y if y else {}
|
||||
|
||||
if 'date' in y:
|
||||
y['date'] = create_date(y['date'], settings.date_formats)
|
||||
|
||||
if 'until' in y:
|
||||
y['until'] = create_date(y['until'], settings.date_formats)
|
||||
|
||||
if 'id' in y:
|
||||
y['id'] = str(y['id'])
|
||||
|
||||
return {
|
||||
**y,
|
||||
'post': m,
|
||||
'images': images_from_html(m)
|
||||
}
|
||||
|
||||
|
||||
def md_from_folder(i):
|
||||
return first(glob(i + '/*.md'))
|
||||
|
||||
|
||||
def get_folder_documents(sync, file_time_formats):
|
||||
# GET EVENT FOLDERS
|
||||
documents = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
documents = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i), file_time_formats
|
||||
),
|
||||
documents
|
||||
)
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
'images': lmap(
|
||||
lambda path: os.path.relpath(path, i['folder']),
|
||||
list_images(i['folder'])
|
||||
)
|
||||
},
|
||||
documents
|
||||
)
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
'post': create_post_from_image_list(i['images'])
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
**map_cases(
|
||||
i,
|
||||
[(
|
||||
lambda i: md_from_folder(i['folder']),
|
||||
lambda i: get_manifest(md_from_folder(i['folder'])),
|
||||
)],
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
# GET EVENT FILES
|
||||
def get_file_documents(sync, file_time_formats):
|
||||
documents = glob('*.md')
|
||||
documents = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i), file_time_formats
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
'file': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
**get_manifest(i['file'])
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def main(dir_documents, sync, file_time_formats):
|
||||
# INIT
|
||||
os.chdir(dir_documents)
|
||||
|
||||
documents = get_folder_documents(sync, file_time_formats)
|
||||
documents += get_file_documents(sync, file_time_formats)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmapfn(documents)(
|
||||
lambda i: i['id']
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = lmap(
|
||||
lambda i: i[0],
|
||||
filter(
|
||||
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
|
||||
lmap(
|
||||
lambda document: (document, sync.read(document)),
|
||||
documents
|
||||
)
|
||||
))
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
documents = lmap(
|
||||
sync.create,
|
||||
documents
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
documents = list(filter(None, documents))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('images', synced_images_ids),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
# SCOPE
|
||||
print('SCOPE:')
|
||||
lprint(scope_documents_ids)
|
||||
|
||||
# DELETE
|
||||
print('DELETE DOCUMENTS:')
|
||||
lprint_json(lmap(
|
||||
lambda i: i['id'],
|
||||
documents_to_remove
|
||||
))
|
||||
|
||||
# DELETE
|
||||
print('UPDATE DOCUMENTS:')
|
||||
lprint_json(lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
))
|
||||
lprint_json(documents)
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
|
||||
main(
|
||||
'/Users/tmshv/Dropbox/Dev/Hud school/Events',
|
||||
SyncPost(
|
||||
'events',
|
||||
None,
|
||||
image_url_base,
|
||||
settings.image_output,
|
||||
settings.event_image_sizes
|
||||
),
|
||||
file_time_formats=['%Y.%m.%d']
|
||||
)
|
||||
|
||||
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
|
||||
main(
|
||||
'/Users/tmshv/Dropbox/Dev/Hud school/News',
|
||||
SyncPost(
|
||||
'timeline',
|
||||
'post',
|
||||
image_url_base,
|
||||
settings.image_output,
|
||||
settings.event_image_sizes
|
||||
),
|
||||
file_time_formats=settings.date_formats_reverse
|
||||
)
|
||||
@@ -15,48 +15,6 @@ class SyncSchedule(Sync):
|
||||
super().__init__()
|
||||
self.collection = db().schedules
|
||||
|
||||
# def sync_collective(self, record):
|
||||
# if isinstance(record, list):
|
||||
# return lmap(
|
||||
# self.sync_collective,
|
||||
# record
|
||||
# )
|
||||
#
|
||||
# q = {'id': record['id']}
|
||||
# try:
|
||||
# collection = db().collective
|
||||
# update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
# i = collection.find_one({'id': record['id']})
|
||||
# return i
|
||||
# except ValueError:
|
||||
# pass
|
||||
#
|
||||
# return None
|
||||
#
|
||||
# def compile(self, profile):
|
||||
# profile['picture'] = create_image(
|
||||
# profile['picture'],
|
||||
# sizes,
|
||||
# lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
|
||||
# dir_local_images
|
||||
# )
|
||||
# return profile
|
||||
#
|
||||
# def read_document(self, i, query_fn=None):
|
||||
# q = query_fn(i) if query_fn else {'id': i['id']}
|
||||
# try:
|
||||
# return self.collection().find_one(q)
|
||||
# except ValueError:
|
||||
# pass
|
||||
#
|
||||
# return None
|
||||
#
|
||||
# def query_documents(self, q):
|
||||
# return self.collection().find(q)
|
||||
#
|
||||
# def delete_document(self, q):
|
||||
# return self.collection().find_one_and_delete(q)
|
||||
|
||||
def create_id(self, document):
|
||||
id = document['id'] if 'id' in document and document['id'] else None
|
||||
if not id:
|
||||
|
||||
+28
-76
@@ -3,8 +3,9 @@ from glob import glob
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync import Sync
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext
|
||||
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
|
||||
from utils.hash import hash_str, hash_file
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
@@ -15,42 +16,11 @@ url_base = settings.teacher_image_url_base
|
||||
dir_local_images = settings.teacher_image_output
|
||||
sizes = settings.teacher_image_sizes
|
||||
|
||||
class Teacher:
|
||||
def collection(self):
|
||||
return db().collective
|
||||
|
||||
def read_teacher_by_id(self, record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
self.read_teacher_by_id,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record}
|
||||
try:
|
||||
return db().collective.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def sync_collective(self, record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
self.sync_collective,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().collective
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
class Teacher(Sync):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.collection = db().collective
|
||||
|
||||
def compile(self, profile):
|
||||
profile['picture'] = create_image(
|
||||
@@ -61,39 +31,27 @@ class Teacher:
|
||||
)
|
||||
return profile
|
||||
|
||||
def read_document(self, i, query_fn=None):
|
||||
q = query_fn(i) if query_fn else {'id': i['id']}
|
||||
try:
|
||||
return self.collection().find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def query_documents(self, q):
|
||||
return self.collection().find(q)
|
||||
|
||||
def delete_document(self, q):
|
||||
return self.collection().find_one_and_delete(q)
|
||||
|
||||
def create_id(self, document):
|
||||
if 'id' in document and document['id']:
|
||||
return document['id']
|
||||
return document
|
||||
|
||||
document['id'] = url_encode_text(document['name'])
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
document['hash'] = hash_str(
|
||||
hash_file(document['file']) + hash_file(document['picture'])
|
||||
)
|
||||
return document
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# ALL DOCUMENTS IDS FOUNDED
|
||||
scope_documents_ids = []
|
||||
|
||||
def main(dir_teachers, sync):
|
||||
os.chdir(dir_teachers)
|
||||
sync = Teacher()
|
||||
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
documents = glob('*.md')
|
||||
|
||||
# # READ TEACHER YAML/MD MANIFEST
|
||||
# READ TEACHER YAML/MD MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: (i,) + read_yaml_md(i),
|
||||
documents
|
||||
@@ -119,20 +77,10 @@ if __name__ == '__main__':
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(
|
||||
lambda i: {**i, 'id': sync.create_id(i)},
|
||||
documents
|
||||
)
|
||||
documents = lmapfn(documents)(sync.create_id)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILES
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i, 'hash': hash_str(
|
||||
hash_file(i['file']) + hash_file(i['picture'])
|
||||
)
|
||||
},
|
||||
documents
|
||||
)
|
||||
documents = lmapfn(documents)(sync.create_hash)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmap(
|
||||
@@ -146,7 +94,7 @@ if __name__ == '__main__':
|
||||
filter(
|
||||
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
|
||||
lmap(
|
||||
lambda document: (document, sync.read_document(document)),
|
||||
lambda document: (document, sync.read(document)),
|
||||
documents
|
||||
)
|
||||
))
|
||||
@@ -157,22 +105,22 @@ if __name__ == '__main__':
|
||||
documents
|
||||
)
|
||||
|
||||
# # REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
documents
|
||||
)
|
||||
|
||||
# # SYNC TEACHER_PROFILE
|
||||
# SYNC TEACHER_PROFILE
|
||||
documents = lmap(
|
||||
sync.sync_collective,
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query_documents({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_remove = lmap(
|
||||
sync.delete_document,
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
@@ -191,3 +139,7 @@ if __name__ == '__main__':
|
||||
print('UPDATE DOCUMENTS:')
|
||||
lprint_json(documents)
|
||||
print('[SYNC DOCUMENTS DONE]')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(dir_teachers, Teacher())
|
||||
Reference in new issue
Block a user