Sync util big update
This commit is contained in:
1 parent
1e80bae401
commit
40321327d6
30 files changed
+1001
-159
No files matched your search
@@ -0,0 +1,59 @@
|
||||
import re
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from markdown import markdown
|
||||
|
||||
|
||||
def create_date(date_str, date_formats=None):
|
||||
"""
|
||||
2016
|
||||
2016.10
|
||||
2016.10.10
|
||||
-> datetime
|
||||
|
||||
:param date_str:
|
||||
:return:
|
||||
"""
|
||||
if not date_formats:
|
||||
date_formats = [
|
||||
'%Y',
|
||||
'%Y.%m',
|
||||
'%Y.%m.%d'
|
||||
]
|
||||
|
||||
for format in date_formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, format)
|
||||
except:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
# 2016 На крыльях бабочек
|
||||
# 2016.10 На крыльях бабочек
|
||||
# 2016.10.10 На крыльях бабочек
|
||||
folder_name_pattern = re.compile('([\d.]+)(.*)')
|
||||
|
||||
|
||||
def create_date_and_title_from_folder_name(folder_name, date_formats=None):
|
||||
m = folder_name_pattern.findall(folder_name)
|
||||
if not m:
|
||||
return None, None
|
||||
|
||||
date_str, title = m[0]
|
||||
title = title.strip()
|
||||
date = create_date(date_str, date_formats)
|
||||
|
||||
if not date:
|
||||
return None, title
|
||||
|
||||
return date, title
|
||||
|
||||
|
||||
create_post_from_image_list = lambda images: markdown(
|
||||
'\n'.join(map(
|
||||
lambda i: ''.format(img=i),
|
||||
images
|
||||
))
|
||||
)
|
||||
@@ -0,0 +1,230 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from models.product import create_product
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, first, map_cases, dir_globber, lprint_json, key_mapper, iterate_iter_over_fns
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text, interpolate, interpolate_swift
|
||||
|
||||
|
||||
def sync_album(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_album,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().events
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_album(doc):
|
||||
sizes = settings.event_image_sizes
|
||||
# url_base = 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'
|
||||
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
url = lambda a, i, s, e: 'https://static.shburg.org/art/image/product-{album}-{id}-{size}{ext}'.format(
|
||||
album=a,
|
||||
id=i,
|
||||
size=s,
|
||||
ext=e
|
||||
)
|
||||
create_url = lambda aid, pid: lambda size, ext: url(aid, pid, size, ext.lower())
|
||||
image_id = lambda i: url_encode_text(
|
||||
os.path.splitext(i)[0]
|
||||
)
|
||||
image_path = lambda i: os.path.join(doc['folder'], i)
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
return None
|
||||
|
||||
create_album_id = lambda date, title: '{date}-{title}'.format(
|
||||
date=date.strftime('%Y'),
|
||||
title=url_encode_text(title)
|
||||
)
|
||||
|
||||
images = []
|
||||
|
||||
doc['id'] = doc['id'] if 'id' in doc else create_album_id(doc['date'], doc['title'])
|
||||
|
||||
if 'album' in doc:
|
||||
# album_context =
|
||||
iter_album = iterate_iter_over_fns([
|
||||
lambda product: {
|
||||
**product,
|
||||
'image': create_image(
|
||||
image_path(product['file']),
|
||||
sizes,
|
||||
create_url(
|
||||
doc['id'],
|
||||
image_id(product['file'])
|
||||
),
|
||||
images_dir,
|
||||
skip_processing=True
|
||||
)
|
||||
},
|
||||
lambda product: {
|
||||
**product,
|
||||
'awards': lmap(
|
||||
lambda award: {
|
||||
**award,
|
||||
'file': os.path.join(doc['folder'], award['file']),
|
||||
'assigned': interpolate(award['assigned'], product, interpolate_swift),
|
||||
# 'awardFor': product
|
||||
},
|
||||
product['awards']
|
||||
)
|
||||
}
|
||||
])
|
||||
|
||||
doc['album'] = lmap(
|
||||
create_product,
|
||||
iter_album(doc['album'])
|
||||
)
|
||||
|
||||
post_html = lxml.html.fromstring(doc['post'])
|
||||
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
img_path = image_path(src)
|
||||
|
||||
if os.path.exists(img_path):
|
||||
image = create_image(
|
||||
img_path,
|
||||
sizes,
|
||||
create_url(
|
||||
doc['id'],
|
||||
image_id(src)
|
||||
),
|
||||
images_dir,
|
||||
skip_processing=True
|
||||
)
|
||||
images.append(image)
|
||||
|
||||
img.set('src', image['data']['big']['url'])
|
||||
img.set('class', settings.album_html_img_class)
|
||||
img.set('data-file', src)
|
||||
|
||||
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
|
||||
doc['images'] = images
|
||||
return doc
|
||||
|
||||
|
||||
def get_manifest(filepath):
|
||||
y, m = read_yaml_md(filepath)
|
||||
|
||||
data = {**y} if y else {}
|
||||
_ = lambda key, d=None: data[key] if key in data else d
|
||||
date = create_date(_('date'), settings.date_formats_direct)
|
||||
if date:
|
||||
data['date'] = date
|
||||
|
||||
id = _('id')
|
||||
if id:
|
||||
data['id'] = str(id)
|
||||
|
||||
post = {'post': m} if m else {}
|
||||
return {**data, **post}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
|
||||
os.chdir(albums_dir)
|
||||
|
||||
# GET ALBUMS FOLDERS
|
||||
albums = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
# print(get_md_manifest('./2012 Техника мокрого валяния/WET FELTING 2012.md'))
|
||||
# print(get_manifest('./2016.02.17 Путешествие вокруг света/AROUND THE WORLD.md'))
|
||||
# exit()
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
albums = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i)
|
||||
),
|
||||
albums
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
glob_in = dir_globber(['*.jpg', '*.JPG', '*.png', '*.PNG'])
|
||||
albums = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
'post': create_post_from_image_list(lmap(
|
||||
lambda path: os.path.relpath(path, i[0]),
|
||||
glob_in(i[0])
|
||||
))
|
||||
},
|
||||
albums
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
md_glob = lambda i: first(glob(i + '/*.md'))
|
||||
albums = lmap(
|
||||
lambda album: {
|
||||
**album,
|
||||
**map_cases(
|
||||
album,
|
||||
[(
|
||||
lambda album: md_glob(album['folder']),
|
||||
lambda album: get_manifest(md_glob(album['folder'])),
|
||||
)],
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
albums
|
||||
)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
albums = lmap(
|
||||
create_album,
|
||||
albums
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
albums = list(filter(None, albums))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
def sync_images(i):
|
||||
return lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(i)
|
||||
)
|
||||
|
||||
|
||||
albums = lmap(
|
||||
key_mapper('images', sync_images),
|
||||
albums
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
# albums = lmap(
|
||||
# sync_album,
|
||||
# albums
|
||||
# )
|
||||
|
||||
lprint_json(albums)
|
||||
print('SYNC ALBUMS DONE')
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
|
||||
|
||||
def sync_collective(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_collective,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().collective
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_profile(profile):
|
||||
sizes = settings.image_sizes
|
||||
images_dir = settings.collective_image_output
|
||||
|
||||
url_base = 'https://static.shburg.org/art/image/teacher-{id}-{size}{ext}'
|
||||
url_fn = lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext)
|
||||
|
||||
profile['picture'] = create_image(profile['picture'], sizes, url_fn, images_dir)
|
||||
return profile
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Collective'
|
||||
os.chdir(dir)
|
||||
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
files = glob('*.md')
|
||||
|
||||
# READ TEACHER YAML/MD MANIFEST
|
||||
collective = lmap(
|
||||
read_yaml_md,
|
||||
files
|
||||
)
|
||||
|
||||
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
|
||||
collective = lmap(
|
||||
lambda i: {'biography': i[1], **i[0]},
|
||||
collective
|
||||
)
|
||||
|
||||
# MAP TEACHER_MANIFEST -> TEACHER_PROFILE
|
||||
collective = lmap(
|
||||
create_profile,
|
||||
collective
|
||||
)
|
||||
|
||||
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
collective = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
collective
|
||||
)
|
||||
|
||||
lprint(collective)
|
||||
print('SYNC COLLECTIVE DONE')
|
||||
@@ -0,0 +1,139 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
from tempfile import mkstemp
|
||||
|
||||
from shutil import copyfile
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.io import read_yaml
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
# Adds specified category to each file in files list
|
||||
unwrap_file_list = lambda files, title: map(
|
||||
lambda profile: {'category': title, **profile},
|
||||
files
|
||||
)
|
||||
|
||||
|
||||
def sync_document(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_document,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'url': record['url']}
|
||||
try:
|
||||
documents = db().documents
|
||||
update_result = documents.update_one(q, {'$set': record}, upsert=True)
|
||||
i = documents.find_one({'url': record['url']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def pdf_to_jpg(pdf):
|
||||
cwd = os.getcwd()
|
||||
abspdf = os.path.join(cwd, pdf)
|
||||
_, temp = mkstemp('.jpg')
|
||||
image_magick_pdf_to_img(abspdf, temp)
|
||||
return temp
|
||||
|
||||
|
||||
def create_preview(pdf, sizes, preview_dir):
|
||||
temp_preview_path = pdf_to_jpg(pdf)
|
||||
|
||||
id = url_encode_text(pdf)
|
||||
url_base = 'https://static.shburg.org/art/image/document-{id}-{size}{ext}'
|
||||
url = lambda size, ext: url_base.format(id=id, size=size, ext=ext)
|
||||
|
||||
img = create_image(temp_preview_path, sizes, url, preview_dir)
|
||||
os.remove(temp_preview_path)
|
||||
return img
|
||||
|
||||
|
||||
def create_document(doc):
|
||||
sizes = settings.image_sizes
|
||||
preview_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
|
||||
doc['url'] = 'https://static.shburg.org/art/uploads/' + url_encode_file(doc['file'])
|
||||
doc['preview'] = create_preview(doc['file'], sizes=sizes, preview_dir=preview_dir)
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def unwrap_manifest(param):
|
||||
if isinstance(param, list):
|
||||
return combine(map(
|
||||
unwrap_manifest,
|
||||
param
|
||||
))
|
||||
|
||||
if 'files' in param:
|
||||
return lmap(
|
||||
unwrap_manifest,
|
||||
unwrap_file_list(param['files'], param['title'])
|
||||
)
|
||||
|
||||
if 'file' not in param:
|
||||
return None
|
||||
|
||||
return param
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
|
||||
|
||||
os.chdir(dir)
|
||||
|
||||
# GET DOCUMENT YAML_MANIFEST FILES
|
||||
files = glob(dir + '/*.yaml')
|
||||
|
||||
# READ YAML_MANIFEST FILES
|
||||
files_data = lmap(
|
||||
read_yaml,
|
||||
files
|
||||
)
|
||||
|
||||
# GET FLAT LIST OF DOCUMENTS
|
||||
documents = unwrap_manifest(files_data)
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
create_document,
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_preview = lambda i: sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('preview', sync_preview),
|
||||
documents
|
||||
)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
out_path = lambda doc: os.path.join('/Users/tmshv/Dropbox/Dev/Hud school/Static/uploads', os.path.basename(doc['url']))
|
||||
lmap(
|
||||
lambda doc : copyfile(
|
||||
doc['file'],
|
||||
out_path(doc)
|
||||
),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync_document,
|
||||
documents
|
||||
)
|
||||
|
||||
lprint(documents)
|
||||
print('SYNC DOCUMENTS DONE')
|
||||
@@ -0,0 +1,169 @@
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
|
||||
import lxml.html
|
||||
from markdown import markdown
|
||||
|
||||
import settings
|
||||
from db import db
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list
|
||||
from sync.image import sync_image
|
||||
from utils.fn import combine, lmap, lprint, first, map_cases, key_mapper
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
def sync_event(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_event,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'id': record['id']}
|
||||
try:
|
||||
collection = db().events
|
||||
update_result = collection.update_one(q, {'$set': record}, upsert=True)
|
||||
i = collection.find_one({'id': record['id']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def create_event(doc):
|
||||
if 'date' not in doc or not doc['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in doc or not doc['title']:
|
||||
return None
|
||||
|
||||
doc['id'] = doc['id'] if 'id' in doc else url_encode_text(doc['title'])
|
||||
|
||||
url_base = 'https://static.shburg.org/art/image/event-{id}-{img}-{size}{ext}'
|
||||
images_dir = '/Users/tmshv/Dropbox/Dev/Hud School/Static/images'
|
||||
sizes = settings.event_image_sizes
|
||||
|
||||
post_html = lxml.html.fromstring(doc['post'])
|
||||
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
img_path = os.path.join(doc['folder'], src)
|
||||
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda size, ext: url_base.format(id=doc['id'], img=img_id, size=size, ext=ext.lower())
|
||||
|
||||
image = create_image(img_path, sizes, url_fn, images_dir)
|
||||
images.append(image)
|
||||
img.set('src', image['data']['big']['url'])
|
||||
|
||||
doc['post'] = lxml.html.tostring(post_html).decode('utf-8')
|
||||
doc['images'] = images
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def get_md_manifest(md_path):
|
||||
y, m = read_yaml_md(md_path)
|
||||
|
||||
y['date'] = datetime.strptime(y['date'], '%d.%m.%Y')
|
||||
y['id'] = str(y['id'])
|
||||
|
||||
return {
|
||||
**y,
|
||||
'post': m
|
||||
}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
|
||||
|
||||
os.chdir(dir)
|
||||
|
||||
# GET EVENT FOLDERS
|
||||
events = lmap(
|
||||
lambda i: i.path,
|
||||
filter(
|
||||
lambda i: i.is_dir(),
|
||||
os.scandir('.')
|
||||
)
|
||||
)
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
events = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(
|
||||
os.path.basename(i), ['%Y.%m.%d']
|
||||
),
|
||||
events
|
||||
)
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
image_types = ['*.jpg', '*.JPG', '*.png', '*.PNG']
|
||||
glob_in = lambda dir: combine(map(
|
||||
lambda type: glob(os.path.join(dir, type)),
|
||||
image_types
|
||||
))
|
||||
events = lmap(
|
||||
lambda i: {
|
||||
'folder': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
'post': create_post_from_image_list(lmap(
|
||||
lambda path: os.path.relpath(path, i[0]),
|
||||
glob_in(i[0])
|
||||
))
|
||||
},
|
||||
events
|
||||
)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
md_glob = lambda i : first(glob(i + '/*.md'))
|
||||
events = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
**map_cases(
|
||||
i,
|
||||
[(
|
||||
lambda i: md_glob(i['folder']),
|
||||
lambda i: get_md_manifest(md_glob(i['folder'])),
|
||||
)],
|
||||
|
||||
lambda i: {}
|
||||
)
|
||||
},
|
||||
events
|
||||
)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
events = lmap(
|
||||
create_event,
|
||||
events
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
events = list(filter(None, events))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
sync_images = lambda i: lmap(
|
||||
lambda image: image['_id'],
|
||||
sync_image(i)
|
||||
)
|
||||
events = lmap(
|
||||
key_mapper('images', sync_images),
|
||||
events
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
events = lmap(
|
||||
sync_event,
|
||||
events
|
||||
)
|
||||
|
||||
lprint(events)
|
||||
print('SYNC EVENTS DONE')
|
||||
@@ -0,0 +1,20 @@
|
||||
from db import db
|
||||
from utils.fn import lmap
|
||||
|
||||
|
||||
def sync_image(record):
|
||||
if isinstance(record, list):
|
||||
return lmap(
|
||||
sync_image,
|
||||
record
|
||||
)
|
||||
|
||||
q = {'hash': record['hash']}
|
||||
try:
|
||||
db().images.update_one(q, {'$set': record}, upsert=True)
|
||||
i = db().images.find_one({'hash': record['hash']})
|
||||
return i
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
Reference in new issue
Block a user