sync util update

This commit is contained in:
tmshv committed 2016-03-07 04:46:14 +03:00
1 parent fe409c04a4
commit 261e842802
5 files changed
+134 -42

No files matched your search

+3
View File
@@ -89,6 +89,9 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
def images_from_html(md):
if not md:
return []
post_html = lxml.html.fromstring(md)
images = []
+24 -4
View File
@@ -1,6 +1,8 @@
import os
import re
from glob import glob
import frontmatter
import lxml.html
from lxml import etree
@@ -13,7 +15,7 @@ from utils.fn import lmap, dir_globber, lprint_json, iterate_iter_over_fns, firs
key_mapper, lprint
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.io import read_yaml_md
from utils.io import read_yaml_md, read_file, parse_yaml_front_matter
from utils.text.transform import url_encode_text
@@ -326,7 +328,25 @@ def main(dir_documents, sync):
if __name__ == '__main__':
image_creator = ImageCreator(cache=None)
alb = SyncAlbum(image_creator=image_creator, images_dir=settings.image_output)
alb.skip_image_processing = True
# alb.skip_image_processing = True
albums_dir = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
main(dir_documents=albums_dir, sync=alb)
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Gallery'
# dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Gallery'
dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/Gallery'
main(dir_documents=dir_docs, sync=alb)
# dir_docs += '/2013 Валяние Зверушки'
# dir_docs += '/Валяние Зверушки.md'
# dir_docs = '/Users/tmshv/Yandex.Disk.localized/Hudozka/Site/News/2015 Выставка 70-летия победы.md'
# print(frontmatter.load(dir_docs).content)
# d = read_file(dir_docs)
# d = d.replace('\u0D0A', '\n')
# print(frontmatter.loads(d).metadata)
# print(parse_yaml_front_matter(d))
#
# print(len(re.split('^---', d, 2, re.MULTILINE)))
# print(re.split('^---', d, 2)[1])
#
# print(read_yaml_md(dir_docs))
+52 -12
View File
@@ -5,8 +5,8 @@ from tempfile import mkstemp
import settings
from db import db
from sync import Sync, untouched, synced_image_id
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper
from sync import Sync, synced_image_id, untouched
from utils.fn import combine, lmap, lprint_json, lprint, key_mapper, lmapfn
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
@@ -15,10 +15,9 @@ from utils.text.transform import url_encode_text, url_encode_file
class SyncDocument(Sync):
def __init__(self, collection_name, document_type, sizes):
def __init__(self, collection_name, sizes):
super().__init__()
self.collection = db()[collection_name]
self.document_type = document_type
self.sizes = sizes
self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
@@ -28,7 +27,7 @@ class SyncDocument(Sync):
file = document['file']
filename = os.path.basename(document['file'])
document['type'] = self.document_type
document['type'] = self.document_type(document)
document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
document['file'] = {
'name': filename,
@@ -37,10 +36,14 @@ class SyncDocument(Sync):
return document
def document_type(self, document):
if document['category'] == 'Награды':
return 'award'
return 'document'
def create_id(self, document):
bn = os.path.basename(document['file'])
document['id'] = url_encode_text('{type}-{category}-{file}'.format(
type=self.document_type,
document['id'] = url_encode_text('{category}-{file}'.format(
category=document['category'],
file=bn
))
@@ -57,11 +60,12 @@ class SyncDocument(Sync):
)
return document
def create_preview(self, pdf, sizes, preview_dir):
temp_preview_path = pdf_to_jpg(pdf)
def create_preview(self, file, sizes, preview_dir):
temp_preview_path = pdf_to_jpg(file)
id = url_encode_text(pdf)
url = lambda size, ext: self.url_base_preview.format(id=id, size=size, ext=ext)
file = os.path.basename(file)
file = url_encode_text(file)
url = lambda size, ext: self.url_base_preview.format(id=file, size=size, ext=ext)
img = create_image(temp_preview_path, sizes, url, preview_dir)
os.remove(temp_preview_path)
@@ -153,12 +157,49 @@ def documents_from_subdirs():
return documents
def get_pdf_title(file):
from PyPDF2 import PdfFileReader
from PyPDF2.generic import TextStringObject
from PyPDF2.generic import IndirectObject
pdf = PdfFileReader(open(file, 'rb'))
info = pdf.getDocumentInfo()
if info:
if type(info.title) == TextStringObject:
return str(info.title)
if type(info.title_raw) == IndirectObject:
o = pdf.getObject(info.title_raw)
return str(o)
return None
def until_none(ls):
better = None
for i in ls:
if (i is not None) and i != '':
better = i
return better
def main(dir_documents, dir_static_uploads, sync):
os.chdir(dir_documents)
# documents = documents_from_yaml()
documents = documents_from_subdirs()
# CHOICE BETTER TITLE
documents = lmapfn(documents)(
lambda i: {
**i,
'title': until_none([i['title'], get_pdf_title(i['file'])])
}
)
# lprint_json(documents)
# return
# CREATE DOCUMENT IDENTITY
documents = lmap(sync.create_id, documents)
@@ -242,7 +283,6 @@ if __name__ == '__main__':
dir_static_uploads,
SyncDocument(
'documents',
'document',
sizes=settings.image_sizes
)
)
+19 -14
View File
@@ -7,7 +7,7 @@ import lxml.html
import settings
from db import db
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \
images_from_html, list_images, synced_images_ids
images_from_html, list_images, synced_images_ids, untouched
from sync.image import sync_image
from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn
from utils.hash import hash_file, hash_str
@@ -78,6 +78,14 @@ class SyncPost(Sync):
document['images'] = images
return document
def create_remove_query(self, query):
if self.type:
query = {
**query,
'type': self.type
}
return query
def get_manifest(path):
y, m = read_yaml_md(path)
@@ -223,15 +231,7 @@ def main(dir_documents, sync, file_time_formats):
)
# SKIP UNTOUCHED DOCUMENTS
documents = lmap(
lambda i: i[0],
filter(
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, sync.read(document)),
documents
)
))
documents = untouched(documents, sync)
# MAP EVENT MANIFEST -> EVENT_OBJECT
documents = lmap(
@@ -254,7 +254,8 @@ def main(dir_documents, sync, file_time_formats):
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = sync.query(remove_query)
documents_to_remove = lmap(
sync.delete,
map(
@@ -280,14 +281,16 @@ def main(dir_documents, sync, file_time_formats):
lambda i: i['id'],
documents
))
lprint_json(documents)
# lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Events'
main(
'/Users/tmshv/Dropbox/Dev/Hud school/Events',
dir_docs,
SyncPost(
'events',
None,
@@ -299,8 +302,10 @@ if __name__ == '__main__':
)
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/News'
dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/News'
main(
'/Users/tmshv/Dropbox/Dev/Hud school/News',
dir_docs,
SyncPost(
'timeline',
'post',