tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

utils update: moving some parameters to settings

Browse Source
This commit is contained in:
tmshv committed 2016-08-04 23:35:39 +03:00
1 parent 8143e21388
commit 24b28e6e4d
5 files changed
+106 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/Sync/settings.py
+17 -7
View File
@@ -1,3 +1,5 @@
import os
__author__ = 'Roman Timashev'
root_dir_gallery = './Gallery'
@@ -5,7 +7,7 @@ root_dir_schedule = './Schedules'
default_description_version = 1
database_uri = 'mongodb://localhost/hudozka'
database_uri = os.environ['MONGO_URI']
CYR_LOWER = 'йцукенгшщзхъфывапролджэячсмитьбю'
CYR_UPPER = 'ЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮ'
@@ -14,7 +16,6 @@ EN_UPPER = 'QWERTYUIOPASDFGHJKLZXCVBNM'
DIGITS = '0123456789'
description_available_chars = CYR_LOWER + CYR_UPPER + EN_LOWER + EN_UPPER + DIGITS + ' _-'
translit_table = (
['а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', '̆'],
['a', 'b', 'v', 'g', 'd', 'e', 'e', 'zh', 'z', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'f', 'h', 'z', 'ch', 'sh', 'sch', '_', 'y', '_', 'e', 'yu', 'ya', 'A', 'B', 'V', 'G', 'D', 'E', 'E', 'ZH', 'Z', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'F', 'H', 'Z', 'CH', 'SH', 'SCH', '_', 'Y', '_', 'E', 'YU', 'YA', '_']
@@ -27,15 +28,11 @@ teachers_names = {
'В.А.Саржин': 'va-sarzhin',
'Р.К.Тимашев': 'rk-timashev',
'А.С.Тимашева': 'as-timasheva',
# '': '',
}
image_processing = True
image_save_dir = './static_images'
image_base_url = 'https://static.shburg.org/art/images/'
image_output = '/Users/tmshv/Desktop/Hudozka Static/images'
album_html_img_class = 'hudozka-product'
image_ext = '.jpg'
@@ -90,4 +87,17 @@ date_formats_reverse = [
'%Y',
'%Y.%m',
'%Y.%m.%d'
]
]
dir_documents = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Documents'
dir_events = '/Users/tmshv/Yandex.Disk/Hudozka/Site/Events'
dir_news = '/Users/tmshv/Yandex.Disk/Hudozka/Site/News'
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
dir_static_images = '/Users/tmshv/Desktop/Hudozka Static/images'
url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
url_base_document = 'https://static.shburg.org/art/uploads/{file}'
image_base_url = 'https://static.shburg.org/art/images/'
image_name_format = '{type}-{id}-{img}-{size}{ext}'
image_output = '/Users/tmshv/Desktop/Hudozka Static/images'
utils/Sync/sync/document.py
+51 -39
View File
@@ -15,13 +15,13 @@ from utils.text.transform import url_encode_text, url_encode_file
class SyncDocument(Sync):
def __init__(self, collection_name, sizes):
def __init__(self, collection_name, sizes, dir_static_previews, url_base_preview, url_base_document):
super().__init__()
self.collection = db()[collection_name]
self.sizes = sizes
self.dir_static_previews = '/Users/tmshv/Desktop/Hudozka Static/images'
self.url_base_preview = 'https://static.shburg.org/art/images/{id}-{size}{ext}'
self.url_base_document = 'https://static.shburg.org/art/uploads/{file}'
self.dir_static_previews = dir_static_previews
self.url_base_preview = url_base_preview
self.url_base_document = url_base_document
def create(self, document):
file = document['file']
@@ -115,10 +115,9 @@ def unwrap_manifest(param):
return param
def documents_from_yaml():
def documents_from_yaml(dirpath):
# GET DOCUMENT YAML_MANIFEST FILES
documents = glob(dir_documents + '/*.yaml')
documents = glob(dirpath + '/*.yaml')
# READ YAML_MANIFEST FILES
documents = lmap(
@@ -183,7 +182,17 @@ def until_none(ls):
return better
def main(dir_documents, dir_static_uploads, sync):
def main(dir_documents, dir_static_uploads, sync, update_documents=True, delete_documents=True):
"""
:param dir_documents:
:param dir_static_uploads:
:param sync:
:param update_documents:
:param delete_documents:
:return:
"""
os.chdir(dir_documents)
# documents = documents_from_yaml()
@@ -197,9 +206,6 @@ def main(dir_documents, dir_static_uploads, sync):
}
)
# lprint_json(documents)
# return
# CREATE DOCUMENT IDENTITY
documents = lmap(sync.create_id, documents)
@@ -241,32 +247,36 @@ def main(dir_documents, dir_static_uploads, sync):
documents
)
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('preview', lambda i: synced_image_id(i)),
documents
)
# SYNC DOCUMENT_OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
if update_documents:
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
documents = lmap(
key_mapper('preview', lambda i: synced_image_id(i)),
documents
)
)
# SYNC DOCUMENT_OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
if delete_documents:
documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
documents_to_delete = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_delete
)
)
else:
documents_to_delete = []
print('SCOPE:', len(scope_documents_ids))
lprint(scope_documents_ids)
print('DELETE DOCUMENTS:', len(documents_to_remove))
lprint_json(documents_to_remove)
print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete))))
lprint_json(documents_to_delete)
print('UPDATE DOCUMENTS:', len(documents))
lprint_json(documents)
@@ -275,16 +285,18 @@ def main(dir_documents, dir_static_uploads, sync):
if __name__ == '__main__':
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Documents'
dir_static_uploads = '/Users/tmshv/Desktop/Hudozka Static/uploads'
main(
dir_documents,
dir_static_uploads,
settings.dir_documents,
settings.dir_static_uploads,
SyncDocument(
'documents',
sizes=settings.image_sizes
)
sizes=settings.image_sizes,
dir_static_previews=settings.dir_static_images,
url_base_preview=settings.url_base_preview,
url_base_document=settings.url_base_document,
),
update_documents=True,
delete_documents=True
)
# DOCUMENT_OBJECT SAMPLE
utils/Sync/sync/image.py
+6
View File
@@ -1,4 +1,5 @@
from db import db
from settings import image_name_format, image_base_url
from utils.fn import lmap
@@ -58,3 +59,8 @@ def sync_image(record):
pass
return None
def image_create_url(file):
url = image_base_url + image_name_format
return url.format(file)
utils/Sync/sync/post.py
+22 -22
View File
@@ -218,7 +218,7 @@ def get_file_documents(sync, file_time_formats):
return documents
def main(dir_documents, sync, file_time_formats):
def main(dir_documents, sync, file_time_formats, update_documents=True, delete_documents=True):
# INIT
os.chdir(dir_documents)
@@ -249,30 +249,34 @@ def main(dir_documents, sync, file_time_formats):
)
# SYNC EVENT_OBJECT WITH DB
documents = lmap(
sync.update,
documents
)
remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = sync.query(remove_query)
documents_to_remove = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_remove
if update_documents:
documents = lmap(
sync.update,
documents
)
)
if delete_documents:
remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
documents_to_delete = sync.query(remove_query)
documents_to_delete = lmap(
sync.delete,
map(
lambda i: {'_id': i['_id']},
documents_to_delete
)
)
else:
documents_to_delete = []
# SCOPE
print('SCOPE:')
lprint(scope_documents_ids)
# DELETE
print('DELETE DOCUMENTS:')
print('DELETE DOCUMENTS: %s' % ('NO' if not delete_documents else str(len(documents_to_delete))))
lprint_json(lmap(
lambda i: i['id'],
documents_to_remove
documents_to_delete
))
# DELETE
@@ -287,10 +291,8 @@ def main(dir_documents, sync, file_time_formats):
if __name__ == '__main__':
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Events'
main(
dir_docs,
settings.dir_events,
SyncPost(
'events',
None,
@@ -302,10 +304,8 @@ if __name__ == '__main__':
)
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/News'
dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/News'
main(
dir_docs,
settings.dir_news,
SyncPost(
'timeline',
'post',
utils/Sync/utils/text/transform.py
+10 -1
View File
@@ -76,7 +76,16 @@ def text_lower(i):
return i.lower()
url_encode_file = iterate_over_fns([translit, space_to_dash, remove_underscores, remove_punctuation, text_lower])
url_encode_file = iterate_over_fns([
translit,
space_to_dash,
short_dash,
remove_underscores,
remove_punctuation,
reduce_dashes,
text_lower
])
url_encode_text = iterate_over_fns([
translit,
space_to_dash,