sync util update

This commit is contained in:
tmshv committed 2016-03-07 04:46:14 +03:00
1 parent fe409c04a4
commit 261e842802
5 files changed
+134 -42

No files matched your search

+19 -14
View File
@@ -7,7 +7,7 @@ import lxml.html
import settings
from db import db
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, Sync, create_date, \
images_from_html, list_images, synced_images_ids
images_from_html, list_images, synced_images_ids, untouched
from sync.image import sync_image
from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn
from utils.hash import hash_file, hash_str
@@ -78,6 +78,14 @@ class SyncPost(Sync):
document['images'] = images
return document
def create_remove_query(self, query):
if self.type:
query = {
**query,
'type': self.type
}
return query
def get_manifest(path):
y, m = read_yaml_md(path)
@@ -223,15 +231,7 @@ def main(dir_documents, sync, file_time_formats):
)
# SKIP UNTOUCHED DOCUMENTS
documents = lmap(
lambda i: i[0],
filter(
lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
lmap(
lambda document: (document, sync.read(document)),
documents
)
))
documents = untouched(documents, sync)
# MAP EVENT MANIFEST -> EVENT_OBJECT
documents = lmap(
@@ -254,7 +254,8 @@ def main(dir_documents, sync, file_time_formats):
documents
)
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
documents_to_remove = sync.query(remove_query)
documents_to_remove = lmap(
sync.delete,
map(
@@ -280,14 +281,16 @@ def main(dir_documents, sync, file_time_formats):
lambda i: i['id'],
documents
))
lprint_json(documents)
# lprint_json(documents)
print('[SYNC DOCUMENTS DONE]')
if __name__ == '__main__':
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/Events'
dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/Events'
main(
'/Users/tmshv/Dropbox/Dev/Hud school/Events',
dir_docs,
SyncPost(
'events',
None,
@@ -299,8 +302,10 @@ if __name__ == '__main__':
)
image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
# dir_docs = '/Users/tmshv/Dropbox/Dev/Hud school/News'
dir_docs = '/Volumes/webdav.yandex.ru/Hudozka/Site/News'
main(
'/Users/tmshv/Dropbox/Dev/Hud school/News',
dir_docs,
SyncPost(
'timeline',
'post',