[Sync] -sync/document
This commit is contained in:
Roman Timashev committed
1 parent
fa71fc5cf1
commit
cd9c790f97
2 files changed
-207
No files matched your search
@@ -1,206 +0,0 @@
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
import settings
|
||||
from sync import synced_image_id, untouched
|
||||
from sync.core.SyncDocument import SyncDocument
|
||||
from sync.data import scan_subdirs
|
||||
from sync.models.Document import Document
|
||||
from utils.fn import combine, lmap, key_mapper
|
||||
|
||||
|
||||
def sync_documents(provider, collection, update=True, delete=True):
|
||||
"""
|
||||
DOCUMENT_OBJECT SAMPLE
|
||||
{
|
||||
"hash": "c6a2aaf9a393c683c2250dd916115b7056ef16f023e0cd348c184a85cde9c9a0",
|
||||
"_id": {
|
||||
"$oid": "56d08411ace9573958e5e497"
|
||||
},
|
||||
"type": "document",
|
||||
"id": "dlya-postupayuschih-dogovor-o-pozhertvovanii-pdf",
|
||||
"category": "Для поступающих",
|
||||
"file": {
|
||||
"size": 53088,
|
||||
"name": "Договор о пожертвовании.pdf"
|
||||
},
|
||||
"url": "https://static.shburg.org/art/uploads/dogovor-o-pozhertvovanii.pdf",
|
||||
"preview": {
|
||||
"$oid": "56d08007ace9573958e5e48f"
|
||||
},
|
||||
"title": "Договор о пожертвовании"
|
||||
}
|
||||
|
||||
:param provider:
|
||||
:param update:
|
||||
:param delete:
|
||||
:return:
|
||||
"""
|
||||
|
||||
static_path = static_path_fn(settings.dir_static_uploads)
|
||||
|
||||
return main(
|
||||
SyncDocument(
|
||||
collection,
|
||||
provider,
|
||||
sizes=settings.image_sizes,
|
||||
dir_static_previews=settings.dir_static_images,
|
||||
url_base_preview=settings.url_base_preview,
|
||||
url_base_document=settings.document_url_template,
|
||||
),
|
||||
static_path,
|
||||
update_documents=update,
|
||||
delete_documents=delete
|
||||
)
|
||||
|
||||
|
||||
def upload(provider, document):
|
||||
f = document['file']
|
||||
p = provider.get_local(f)
|
||||
n = os.path.basename(document['url'])
|
||||
|
||||
files = {'file': open(p, 'rb')}
|
||||
# values = {'DB': 'photcat', 'OUT': 'csv', 'SHORT': 'short'}
|
||||
|
||||
r = requests.put(
|
||||
url='https://static.shlisselburg.org/upload/art/uploads/{}'.format(n),
|
||||
files=files,
|
||||
auth=('uploader', '1207690962638465')
|
||||
)
|
||||
if r.status_code == 200:
|
||||
print('Document {} uploaded'.format(document['url']))
|
||||
else:
|
||||
print('Failed to upload document {}'.format(document['url']))
|
||||
print('https://static.shlisselburg.org/upload/art/uploads/{}'.format(n))
|
||||
print(r.request.headers)
|
||||
print(r.headers)
|
||||
print(r.text)
|
||||
print()
|
||||
|
||||
|
||||
def main(sync, static_path, update_documents, delete_documents):
|
||||
documents = scan_subdirs(sync.provider, '.pdf')
|
||||
documents = lmap(
|
||||
lambda document: read_document(sync, document),
|
||||
documents
|
||||
)
|
||||
documents_id = lmap(
|
||||
lambda document: document['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# CHECKING
|
||||
file_names = set(lmap(
|
||||
lambda document: document['file'],
|
||||
documents
|
||||
))
|
||||
if len(file_names) != len(documents):
|
||||
raise Exception('File names should be unique')
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# COPY FILE -> STATIC_DIR/URL_FILENAME
|
||||
for i in documents:
|
||||
sync.provider.copy(
|
||||
i['file'],
|
||||
static_path(i)
|
||||
)
|
||||
|
||||
for i in documents:
|
||||
upload(sync.provider, i)
|
||||
|
||||
# MAP DOCUMENT PROFILE FROM MANIFEST TO DOCUMENT_OBJECT
|
||||
documents = lmap(
|
||||
sync.create,
|
||||
documents
|
||||
)
|
||||
|
||||
if update_documents:
|
||||
# REPLACE PREVIEW_OBJECT WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('preview', lambda img: synced_image_id(img)),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC DOCUMENT_OBJECT WITH DB
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
# DELETING
|
||||
documents_delete = sync.query({'id': {'$nin': documents_id}})
|
||||
if delete_documents:
|
||||
documents_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda document: {'_id': document['_id']},
|
||||
documents_delete
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_delete
|
||||
|
||||
|
||||
def read_document(sync, path):
|
||||
name = os.path.basename(path)
|
||||
|
||||
document = {
|
||||
'file': path,
|
||||
'category': os.path.dirname(path),
|
||||
'title': os.path.splitext(name)[0]
|
||||
}
|
||||
|
||||
document = sync.create_id(document)
|
||||
document = sync.create_url(document)
|
||||
document = sync.create_hash(document)
|
||||
|
||||
return document
|
||||
|
||||
|
||||
def create_document(provider, m):
|
||||
# dir_static_previews = settings.dir_static_images,
|
||||
# url_base_preview = settings.url_base_preview,
|
||||
# url_base_document = settings.url_base_document
|
||||
return Document(provider, m, {})
|
||||
|
||||
|
||||
def static_path_fn(dir_path):
|
||||
return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
|
||||
|
||||
|
||||
def categorize_files_list(files, title):
|
||||
"""
|
||||
Adds specified category to each file in files list
|
||||
:param files:
|
||||
:param title:
|
||||
:return:
|
||||
"""
|
||||
return map(
|
||||
lambda profile: {
|
||||
**profile,
|
||||
'category': title
|
||||
},
|
||||
files
|
||||
)
|
||||
|
||||
|
||||
def unwrap_manifest(param):
|
||||
if isinstance(param, list):
|
||||
return combine(map(
|
||||
unwrap_manifest,
|
||||
param
|
||||
))
|
||||
|
||||
if 'files' in param:
|
||||
return lmap(
|
||||
unwrap_manifest,
|
||||
categorize_files_list(param['files'], param['title'])
|
||||
)
|
||||
|
||||
if 'file' not in param:
|
||||
return None
|
||||
|
||||
return param
|
||||
@@ -3,7 +3,6 @@ import unittest
|
||||
import settings
|
||||
from sync.data import scan_subdirs
|
||||
from sync.data.fs import FSProvider
|
||||
from sync.document import create_document
|
||||
from sync.models.Document import Document
|
||||
from utils.fn import lmap
|
||||
|
||||
|
||||
Reference in new issue
Block a user