[Sync] + new Person sync
This commit is contained in:
Roman Timashev committed
1 parent
1da289802c
commit
953ef0bc28
8 files changed
+210
-170
No files matched your search
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Person import Person
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
|
||||
|
||||
class SyncPerson(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Person.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build(
|
||||
sizes=settings.image_sizes,
|
||||
url_factory=get_image_url_fn(document.id),
|
||||
)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
|
||||
|
||||
def get_image_url_fn(page_id: str):
|
||||
image_id = lambda filename: url_encode_text(
|
||||
os.path.splitext(os.path.basename(filename))[0]
|
||||
)
|
||||
|
||||
def img_url_fn(file, size, ext):
|
||||
return settings.page_url_base.format(
|
||||
page=page_id,
|
||||
id=image_id(file),
|
||||
size=size,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
return img_url_fn
|
||||
@@ -1,39 +0,0 @@
|
||||
import settings
|
||||
from sync.core import Sync
|
||||
from utils.hash import hash_str
|
||||
from utils.image import create_image
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
url_base = settings.teacher_image_url_base
|
||||
sizes = settings.teacher_image_sizes
|
||||
|
||||
|
||||
class SyncPerson(Sync):
|
||||
def __init__(self, collection, provider, dir_local_images):
|
||||
super().__init__()
|
||||
self.collection = collection
|
||||
self.provider = provider
|
||||
self.dir_local_images = dir_local_images
|
||||
|
||||
def compile(self, profile):
|
||||
image_path = self.provider.get_abs(profile['picture'])
|
||||
profile['picture'] = create_image(
|
||||
image_path,
|
||||
sizes,
|
||||
lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
|
||||
self.dir_local_images
|
||||
)
|
||||
return profile
|
||||
|
||||
def create_id(self, document):
|
||||
if 'id' in document and document['id']:
|
||||
return document
|
||||
|
||||
document['id'] = url_encode_text(document['name'])
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
document['hash'] = hash_str(
|
||||
self.provider.hash(document['file']) + self.provider.hash(document['picture'])
|
||||
)
|
||||
return document
|
||||
@@ -0,0 +1,109 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import Provider
|
||||
from sync.models import Model
|
||||
from sync.models.Image import Image
|
||||
|
||||
from utils.fn import swap_ext
|
||||
from utils.hash import hash_str
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
store = collection(settings.collection_collective)
|
||||
|
||||
|
||||
def find(item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Person(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
documents = provider.type_filter('', '.md')
|
||||
documents = [Person.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
params = get_manifest(provider, path)
|
||||
|
||||
return Person(
|
||||
provider,
|
||||
path,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
self.__hash_keys = ['id', 'url']
|
||||
|
||||
self.__id_template: str = '{category}-{file}'
|
||||
self.__url_template: str = settings.document_url_template
|
||||
self.__url_preview_template: str = settings.document_url_preview_template
|
||||
|
||||
self.params = params if params else {}
|
||||
self.picture = None
|
||||
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
async def build(self, **kwargs):
|
||||
sizes = kwargs['sizes']
|
||||
image_path = self.provider.get_abs(self.params['picture'])
|
||||
|
||||
self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id))
|
||||
|
||||
def bake(self):
|
||||
return {
|
||||
**self.params,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
'picture': self.picture.id,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
if 'id' in self.params:
|
||||
self.id = self.params['id']
|
||||
else:
|
||||
self.id = url_encode_text(self.params['name'])
|
||||
|
||||
def __set_hash(self):
|
||||
self.hash = hash_str(
|
||||
self.provider.hash(self.file) + self.provider.hash(self.params['picture'])
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return '<Person hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def get_manifest(provider: Provider, path: str) -> dict:
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
|
||||
manifest, body = read_yaml_md(data)
|
||||
manifest = {
|
||||
**manifest,
|
||||
'file': path,
|
||||
'biography': body,
|
||||
'picture': manifest['picture'] if 'picture' in manifest else swap_ext('.jpg')(path),
|
||||
}
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
def url_factory(id):
|
||||
def url(file, size, ext):
|
||||
return settings.person_image_url_template.format(id=id, size=size, ext=ext)
|
||||
|
||||
return url
|
||||
@@ -1,3 +1,6 @@
|
||||
import os
|
||||
|
||||
|
||||
def find(store, item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
@@ -40,5 +43,16 @@ class Model:
|
||||
|
||||
return not (self.hash == item['hash'])
|
||||
|
||||
def _filename(self):
|
||||
return os.path.basename(self.file)
|
||||
|
||||
async def save(self):
|
||||
pass
|
||||
document = self.bake()
|
||||
|
||||
query = {'id': document['id']}
|
||||
try:
|
||||
self.store.update_one(query, {'$set': document}, upsert=True)
|
||||
return self
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
@@ -1,119 +0,0 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core.person import SyncPerson
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
|
||||
from utils.io import read_yaml_md
|
||||
|
||||
url_base = settings.teacher_image_url_base
|
||||
dir_local_images = settings.teacher_image_output
|
||||
sizes = settings.teacher_image_sizes
|
||||
|
||||
|
||||
def sync_persons(provider, collection, update=True, delete=True):
|
||||
"""
|
||||
:param collection:
|
||||
:param provider:
|
||||
:param update:
|
||||
:param delete:
|
||||
:return:
|
||||
"""
|
||||
|
||||
return main(
|
||||
SyncPerson(
|
||||
collection,
|
||||
provider,
|
||||
dir_local_images
|
||||
),
|
||||
update_documents=update,
|
||||
delete_documents=delete
|
||||
)
|
||||
|
||||
|
||||
def make_person(provider, file):
|
||||
pass
|
||||
|
||||
|
||||
def main(sync, update_documents=False, delete_documents=False):
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
documents = sync.provider.type_filter('', '.md')
|
||||
|
||||
# READ TEACHER YAML/MD MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: (i,) + get_manifest(sync.provider, i),
|
||||
documents
|
||||
)
|
||||
|
||||
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i[1],
|
||||
'file': i[0],
|
||||
'biography': i[2]
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# DEFINE PROFILE PICTURE
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmapfn(documents)(sync.create_id)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILES
|
||||
documents = lmapfn(documents)(sync.create_hash)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# DO HEAVY PROCESS WITH DOCUMENTS
|
||||
documents = lmap(
|
||||
sync.compile,
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC TEACHER_PROFILE
|
||||
if update_documents:
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
if delete_documents:
|
||||
documents_to_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_delete
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_to_delete
|
||||
|
||||
|
||||
def get_manifest(provider, path):
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
return read_yaml_md(data)
|
||||
Reference in new issue
Block a user