[Sync] + new Person sync
This commit is contained in:
Roman Timashev committed
1 parent
1da289802c
commit
953ef0bc28
8 files changed
+209
-169
No files matched your search
@@ -88,14 +88,6 @@ event_image_sizes = [
|
||||
('big', 1200, 800)
|
||||
]
|
||||
|
||||
teacher_image_url_base = 'https://static.shlisselburg.org/art/images/teacher-{id}-{size}{ext}'
|
||||
teacher_image_output = '/Users/tmshv/Desktop/Hudozka Static/images'
|
||||
teacher_image_sizes = [
|
||||
('big', 1200, 800),
|
||||
('medium', 640, 480),
|
||||
('preview', 50, 50),
|
||||
]
|
||||
|
||||
album_image_sizes = [
|
||||
('big', 1200, 800),
|
||||
('medium', 640, 480),
|
||||
@@ -156,6 +148,8 @@ image_url_upload = 'https://static.shlisselburg.org/upload/art/images/'
|
||||
image_url_base = 'https://static.shlisselburg.org/art/images/'
|
||||
image_name_format = '{type}-{id}-{img}-{size}{ext}'
|
||||
|
||||
person_image_url_template = 'https://static.shlisselburg.org/art/images/person-{id}-{size}{ext}'
|
||||
|
||||
yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN')
|
||||
|
||||
update_enabled = value('update_enabled')
|
||||
|
||||
+7
-1
@@ -3,6 +3,7 @@ from db import collection
|
||||
from sync.core import Sync
|
||||
from sync.core.SyncDocument import SyncDocument
|
||||
from sync.core.SyncPage import SyncPage
|
||||
from sync.core.SyncPerson import SyncPerson
|
||||
from sync.data.fs import FSProvider
|
||||
from sync.data.yandexdisk import YDProvider
|
||||
|
||||
@@ -51,17 +52,22 @@ async def run(run_interval=0):
|
||||
collection=c(settings.collection_pages),
|
||||
)
|
||||
|
||||
persons = SyncPerson(
|
||||
provider=io(settings.dir_collective),
|
||||
collection=c(settings.collection_collective),
|
||||
)
|
||||
|
||||
while True:
|
||||
# main(io(settings.dir_documents), sync_documents, settings.collection_documents)
|
||||
# main(io(settings.dir_awards), sync_documents, settings.collection_awards)
|
||||
# main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
|
||||
# main(io(settings.dir_articles), sync_posts, settings.collection_articles)
|
||||
# main(io(settings.dir_collective), sync_persons, settings.collection_collective)
|
||||
# main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
|
||||
|
||||
await asyncio.wait([
|
||||
main(documents),
|
||||
main(pages),
|
||||
main(persons),
|
||||
])
|
||||
|
||||
if run_interval == 0:
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Person import Person
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
|
||||
|
||||
class SyncPerson(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Person.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build(
|
||||
sizes=settings.image_sizes,
|
||||
url_factory=get_image_url_fn(document.id),
|
||||
)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
|
||||
|
||||
def get_image_url_fn(page_id: str):
|
||||
image_id = lambda filename: url_encode_text(
|
||||
os.path.splitext(os.path.basename(filename))[0]
|
||||
)
|
||||
|
||||
def img_url_fn(file, size, ext):
|
||||
return settings.page_url_base.format(
|
||||
page=page_id,
|
||||
id=image_id(file),
|
||||
size=size,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
return img_url_fn
|
||||
@@ -1,39 +0,0 @@
|
||||
import settings
|
||||
from sync.core import Sync
|
||||
from utils.hash import hash_str
|
||||
from utils.image import create_image
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
url_base = settings.teacher_image_url_base
|
||||
sizes = settings.teacher_image_sizes
|
||||
|
||||
|
||||
class SyncPerson(Sync):
|
||||
def __init__(self, collection, provider, dir_local_images):
|
||||
super().__init__()
|
||||
self.collection = collection
|
||||
self.provider = provider
|
||||
self.dir_local_images = dir_local_images
|
||||
|
||||
def compile(self, profile):
|
||||
image_path = self.provider.get_abs(profile['picture'])
|
||||
profile['picture'] = create_image(
|
||||
image_path,
|
||||
sizes,
|
||||
lambda size, ext: url_base.format(id=profile['id'], size=size, ext=ext),
|
||||
self.dir_local_images
|
||||
)
|
||||
return profile
|
||||
|
||||
def create_id(self, document):
|
||||
if 'id' in document and document['id']:
|
||||
return document
|
||||
|
||||
document['id'] = url_encode_text(document['name'])
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
document['hash'] = hash_str(
|
||||
self.provider.hash(document['file']) + self.provider.hash(document['picture'])
|
||||
)
|
||||
return document
|
||||
@@ -0,0 +1,109 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import Provider
|
||||
from sync.models import Model
|
||||
from sync.models.Image import Image
|
||||
|
||||
from utils.fn import swap_ext
|
||||
from utils.hash import hash_str
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
store = collection(settings.collection_collective)
|
||||
|
||||
|
||||
def find(item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Person(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
documents = provider.type_filter('', '.md')
|
||||
documents = [Person.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
params = get_manifest(provider, path)
|
||||
|
||||
return Person(
|
||||
provider,
|
||||
path,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
self.__hash_keys = ['id', 'url']
|
||||
|
||||
self.__id_template: str = '{category}-{file}'
|
||||
self.__url_template: str = settings.document_url_template
|
||||
self.__url_preview_template: str = settings.document_url_preview_template
|
||||
|
||||
self.params = params if params else {}
|
||||
self.picture = None
|
||||
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
async def build(self, **kwargs):
|
||||
sizes = kwargs['sizes']
|
||||
image_path = self.provider.get_abs(self.params['picture'])
|
||||
|
||||
self.picture = await Image.new(self.provider, image_path, sizes, url_factory(self.id))
|
||||
|
||||
def bake(self):
|
||||
return {
|
||||
**self.params,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
'picture': self.picture.id,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
if 'id' in self.params:
|
||||
self.id = self.params['id']
|
||||
else:
|
||||
self.id = url_encode_text(self.params['name'])
|
||||
|
||||
def __set_hash(self):
|
||||
self.hash = hash_str(
|
||||
self.provider.hash(self.file) + self.provider.hash(self.params['picture'])
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return '<Person hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def get_manifest(provider: Provider, path: str) -> dict:
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
|
||||
manifest, body = read_yaml_md(data)
|
||||
manifest = {
|
||||
**manifest,
|
||||
'file': path,
|
||||
'biography': body,
|
||||
'picture': manifest['picture'] if 'picture' in manifest else swap_ext('.jpg')(path),
|
||||
}
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
def url_factory(id):
|
||||
def url(file, size, ext):
|
||||
return settings.person_image_url_template.format(id=id, size=size, ext=ext)
|
||||
|
||||
return url
|
||||
@@ -1,3 +1,6 @@
|
||||
import os
|
||||
|
||||
|
||||
def find(store, item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
@@ -40,5 +43,16 @@ class Model:
|
||||
|
||||
return not (self.hash == item['hash'])
|
||||
|
||||
def _filename(self):
|
||||
return os.path.basename(self.file)
|
||||
|
||||
async def save(self):
|
||||
document = self.bake()
|
||||
|
||||
query = {'id': document['id']}
|
||||
try:
|
||||
self.store.update_one(query, {'$set': document}, upsert=True)
|
||||
return self
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
@@ -1,119 +0,0 @@
|
||||
import os
|
||||
from glob import glob
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core.person import SyncPerson
|
||||
from sync.image import sync_image
|
||||
from utils.fn import lmap, lprint, key_mapper, lprint_json, ext, lmapfn
|
||||
from utils.io import read_yaml_md
|
||||
|
||||
url_base = settings.teacher_image_url_base
|
||||
dir_local_images = settings.teacher_image_output
|
||||
sizes = settings.teacher_image_sizes
|
||||
|
||||
|
||||
def sync_persons(provider, collection, update=True, delete=True):
|
||||
"""
|
||||
:param collection:
|
||||
:param provider:
|
||||
:param update:
|
||||
:param delete:
|
||||
:return:
|
||||
"""
|
||||
|
||||
return main(
|
||||
SyncPerson(
|
||||
collection,
|
||||
provider,
|
||||
dir_local_images
|
||||
),
|
||||
update_documents=update,
|
||||
delete_documents=delete
|
||||
)
|
||||
|
||||
|
||||
def make_person(provider, file):
|
||||
pass
|
||||
|
||||
|
||||
def main(sync, update_documents=False, delete_documents=False):
|
||||
# GET TEACHER YAML/MD MANIFEST FILES
|
||||
documents = sync.provider.type_filter('', '.md')
|
||||
|
||||
# READ TEACHER YAML/MD MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: (i,) + get_manifest(sync.provider, i),
|
||||
documents
|
||||
)
|
||||
|
||||
# COMBINE MD DATA WITH YAML MANIFEST AS BIOGRAPHY
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i[1],
|
||||
'file': i[0],
|
||||
'biography': i[2]
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# DEFINE PROFILE PICTURE
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
'picture': i['picture'] if 'picture' in i else ext('.jpg')(i['file']),
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmapfn(documents)(sync.create_id)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILES
|
||||
documents = lmapfn(documents)(sync.create_hash)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# DO HEAVY PROCESS WITH DOCUMENTS
|
||||
documents = lmap(
|
||||
sync.compile,
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE PICTURE_OBJECT WITH IT _ID IN MONGODB
|
||||
sync_picture = lambda i: None if i is None else sync_image(i)['_id']
|
||||
documents = lmap(
|
||||
key_mapper('picture', sync_picture),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC TEACHER_PROFILE
|
||||
if update_documents:
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_delete = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
if delete_documents:
|
||||
documents_to_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_delete
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_to_delete
|
||||
|
||||
|
||||
def get_manifest(provider, path):
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
return read_yaml_md(data)
|
||||
@@ -49,8 +49,8 @@ lprint_json = lambda i: lprint(lmap(to_json, i))
|
||||
first = lambda i: i[0] if len(i) > 0 else None
|
||||
|
||||
|
||||
def ext(ext):
|
||||
return lambda i: os.path.splitext(i)[0] + ext
|
||||
def swap_ext(extension: str):
|
||||
return lambda i: os.path.splitext(i)[0] + extension
|
||||
|
||||
|
||||
def map_item_key(item, key, map_fn):
|
||||
|
||||
Reference in new issue
Block a user