[Sync] + new Schedule sync
This commit is contained in:
Roman Timashev committed
1 parent
1a8adaf524
commit
e54c448704
6 files changed
+170
-136
No files matched your search
+7
-1
@@ -5,6 +5,7 @@ from sync.core.SyncArticle import SyncArticle
|
||||
from sync.core.SyncDocument import SyncDocument
|
||||
from sync.core.SyncPage import SyncPage
|
||||
from sync.core.SyncPerson import SyncPerson
|
||||
from sync.core.SyncSchedule import SyncSchedule
|
||||
from sync.data.fs import FSProvider
|
||||
from sync.data.yandexdisk import YDProvider
|
||||
|
||||
@@ -63,8 +64,12 @@ async def run(run_interval=0):
|
||||
collection=c(settings.collection_articles),
|
||||
)
|
||||
|
||||
schedules = SyncSchedule(
|
||||
provider=io(settings.dir_schedules),
|
||||
collection=c(settings.collection_schedules),
|
||||
)
|
||||
|
||||
while True:
|
||||
# main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
|
||||
# main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
|
||||
|
||||
await asyncio.wait([
|
||||
@@ -72,6 +77,7 @@ async def run(run_interval=0):
|
||||
main(pages),
|
||||
main(persons),
|
||||
main(articles),
|
||||
main(schedules),
|
||||
])
|
||||
|
||||
if run_interval == 0:
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.models.Schedule import Schedule
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Schedule')
|
||||
|
||||
|
||||
class SyncSchedule(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Schedule.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build()
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,38 +0,0 @@
|
||||
import os
|
||||
from tempfile import mkstemp
|
||||
|
||||
from sync.core import Sync
|
||||
from utils.fn import last_good, lmap
|
||||
from utils.hash import hash_str, hash_file
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
|
||||
class SyncSchedule(Sync):
|
||||
def __init__(self, collection, provider):
|
||||
super().__init__()
|
||||
self.collection = collection
|
||||
self.provider = provider
|
||||
|
||||
def create_id(self, document):
|
||||
period = get_period(document['period'])
|
||||
doc_id = document['id'] if 'id' in document and document['id'] else None
|
||||
if not doc_id:
|
||||
doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=document['semester'])
|
||||
document['id'] = doc_id
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
return {
|
||||
**document,
|
||||
'hash': self.provider.hash(document['file'])
|
||||
}
|
||||
|
||||
|
||||
def get_period(value):
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
|
||||
if isinstance(value, list):
|
||||
return '-'.join(lmap(str, value))
|
||||
@@ -0,0 +1,109 @@
|
||||
import logging
|
||||
|
||||
from toml import loads as toml_str
|
||||
from yaml import load as yaml
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import Provider, get_data
|
||||
from sync.models import Model
|
||||
|
||||
from utils.fn import constant
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
store = collection(settings.collection_schedules)
|
||||
|
||||
|
||||
def find(item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Schedule(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
documents = provider.type_filter('.', '.yaml')
|
||||
documents += provider.type_filter('.', '.toml')
|
||||
|
||||
documents = [Schedule.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
params = get_manifest(provider, path)
|
||||
|
||||
return Schedule(
|
||||
provider,
|
||||
path,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
self.params = params if params else {}
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
def bake(self):
|
||||
return {
|
||||
**self.params,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None
|
||||
if not doc_id:
|
||||
period = self.__get_period()
|
||||
doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester'])
|
||||
self.id = doc_id
|
||||
|
||||
def __set_hash(self):
|
||||
self.hash = self.provider.hash(self.file)
|
||||
|
||||
def __get_period(self):
|
||||
value = self.params['period']
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
|
||||
if isinstance(value, list):
|
||||
return '-'.join(list(map(str, value)))
|
||||
|
||||
def __str__(self):
|
||||
return '<Schedule hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def combine(fns):
|
||||
def call(value):
|
||||
data = value.read()
|
||||
for f in fns:
|
||||
try:
|
||||
return f(data)
|
||||
except Exception as e:
|
||||
pass
|
||||
return None
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def toml(bytes_value):
|
||||
s = bytes_value.decode('utf-8')
|
||||
return toml_str(s)
|
||||
|
||||
|
||||
def get_manifest(provider: Provider, path: str) -> dict:
|
||||
transform = combine([yaml, toml, constant({})])
|
||||
manifest = get_data(provider, path, transform)
|
||||
|
||||
return {
|
||||
'file': path,
|
||||
**manifest,
|
||||
}
|
||||
@@ -27,6 +27,9 @@ class Model:
|
||||
def init(self):
|
||||
pass
|
||||
|
||||
async def build(self, **kwargs):
|
||||
pass
|
||||
|
||||
def bake(self):
|
||||
return None
|
||||
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
from toml import loads as toml_str
|
||||
from yaml import load as yaml
|
||||
|
||||
from sync import untouched
|
||||
from sync.core.schedule import SyncSchedule
|
||||
from sync.data import get_data
|
||||
from utils.fn import lmap, constant
|
||||
|
||||
dir_documents = '/Users/tmshv/Dropbox/Dev/Hud school/Schedules'
|
||||
|
||||
|
||||
def sync_schedules(provider, collection, update=True, delete=True):
|
||||
"""
|
||||
:param collection:
|
||||
:param provider:
|
||||
:param update:
|
||||
:param delete:
|
||||
:return:
|
||||
"""
|
||||
|
||||
return main(
|
||||
SyncSchedule(
|
||||
collection,
|
||||
provider
|
||||
),
|
||||
update_documents=update,
|
||||
delete_documents=delete
|
||||
)
|
||||
|
||||
|
||||
def combine(fns):
|
||||
def call(value):
|
||||
data = value.read()
|
||||
for f in fns:
|
||||
try:
|
||||
return f(data)
|
||||
except Exception as e:
|
||||
pass
|
||||
return None
|
||||
return call
|
||||
|
||||
|
||||
def toml(bytes_value):
|
||||
s = bytes_value.decode('utf-8')
|
||||
return toml_str(s)
|
||||
|
||||
|
||||
def main(sync, update_documents=False, delete_documents=False):
|
||||
# COLLECT YAML/MD/JPG/PDF MANIFEST FILES
|
||||
documents = sync.provider.type_filter('.', '.yaml')
|
||||
documents += sync.provider.type_filter('.', '.toml')
|
||||
|
||||
# READ MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
'file': i,
|
||||
**get_data(sync.provider, i, combine([yaml, toml, constant({})]))
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE DOCUMENT HASH
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
# ALL DOCUMENTS IDS FOUNDED
|
||||
scope_documents_ids = lmap(
|
||||
lambda i: i['id'],
|
||||
documents
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# DO HEAVY PROCESS WITH DOCUMENTS
|
||||
|
||||
# SYNC SCHEDULE
|
||||
if update_documents:
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_remove = sync.query({'id': {'$nin': scope_documents_ids}})
|
||||
if delete_documents:
|
||||
documents_to_remove = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_remove
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_to_remove
|
||||
Reference in new issue
Block a user