rm schedule, settings and other from sync
This commit is contained in:
tmshv committed
1 parent
14a5804991
commit
d1fc26fcf3
5 files changed
-239
No files matched your search
@@ -76,29 +76,23 @@ date_formats = [
|
||||
|
||||
origin = value('origin')
|
||||
|
||||
collection_settings = 'settings'
|
||||
collection_images = 'images'
|
||||
collection_documents = 'documents'
|
||||
collection_schedules = 'schedules'
|
||||
collection_articles = 'articles'
|
||||
collection_pages = 'pages'
|
||||
|
||||
hash_salt_articles = '6'
|
||||
hash_salt_pages = '4'
|
||||
hash_salt_documents = '4'
|
||||
hash_salt_images = '5'
|
||||
|
||||
provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs')
|
||||
provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs')
|
||||
|
||||
f = abs_fn(provider_root)
|
||||
|
||||
dir_settings = './Settings'
|
||||
dir_documents = './Documents'
|
||||
dir_schedules = './Schedules'
|
||||
dir_articles = './Articles'
|
||||
dir_pages = './Pages'
|
||||
dir_images = './Images'
|
||||
|
||||
document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
|
||||
document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
|
||||
|
||||
@@ -7,11 +7,8 @@ from sync.data.yandexdisk import YDProvider
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sync.models.Image import Image
|
||||
from sync.models.Article import Article
|
||||
from sync.models.Page import Page
|
||||
from sync.models.Schedule import Schedule
|
||||
from sync.models.Settings import Settings
|
||||
|
||||
|
||||
def init_logger(name: str, file: str):
|
||||
@@ -63,36 +60,11 @@ async def run(run_interval=0):
|
||||
sync_articles.set_options(True, True, True)
|
||||
sync_articles.validate_urls = False
|
||||
|
||||
sync_schedules = Sync(
|
||||
provider=io(settings.provider_root),
|
||||
model=Schedule,
|
||||
)
|
||||
sync_schedules.set_options(True, True, True)
|
||||
sync_schedules.validate_urls = False
|
||||
|
||||
sync_settings = Sync(
|
||||
provider=io(settings.provider_root),
|
||||
model=Settings,
|
||||
)
|
||||
sync_settings.set_options(True, True, True)
|
||||
sync_settings.validate_urls = False
|
||||
|
||||
sync_images = Sync(
|
||||
provider=io(settings.provider_root),
|
||||
model=Image,
|
||||
)
|
||||
sync_images.set_options(True, True, True)
|
||||
sync_images.strict_origin = True
|
||||
sync_images.validate_urls = False
|
||||
|
||||
while True:
|
||||
await asyncio.wait([
|
||||
run_sync(sync_documents),
|
||||
run_sync(sync_pages),
|
||||
run_sync(sync_articles),
|
||||
run_sync(sync_schedules),
|
||||
run_sync(sync_settings),
|
||||
run_sync(sync_images),
|
||||
])
|
||||
|
||||
if run_interval == 0:
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
from toml import loads as toml_str
|
||||
from yaml import load as yaml
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import Provider, get_data
|
||||
from sync.models import Model
|
||||
|
||||
from utils.fn import constant
|
||||
|
||||
store = collection(settings.collection_schedules)
|
||||
|
||||
|
||||
def find(item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Schedule(Model):
|
||||
@staticmethod
|
||||
async def find(query):
|
||||
return store.find(query)
|
||||
|
||||
@staticmethod
|
||||
async def delete(query):
|
||||
return store.find_one_and_delete(query)
|
||||
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
scan_dir = settings.dir_schedules
|
||||
documents = provider.type_filter(scan_dir, '.yaml')
|
||||
documents += provider.type_filter(scan_dir, '.toml')
|
||||
|
||||
documents = [Schedule.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
params = get_manifest(provider, path)
|
||||
|
||||
return Schedule(
|
||||
provider,
|
||||
path,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
super().__init__(provider, store, file, params=params)
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
def bake(self):
|
||||
return {
|
||||
**self.params,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None
|
||||
if not doc_id:
|
||||
period = self.__get_period()
|
||||
doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester'])
|
||||
self.id = doc_id
|
||||
|
||||
def __set_hash(self):
|
||||
self.hash = self.provider.hash(self.file)
|
||||
|
||||
def __get_period(self):
|
||||
value = self.params['period']
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
|
||||
if isinstance(value, list):
|
||||
return '-'.join(list(map(str, value)))
|
||||
|
||||
def __str__(self):
|
||||
return '<Schedule hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def combine(fns):
|
||||
def call(value):
|
||||
data = value.read()
|
||||
for f in fns:
|
||||
try:
|
||||
return f(data)
|
||||
except Exception as e:
|
||||
pass
|
||||
return None
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def toml(bytes_value):
|
||||
s = bytes_value.decode('utf-8')
|
||||
return toml_str(s)
|
||||
|
||||
|
||||
def get_manifest(provider: Provider, path: str) -> dict:
|
||||
transform = combine([yaml, toml, constant({})])
|
||||
manifest = get_data(provider, path, transform)
|
||||
|
||||
return {
|
||||
'file': path,
|
||||
**manifest,
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
from toml import loads as toml_str
|
||||
from yaml import load as yaml
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import Provider, get_data
|
||||
from sync.models import Model
|
||||
|
||||
from utils.fn import constant
|
||||
|
||||
store = collection(settings.collection_settings)
|
||||
|
||||
|
||||
class Settings(Model):
|
||||
@staticmethod
|
||||
async def find(query):
|
||||
return store.find(query)
|
||||
|
||||
@staticmethod
|
||||
async def delete(query):
|
||||
return store.find_one_and_delete(query)
|
||||
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
scan_dir = settings.dir_settings
|
||||
documents = provider.type_filter(scan_dir, '.yaml')
|
||||
|
||||
documents = [Settings.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
params = yaml(provider.read(path))
|
||||
|
||||
return Settings(
|
||||
provider,
|
||||
path,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
super().__init__(provider, store, file, params=params)
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
def bake(self):
|
||||
return {
|
||||
**self.params,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
self.id = 'settings'
|
||||
|
||||
def __set_hash(self):
|
||||
self.hash = self.provider.hash(self.file)
|
||||
|
||||
def __str__(self):
|
||||
return f'<Settings file={self.file} id={self.id}>'
|
||||
@@ -1,27 +0,0 @@
|
||||
import settings
|
||||
from db import collection
|
||||
from utils.text.transform import interpolate_swift
|
||||
|
||||
|
||||
def resolve_value(param, store):
|
||||
"""
|
||||
|
||||
:param param: \(nv-andreeva)
|
||||
:param store: function should return value by key nv-andreeva
|
||||
:return:
|
||||
"""
|
||||
i = interpolate_swift(param)
|
||||
return store(i)
|
||||
|
||||
|
||||
|
||||
def create_product(doc):
|
||||
# author, image, title, awards, teacher, date
|
||||
|
||||
# unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
|
||||
# unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
|
||||
|
||||
product = {**doc}
|
||||
# unpop_image(product)
|
||||
|
||||
return product
|
||||
Reference in new issue
Block a user