From d1fc26fcf345a58e00267122a5a72d281a76f27a Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Thu, 27 Feb 2020 17:51:21 +0300 Subject: [PATCH] rm schedule, settings and other from sync --- modules/sync/settings.py | 6 -- modules/sync/sync.py | 28 ------- modules/sync/sync/models/Schedule.py | 114 --------------------------- modules/sync/sync/models/Settings.py | 64 --------------- modules/sync/sync/models/product.py | 27 ------- 5 files changed, 239 deletions(-) delete mode 100644 modules/sync/sync/models/Schedule.py delete mode 100644 modules/sync/sync/models/Settings.py delete mode 100644 modules/sync/sync/models/product.py diff --git a/modules/sync/settings.py b/modules/sync/settings.py index 304c405e..4317e51c 100644 --- a/modules/sync/settings.py +++ b/modules/sync/settings.py @@ -76,29 +76,23 @@ date_formats = [ origin = value('origin') -collection_settings = 'settings' collection_images = 'images' collection_documents = 'documents' -collection_schedules = 'schedules' collection_articles = 'articles' collection_pages = 'pages' hash_salt_articles = '6' hash_salt_pages = '4' hash_salt_documents = '4' -hash_salt_images = '5' provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs') provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') f = abs_fn(provider_root) -dir_settings = './Settings' dir_documents = './Documents' -dir_schedules = './Schedules' dir_articles = './Articles' dir_pages = './Pages' -dir_images = './Images' document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}' document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' diff --git a/modules/sync/sync.py b/modules/sync/sync.py index 9f7cdef5..b15dc929 100644 --- a/modules/sync/sync.py +++ b/modules/sync/sync.py @@ -7,11 +7,8 @@ from sync.data.yandexdisk import YDProvider import asyncio import logging -from sync.models.Image import Image from sync.models.Article import Article from sync.models.Page import Page -from sync.models.Schedule import Schedule -from sync.models.Settings import Settings def init_logger(name: str, file: str): @@ -63,36 +60,11 @@ async def run(run_interval=0): sync_articles.set_options(True, True, True) sync_articles.validate_urls = False - sync_schedules = Sync( - provider=io(settings.provider_root), - model=Schedule, - ) - sync_schedules.set_options(True, True, True) - sync_schedules.validate_urls = False - - sync_settings = Sync( - provider=io(settings.provider_root), - model=Settings, - ) - sync_settings.set_options(True, True, True) - sync_settings.validate_urls = False - - sync_images = Sync( - provider=io(settings.provider_root), - model=Image, - ) - sync_images.set_options(True, True, True) - sync_images.strict_origin = True - sync_images.validate_urls = False - while True: await asyncio.wait([ run_sync(sync_documents), run_sync(sync_pages), run_sync(sync_articles), - run_sync(sync_schedules), - run_sync(sync_settings), - run_sync(sync_images), ]) if run_interval == 0: diff --git a/modules/sync/sync/models/Schedule.py b/modules/sync/sync/models/Schedule.py deleted file mode 100644 index a37a6889..00000000 --- a/modules/sync/sync/models/Schedule.py +++ /dev/null @@ -1,114 +0,0 @@ -from toml import loads as toml_str -from yaml import load as yaml - -import settings -from db import collection -from sync.data import Provider, get_data -from sync.models import Model - -from utils.fn import constant - -store = collection(settings.collection_schedules) - - -def find(item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None - - -class Schedule(Model): - @staticmethod - async def find(query): - return store.find(query) - - @staticmethod - async def delete(query): - return store.find_one_and_delete(query) - - @staticmethod - async def scan(provider): - scan_dir = settings.dir_schedules - documents = provider.type_filter(scan_dir, '.yaml') - documents += provider.type_filter(scan_dir, '.toml') - - documents = [Schedule.read(provider, i) for i in documents] - return documents - - @staticmethod - def read(provider, path): - params = get_manifest(provider, path) - - return Schedule( - provider, - path, - params=params, - ) - - def __init__(self, provider, file, params=None): - super().__init__(provider, store, file, params=params) - - def init(self): - pass - self.__set_id() - self.__set_hash() - - def bake(self): - return { - **self.params, - 'id': self.id, - 'hash': self.hash, - 'file': self.file, - } - - def __set_id(self): - doc_id = self.params['id'] if 'id' in self.params and self.params['id'] else None - if not doc_id: - period = self.__get_period() - doc_id = 'schedule-{period}-{semester}'.format(period=period, semester=self.params['semester']) - self.id = doc_id - - def __set_hash(self): - self.hash = self.provider.hash(self.file) - - def __get_period(self): - value = self.params['period'] - if isinstance(value, str): - return value - - if isinstance(value, list): - return '-'.join(list(map(str, value))) - - def __str__(self): - return ''.format(self.hash, self.file, self.id) - - -def combine(fns): - def call(value): - data = value.read() - for f in fns: - try: - return f(data) - except Exception as e: - pass - return None - - return call - - -def toml(bytes_value): - s = bytes_value.decode('utf-8') - return toml_str(s) - - -def get_manifest(provider: Provider, path: str) -> dict: - transform = combine([yaml, toml, constant({})]) - manifest = get_data(provider, path, transform) - - return { - 'file': path, - **manifest, - } diff --git a/modules/sync/sync/models/Settings.py b/modules/sync/sync/models/Settings.py deleted file mode 100644 index b7323c22..00000000 --- a/modules/sync/sync/models/Settings.py +++ /dev/null @@ -1,64 +0,0 @@ -from toml import loads as toml_str -from yaml import load as yaml - -import settings -from db import collection -from sync.data import Provider, get_data -from sync.models import Model - -from utils.fn import constant - -store = collection(settings.collection_settings) - - -class Settings(Model): - @staticmethod - async def find(query): - return store.find(query) - - @staticmethod - async def delete(query): - return store.find_one_and_delete(query) - - @staticmethod - async def scan(provider): - scan_dir = settings.dir_settings - documents = provider.type_filter(scan_dir, '.yaml') - - documents = [Settings.read(provider, i) for i in documents] - return documents - - @staticmethod - def read(provider, path): - params = yaml(provider.read(path)) - - return Settings( - provider, - path, - params=params, - ) - - def __init__(self, provider, file, params=None): - super().__init__(provider, store, file, params=params) - - def init(self): - pass - self.__set_id() - self.__set_hash() - - def bake(self): - return { - **self.params, - 'id': self.id, - 'hash': self.hash, - 'file': self.file, - } - - def __set_id(self): - self.id = 'settings' - - def __set_hash(self): - self.hash = self.provider.hash(self.file) - - def __str__(self): - return f'' diff --git a/modules/sync/sync/models/product.py b/modules/sync/sync/models/product.py deleted file mode 100644 index 0eb373fd..00000000 --- a/modules/sync/sync/models/product.py +++ /dev/null @@ -1,27 +0,0 @@ -import settings -from db import collection -from utils.text.transform import interpolate_swift - - -def resolve_value(param, store): - """ - - :param param: \(nv-andreeva) - :param store: function should return value by key nv-andreeva - :return: - """ - i = interpolate_swift(param) - return store(i) - - - -def create_product(doc): - # author, image, title, awards, teacher, date - - # unpop_image = key_mapper('image', lambda i: sync_image(i)['_id']) - # unpop_teacher = key_mapper('teacher', lambda i: i['_id']) - - product = {**doc} - # unpop_image(product) - - return product