diff --git a/modules/sync/settings.py b/modules/sync/settings.py index 547f70a1..bff13c45 100644 --- a/modules/sync/settings.py +++ b/modules/sync/settings.py @@ -78,10 +78,8 @@ origin = value('origin') collection_images = 'images' collection_documents = 'documents' -collection_articles = 'articles' collection_pages = 'pages' -hash_salt_articles = '8' hash_salt_pages = '4' hash_salt_documents = '4' hash_salt_images = '5' @@ -91,7 +89,6 @@ provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') f = abs_fn(provider_root) -dir_articles = './Articles' dir_pages = './Pages' document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}' diff --git a/modules/sync/sync.py b/modules/sync/sync.py index 5b59c929..0e14cdde 100644 --- a/modules/sync/sync.py +++ b/modules/sync/sync.py @@ -6,7 +6,6 @@ from sync.data.yandexdisk import YDProvider import asyncio import logging -from sync.models.Article import Article from sync.models.Page import Page @@ -46,17 +45,9 @@ async def run(run_interval=0): ) sync_pages.set_options(True, True, True) - sync_articles = Sync( - provider=io(settings.provider_root), - model=Article, - ) - sync_articles.set_options(True, True, True) - sync_articles.validate_urls = False - while True: await asyncio.wait([ run_sync(sync_pages), - run_sync(sync_articles), ]) if run_interval == 0: diff --git a/modules/sync/sync/models/Article.py b/modules/sync/sync/models/Article.py deleted file mode 100644 index 40d003ce..00000000 --- a/modules/sync/sync/models/Article.py +++ /dev/null @@ -1,186 +0,0 @@ -import logging - -import os - -import kazimir -import settings -from db import collection -from markdown import markdown -from sync import create_date_and_title_from_folder_name, create_date, create_post -from sync.data import list_images -from sync.models import Model -from sync.models.Image import Image - -from utils.fn import first, combine -from utils.hash import hash_str -from utils.io import parse_yaml_front_matter -from utils.text.transform import url_encode_text - -logger = logging.getLogger(settings.name + '.Article') -store = collection(settings.collection_articles) - - -class Article(Model): - @staticmethod - async def find(query): - return store.find(query) - - @staticmethod - async def delete(query): - return store.find_one_and_delete(query) - - @staticmethod - async def scan(provider): - scan_dir = settings.dir_articles - documents = [i for i in provider.scan(scan_dir) if provider.is_dir(i)] - documents = [Article.read(provider, i) for i in documents] - return documents - - @staticmethod - def read(provider, path): - file_time_formats = ['%Y.%m.%d'] - - # PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE - doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats) - document = { - 'folder': path, - 'date': doc_date, - 'title': doc_title, - } - - # TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST - manifest_file = md_from_folder(provider, path) - if manifest_file: - manifest = read_manifest(provider, manifest_file) - else: - manifest = {} - - return Article( - provider, - path, - params={ - **document, - **manifest, - }, - ) - - def __init__(self, provider, file, params=None): - self.hash_salt = settings.hash_salt_articles - self.version = '2' - self.post = None - self.preview = None - self.images = None - self.documents = None - self.tokens = None - super().__init__(provider, store, file, params=params) - - def init(self): - if self.has_param('content'): - images, documents = kazimir.extract_files(self.get_param('content')) - else: - images = list_images(self.provider, self.file) - images = [os.path.relpath(i, self.file) for i in images] - documents = [] - self.params['images'] = images - self.params['documents'] = documents - - self.__set_id() - self.__set_hash() - self.origin = settings.origin - - def validate(self): - document = self.params - if 'date' not in document or not document['date']: - return None - - if 'title' not in document or not document['title']: - return None - - async def build(self, **kwargs): - sizes = kwargs['sizes'] - - if self.has_param('content'): - markdown_post = self.get_param('content') - else: - markdown_post = create_post_from_image_list(self.get_param('images')) - - post, images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) - self.post = post - self.images = images - self.documents = documents - self.tokens = tokens - - if self.has_param('preview'): - preview = self.get_param('preview') - preview = self._get_relpath(preview) - self.preview = await Image.new(self.provider, preview, sizes) - - def bake(self): - return { - **self.params, - 'images': [x.ref for x in self.images], - 'documents': [x.ref for x in self.documents], - 'post': self.post, - 'id': self.id, - 'hash': self.hash, - 'file': self.file, - 'origin': self.origin, - 'version': self.version, - 'tokens': self.tokens, - 'preview': self.preview.ref if self.preview else None, - } - - def __set_id(self): - if 'id' in self.params: - self.id = self.params['id'] - else: - self.id = url_encode_text(self.params['title']) - - def __set_hash(self): - files = [] - files += sorted(self.get_param('images')) - files += sorted(self.get_param('documents')) - if self.has_param('preview'): - files.append(self.get_param('preview')) - - folder = self.get_param('folder') - files = [os.path.join(folder, x) for x in files] - files = [self.provider.hash(x) for x in files] - - self.hash = hash_str( - combine([self.hash_salt] + [hash_str(self.params)] + files) - ) - - def __str__(self): - return f'
' - - -def md_from_folder(provider, i): - return first(provider.type_filter(i, '.md')) - - -def read_manifest(provider, path): - data = provider.read(path).read().decode('utf-8') - manifest = parse_yaml_front_matter(data) - - if 'date' in manifest: - manifest['date'] = create_date(manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date(manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - return { - **manifest, - } - - -def create_post_from_image_list(images): - return markdown( - '\n'.join(map( - lambda i: '![]({img})'.format(img=i), - images - )) - )