diff --git a/modules/sync/kazimir/Token.py b/modules/sync/kazimir/Token.py index dcdc2963..aa5612ff 100644 --- a/modules/sync/kazimir/Token.py +++ b/modules/sync/kazimir/Token.py @@ -121,10 +121,12 @@ class ImageToken(Token): return data async def compile(self): + s = 1500 # ('big', 1500, 667) img = await self.get_data() alt = img['alt'] caption = markdown_to_html(img['caption']) src = img['src'] + src = f'https://images.weserv.nl/?url=${src}&w=${s}&h=${s}' tpl = f'''
diff --git a/modules/sync/sync.py b/modules/sync/sync.py index 0e14cdde..d9c230fb 100644 --- a/modules/sync/sync.py +++ b/modules/sync/sync.py @@ -6,7 +6,7 @@ from sync.data.yandexdisk import YDProvider import asyncio import logging -from sync.models.Page import Page +from sync.models.Page import Page, PageController def init_logger(name: str, file: str): @@ -37,11 +37,12 @@ async def run_sync(sync: Sync, **kwargs): async def run(run_interval=0): - io = lambda root: get_provider(settings.provider_name, root) + provider = get_provider(settings.provider_name, settings.provider_root) + controller = PageController(provider) sync_pages = Sync( - provider=io(settings.provider_root), - model=Page, + ctrl=controller, + name='Page' ) sync_pages.set_options(True, True, True) diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py index 4886435d..ab8dc7fc 100644 --- a/modules/sync/sync/__init__.py +++ b/modules/sync/sync/__init__.py @@ -98,22 +98,21 @@ async def create_post(provider: Provider, folder: str, md: str, sizes): } async def build_image(data): - src = data['file'] - relative_image_path = os.path.join(folder, src) - - image_path = provider.get_local(relative_image_path) + filename = data['file'] + relative_path = os.path.join(folder, filename) + image_path = provider.get_local(relative_path) if os.path.exists(image_path): - image = await Image.new(provider, relative_image_path, sizes) + image = await Image.new(provider, relative_path, sizes) if image: - url = image.get_size('big')['url'] images.append(image) + src = image.get_src() return { - 'src': url, + 'src': src, 'alt': data['caption'], 'caption': data['caption'], } - raise Exception('Fail to get Image', relative_image_path) + raise Exception('Fail to get Image', relative_path) async def read_file(data): filepath = data diff --git a/modules/sync/sync/core/__init__.py b/modules/sync/sync/core/__init__.py index 03be9576..854dbfc2 100644 --- a/modules/sync/sync/core/__init__.py +++ b/modules/sync/sync/core/__init__.py @@ -1,12 +1,10 @@ import logging - import settings from sync import untouched -from sync.data import Provider class Sync: - def __init__(self, provider: Provider, model): + def __init__(self, ctrl, name: str): super().__init__() self.validate_urls = True @@ -16,11 +14,9 @@ class Sync: } self.__delete_query = lambda item: {'_id': item['_id']} - name = model.__name__ self.logger = logging.getLogger('%s.%s' % (settings.name, name)) - self.provider = provider - self.model = model + self.ctrl = ctrl self.filter_item_fn = lambda x: True self.skip_unchanged = True @@ -37,9 +33,6 @@ class Sync: self.filter_item_fn = fn return self - async def clean(self): - pass - async def build(self, item): args = self._get_build_args() await item.build(**args) @@ -83,7 +76,7 @@ class Sync: """ self.logger.info('Checking for update') - items = await self.model.scan(self.provider) + items = await self.ctrl.get_items() self.logger.info('Found {} Item(s)'.format(len(items))) all_items = items @@ -116,14 +109,10 @@ class Sync: # DELETING if self.delete_enabled: - obsolete_items = await self.model.find(self.__items_to_delete_query(all_items)) - obsolete_items = list(obsolete_items) - + obsolete_items = await self.ctrl.find(self.__items_to_delete_query(all_items)) for item in obsolete_items: self.logger.info('Deleting Item {}'.format(item['id'])) - await self.model.delete(self.__delete_query(item)) - - await self.clean() + await self.ctrl.delete_item(self.__delete_query(item)) def check_urls_uniqueness(self, items): urls = [x.url for x in items] @@ -132,6 +121,3 @@ class Sync: if len(urls) != len(unique_urls): self.logger.error('URLs is not unique') raise Exception('URLs is not unique') - - async def upload(self): - pass diff --git a/modules/sync/sync/models/Image.py b/modules/sync/sync/models/Image.py index c4954474..1915f44c 100644 --- a/modules/sync/sync/models/Image.py +++ b/modules/sync/sync/models/Image.py @@ -129,6 +129,9 @@ class Image(Model): return self.data[size] return None + def get_src(self): + return self.get_size('original')['url'] + def _get_image_file(self): return self._get_abs(self.file) diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py index 4b72439c..aa2b22e6 100644 --- a/modules/sync/sync/models/Page.py +++ b/modules/sync/sync/models/Page.py @@ -1,20 +1,16 @@ import os - import logging - from lxml import etree import lxml.html - import kazimir import settings from db import collection from sync.data import request from sync.models import Model - from sync import create_date, title_from_html, create_post from sync.models.Image import Image from utils.hash import hash_str -from utils.io import parse_yaml_front_matter +from utils.front_matter import parse_yaml_front_matter from utils.text.transform import url_encode_text logger = logging.getLogger(settings.name + '.Page') @@ -30,65 +26,77 @@ def find(store, item_id: str): return None -class Page(Model): - @staticmethod - async def find(query): +class PageController: + def __init__(self, provider): + self.provider = provider + + async def find(self, query): return store.find(query) - @staticmethod - async def delete(query): + async def delete_item(self, query): return store.find_one_and_delete(query) - @staticmethod - async def scan(provider): + async def get_items(self): scan_path = settings.dir_pages - items = [ - Page.read(provider, path) - for path in provider.scan(scan_path) - if provider.is_dir(path) - ] - items = [i for i in items if i] - + items = [] + for path in self.provider.scan(scan_path): + if not self.provider.is_dir(path): + continue + item = await self.create_item(path) + if not item: + continue + items.append(item) return items - @staticmethod - def read(provider, path): + async def create_item(self, path: str): """ - :param provider: :param path: path to page manifest file. For example: /Pages//.md :return: """ dir_name = os.path.basename(path) - - manifest_file = lambda ext: os.path.join(path, dir_name + ext) - manifest_path = manifest_file('.md') - - if not provider.exists(manifest_path): + manifest_path = os.path.join(path, f'{dir_name}.md') + if not self.provider.exists(manifest_path): logger.warning(f'Manifest not found on path {manifest_path}') return None - params = { - 'file': os.path.basename(manifest_path), - 'folder': os.path.dirname(manifest_path), - 'title': os.path.basename(manifest_path), - } - - manifest = get_manifest(provider, manifest_path) - params = { - **params, - **manifest, - } - - if 'url' not in params: - logger.warning('URL not specified for Page {}'.format(manifest_path)) + manifest = await self.get_manifest(manifest_path) + if not manifest: + logger.warning(f'Cannot define manifest. Skip {manifest_path}') return None return Page( - provider, + self.provider, manifest_path, - params=params, + params=manifest, ) + async def get_manifest(self, path: str): + data = self.provider.read(path).read().decode('utf-8') + manifest = parse_yaml_front_matter(data) + + if 'url' not in manifest: + logger.warning(f'URL not specified for Page {path}') + return None + + if 'date' in manifest: + manifest['date'] = create_date( + manifest['date'], settings.date_formats) + + if 'until' in manifest: + manifest['until'] = create_date( + manifest['until'], settings.date_formats) + + if 'id' in manifest: + manifest['id'] = str(manifest['id']) + + return { + **manifest, + 'file': os.path.basename(path), + 'folder': os.path.dirname(path), + } + + +class Page(Model): def __init__(self, provider, file, params=None): self.__hash_salt = settings.hash_salt_pages self.__hash_keys = ['id', 'url'] @@ -183,19 +191,3 @@ class Page(Model): def __str__(self): return ''.format(self.file, self.id) - - -def get_manifest(provider, path): - data = provider.read(path).read().decode('utf-8') - manifest = parse_yaml_front_matter(data) - - if 'date' in manifest: - manifest['date'] = create_date(manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date(manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - return manifest diff --git a/modules/sync/utils/io.py b/modules/sync/utils/front_matter.py similarity index 78% rename from modules/sync/utils/io.py rename to modules/sync/utils/front_matter.py index 0c94a6cd..6d3977bf 100644 --- a/modules/sync/utils/io.py +++ b/modules/sync/utils/front_matter.py @@ -1,12 +1,9 @@ import re import yaml - FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE) - - def parse_yaml_front_matter(data: str): fm, content = split_yaml_front_matter(data) fm = fm if fm else {} @@ -16,7 +13,7 @@ def parse_yaml_front_matter(data: str): } -def split_yaml_front_matter(text, **defaults): +def split_yaml_front_matter(text: str): """ Parse text with YAML frontmatter, return metadata and content. Pass in optional metadata defaults as keyword args. @@ -25,19 +22,15 @@ def split_yaml_front_matter(text, **defaults): and original text content. """ - # metadata starts with defaults - metadata = defaults.copy() - # split on the first two triple-dashes try: _, fm, content = FM_BOUNDARY.split(text, 2) except ValueError: # if we can't split, bail - return metadata, text + return None, None # parse yaml, now that we have frontmatter fm = yaml.safe_load(fm) if isinstance(fm, dict): - metadata.update(fm) - - return metadata, content.strip() + return fm, content.strip() + return None, None