tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

Merge pull request #41 from tmshv/fix/sync-update

Browse Source 
Sync update
This commit is contained in:
tmshv authored and GitHub committed 2020-03-10 18:34:01 +03:00
2 parents 96040267bb ffd8396fa0
commit 6c6c9f91ca
7 files changed
+77 -101
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

modules/sync/kazimir/Token.py
+2
View File
@@ -121,10 +121,12 @@ class ImageToken(Token):
return data
async def compile(self):
s = 1500 # ('big', 1500, 667)
img = await self.get_data()
alt = img['alt']
caption = markdown_to_html(img['caption'])
src = img['src']
src = f'https://images.weserv.nl/?url=${src}&w=${s}&h=${s}'
tpl = f'''
<div class="kazimir__image">
modules/sync/sync.py
+5 -4
View File
@@ -6,7 +6,7 @@ from sync.data.yandexdisk import YDProvider
import asyncio
import logging
from sync.models.Page import Page
from sync.models.Page import Page, PageController
def init_logger(name: str, file: str):
@@ -37,11 +37,12 @@ async def run_sync(sync: Sync, **kwargs):
async def run(run_interval=0):
io = lambda root: get_provider(settings.provider_name, root)
provider = get_provider(settings.provider_name, settings.provider_root)
controller = PageController(provider)
sync_pages = Sync(
provider=io(settings.provider_root),
model=Page,
ctrl=controller,
name='Page'
)
sync_pages.set_options(True, True, True)
modules/sync/sync/__init__.py
+7 -8
View File
@@ -98,22 +98,21 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
}
async def build_image(data):
src = data['file']
relative_image_path = os.path.join(folder, src)
image_path = provider.get_local(relative_image_path)
filename = data['file']
relative_path = os.path.join(folder, filename)
image_path = provider.get_local(relative_path)
if os.path.exists(image_path):
image = await Image.new(provider, relative_image_path, sizes)
image = await Image.new(provider, relative_path, sizes)
if image:
url = image.get_size('big')['url']
images.append(image)
src = image.get_src()
return {
'src': url,
'src': src,
'alt': data['caption'],
'caption': data['caption'],
}
raise Exception('Fail to get Image', relative_image_path)
raise Exception('Fail to get Image', relative_path)
async def read_file(data):
filepath = data
modules/sync/sync/core/__init__.py
+5 -19
View File
@@ -1,12 +1,10 @@
import logging
import settings
from sync import untouched
from sync.data import Provider
class Sync:
def __init__(self, provider: Provider, model):
def __init__(self, ctrl, name: str):
super().__init__()
self.validate_urls = True
@@ -16,11 +14,9 @@ class Sync:
}
self.__delete_query = lambda item: {'_id': item['_id']}
name = model.__name__
self.logger = logging.getLogger('%s.%s' % (settings.name, name))
self.provider = provider
self.model = model
self.ctrl = ctrl
self.filter_item_fn = lambda x: True
self.skip_unchanged = True
@@ -37,9 +33,6 @@ class Sync:
self.filter_item_fn = fn
return self
async def clean(self):
pass
async def build(self, item):
args = self._get_build_args()
await item.build(**args)
@@ -83,7 +76,7 @@ class Sync:
"""
self.logger.info('Checking for update')
items = await self.model.scan(self.provider)
items = await self.ctrl.get_items()
self.logger.info('Found {} Item(s)'.format(len(items)))
all_items = items
@@ -116,14 +109,10 @@ class Sync:
# DELETING
if self.delete_enabled:
obsolete_items = await self.model.find(self.__items_to_delete_query(all_items))
obsolete_items = list(obsolete_items)
obsolete_items = await self.ctrl.find(self.__items_to_delete_query(all_items))
for item in obsolete_items:
self.logger.info('Deleting Item {}'.format(item['id']))
await self.model.delete(self.__delete_query(item))
await self.clean()
await self.ctrl.delete_item(self.__delete_query(item))
def check_urls_uniqueness(self, items):
urls = [x.url for x in items]
@@ -132,6 +121,3 @@ class Sync:
if len(urls) != len(unique_urls):
self.logger.error('URLs is not unique')
raise Exception('URLs is not unique')
async def upload(self):
pass
modules/sync/sync/models/Image.py
+3
View File
@@ -129,6 +129,9 @@ class Image(Model):
return self.data[size]
return None
def get_src(self):
return self.get_size('original')['url']
def _get_image_file(self):
return self._get_abs(self.file)
modules/sync/sync/models/Page.py
+51 -59
View File
@@ -1,20 +1,16 @@
import os
import logging
from lxml import etree
import lxml.html
import kazimir
import settings
from db import collection
from sync.data import request
from sync.models import Model
from sync import create_date, title_from_html, create_post
from sync.models.Image import Image
from utils.hash import hash_str
from utils.io import parse_yaml_front_matter
from utils.front_matter import parse_yaml_front_matter
from utils.text.transform import url_encode_text
logger = logging.getLogger(settings.name + '.Page')
@@ -30,65 +26,77 @@ def find(store, item_id: str):
return None
class Page(Model):
@staticmethod
async def find(query):
class PageController:
def __init__(self, provider):
self.provider = provider
async def find(self, query):
return store.find(query)
@staticmethod
async def delete(query):
async def delete_item(self, query):
return store.find_one_and_delete(query)
@staticmethod
async def scan(provider):
async def get_items(self):
scan_path = settings.dir_pages
items = [
Page.read(provider, path)
for path in provider.scan(scan_path)
if provider.is_dir(path)
]
items = [i for i in items if i]
items = []
for path in self.provider.scan(scan_path):
if not self.provider.is_dir(path):
continue
item = await self.create_item(path)
if not item:
continue
items.append(item)
return items
@staticmethod
def read(provider, path):
async def create_item(self, path: str):
"""
:param provider:
:param path: path to page manifest file. For example: /Pages/<Some_Page>/<Some_Page>.md
:return:
"""
dir_name = os.path.basename(path)
manifest_file = lambda ext: os.path.join(path, dir_name + ext)
manifest_path = manifest_file('.md')
if not provider.exists(manifest_path):
manifest_path = os.path.join(path, f'{dir_name}.md')
if not self.provider.exists(manifest_path):
logger.warning(f'Manifest not found on path {manifest_path}')
return None
params = {
'file': os.path.basename(manifest_path),
'folder': os.path.dirname(manifest_path),
'title': os.path.basename(manifest_path),
}
manifest = get_manifest(provider, manifest_path)
params = {
**params,
**manifest,
}
if 'url' not in params:
logger.warning('URL not specified for Page {}'.format(manifest_path))
manifest = await self.get_manifest(manifest_path)
if not manifest:
logger.warning(f'Cannot define manifest. Skip {manifest_path}')
return None
return Page(
provider,
self.provider,
manifest_path,
params=params,
params=manifest,
)
async def get_manifest(self, path: str):
data = self.provider.read(path).read().decode('utf-8')
manifest = parse_yaml_front_matter(data)
if 'url' not in manifest:
logger.warning(f'URL not specified for Page {path}')
return None
if 'date' in manifest:
manifest['date'] = create_date(
manifest['date'], settings.date_formats)
if 'until' in manifest:
manifest['until'] = create_date(
manifest['until'], settings.date_formats)
if 'id' in manifest:
manifest['id'] = str(manifest['id'])
return {
**manifest,
'file': os.path.basename(path),
'folder': os.path.dirname(path),
}
class Page(Model):
def __init__(self, provider, file, params=None):
self.__hash_salt = settings.hash_salt_pages
self.__hash_keys = ['id', 'url']
@@ -183,19 +191,3 @@ class Page(Model):
def __str__(self):
return '<Page file={} id={}>'.format(self.file, self.id)
def get_manifest(provider, path):
data = provider.read(path).read().decode('utf-8')
manifest = parse_yaml_front_matter(data)
if 'date' in manifest:
manifest['date'] = create_date(manifest['date'], settings.date_formats)
if 'until' in manifest:
manifest['until'] = create_date(manifest['until'], settings.date_formats)
if 'id' in manifest:
manifest['id'] = str(manifest['id'])
return manifest
modules/sync/utils/io.py → modules/sync/utils/front_matter.py
+4 -11
View File
@@ -1,12 +1,9 @@
import re
import yaml
FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
def parse_yaml_front_matter(data: str):
fm, content = split_yaml_front_matter(data)
fm = fm if fm else {}
@@ -16,7 +13,7 @@ def parse_yaml_front_matter(data: str):
}
def split_yaml_front_matter(text, **defaults):
def split_yaml_front_matter(text: str):
"""
Parse text with YAML frontmatter, return metadata and content.
Pass in optional metadata defaults as keyword args.
@@ -25,19 +22,15 @@ def split_yaml_front_matter(text, **defaults):
and original text content.
"""
# metadata starts with defaults
metadata = defaults.copy()
# split on the first two triple-dashes
try:
_, fm, content = FM_BOUNDARY.split(text, 2)
except ValueError:
# if we can't split, bail
return metadata, text
return None, None
# parse yaml, now that we have frontmatter
fm = yaml.safe_load(fm)
if isinstance(fm, dict):
metadata.update(fm)
return metadata, content.strip()
return fm, content.strip()
return None, None