[Sync] + new Page sync
This commit is contained in:
Roman Timashev committed
1 parent
cd9c790f97
commit
8bf1ea37c8
4 files changed
+287
-227
No files matched your search
+7
-1
@@ -4,6 +4,7 @@ from sync.album import sync_albums
|
||||
from sync.core import Sync
|
||||
from sync.core.SyncDocument import SyncDocument
|
||||
from sync.data import Provider
|
||||
from sync.core.SyncPage import SyncPage
|
||||
from sync.data.fs import FSProvider
|
||||
from sync.data.yandexdisk import YDProvider
|
||||
from sync.document import sync_documents
|
||||
@@ -55,6 +56,11 @@ async def run(run_interval=0):
|
||||
sizes=settings.image_sizes,
|
||||
)
|
||||
|
||||
pages = SyncPage(
|
||||
provider=io(settings.dir_pages),
|
||||
collection=c(settings.collection_pages),
|
||||
)
|
||||
|
||||
while True:
|
||||
# main(io(settings.dir_documents), sync_documents, settings.collection_documents)
|
||||
# main(io(settings.dir_awards), sync_documents, settings.collection_awards)
|
||||
@@ -62,10 +68,10 @@ async def run(run_interval=0):
|
||||
# main(io(settings.dir_articles), sync_posts, settings.collection_articles)
|
||||
# main(io(settings.dir_collective), sync_persons, settings.collection_collective)
|
||||
# main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
|
||||
# main(io(settings.dir_pages), sync_pages, settings.collection_pages)
|
||||
|
||||
await asyncio.wait([
|
||||
main(documents),
|
||||
main(pages),
|
||||
])
|
||||
|
||||
if run_interval == 0:
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Page import Page
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Page')
|
||||
|
||||
|
||||
class SyncPage(Sync):
|
||||
def __init__(self, provider: Provider, collection):
|
||||
super().__init__(provider, collection)
|
||||
|
||||
async def clean(self):
|
||||
pass
|
||||
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Page.scan(self.provider)
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Page(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Pages(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Pages to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Page {}'.format(document.id))
|
||||
|
||||
await document.setup_images(settings.image_sizes, get_image_url_fn(document.id))
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Page {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Pages to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Page {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
await self.clean()
|
||||
|
||||
|
||||
def get_image_url_fn(page_id: str):
|
||||
image_id = lambda filename: url_encode_text(
|
||||
os.path.splitext(filename)[0]
|
||||
)
|
||||
|
||||
def img_url_fn(file, size, ext):
|
||||
return settings.page_url_base.format(
|
||||
page=page_id,
|
||||
id=image_id(file),
|
||||
size=size,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
return img_url_fn
|
||||
@@ -0,0 +1,203 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
from lxml import etree
|
||||
import lxml.html
|
||||
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.data import request
|
||||
from sync.models import Model
|
||||
|
||||
from sync import create_date, images_from_html, title_from_html
|
||||
from sync.models.Image import Image
|
||||
from utils.hash import hash_str
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Page')
|
||||
|
||||
|
||||
def find(store, item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Page(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
documents = [i for i in provider.scan('.') if provider.is_dir(i)]
|
||||
documents = [provider.type_filter(i, '.md') for i in documents]
|
||||
documents = [i[0] for i in documents if len(i)]
|
||||
documents = [Page.read_path(provider, i) for i in documents]
|
||||
documents = [i for i in documents if i]
|
||||
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read_path(provider, path):
|
||||
params = {
|
||||
'file': os.path.basename(path),
|
||||
'folder': os.path.dirname(path),
|
||||
'title': os.path.basename(path),
|
||||
}
|
||||
|
||||
manifest = get_manifest(provider, path)
|
||||
params = {
|
||||
**params,
|
||||
**manifest,
|
||||
}
|
||||
|
||||
if 'url' not in params:
|
||||
logger.warning('URL not specified for Page {}'.format(path))
|
||||
return None
|
||||
|
||||
return Page(
|
||||
provider,
|
||||
path,
|
||||
params=params,
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
store = collection(settings.collection_pages)
|
||||
|
||||
self.__hash_keys = ['id', 'url']
|
||||
|
||||
self.__id_template: str = '{category}-{file}'
|
||||
self.__url_template: str = settings.document_url_template
|
||||
self.__url_preview_template: str = settings.document_url_preview_template
|
||||
|
||||
self.params = params if params else {}
|
||||
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
self.__set_id()
|
||||
self.__set_url()
|
||||
self.__set_hash()
|
||||
|
||||
async def is_changed(self):
|
||||
i = find(self.store, self.id)
|
||||
if not i:
|
||||
return True
|
||||
|
||||
if 'hash' not in i:
|
||||
return True
|
||||
|
||||
return not (self.hash == i['hash'])
|
||||
|
||||
async def save(self):
|
||||
document = self.bake()
|
||||
|
||||
query = {'id': document['id']}
|
||||
try:
|
||||
self.store.update_one(query, {'$set': document}, upsert=True)
|
||||
return self
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
async def setup_images(self, sizes, url_factory):
|
||||
folder = self.params['folder']
|
||||
images = []
|
||||
for i in self.params['images']:
|
||||
img_file = os.path.join(folder, i)
|
||||
|
||||
img = await Image.new(self.provider, img_file, sizes, url_factory)
|
||||
if img:
|
||||
images.append(img)
|
||||
self.images = images
|
||||
|
||||
def bake(self):
|
||||
images = [image.id for image in self.images]
|
||||
|
||||
data = create_post(self.params['data'], lambda src: self.__image_url_by_src(src))
|
||||
|
||||
return {
|
||||
**self.params,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'url': self.url,
|
||||
'file': self.file,
|
||||
'data': data,
|
||||
'images': images,
|
||||
}
|
||||
|
||||
def __image_url_by_src(self, src):
|
||||
for img in self.images:
|
||||
if img.file.endswith(src):
|
||||
s = img.get_size('big')
|
||||
if s:
|
||||
return s['url']
|
||||
else:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def __filename(self):
|
||||
return os.path.basename(self.file)
|
||||
|
||||
def __set_id(self):
|
||||
if 'id' in self.params:
|
||||
self.id = self.params['id']
|
||||
else:
|
||||
self.id = url_encode_text(self.params['url'])
|
||||
|
||||
def __set_url(self):
|
||||
self.url = self.params['url']
|
||||
|
||||
def __set_hash(self):
|
||||
images = [os.path.join(self.params['folder'], i) for i in self.params['images']]
|
||||
files = sorted([self.file] + images)
|
||||
hashes = [self.provider.hash(i) for i in files]
|
||||
|
||||
self.hash = hash_str(''.join(hashes))
|
||||
|
||||
def __str__(self):
|
||||
return '<Page hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def create_post(md, image_path_fn):
|
||||
html = lxml.html.fromstring(md)
|
||||
|
||||
for img in html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
path = image_path_fn(src)
|
||||
|
||||
if path:
|
||||
img.set('src', path)
|
||||
img.set('class', settings.album_html_img_class)
|
||||
img.set('data-file', src)
|
||||
return etree.tounicode(html)
|
||||
|
||||
|
||||
def get_manifest(provider, path):
|
||||
dirname = os.path.dirname(path)
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
manifest, body = read_yaml_md(data)
|
||||
manifest = manifest if manifest else {}
|
||||
|
||||
if 'date' in manifest:
|
||||
manifest['date'] = create_date(manifest['date'], settings.date_formats)
|
||||
|
||||
if 'until' in manifest:
|
||||
manifest['until'] = create_date(manifest['until'], settings.date_formats)
|
||||
|
||||
if 'id' in manifest:
|
||||
manifest['id'] = str(manifest['id'])
|
||||
|
||||
if 'content' in manifest:
|
||||
body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8')
|
||||
del manifest['content']
|
||||
|
||||
return {
|
||||
**manifest,
|
||||
'data': body,
|
||||
'title': title_from_html(body),
|
||||
'images': images_from_html(body)
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
import os
|
||||
|
||||
import settings
|
||||
from sync.core import Sync
|
||||
from sync.data import list_images, Provider
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
|
||||
synced_images_ids, untouched, title_from_html, create_post
|
||||
from sync.core.post import SyncPost
|
||||
from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
|
||||
from utils.hash import hash_str
|
||||
from utils.image import create_image
|
||||
from utils.io import read_yaml_md
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
|
||||
class ImageCreator:
|
||||
def __init__(self, provider: Provider, sizes: list, output_dir: str, cache: dict):
|
||||
super().__init__()
|
||||
self.provider = provider
|
||||
self.sizes = sizes
|
||||
self.output_dir = output_dir
|
||||
self.cache = cache if cache else set()
|
||||
|
||||
def create(self, file, name_fn):
|
||||
filehash = self.provider.hash(file)
|
||||
cached = self.cached(filehash)
|
||||
if cached:
|
||||
return cached
|
||||
return create_image(self.provider.get_abs(file), self.sizes, name_fn, self.output_dir, skip_processing=False)
|
||||
|
||||
def cached(self, name):
|
||||
return self.cache[name] if name in self.cache else None
|
||||
|
||||
|
||||
def create_document_from_folder(provider, path):
|
||||
document = {
|
||||
'file': os.path.basename(path),
|
||||
'folder': os.path.dirname(path),
|
||||
'title': os.path.basename(path),
|
||||
}
|
||||
|
||||
manifest = get_manifest(provider, path)
|
||||
document = {
|
||||
**document,
|
||||
**manifest,
|
||||
}
|
||||
|
||||
if 'url' not in document:
|
||||
return None
|
||||
|
||||
document = create_id(document)
|
||||
document = create_hash(provider, document)
|
||||
return document
|
||||
|
||||
|
||||
def get_manifest(provider, path):
|
||||
dirname = os.path.dirname(path)
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
manifest, body = read_yaml_md(data)
|
||||
manifest = manifest if manifest else {}
|
||||
|
||||
if 'date' in manifest:
|
||||
manifest['date'] = create_date(manifest['date'], settings.date_formats)
|
||||
|
||||
if 'until' in manifest:
|
||||
manifest['until'] = create_date(manifest['until'], settings.date_formats)
|
||||
|
||||
if 'id' in manifest:
|
||||
manifest['id'] = str(manifest['id'])
|
||||
|
||||
if 'content' in manifest:
|
||||
body = provider.read(os.path.join(dirname, manifest['content'])).read().decode('utf-8')
|
||||
del manifest['content']
|
||||
|
||||
return {
|
||||
**manifest,
|
||||
'data': body,
|
||||
'title': title_from_html(body),
|
||||
'images': images_from_html(body)
|
||||
}
|
||||
|
||||
|
||||
def create_id(document):
|
||||
new_id = url_encode_text(document['url'])
|
||||
return {
|
||||
**document,
|
||||
'id': document['id'] if 'id' in document else new_id
|
||||
}
|
||||
|
||||
|
||||
def create_hash(provider, document):
|
||||
files = sorted([document['file']] + document['images'])
|
||||
hashes = lmap(
|
||||
provider.hash,
|
||||
lmap(
|
||||
lambda x: os.path.join(document['folder'], x),
|
||||
files
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
**document,
|
||||
'hash': hash_str(''.join(hashes))
|
||||
}
|
||||
|
||||
|
||||
def create_page(provider: Provider, img: ImageCreator, document):
|
||||
image_path = lambda x: os.path.join(document['folder'], x)
|
||||
image_id = lambda filename: url_encode_text(
|
||||
os.path.splitext(filename)[0]
|
||||
)
|
||||
|
||||
def process_image(src: str):
|
||||
path = image_path(src)
|
||||
|
||||
if provider.exists(path):
|
||||
url = lambda s, e: settings.url_page_base_preview.format(
|
||||
page=document['id'],
|
||||
id=image_id(src),
|
||||
size=s,
|
||||
ext=e
|
||||
)
|
||||
|
||||
image = img.create(path, url)
|
||||
if image:
|
||||
images.append(image)
|
||||
return image['data']['big']['url']
|
||||
return None
|
||||
|
||||
images = []
|
||||
post_html = create_post(document['data'], process_image)
|
||||
|
||||
document['data'] = post_html
|
||||
document['images'] = images
|
||||
return document
|
||||
|
||||
|
||||
def check_url(documents):
|
||||
urls = list(map(
|
||||
lambda i: i['url'],
|
||||
documents
|
||||
))
|
||||
|
||||
unique_urls = set(urls)
|
||||
if len(urls) != len(unique_urls):
|
||||
raise Exception('URL of pages should be unique')
|
||||
|
||||
|
||||
def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True):
|
||||
img_sizes = settings.album_image_sizes
|
||||
output_dir = settings.dir_static_images
|
||||
|
||||
sync = Sync()
|
||||
sync.collection = collection
|
||||
image_creator = ImageCreator(provider, img_sizes, output_dir, {})
|
||||
|
||||
documents = list(filter(
|
||||
lambda x: provider.is_dir(x),
|
||||
provider.scan('.')
|
||||
))
|
||||
|
||||
documents = lmap(
|
||||
lambda x: provider.type_filter(x, '.md'),
|
||||
documents
|
||||
)
|
||||
|
||||
documents = list(filter(
|
||||
lambda x: len(x),
|
||||
documents
|
||||
))
|
||||
|
||||
documents = lmap(
|
||||
lambda x: create_document_from_folder(provider, x[0]),
|
||||
documents
|
||||
)
|
||||
|
||||
documents = list(filter(None, documents))
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
documents_ids = lmapfn(documents)(
|
||||
lambda i: i['id']
|
||||
)
|
||||
|
||||
check_url(documents)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
if skip_unchanged:
|
||||
documents = untouched(documents, sync)
|
||||
|
||||
# # MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
# documents = lmap(
|
||||
# sync.create,
|
||||
# documents
|
||||
# )
|
||||
|
||||
documents = lmap(
|
||||
lambda document: create_page(provider, image_creator, document),
|
||||
documents
|
||||
)
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('images', synced_images_ids),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
if update:
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_delete = []
|
||||
if delete:
|
||||
remove_query = sync.create_remove_query({'id': {'$nin': documents_ids}})
|
||||
documents_to_delete = sync.query(remove_query)
|
||||
documents_to_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_delete
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_to_delete
|
||||
Reference in new issue
Block a user