[Sync] + new Article sync
This commit is contained in:
Roman Timashev committed
1 parent
953ef0bc28
commit
2e5aa4d21e
5 files changed
+257
-285
No files matched your search
+7
-1
@@ -1,6 +1,7 @@
|
||||
import settings
|
||||
from db import collection
|
||||
from sync.core import Sync
|
||||
from sync.core.SyncArticle import SyncArticle
|
||||
from sync.core.SyncDocument import SyncDocument
|
||||
from sync.core.SyncPage import SyncPage
|
||||
from sync.core.SyncPerson import SyncPerson
|
||||
@@ -57,17 +58,22 @@ async def run(run_interval=0):
|
||||
collection=c(settings.collection_collective),
|
||||
)
|
||||
|
||||
articles = SyncArticle(
|
||||
provider=io(settings.dir_articles),
|
||||
collection=c(settings.collection_articles),
|
||||
)
|
||||
|
||||
while True:
|
||||
# main(io(settings.dir_documents), sync_documents, settings.collection_documents)
|
||||
# main(io(settings.dir_awards), sync_documents, settings.collection_awards)
|
||||
# main(io(settings.dir_schedules), sync_schedules, settings.collection_schedules)
|
||||
# main(io(settings.dir_articles), sync_posts, settings.collection_articles)
|
||||
# main(io(settings.dir_gallery), sync_albums, settings.collection_albums)
|
||||
|
||||
await asyncio.wait([
|
||||
main(documents),
|
||||
main(pages),
|
||||
main(persons),
|
||||
main(articles),
|
||||
])
|
||||
|
||||
if run_interval == 0:
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Article import Article
|
||||
from sync.models.Person import Person
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Article')
|
||||
|
||||
|
||||
class SyncArticle(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Article.scan(self.provider)
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build(
|
||||
sizes=settings.image_sizes,
|
||||
)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,92 +0,0 @@
|
||||
import os
|
||||
from tempfile import mkstemp
|
||||
|
||||
import lxml.html
|
||||
|
||||
import kazimir
|
||||
from sync.core import Sync
|
||||
from utils.fn import last_good, lmap, combine
|
||||
from utils.hash import hash_str, hash_file
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
|
||||
class SyncPost(Sync):
|
||||
def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
|
||||
super().__init__()
|
||||
self.collection = collection
|
||||
self.provider = provider
|
||||
self.sizes = sizes
|
||||
self.type = document_type
|
||||
self.image_url_base = image_url_base
|
||||
self.dir_local_images = dir_local_images
|
||||
self.origin = origin
|
||||
|
||||
def create_id(self, document):
|
||||
new_id = url_encode_text(document['title'])
|
||||
document['id'] = document['id'] if 'id' in document else new_id
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
files = sorted(document['images'])
|
||||
hashes = lmap(
|
||||
self.provider.hash,
|
||||
lmap(
|
||||
lambda i: os.path.join(document['folder'], i),
|
||||
files
|
||||
)
|
||||
)
|
||||
|
||||
document['hash'] = hash_str(
|
||||
combine([hash_str(document)] + hashes)
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def create(self, document):
|
||||
if 'date' not in document or not document['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in document or not document['title']:
|
||||
return None
|
||||
|
||||
if self.type:
|
||||
document['type'] = self.type
|
||||
|
||||
post_html = lxml.html.fromstring(document['post'])
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
relative_image_path = os.path.join(document['folder'], src)
|
||||
|
||||
img_path = self.provider.get_local(relative_image_path)
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda size, ext: self.image_url_base.format(
|
||||
id=document['id'],
|
||||
img=img_id,
|
||||
size=size,
|
||||
ext=ext.lower()
|
||||
)
|
||||
|
||||
image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
|
||||
if image:
|
||||
images.append(image)
|
||||
img.set('src', image['data']['big']['url'])
|
||||
else:
|
||||
print('fail: ', document['folder'], src)
|
||||
|
||||
document['post'] = kazimir.html_from_tree(post_html)
|
||||
document['images'] = images
|
||||
document['origin'] = self.origin
|
||||
return document
|
||||
|
||||
def create_remove_query(self, query):
|
||||
if self.type:
|
||||
query = {
|
||||
**query,
|
||||
'type': self.type,
|
||||
'origin': self.origin
|
||||
}
|
||||
return query
|
||||
@@ -0,0 +1,192 @@
|
||||
import logging
|
||||
|
||||
import os
|
||||
|
||||
import lxml.html
|
||||
|
||||
import kazimir
|
||||
import settings
|
||||
from db import collection
|
||||
from kazimir import kazimir_to_html
|
||||
from sync import create_date_and_title_from_folder_name, images_from_html, create_post_from_image_list, create_date
|
||||
from sync.data import Provider, list_images
|
||||
from sync.models import Model
|
||||
from sync.models.Image import Image
|
||||
|
||||
from utils.fn import swap_ext, first, map_cases, constant, combine
|
||||
from utils.hash import hash_str
|
||||
from utils.io import read_yaml_md, parse_yaml_front_matter
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
store = collection(settings.collection_articles)
|
||||
|
||||
|
||||
def find(item_id: str):
|
||||
q = {'id': item_id}
|
||||
try:
|
||||
return store.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Article(Model):
|
||||
@staticmethod
|
||||
async def scan(provider):
|
||||
documents = [i for i in provider.scan('.') if provider.is_dir(i)]
|
||||
documents = [Article.read(provider, i) for i in documents]
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def read(provider, path):
|
||||
file_time_formats = ['%Y.%m.%d']
|
||||
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
|
||||
document = {
|
||||
'folder': path,
|
||||
'date': doc_date,
|
||||
'title': doc_title,
|
||||
}
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
images = list_images(provider, path)
|
||||
images = [os.path.relpath(i, path) for i in images]
|
||||
document['images'] = images
|
||||
document['post'] = create_post_from_image_list(images)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
manifest_file = md_from_folder(provider, path)
|
||||
if manifest_file:
|
||||
manifest = get_manifest(provider, manifest_file)
|
||||
else:
|
||||
manifest = {}
|
||||
|
||||
return Article(
|
||||
provider,
|
||||
path,
|
||||
params={
|
||||
**document,
|
||||
**manifest,
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, provider, file, params=None):
|
||||
# self.__id_template: str = '{category}-{file}'
|
||||
# self.__url_template: str = settings.document_url_template
|
||||
# self.__url_preview_template: str = settings.document_url_preview_template
|
||||
|
||||
self.params = params if params else {}
|
||||
self.origin = settings.origin
|
||||
self.version = '2'
|
||||
self.post = None
|
||||
self.images = None
|
||||
super().__init__(provider, store, file)
|
||||
|
||||
def init(self):
|
||||
self.__set_id()
|
||||
self.__set_hash()
|
||||
|
||||
def validate(self):
|
||||
document = self.params
|
||||
if 'date' not in document or not document['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in document or not document['title']:
|
||||
return None
|
||||
|
||||
async def build(self, **kwargs):
|
||||
sizes = kwargs['sizes']
|
||||
post, images = await create_post(self.provider, self.id, self.params, sizes)
|
||||
self.post = post
|
||||
self.images = images
|
||||
|
||||
post, images = await create_post(self.provider, self.id, self.params, sizes)
|
||||
self.post = kazimir.html_from_tree(post)
|
||||
self.images = images
|
||||
|
||||
def bake(self):
|
||||
return {
|
||||
**self.params,
|
||||
'images': [i.id for i in self.images],
|
||||
'post': self.post,
|
||||
'id': self.id,
|
||||
'hash': self.hash,
|
||||
'file': self.file,
|
||||
'origin': self.origin,
|
||||
'version': self.version,
|
||||
}
|
||||
|
||||
def __set_id(self):
|
||||
if 'id' in self.params:
|
||||
self.id = self.params['id']
|
||||
else:
|
||||
self.id = url_encode_text(self.params['title'])
|
||||
|
||||
def __set_hash(self):
|
||||
f = self.params['folder']
|
||||
images = sorted(self.params['images'])
|
||||
images = [os.path.join(f, i) for i in images]
|
||||
images = [self.provider.hash(i) for i in images]
|
||||
|
||||
self.hash = hash_str(
|
||||
combine([hash_str(self.params)] + images)
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return '<Article hash={} file={} id={}>'.format(self.hash, self.file, self.id)
|
||||
|
||||
|
||||
def md_from_folder(provider, i):
|
||||
return first(provider.type_filter(i, '.md'))
|
||||
|
||||
|
||||
def get_manifest(provider, path):
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
manifest = parse_yaml_front_matter(data)
|
||||
|
||||
if 'date' in manifest:
|
||||
manifest['date'] = create_date(manifest['date'], settings.date_formats)
|
||||
|
||||
if 'until' in manifest:
|
||||
manifest['until'] = create_date(manifest['until'], settings.date_formats)
|
||||
|
||||
if 'id' in manifest:
|
||||
manifest['id'] = str(manifest['id'])
|
||||
|
||||
html = kazimir_to_html(manifest['content'])
|
||||
return {
|
||||
**manifest,
|
||||
'post': html,
|
||||
'images': images_from_html(html) # Add images for hashing
|
||||
}
|
||||
|
||||
|
||||
async def create_post(provider, doc_id, document, sizes):
|
||||
image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
|
||||
|
||||
post_html = lxml.html.fromstring(document['post'])
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
relative_image_path = os.path.join(document['folder'], src)
|
||||
|
||||
img_path = provider.get_local(relative_image_path)
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda file, size, ext: image_url_base.format(
|
||||
id=doc_id,
|
||||
img=img_id,
|
||||
size=size,
|
||||
ext=ext.lower()
|
||||
)
|
||||
|
||||
image = await Image.new(provider, img_path, sizes, url_fn)
|
||||
if image:
|
||||
url = image.get_size('big')['url']
|
||||
images.append(image)
|
||||
img.set('src', url)
|
||||
else:
|
||||
logger.error('Fail to get Image', document['folder'], src)
|
||||
return post_html, images
|
||||
@@ -1,192 +0,0 @@
|
||||
import os
|
||||
|
||||
import settings
|
||||
from kazimir import kazimir_to_html
|
||||
from sync.data import list_images
|
||||
from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
|
||||
synced_images_ids, untouched
|
||||
from sync.core.post import SyncPost
|
||||
from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant
|
||||
from utils.io import read_yaml_md, parse_yaml_front_matter
|
||||
|
||||
|
||||
def create_document_from_folder(sync, path, file_time_formats):
|
||||
# PARSE FOLDER NAME -> GET OPTIONAL DATE/TITLE
|
||||
doc_date, doc_title = create_date_and_title_from_folder_name(path, file_time_formats)
|
||||
document = {
|
||||
'folder': path,
|
||||
'date': doc_date,
|
||||
'title': doc_title,
|
||||
}
|
||||
|
||||
# CREATE BASIC_MANIFEST BASED ON FOLDER NAME AND IMAGE CONTENT
|
||||
images = lmap(
|
||||
lambda img: os.path.relpath(img, path),
|
||||
list_images(sync.provider, path)
|
||||
)
|
||||
post = create_post_from_image_list(images)
|
||||
|
||||
# TRY TO FILL UP BASIC_MANIFEST WITH MD_MANIFEST
|
||||
params = map_cases(
|
||||
document,
|
||||
[(
|
||||
lambda doc: md_from_folder(sync.provider, path),
|
||||
lambda doc: get_manifest(sync.provider, md_from_folder(sync.provider, path)),
|
||||
)],
|
||||
constant({})
|
||||
)
|
||||
|
||||
document = {
|
||||
'images': images,
|
||||
'post': post,
|
||||
**document,
|
||||
**params,
|
||||
}
|
||||
|
||||
document = sync.create_id(document)
|
||||
document = sync.create_hash(document)
|
||||
return document
|
||||
|
||||
|
||||
def get_manifest(provider, path):
|
||||
data = provider.read(path).read().decode('utf-8')
|
||||
manifest = parse_yaml_front_matter(data)
|
||||
|
||||
if 'date' in manifest:
|
||||
manifest['date'] = create_date(manifest['date'], settings.date_formats)
|
||||
|
||||
if 'until' in manifest:
|
||||
manifest['until'] = create_date(manifest['until'], settings.date_formats)
|
||||
|
||||
if 'id' in manifest:
|
||||
manifest['id'] = str(manifest['id'])
|
||||
|
||||
html = kazimir_to_html(manifest['content'])
|
||||
return {
|
||||
**manifest,
|
||||
'post': html,
|
||||
'images': images_from_html(html) # Add images for hashing
|
||||
}
|
||||
|
||||
|
||||
def md_from_folder(provider, i):
|
||||
return first(provider.type_filter(i, '.md'))
|
||||
|
||||
|
||||
def get_folder_documents(sync, file_time_formats):
|
||||
# GET EVENT FOLDERS
|
||||
documents = lmap(
|
||||
lambda i: i,
|
||||
filter(
|
||||
lambda i: sync.provider.is_dir(i),
|
||||
sync.provider.scan('.')
|
||||
)
|
||||
)
|
||||
|
||||
return lmap(
|
||||
lambda doc: create_document_from_folder(sync, doc, file_time_formats),
|
||||
documents
|
||||
)
|
||||
|
||||
|
||||
# GET EVENT FILES
|
||||
def get_file_documents(sync, file_time_formats):
|
||||
documents = sync.provider.type_filter('.', '.md')
|
||||
documents = lmap(
|
||||
lambda i: (i,) + create_date_and_title_from_folder_name(i, file_time_formats),
|
||||
documents
|
||||
)
|
||||
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
'file': i[0],
|
||||
'date': i[1],
|
||||
'title': i[2],
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# OVERWRITE BASIC_MANIFEST WITH MD_MANIFEST
|
||||
documents = lmap(
|
||||
lambda i: {
|
||||
**i,
|
||||
**get_manifest(sync.provider, i['file'])
|
||||
},
|
||||
documents
|
||||
)
|
||||
|
||||
# CREATE DOCUMENT IDENTITY
|
||||
documents = lmap(sync.create_id, documents)
|
||||
|
||||
# CREATE HASH OF DOCUMENT FILE
|
||||
documents = lmap(sync.create_hash, documents)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def main(sync, file_time_formats, update_documents=True, delete_documents=True):
|
||||
documents = get_folder_documents(sync, file_time_formats)
|
||||
documents += get_file_documents(sync, file_time_formats)
|
||||
|
||||
# CREATE SCOPE OF CURRENT SESSION
|
||||
scope_documents_ids = lmapfn(documents)(
|
||||
lambda i: i['id']
|
||||
)
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
# documents = untouched(documents, sync)
|
||||
|
||||
# MAP EVENT MANIFEST -> EVENT_OBJECT
|
||||
documents = lmap(
|
||||
sync.create,
|
||||
documents
|
||||
)
|
||||
|
||||
# FILTER INCORRECT EVENTS
|
||||
documents = list(filter(None, documents))
|
||||
|
||||
# REPLACE IMAGES OBJECTS WITH IT _ID IN MONGODB
|
||||
documents = lmap(
|
||||
key_mapper('images', synced_images_ids),
|
||||
documents
|
||||
)
|
||||
|
||||
# SYNC EVENT_OBJECT WITH DB
|
||||
if update_documents:
|
||||
documents = lmap(
|
||||
sync.update,
|
||||
documents
|
||||
)
|
||||
|
||||
documents_to_delete = []
|
||||
if delete_documents:
|
||||
remove_query = sync.create_remove_query({'id': {'$nin': scope_documents_ids}})
|
||||
documents_to_delete = sync.query(remove_query)
|
||||
documents_to_delete = lmap(
|
||||
sync.delete,
|
||||
map(
|
||||
lambda i: {'_id': i['_id']},
|
||||
documents_to_delete
|
||||
)
|
||||
)
|
||||
|
||||
return documents, documents_to_delete
|
||||
|
||||
|
||||
def sync_posts(provider, collection, update=True, delete=True):
|
||||
image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
|
||||
|
||||
return main(
|
||||
SyncPost(
|
||||
collection,
|
||||
provider,
|
||||
'post',
|
||||
image_url_base,
|
||||
settings.dir_static_images,
|
||||
settings.event_image_sizes,
|
||||
origin=settings.origin
|
||||
),
|
||||
file_time_formats=['%Y.%m.%d'],
|
||||
update_documents=update,
|
||||
delete_documents=delete
|
||||
)
|
||||
Reference in new issue
Block a user