[Sync] + new Article sync
This commit is contained in:
1 parent
953ef0bc28
commit
2e5aa4d21e
5 files changed
+257
-285
No files matched your search
@@ -0,0 +1,58 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Article import Article
|
||||
from sync.models.Person import Person
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Article')
|
||||
|
||||
|
||||
class SyncArticle(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Article.scan(self.provider)
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build(
|
||||
sizes=settings.image_sizes,
|
||||
)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,92 +0,0 @@
|
||||
import os
|
||||
from tempfile import mkstemp
|
||||
|
||||
import lxml.html
|
||||
|
||||
import kazimir
|
||||
from sync.core import Sync
|
||||
from utils.fn import last_good, lmap, combine
|
||||
from utils.hash import hash_str, hash_file
|
||||
from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
|
||||
class SyncPost(Sync):
|
||||
def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
|
||||
super().__init__()
|
||||
self.collection = collection
|
||||
self.provider = provider
|
||||
self.sizes = sizes
|
||||
self.type = document_type
|
||||
self.image_url_base = image_url_base
|
||||
self.dir_local_images = dir_local_images
|
||||
self.origin = origin
|
||||
|
||||
def create_id(self, document):
|
||||
new_id = url_encode_text(document['title'])
|
||||
document['id'] = document['id'] if 'id' in document else new_id
|
||||
return document
|
||||
|
||||
def create_hash(self, document):
|
||||
files = sorted(document['images'])
|
||||
hashes = lmap(
|
||||
self.provider.hash,
|
||||
lmap(
|
||||
lambda i: os.path.join(document['folder'], i),
|
||||
files
|
||||
)
|
||||
)
|
||||
|
||||
document['hash'] = hash_str(
|
||||
combine([hash_str(document)] + hashes)
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def create(self, document):
|
||||
if 'date' not in document or not document['date']:
|
||||
return None
|
||||
|
||||
if 'title' not in document or not document['title']:
|
||||
return None
|
||||
|
||||
if self.type:
|
||||
document['type'] = self.type
|
||||
|
||||
post_html = lxml.html.fromstring(document['post'])
|
||||
images = []
|
||||
for img in post_html.cssselect('img'):
|
||||
src = img.get('src')
|
||||
relative_image_path = os.path.join(document['folder'], src)
|
||||
|
||||
img_path = self.provider.get_local(relative_image_path)
|
||||
if os.path.exists(img_path):
|
||||
img_id = url_encode_text(os.path.splitext(src)[0])
|
||||
url_fn = lambda size, ext: self.image_url_base.format(
|
||||
id=document['id'],
|
||||
img=img_id,
|
||||
size=size,
|
||||
ext=ext.lower()
|
||||
)
|
||||
|
||||
image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
|
||||
if image:
|
||||
images.append(image)
|
||||
img.set('src', image['data']['big']['url'])
|
||||
else:
|
||||
print('fail: ', document['folder'], src)
|
||||
|
||||
document['post'] = kazimir.html_from_tree(post_html)
|
||||
document['images'] = images
|
||||
document['origin'] = self.origin
|
||||
return document
|
||||
|
||||
def create_remove_query(self, query):
|
||||
if self.type:
|
||||
query = {
|
||||
**query,
|
||||
'type': self.type,
|
||||
'origin': self.origin
|
||||
}
|
||||
return query
|
||||
Reference in new issue
Block a user