[Sync] + new Article sync

This commit is contained in:
Roman Timashev committed 2017-09-13 14:06:59 +03:00
1 parent 953ef0bc28
commit 2e5aa4d21e
5 files changed
+257 -285

No files matched your search

+58
View File
@@ -0,0 +1,58 @@
import os
import logging
import settings
from sync import untouched
from sync.core import Sync
from sync.data import Provider
from sync.models.Article import Article
from sync.models.Person import Person
from utils.text.transform import url_encode_text
logger = logging.getLogger(settings.name + '.Article')
class SyncArticle(Sync):
async def run(self):
logger.info('Checking for update')
documents = await Article.scan(self.provider)
documents_id = [doc.id for doc in documents]
# self.validate(documents)
logger.info('Found {} Item(s)'.format(len(documents)))
# SKIP UNTOUCHED DOCUMENTS
documents = await untouched(documents)
logger.info('Changed {} Items(s)'.format(len(documents)))
# UPDATING
if settings.update_enabled:
if len(documents) == 0:
logger.info('No Items to update')
for document in documents:
logger.info('Updating Item {}'.format(document.id))
await document.build(
sizes=settings.image_sizes,
)
await document.save()
logger.info('Updated Item {}'.format(document.id))
# DELETING
if settings.delete_enabled:
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
if len(documents_delete) == 0:
logger.info('No Items to delete')
else:
for document in documents_delete:
logger.info('Deleting Item {}'.format(document['id']))
q = {'_id': document['_id']}
self.delete(q)
# await self.clean()
-92
View File
@@ -1,92 +0,0 @@
import os
from tempfile import mkstemp
import lxml.html
import kazimir
from sync.core import Sync
from utils.fn import last_good, lmap, combine
from utils.hash import hash_str, hash_file
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
from utils.text.transform import url_encode_text, url_encode_file
class SyncPost(Sync):
def __init__(self, collection, provider, document_type, image_url_base, dir_local_images, sizes, origin):
super().__init__()
self.collection = collection
self.provider = provider
self.sizes = sizes
self.type = document_type
self.image_url_base = image_url_base
self.dir_local_images = dir_local_images
self.origin = origin
def create_id(self, document):
new_id = url_encode_text(document['title'])
document['id'] = document['id'] if 'id' in document else new_id
return document
def create_hash(self, document):
files = sorted(document['images'])
hashes = lmap(
self.provider.hash,
lmap(
lambda i: os.path.join(document['folder'], i),
files
)
)
document['hash'] = hash_str(
combine([hash_str(document)] + hashes)
)
return document
def create(self, document):
if 'date' not in document or not document['date']:
return None
if 'title' not in document or not document['title']:
return None
if self.type:
document['type'] = self.type
post_html = lxml.html.fromstring(document['post'])
images = []
for img in post_html.cssselect('img'):
src = img.get('src')
relative_image_path = os.path.join(document['folder'], src)
img_path = self.provider.get_local(relative_image_path)
if os.path.exists(img_path):
img_id = url_encode_text(os.path.splitext(src)[0])
url_fn = lambda size, ext: self.image_url_base.format(
id=document['id'],
img=img_id,
size=size,
ext=ext.lower()
)
image = create_image(img_path, self.sizes, url_fn, self.dir_local_images)
if image:
images.append(image)
img.set('src', image['data']['big']['url'])
else:
print('fail: ', document['folder'], src)
document['post'] = kazimir.html_from_tree(post_html)
document['images'] = images
document['origin'] = self.origin
return document
def create_remove_query(self, query):
if self.type:
query = {
**query,
'type': self.type,
'origin': self.origin
}
return query