[Sync] update: removed unnecesarry Sync subclasses
This commit is contained in:
Roman Timashev committed
1 parent
84745ead6d
commit
6fa8564cc8
15 files changed
+163
-471
No files matched your search
@@ -1,51 +0,0 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.models.Album import Album
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Album')
|
||||
|
||||
|
||||
class SyncAlbum(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Album.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build()
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,58 +0,0 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Article import Article
|
||||
from sync.models.Person import Person
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Article')
|
||||
|
||||
|
||||
class SyncArticle(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Article.scan(self.provider)
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build(
|
||||
sizes=settings.image_sizes,
|
||||
)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,8 +1,6 @@
|
||||
import os
|
||||
from tempfile import mkstemp
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
@@ -15,65 +13,16 @@ from utils.image import create_image
|
||||
from utils.image.resize import image_magick_pdf_to_img
|
||||
from utils.text.transform import url_encode_text, url_encode_file
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Document')
|
||||
|
||||
|
||||
class SyncDocument(Sync):
|
||||
def __init__(self, provider: Provider, collection, sizes):
|
||||
super().__init__(provider, collection)
|
||||
def __init__(self, provider: Provider, sizes):
|
||||
super().__init__(provider, Document)
|
||||
self.sizes = sizes
|
||||
self.dir_static_previews = ''
|
||||
|
||||
self.temp_dir = '/Users/tmshv/HudozkaSyncTemp'
|
||||
|
||||
async def clean(self):
|
||||
pass
|
||||
|
||||
async def run(self):
|
||||
logger.info('Checking documents for update')
|
||||
|
||||
documents = await Document.scan(self.provider)
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Document(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
# documents = untouched(documents, self)
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Document(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Documents to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Document {}'.format(document.id))
|
||||
|
||||
await document.upload()
|
||||
await document.create_preview(self.sizes)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Document {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Documents to delete')
|
||||
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
await self.clean()
|
||||
|
||||
|
||||
async def get_assets(self, document) -> []:
|
||||
return []
|
||||
def _get_build_args(self):
|
||||
return {
|
||||
'sizes': self.sizes,
|
||||
}
|
||||
|
||||
|
||||
def validate(self, documents):
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Page import Page
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Page')
|
||||
|
||||
|
||||
class SyncPage(Sync):
|
||||
def __init__(self, provider: Provider, collection):
|
||||
super().__init__(provider, collection)
|
||||
|
||||
async def clean(self):
|
||||
pass
|
||||
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Page.scan(self.provider)
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Page(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Pages(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Pages to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Page {}'.format(document.id))
|
||||
|
||||
await document.setup_images(settings.image_sizes, get_image_url_fn(document.id))
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Page {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Pages to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Page {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
await self.clean()
|
||||
|
||||
|
||||
def get_image_url_fn(page_id: str):
|
||||
image_id = lambda filename: url_encode_text(
|
||||
os.path.splitext(os.path.basename(filename))[0]
|
||||
)
|
||||
|
||||
def img_url_fn(file, size, ext):
|
||||
return settings.page_url_base.format(
|
||||
page=page_id,
|
||||
id=image_id(file),
|
||||
size=size,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
return img_url_fn
|
||||
@@ -1,75 +0,0 @@
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.data import Provider
|
||||
from sync.models.Person import Person
|
||||
|
||||
from utils.text.transform import url_encode_text
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Person')
|
||||
|
||||
|
||||
class SyncPerson(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Person.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build(
|
||||
sizes=settings.image_sizes,
|
||||
url_factory=get_image_url_fn(document.id),
|
||||
)
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
|
||||
|
||||
def get_image_url_fn(page_id: str):
|
||||
image_id = lambda filename: url_encode_text(
|
||||
os.path.splitext(os.path.basename(filename))[0]
|
||||
)
|
||||
|
||||
def img_url_fn(file, size, ext):
|
||||
return settings.page_url_base.format(
|
||||
page=page_id,
|
||||
id=image_id(file),
|
||||
size=size,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
return img_url_fn
|
||||
@@ -1,51 +0,0 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched
|
||||
from sync.core import Sync
|
||||
from sync.models.Schedule import Schedule
|
||||
|
||||
logger = logging.getLogger(settings.name + '.Schedule')
|
||||
|
||||
|
||||
class SyncSchedule(Sync):
|
||||
async def run(self):
|
||||
logger.info('Checking for update')
|
||||
|
||||
documents = await Schedule.scan(self.provider)
|
||||
|
||||
documents_id = [doc.id for doc in documents]
|
||||
# self.validate(documents)
|
||||
logger.info('Found {} Item(s)'.format(len(documents)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
documents = await untouched(documents)
|
||||
logger.info('Changed {} Items(s)'.format(len(documents)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(documents) == 0:
|
||||
logger.info('No Items to update')
|
||||
|
||||
for document in documents:
|
||||
logger.info('Updating Item {}'.format(document.id))
|
||||
|
||||
await document.build()
|
||||
await document.save()
|
||||
|
||||
logger.info('Updated Item {}'.format(document.id))
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
documents_delete = list(self.query({'id': {'$nin': documents_id}}))
|
||||
|
||||
if len(documents_delete) == 0:
|
||||
logger.info('No Items to delete')
|
||||
else:
|
||||
for document in documents_delete:
|
||||
logger.info('Deleting Item {}'.format(document['id']))
|
||||
|
||||
q = {'_id': document['_id']}
|
||||
self.delete(q)
|
||||
|
||||
# await self.clean()
|
||||
@@ -1,3 +1,7 @@
|
||||
import logging
|
||||
|
||||
import settings
|
||||
from sync import untouched, Model
|
||||
from sync.data import Provider
|
||||
|
||||
|
||||
@@ -9,15 +13,35 @@ def get_id(item):
|
||||
|
||||
|
||||
class Sync:
|
||||
@staticmethod
|
||||
def compile(document):
|
||||
return document.bake()
|
||||
|
||||
def __init__(self, provider: Provider, collection):
|
||||
def __init__(self, provider: Provider, model):
|
||||
super().__init__()
|
||||
|
||||
self.__default_build_args = {
|
||||
'sizes': settings.image_sizes,
|
||||
}
|
||||
self.__items_to_delete_query = lambda items_id: {'id': {'$nin': items_id}}
|
||||
self.__delete_query = lambda item: {'_id': item['_id']}
|
||||
|
||||
name = model.__name__
|
||||
self.logger = logging.getLogger('%s.%s' % (settings.name, name))
|
||||
|
||||
self.provider = provider
|
||||
self.collection = collection
|
||||
self.model = model
|
||||
|
||||
async def clean(self):
|
||||
pass
|
||||
|
||||
async def update(self, item):
|
||||
args = self._get_build_args()
|
||||
|
||||
await item.build(**args)
|
||||
await item.upload()
|
||||
await item.save()
|
||||
|
||||
def _get_build_args(self):
|
||||
return {
|
||||
**self.__default_build_args
|
||||
}
|
||||
|
||||
async def run(self):
|
||||
"""
|
||||
@@ -30,33 +54,38 @@ class Sync:
|
||||
|
||||
:return:
|
||||
"""
|
||||
return None, None
|
||||
self.logger.info('Checking for update')
|
||||
|
||||
items = await self.model.scan(self.provider)
|
||||
items_id = [doc.id for doc in items]
|
||||
self.logger.info('Found {} Item(s)'.format(len(items)))
|
||||
|
||||
# SKIP UNTOUCHED DOCUMENTS
|
||||
items = await untouched(items)
|
||||
self.logger.info('Changed {} Items(s)'.format(len(items)))
|
||||
|
||||
# UPDATING
|
||||
if settings.update_enabled:
|
||||
if len(items) == 0:
|
||||
self.logger.info('No Items to update')
|
||||
|
||||
for item in items:
|
||||
self.logger.info('Updating Item {}'.format(item.id))
|
||||
await self.update(item)
|
||||
|
||||
# DELETING
|
||||
if settings.delete_enabled:
|
||||
items = await self.model.find(self.__items_to_delete_query(items_id))
|
||||
items = list(items)
|
||||
|
||||
if len(items) == 0:
|
||||
self.logger.info('No Items to delete')
|
||||
else:
|
||||
for item in items:
|
||||
self.logger.info('Deleting Item {}'.format(item['id']))
|
||||
await self.model.delete(self.__delete_query(item))
|
||||
|
||||
await self.clean()
|
||||
|
||||
async def upload(self):
|
||||
pass
|
||||
|
||||
def read(self, document: dict):
|
||||
q = {'id': document['id']}
|
||||
try:
|
||||
return self.collection.find_one(q)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
def update(self, document: dict):
|
||||
query = {'id': document['id']}
|
||||
try:
|
||||
self.collection.update_one(query, {'$set': document}, upsert=True)
|
||||
return self.read(document)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
def query(self, q):
|
||||
return self.collection.find(q)
|
||||
|
||||
def delete(self, q):
|
||||
return self.collection.find_one_and_delete(q)
|
||||
|
||||
def create_remove_query(self, query):
|
||||
return query
|
||||
Reference in new issue
Block a user