[sync] updated: config based; async loop; Documents adopted to new system

utils/Sync/sync/__init__.py

@@ -9,6 +9,7 @@ from markdown import markdown
 
 import settings
 from sync.image import sync_image
+from sync.models import Model
 from utils.fn import lmap, combine
 
 
@@ -43,26 +44,30 @@ def create_date(date_str, date_formats=None):
 folder_name_pattern = re.compile('([\d.]+)(.*)')
 
 
-def untouched(documents, store):
+async def untouched(items: [Model]) -> [Model]:
     if settings.skip_unchanged:
-        return documents
-    return lmap(
-        lambda i: i[0],
-        filter(
-            lambda i: (i[1] is None) or ('hash' not in i[1]) or (i[0]['hash'] != i[1]['hash']),
-            map(
-                lambda i: (i, store.read(i)),
-                documents
-            )
-        ))
+        return items
+
+    result = []
+    for item in items:
+        status = await item.is_changed()
+        if status:
+            result.append(item)
+    return result
+
+    # def is_equals(e, n):
+    #     return (n is None) or ('hash' not in n) or (e.hash != n['hash'])
+    #
+    # doc = lambda i: {'id': i.id, 'hash': i.hash}
+    #
+    # stored_documents = [(i, store.read(doc(i))) for i in documents]
+    # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)]
+
+    # return filtered_documents
 
 
 def synced_images_ids(images):
-    ids = lmap(
-        lambda image: image['_id'],
-        sync_image(images)
-    )
-    return ids
+    return [i['_id'] for i in sync_image(images)]
 
 
 def synced_image_id(image):

utils/Sync/sync/core/SyncDocument.py

@@ -0,0 +1,188 @@
+import os
+from tempfile import mkstemp
+
+import logging
+
+import settings
+from sync import untouched
+from sync.core import Sync
+from sync.data import scan_subdirs, Provider
+from sync.models.Document import Document
+
+from sync.models import Model
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+logger = logging.getLogger(settings.name + '.Document')
+
+
+class SyncDocument(Sync):
+    def __init__(self, provider: Provider, collection, sizes):
+        super().__init__()
+        self.collection = collection
+        self.provider = provider
+        self.sizes = sizes
+        self.dir_static_previews = ''
+
+        self.temp_dir = '/Users/tmshv/HudozkaSyncTemp'
+
+    async def clean(self):
+        pass
+
+    async def run(self) -> ([Model], [Model]):
+        logger.info('Checking documents for update')
+
+        documents = await Document.scan(self.provider)
+        documents_id = [doc.id for doc in documents]
+        # self.validate(documents)
+        logger.info('Found {} Document(s)'.format(len(documents)))
+
+        # SKIP UNTOUCHED DOCUMENTS
+        # documents = untouched(documents, self)
+        documents = await untouched(documents)
+        logger.info('Changed {} Document(s)'.format(len(documents)))
+
+        # UPDATING
+        if settings.update_enabled:
+            if len(documents) == 0:
+                logger.info('No Documents to update')
+
+            for document in documents:
+                logger.info('Updating Document {}'.format(document.id))
+
+                await document.upload()
+                await document.create_preview(self.sizes)
+                await document.save()
+
+                logger.info('Updated Document {}'.format(document.id))
+
+        # DELETING
+        if settings.delete_enabled:
+            documents_delete = list(self.query({'id': {'$nin': documents_id}}))
+
+            if len(documents_delete) == 0:
+                logger.info('No Documents to delete')
+
+            for document in documents_delete:
+                logger.info('Deleting {}'.format(document['id']))
+
+                q = {'_id': document['_id']}
+                self.delete(q)
+
+        await self.clean()
+
+
+async def get_assets(self, document) -> []:
+    return []
+
+
+def validate(self, documents):
+    files = [doc['file'] for doc in documents]
+    file_names = [os.path.basename(f) for f in files]
+    file_names = set(file_names)
+
+    if len(file_names) != len(documents):
+        raise Exception('File names should be unique')
+
+
+def create(self, document):
+    file = document['file']
+    filename = os.path.basename(file)
+
+    document['type'] = document_type(document)
+    document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
+    document['title'] = last_good([
+        document['title'],
+        get_pdf_title(self.provider, document['file'])
+    ])
+    document['file'] = {
+        'name': filename,
+        'size': self.provider.size(file)
+    }
+
+    return document
+
+
+def create_id(self, document: dict) -> dict:
+    filepath = document['file']
+    filename = os.path.basename(filepath)
+
+    return {
+        **document,
+        'id': self.id_template.format(
+            category=url_encode_text(document['category']),
+            file=url_encode_text(filename)
+        ),
+    }
+
+
+def create_url(self, document: dict) -> dict:
+    filename = os.path.basename(document['file'])
+
+    return {
+        **document,
+        'url': self.url_template.format(
+            file=url_encode_file(filename)
+        ),
+    }
+
+
+def create_hash(self, document: dict) -> dict:
+    hash_doc = [document[key] for key in self.hash_keys]
+
+    return {
+        **document,
+        'hash': hash_str(
+            hash_str(hash_doc) + self.provider.hash(document['file'])
+        ),
+    }
+
+
+def create_preview(self, file, sizes, preview_dir):
+    def url(size, ext):
+        return self.url_base_preview.format(id=file, size=size, ext=ext)
+
+    temp_preview_path = pdf_to_jpg(self.provider, file)
+    file = os.path.basename(file)
+    file = url_encode_text(file)
+    img = create_image(temp_preview_path, sizes, url, preview_dir)
+    os.remove(temp_preview_path)
+    return img
+
+
+def document_type(document):
+    if document['category'] == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/core/__init__.py

@@ -2,6 +2,13 @@ from sync.models import Model
 from utils.fn import lmap
 
 
+def get_id(item):
+    try:
+        return item.id
+    except AttributeError:
+        return item['id']
+
+
 class Sync:
     @staticmethod
     def compile(document):
@@ -12,10 +19,23 @@ class Sync:
 
         self.collection = None
 
-    def read(self, document):
-        if isinstance(document, list):
-            return lmap(self.read, document)
+    async def run(self) -> ([Model], [Model]):
+        """
+        # Get scope files
+        # Validate these files. Raise an error
+        # calc_hashes
+        # make diff with previous run
+        # compile objects
+        # upload
 
+        :return:
+        """
+        return None, None
+
+    async def upload(self):
+        pass
+
+    def read(self, document: dict):
         q = {'id': document['id']}
         try:
             return self.collection.find_one(q)
@@ -23,16 +43,10 @@ class Sync:
             pass
         return None
 
-    def update(self, document):
-        if isinstance(document, list):
-            return lmap(self.update, document)
-
-        if isinstance(document, Model):
-            return self.update(self.compile(document))
-
-        q = {'id': document['id']}
+    def update(self, document: dict):
+        query = {'id': document['id']}
         try:
-            self.collection.update_one(q, {'$set': document}, upsert=True)
+            self.collection.update_one(query, {'$set': document}, upsert=True)
             return self.read(document)
         except ValueError:
             pass

utils/Sync/sync/core/document.py

@@ -1,107 +0,0 @@
-import os
-from tempfile import mkstemp
-
-from sync.core import Sync
-from utils.fn import last_good, lmap
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class SyncDocument(Sync):
-    def __init__(self, collection, provider, sizes, dir_static_previews, url_base_preview, url_base_document):
-        super().__init__()
-        self.collection = collection
-        self.provider = provider
-        self.sizes = sizes
-        self.dir_static_previews = dir_static_previews
-        self.url_base_preview = url_base_preview
-        self.url_base_document = url_base_document
-
-    def create(self, document):
-        file = document['file']
-        filename = os.path.basename(file)
-
-        document['type'] = document_type(document)
-        document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
-        document['title'] = last_good([
-            document['title'],
-            get_pdf_title(self.provider, document['file'])
-        ])
-        document['file'] = {
-            'name': filename,
-            'size': self.provider.size(file)
-        }
-
-        return document
-
-    def create_id(self, document):
-        bn = os.path.basename(document['file'])
-        document['id'] = url_encode_text('{category}-{file}'.format(
-            category=document['category'],
-            file=bn
-        ))
-        return document
-
-    def create_url(self, document):
-        filename = os.path.basename(document['file'])
-        document['url'] = self.url_base_document.format(file=url_encode_file(filename))
-        return document
-
-    def create_hash(self, document):
-        hash_keys = ['id', 'url']
-        hash_doc = lmap(
-            lambda key: document[key],
-            hash_keys
-        )
-        document['hash'] = hash_str(
-            hash_str(hash_doc) + self.provider.hash(document['file'])
-        )
-        return document
-
-    def create_preview(self, file, sizes, preview_dir):
-        def url(size, ext):
-            return self.url_base_preview.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = pdf_to_jpg(self.provider, file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, sizes, url, preview_dir)
-        os.remove(temp_preview_path)
-        return img
-
-
-def document_type(document):
-    if document['category'] == 'Награды':
-        return 'award'
-    return 'document'
-
-
-def pdf_to_jpg(provider, pdf):
-    _, temp_in = mkstemp(suffix='.pdf')
-    _, temp_out = mkstemp(suffix='.jpg')
-
-    abspdf = provider.copy(pdf, temp_in)
-    image_magick_pdf_to_img(abspdf, temp_out)
-
-    os.remove(temp_in)
-    return temp_out
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None

utils/Sync/sync/data/__init__.py

@@ -3,6 +3,7 @@ from io import BytesIO
 
 import requests
 
+import settings
 from utils.fn import combine, lmap
 
 
@@ -96,6 +97,10 @@ def upload(file_path, relative_path):
     """
     files = {'file': open(file_path, 'rb')}
     url = 'https://static.shlisselburg.org/upload/art{}'.format(relative_path)
+
+    if not settings.upload_enabled:
+        return url
+
     r = requests.put(
         url=url,
         files=files,

utils/Sync/sync/data/fs.py

@@ -13,13 +13,9 @@ class FSProvider(Provider):
         return open(self.get_abs(path), 'rb')
 
     def scan(self, path):
-        return lmap(
-            self.get_rel,
-            map(
-                lambda i: i.path,
-                os.scandir(self.get_abs(path))
-            )
-        )
+        paths = [i.path for i in os.scandir(self.get_abs(path))]
+
+        return [self.get_rel(path) for path in paths]
 
     def glob(self, pattr):
         return lmap(self.get_rel, glob(self.get_abs(pattr)))

utils/Sync/sync/data/request.py

@@ -0,0 +1,41 @@
+import aiohttp
+import requests
+
+import settings
+
+
+async def upload(url, path):
+    """
+    :param url:
+    :param path:
+    :return:
+    """
+
+    if not settings.upload_enabled:
+        return url
+
+    # urlf = 'https://static.shlisselburg.org/upload/art{}'
+
+    files = {'file': open(path, 'rb')}
+    r = requests.put(
+        url=url,
+        files=files,
+        # auth=settings.upload_auth
+        auth=('uploader', '1207690962638465')
+    )
+    if r.status_code == 200:
+        return url
+    else:
+        return None
+
+
+async def get(url):
+    async with aiohttp.ClientSession as session:
+        res = session.get(url)
+        return await res.text()
+
+
+async def put(url, data):
+    async with aiohttp.ClientSession as session:
+        res = session.put(url, data=data)
+        return await res.text()

utils/Sync/sync/document.py

@@ -4,9 +4,9 @@ import requests
 
 import settings
 from sync import synced_image_id, untouched
-from sync.core.document import SyncDocument
+from sync.core.SyncDocument import SyncDocument
 from sync.data import scan_subdirs
-from sync.models.document import Document
+from sync.models.Document import Document
 from utils.fn import combine, lmap, key_mapper
 
 
@@ -47,7 +47,7 @@ def sync_documents(provider, collection, update=True, delete=True):
             sizes=settings.image_sizes,
             dir_static_previews=settings.dir_static_images,
             url_base_preview=settings.url_base_preview,
-            url_base_document=settings.url_base_document,
+            url_base_document=settings.document_url_template,
         ),
         static_path,
         update_documents=update,

utils/Sync/sync/image.py

@@ -1,5 +1,5 @@
 from db import db
-from settings import image_name_format, image_base_url
+from settings import image_name_format, image_url_base
 from utils.fn import lmap
 
 
@@ -62,5 +62,5 @@ def sync_image(record):
 
 
 def image_create_url(file):
-    url = image_base_url + image_name_format
+    url = image_url_base + image_name_format
     return url.format(file)

utils/Sync/sync/models/Document.py

@@ -0,0 +1,188 @@
+import os
+from tempfile import mkstemp
+
+import requests
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from sync.models.Image import Image
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image import create_image
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+
+
+def find(store, item_id: str):
+    q = {'id': item_id}
+    try:
+        return store.find_one(q)
+    except ValueError:
+        pass
+    return None
+
+
+class Document(Model):
+    @staticmethod
+    async def scan(provider):
+        paths = scan_subdirs(provider, '.pdf')
+        return [Document.read_path(provider, path) for path in paths]
+
+    @staticmethod
+    def read_path(provider, path):
+        name = os.path.basename(path)
+
+        document = {
+            'category': os.path.dirname(path),
+            'title': os.path.splitext(name)[0]
+        }
+
+        return Document(
+            provider,
+            path,
+            **document
+        )
+
+    def __init__(self, provider, file, category=None, title=None):
+        store = collection(settings.collection_documents)
+
+        self.__hash_keys = ['id', 'url']
+
+        self.__id_template: str = '{category}-{file}'
+        self.__url_template: str = settings.document_url_template
+        self.__url_preview_template: str = settings.document_url_preview_template
+
+        self.category = category
+        self.title = title
+        self.type = None
+        self.preview = None
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.__set_id()
+        self.__set_url()
+        self.__set_hash()
+
+    async def is_changed(self):
+        i = find(self.store, self.id)
+        if not i:
+            return True
+
+        if 'hash' not in i:
+            return True
+
+        return not (self.hash == i['hash'])
+
+    async def save(self):
+        document = self.bake()
+
+        query = {'id': document['id']}
+        try:
+            self.store.update_one(query, {'$set': document}, upsert=True)
+            # return self.read(document)
+            return self
+        except ValueError:
+            pass
+        return None
+
+    async def upload(self):
+        filepath = self.provider.get_local(self.file)
+        name = os.path.basename(self.url)
+        url = settings.document_url_upload_template.format(file=name)
+
+        await request.upload(url, filepath)
+
+    def bake(self):
+        file = self.file
+        filename = os.path.basename(file)
+
+        return {
+            'id': self.id,
+            'hash': self.hash,
+            'url': self.url,
+            'type': document_type(self.type),
+            'file': {
+                'name': filename,
+                'size': self.provider.size(file)
+            },
+            'preview': self.preview,
+            'title': last_good([
+                self.title,
+                get_pdf_title(self.provider, self.file)
+            ]),
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __set_id(self):
+        self.id = self.__id_template.format(
+            category=url_encode_text(self.category),
+            file=url_encode_text(self.__filename())
+        )
+
+    def __set_url(self):
+        self.url = self.__url_template.format(
+            file=url_encode_file(self.__filename())
+        )
+
+    def __set_hash(self):
+        doc = [getattr(self, key) for key in self.__hash_keys]
+
+        self.hash = hash_str(
+            hash_str(doc) + self.provider.hash(self.file)
+        )
+
+    async def create_preview(self, sizes):
+        def url(_, size, ext):
+            file = url_encode_text(self.__filename())
+            return self.__url_preview_template.format(id=file, size=size, ext=ext)
+
+        temp_preview_path = pdf_to_jpg(self.provider, self.file)
+        preview = await Image.new(self.provider, temp_preview_path, sizes, url)
+
+        if preview:
+            self.preview = preview.id
+
+        os.remove(temp_preview_path)
+
+    def __str__(self):
+        return '<Document hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None

utils/Sync/sync/models/Image.py

@@ -0,0 +1,207 @@
+import os
+import tempfile
+from tempfile import mkstemp
+from typing import Optional, List
+
+import requests
+
+import settings
+from db import collection
+from sync.data import scan_subdirs, request
+from sync.models import Model
+from utils.fn import last_good, lmap
+from utils.hash import hash_str
+from utils.image.resize import image_magick_pdf_to_img
+from utils.text.transform import url_encode_text, url_encode_file
+import os
+
+import logging
+
+import settings
+from sync.data import upload
+from utils.hash import hash_file
+from utils.image.resize import optimize, thumbnail, orient
+
+logger = logging.getLogger(settings.name)
+
+
+def get_upload_url(url):
+    return os.path.join(settings.image_url_upload, os.path.basename(url))
+
+
+class Image(Model):
+    @staticmethod
+    async def new(provider, file, sizes, url_factory):
+        img = Image(provider, file, url_factory)
+        await img.compile(sizes)
+        await img.upload()
+        await img.sync()
+        return img
+
+    def __init__(self, provider, file, url_factory):
+        store = collection(settings.collection_images)
+        self.url_factory = url_factory
+
+        super().__init__(provider, store, file)
+
+    def init(self):
+        self.hash = self.__get_hash()
+
+    async def compile(self, sizes):
+        img_out = tempfile.mkdtemp()
+        self.image = create_image(self.file, sizes, self.url_factory, img_out)
+
+    async def upload(self):
+        if self.image:
+            for i in self.image:
+                url = get_upload_url(i['url'])
+                file = i['file']
+
+                logger.info('Uploading image {}'.format(file))
+
+                await request.upload(url, file)
+
+    async def sync(self):
+        c = sync_image(self.bake())
+        self.id = c['_id']
+
+    def bake(self):
+        data = {}
+        if self.image:
+            for i in self.image:
+                size_name = i['size']
+                data[size_name] = {
+                    'url': i['url'],
+                    'size': size_name,
+                    'width': i['width'],
+                    'height': i['height'],
+                }
+
+        return {
+            'file': self.file,
+            'hash': self.hash,
+            'data': data,
+        }
+
+    def __filename(self):
+        return os.path.basename(self.file)
+
+    def __get_hash(self):
+        return self.provider.hash(self.file)
+
+    def __str__(self):
+        return '<Image hash={} file={}>'.format(self.hash, self.file)
+
+
+def sync_image(record):
+    from db import db
+
+    q = {'hash': record['hash']}
+    try:
+        db().images.update_one(q, {'$set': record}, upsert=True)
+        i = db().images.find_one({'hash': record['hash']})
+        return i
+    except ValueError:
+        pass
+
+    return None
+
+
+def document_type(doctype):
+    if doctype == 'Награды':
+        return 'award'
+    return 'document'
+
+
+def pdf_to_jpg(provider, pdf):
+    _, temp_in = mkstemp(suffix='.pdf')
+    _, temp_out = mkstemp(suffix='.jpg')
+
+    abspdf = provider.copy(pdf, temp_in)
+    image_magick_pdf_to_img(abspdf, temp_out)
+
+    os.remove(temp_in)
+    return temp_out
+
+
+def get_pdf_title(provider, file):
+    from PyPDF2 import PdfFileReader
+    from PyPDF2.generic import TextStringObject
+    from PyPDF2.generic import IndirectObject
+
+    pdf = PdfFileReader(provider.read(file))
+    info = pdf.getDocumentInfo()
+
+    if info:
+        if type(info.title) == TextStringObject:
+            return str(info.title)
+
+        if type(info.title_raw) == IndirectObject:
+            o = pdf.getObject(info.title_raw)
+            return str(o)
+    return None
+
+
+def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
+    """
+
+    :param file: Image path to process
+    :param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
+    :param url_fn: function (size_name, ext) for creating image url
+    :param output_dir: path to local folder for thumbs storing
+    :return:
+    """
+    if not os.path.exists(file):
+        return None
+
+    _, ext = os.path.splitext(file)
+
+    def fn(size):
+        size_name, width, height = size
+
+        image_url = url_fn(file, size_name, ext)
+        image_filename = os.path.basename(image_url)
+        local_image_path = os.path.join(output_dir, image_filename)
+
+        if settings.image_processing_enabled:
+            image = process_image(file, local_image_path, size)
+            if not image:
+                logger.warning('Failed to process image {}'.format(file))
+                return None
+            width, height = image.size
+
+        return {
+            'file': local_image_path,
+            'url': image_url,
+            'size': size_name,
+            'width': width,
+            'height': height
+        }
+
+    return [fn(s) for s in sizes]
+
+
+def process_image(input_file, output_file, size):
+    size_name, width, height = size
+
+    if size_name == 'original':
+        return optimize(input_file, output_file, quality=90)
+    else:
+        image = read_image(input_file)
+        if not image:
+            return None
+        image_width, image_height = image.size
+        if image_height > image_width:
+            width, height = height, width
+        return thumbnail(input_file, output_file, (width, height))
+
+
+def read_image(src):
+    from PIL import Image
+
+    try:
+        i = Image.open(src)
+        return i
+    except Exception as e:
+        print(src, e)
+        return None

utils/Sync/sync/models/__init__.py

@@ -1,6 +1,25 @@
 class Model:
-    def __init__(self):
+    def __init__(self, provider, store, file):
         super().__init__()
 
+        self.provider = provider
+        self.store = store
+        self.file = file
+
+        self.id = None
+        self.url = None
+        self.hash = None
+
+        self.init()
+
+    def init(self):
+        pass
+
     def bake(self):
         return None
+
+    async def is_changed(self):
+        return False
+
+    async def save(self):
+        pass

utils/Sync/sync/models/document.py

@@ -1,124 +0,0 @@
-import os
-from tempfile import mkstemp
-
-import settings
-from sync.models import Model
-from utils.fn import last_good, lmap
-from utils.hash import hash_str
-from utils.image import create_image
-from utils.image.resize import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
-
-
-class Document(Model):
-    @staticmethod
-    def read(path):
-        name = os.path.basename(path)
-
-        document = {
-            'file': path,
-            'category': os.path.dirname(path),
-            'title': os.path.splitext(name)[0]
-        }
-
-        return document
-
-    def __init__(self, provider, path, url_base, preview_compiler):
-        super().__init__()
-
-        self.path = path
-
-        self.category = None
-        self.file = None
-        self.id = None
-        self.type = None
-        self.url = None
-        self.title = None
-        self.preview = None
-
-        for k, v in m.items():
-            setattr(self, k, v)
-
-        self.provider = provider
-
-    def bake(self):
-        file = self.file
-        filename = os.path.basename(file)
-
-        self.type = document_type(self.type)
-        self.preview = self.create_preview(file)
-        self.title = last_good([
-            self.title,
-            get_pdf_title(self.provider, self.file)
-        ])
-        self.file = {
-            'name': filename,
-            'size': self.provider.size(file)
-        }
-
-    def __set_id(self):
-        bn = os.path.basename(self.file)
-        self.id = url_encode_text('{category}-{file}'.format(
-            category=self.category,
-            file=bn
-        ))
-
-    def __set_url(self):
-        filename = os.path.basename(self.file)
-        self.url = self.url_base_document.format(file=url_encode_file(filename))
-
-    def __set_hash(self):
-        hash_keys = ['id', 'url']
-        hash_doc = lmap(
-            lambda key: getattr(self, key),
-            hash_keys
-        )
-        self.hash = hash_str(
-            hash_str(hash_doc) + self.provider.hash(self.file)
-        )
-
-    def create_preview(self, file):
-        def url(size, ext):
-            return self.url_base_preview.format(id=file, size=size, ext=ext)
-
-        temp_preview_path = pdf_to_jpg(self.provider, file)
-        file = os.path.basename(file)
-        file = url_encode_text(file)
-        img = create_image(temp_preview_path, self.sizes, url, settings.dir_static_images)
-        os.remove(temp_preview_path)
-        return img
-
-
-def document_type(doctype):
-    if doctype == 'Награды':
-        return 'award'
-    return 'document'
-
-
-def pdf_to_jpg(provider, pdf):
-    _, temp_in = mkstemp(suffix='.pdf')
-    _, temp_out = mkstemp(suffix='.jpg')
-
-    abspdf = provider.copy(pdf, temp_in)
-    image_magick_pdf_to_img(abspdf, temp_out)
-
-    os.remove(temp_in)
-    return temp_out
-
-
-def get_pdf_title(provider, file):
-    from PyPDF2 import PdfFileReader
-    from PyPDF2.generic import TextStringObject
-    from PyPDF2.generic import IndirectObject
-
-    pdf = PdfFileReader(provider.read(file))
-    info = pdf.getDocumentInfo()
-
-    if info:
-        if type(info.title) == TextStringObject:
-            return str(info.title)
-
-        if type(info.title_raw) == IndirectObject:
-            o = pdf.getObject(info.title_raw)
-            return str(o)
-    return None

utils/Sync/sync/page.py

@@ -135,6 +135,17 @@ def create_page(provider: Provider, img: ImageCreator, document):
     return document
 
 
+def check_url(documents):
+    urls = list(map(
+        lambda i: i['url'],
+        documents
+    ))
+
+    unique_urls = set(urls)
+    if len(urls) != len(unique_urls):
+        raise Exception('URL of pages should be unique')
+
+
 def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=True):
     img_sizes = settings.album_image_sizes
     output_dir = settings.dir_static_images
@@ -170,6 +181,8 @@ def sync_pages(provider, collection, update=True, delete=True, skip_unchanged=Tr
         lambda i: i['id']
     )
 
+    check_url(documents)
+
     # SKIP UNTOUCHED DOCUMENTS
     if skip_unchanged:
         documents = untouched(documents, sync)

utils/Sync/sync/post.py

@@ -1,7 +1,7 @@
 import os
 
 import settings
-from kazimir import markdown_to_html
+from kazimir import kazimir_to_html
 from sync.data import list_images
 from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \
     synced_images_ids, untouched
@@ -61,7 +61,7 @@ def get_manifest(provider, path):
     if 'id' in manifest:
         manifest['id'] = str(manifest['id'])
 
-    html = markdown_to_html(manifest['content'])
+    html = kazimir_to_html(manifest['content'])
     return {
         **manifest,
         'post': html,
@@ -134,7 +134,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
     )
 
     # SKIP UNTOUCHED DOCUMENTS
-    documents = untouched(documents, sync)
+    # documents = untouched(documents, sync)
 
     # MAP EVENT MANIFEST -> EVENT_OBJECT
     documents = lmap(
@@ -174,7 +174,7 @@ def main(sync, file_time_formats, update_documents=True, delete_documents=True):
 
 
 def sync_posts(provider, collection, update=True, delete=True):
-    image_url_base = settings.image_base_url + 'post-{id}-{img}-{size}{ext}'
+    image_url_base = settings.image_url_base + 'post-{id}-{img}-{size}{ext}'
 
     return main(
         SyncPost(