[Sync] Document refactoring + attaching to post

utils/Sync/settings.py

@@ -132,6 +132,7 @@ collection_pages = 'pages'
 hash_salt_articles = '3'
 hash_salt_pages = '1'
 hash_salt_albums = '0'
+hash_salt_documents = '0'
 
 provider_name = value('provider')['name']  # env('SYNC_PROVIDER', 'fs')
 provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
@@ -139,7 +140,7 @@ provider_root = value('provider')['root']  # env('SYNC_PROVIDER', 'fs')
 f = abs_fn(provider_root)
 
 dir_settings = f('Settings')
-dir_documents = f('Documents')
+dir_documents = './Documents'
 dir_awards = f('Awards')
 dir_schedules = f('Schedules')
 dir_articles = f('Articles')

utils/Sync/sync.py

@@ -47,7 +47,7 @@ async def run(run_interval=0):
     io = lambda root: get_provider(settings.provider_name, root)
 
     sync_documents = SyncDocument(
-        provider=io(settings.dir_documents),
+        provider=io(settings.provider_root),
         sizes=settings.image_sizes,
     )
 

utils/Sync/sync/__init__.py

@@ -7,9 +7,16 @@ from markdown import markdown
 
 import kazimir
 import settings
+from kazimir.YoutubeToken import YoutubeToken
+from kazimir.InstagramToken import InstagramToken
+from kazimir.Marker import Marker
+from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, DocumentToken
 from sync.data import Provider
 from sync.models import Model
+from sync.models.Document import Document
 from sync.models.Image import Image
+from utils.text.file import get_size
+from utils.text.file import get_ext
 
 
 def create_date(date_str, date_formats=None):
@@ -82,10 +89,34 @@ def create_date_and_title_from_folder_name(folder_name, date_formats=None):
 
 
 async def create_post(provider: Provider, folder: str, md: str, sizes):
-    html = kazimir.create_tree(md)
+    from kazimir.fix_links_quotes import fix_links_quotes
+
     images = []
-    for img in html.cssselect('img'):
-        src = img.get('src')
+    documents = []
+
+    async def build_document(data):
+        filepath = os.path.join(folder, data['file'])
+        document = await Document.new(provider, filepath, settings.image_sizes)
+
+        if not document:
+            raise Exception(f'Failed to build Document {filepath}')
+
+        text = data['caption'] if data['caption'] else data['file']
+        url = f'/document/{document.id}'
+
+        image_url = document.preview.get_size('small')['url']
+
+        return {
+            'url': url,
+            'image_url': image_url,
+            'file_url': document.url,
+            'title': document.title,
+            'file_size': get_size(document.file_size, 1),
+            'file_format': get_ext(data['file']),
+        }
+
+    async def build_image(data):
+        src = data['file']
         relative_image_path = os.path.join(folder, src)
 
         image_path = provider.get_local(relative_image_path)
@@ -94,14 +125,30 @@ async def create_post(provider: Provider, folder: str, md: str, sizes):
             if image:
                 url = image.get_size('big')['url']
                 images.append(image)
-                img.set('src', url)
-            else:
-                raise Exception('Fail to get Image', folder, src)
+
+                return {
+                    'src': url,
+                    'alt': data['caption'],
+                    'caption': data['caption'],
+                }
+        raise Exception('Fail to get Image', relative_image_path)
+
+    m = Marker()
+    m.add_token_factory(TokenFactory(SplitToken))
+    m.add_token_factory(TokenFactory(YoutubeToken))
+    m.add_token_factory(TokenFactory(InstagramToken))
+    m.add_token_factory(TokenFactory(UrlToken))
+    m.add_token_factory(BuildTokenFactory(ImageToken, build=build_image))
+    m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
+    # m.add_token_factory(TokenFactory(FileToken))
+    m.add_token_factory(TokenFactory(TextToken))
+    m.add_tree_middleware(fix_links_quotes)
+    html = await m.create_tree(md)
 
     post = kazimir.html_from_tree(html)
     post = await kazimir.typo(post)
 
-    return post, images, []
+    return post, images, documents
 
 
 def images_from_html(md):

utils/Sync/sync/models/Document.py

@@ -6,14 +6,17 @@ from db import collection
 from sync.data import scan_subdirs, request
 from sync.models import Model
 from sync.models.Image import Image
-from utils.hash import hash_str
+from utils.hash import hash_str, md5
 from utils.image import image_magick_pdf_to_img
-from utils.text.transform import url_encode_text, url_encode_file
 
 store = collection(settings.collection_documents)
 
 
 class Document(Model):
+    @staticmethod
+    async def find_one(query):
+        return store.find_one(query)
+
     @staticmethod
     async def find(query):
         return store.find(query)
@@ -24,7 +27,8 @@ class Document(Model):
 
     @staticmethod
     async def scan(provider):
-        paths = scan_subdirs(provider, '.pdf')
+        scan_dir = settings.dir_documents
+        paths = scan_subdirs(provider, '.pdf', scan_dir)
         return [Document.read_path(provider, path) for path in paths]
 
     @staticmethod
@@ -39,28 +43,53 @@ class Document(Model):
         return Document(
             provider,
             path,
-            **document
+            document
         )
 
-    def __init__(self, provider, file, category=None, title=None):
+    @staticmethod
+    async def new(provider, file, sizes):
+        document = Document.read_path(provider, file)
+        changed = await document.is_changed()
+        if changed:
+            await document.build(sizes=sizes)
+            await document.upload()
+            await document.save()
+        else:
+            doc = await Document.find_one({'file': file})
+            document = Document(provider, file, doc)
+            document.ref = doc['_id']
+            await document.create_preview(sizes)
+
+        return document
+
+    def __init__(self, provider, file, params=None):
+        self.__hash_salt = settings.hash_salt_documents
         self.__hash_keys = ['id', 'url']
 
         self.__id_template: str = '{category}-{file}'
         self.__url_template: str = settings.document_url_template
         self.__url_preview_template: str = settings.document_url_preview_template
 
-        self.category = category
-        self.title = title
+        self.title = None
+        self.file_name = None
+        self.file_size = None
         self.type = 'document'  # 'award'
         self.preview = None
 
-        super().__init__(provider, store, file)
+        super().__init__(provider, store, file, params=params)
 
     def init(self):
         self.__set_id()
         self.__set_url()
         self.__set_hash()
 
+        filename = os.path.basename(self.file)
+        self.file_name = filename
+        self.file_size = self.provider.size(self.file)
+
+        if self.has_param('title'):
+            self.title = self.get_param('title')
+
     async def build(self, **kwargs):
         sizes = kwargs['sizes']
         await self.create_preview(sizes)
@@ -75,7 +104,10 @@ class Document(Model):
         query = {'id': document['id']}
         try:
             self.store.update_one(query, {'$set': document}, upsert=True)
-            # return self.read(document)
+
+            doc = await Document.find_one(query)
+            self.ref = doc['_id']
+
             return self
         except ValueError:
             pass
@@ -89,58 +121,59 @@ class Document(Model):
         await request.upload(url, filepath)
 
     def bake(self):
-        file = self.file
-        filename = os.path.basename(file)
-
         return {
+            **self.params,
             'id': self.id,
+            'file': self.file,
             'hash': self.hash,
             'url': self.url,
             'type': self.type,
-            'file': {
-                'name': filename,
-                'size': self.provider.size(file)
-            },
-            'preview': self.preview,
             'title': self.title,
+            'preview': self.preview.ref,
+            'fileInfo': {
+                'name': self.file_name,
+                'size': self.file_size,
+            },
         }
 
     def __filename(self):
         return os.path.basename(self.file)
 
     def __set_id(self):
-        self.id = self.__id_template.format(
-            category=url_encode_text(self.category),
-            file=url_encode_text(self.__filename())
-        )
+        self.id = md5(self.file)
 
     def __set_url(self):
+        s = os.path.splitext(self.file)
+        file = f'{self.id}{s[1]}'
+
         self.url = self.__url_template.format(
-            file=url_encode_file(self.__filename())
+            file=file
         )
 
     def __set_hash(self):
         doc = [getattr(self, key) for key in self.__hash_keys]
 
         self.hash = hash_str(
-            hash_str(doc) + self.provider.hash(self.file)
+            self.__hash_salt + hash_str(doc) + self.provider.hash(self.file)
         )
 
     async def create_preview(self, sizes):
-        def url(_, size, ext):
-            file = url_encode_text(self.__filename())
-            return self.__url_preview_template.format(id=file, size=size, ext=ext)
+        if False and self.has_param('preview'):
+            preview = self.get_param('preview')
+            preview = await Image.find_one({'_id': preview})
+        else:
+            def url(_, size, ext):
+                return self.__url_preview_template.format(id=self.id, size=size, ext=ext)
 
-        temp_preview_path = await pdf_to_jpg(self.provider, self.file)
-        preview = await Image.new(self.provider, temp_preview_path, sizes, url)
+            temp_preview_path = await pdf_to_jpg(self.provider, self.file)
+            preview = await Image.new(self.provider, temp_preview_path, sizes, url)
+            os.remove(temp_preview_path)
 
         if preview:
-            self.preview = preview.id
-
-        os.remove(temp_preview_path)
+            self.preview = preview
 
     def __str__(self):
-        return '<Document hash={} file={} id={}>'.format(self.hash, self.file, self.id)
+        return f'<Document file={self.file} id={self.id}>'
 
 
 async def pdf_to_jpg(provider, pdf):