[Sync] refactoring, removed old code, cleanup

utils/Sync/sync/__init__.py

@@ -1,16 +1,13 @@
 import os
 import re
 from datetime import datetime
-from glob import glob
 from lxml import etree
 
 import lxml.html
 from markdown import markdown
 
 import settings
-from sync.image import sync_image
 from sync.models import Model
-from utils.fn import lmap, combine
 
 
 def create_date(date_str, date_formats=None):
@@ -66,14 +63,6 @@ async def untouched(items: [Model]) -> [Model]:
     # return filtered_documents
 
 
-def synced_images_ids(images):
-    return [i['_id'] for i in sync_image(images)]
-
-
-def synced_image_id(image):
-    return sync_image(image)['_id']
-
-
 def create_date_and_title_from_folder_name(folder_name, date_formats=None):
     m = folder_name_pattern.findall(folder_name)
     if not m:

utils/Sync/sync/core/SyncDocument.py

@@ -6,11 +6,10 @@ import logging
 import settings
 from sync import untouched
 from sync.core import Sync
-from sync.data import scan_subdirs, Provider
+from sync.data import Provider
 from sync.models.Document import Document
 
-from sync.models import Model
-from utils.fn import last_good, lmap
+from utils.fn import last_good
 from utils.hash import hash_str
 from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img

utils/Sync/sync/core/__init__.py

@@ -1,6 +1,4 @@
 from sync.data import Provider
-from sync.models import Model
-from utils.fn import lmap
 
 
 def get_id(item):
@@ -60,11 +58,5 @@ class Sync:
     def delete(self, q):
         return self.collection.find_one_and_delete(q)
 
-    def create_id(self, document):
-        return None
-
-    def create_hash(self, document):
-        return None
-
     def create_remove_query(self, query):
         return query

utils/Sync/sync/data/__init__.py

@@ -1,10 +1,7 @@
 import os
 from io import BytesIO
 
-import requests
-
-import settings
-from utils.fn import combine, lmap
+from utils.fn import combine
 
 
 class Provider:
@@ -61,16 +58,11 @@ def scan_subdirs(provider, ext, path='.'):
     :return: list of files relative to provider.root
     """
     items = provider.scan(path)
-    items = list(filter(
+    items = filter(
         lambda i: provider.is_dir(i),
         items
-    ))
-
-    items = lmap(
-        lambda folder: provider.type_filter(folder, ext),
-        items
     )
-
+    items = [provider.type_filter(folder, ext) for folder in items]
     items = combine(items)
     return items
 
@@ -82,7 +74,5 @@ def get_data(provider, path, transform):
 
 def list_images(provider, path):
     image_types = ['.jpg', '.JPG', '.png', '.PNG']
-    return combine(lmap(
-        lambda ext: provider.type_filter(path, ext),
-        image_types
-    ))
+    images = [provider.type_filter(path, ext) for ext in image_types]
+    return combine(images)

utils/Sync/sync/data/fs.py

@@ -4,7 +4,6 @@ from glob import glob
 from shutil import copyfile
 
 from sync.data import Provider
-from utils.fn import lmap
 from utils.hash import hash_file
 
 
@@ -18,7 +17,8 @@ class FSProvider(Provider):
         return [self.get_rel(path) for path in paths]
 
     def glob(self, pattr):
-        return lmap(self.get_rel, glob(self.get_abs(pattr)))
+        items = glob(self.get_abs(pattr))
+        return [self.get_rel(i) for i in items]
 
     def type_filter(self, path, ext):
         return list(filter(

utils/Sync/sync/data/yandexdisk.py

@@ -8,7 +8,6 @@ from tempfile import mkstemp
 import requests
 
 from sync.data import Provider
-from utils.fn import lmap
 from yandexdisk import YDClient
 
 
@@ -34,7 +33,7 @@ class YDProvider(Provider):
             lambda fp: re.sub('disk:', '', fp),
             files
         )
-        return lmap(self.get_rel, files)
+        return [self.get_rel(i) for i in files]
 
     def glob(self, pattr):
         return []

utils/Sync/sync/image.py

@@ -1,20 +0,0 @@
-from db import db
-from utils.fn import lmap
-
-
-def sync_image(record):
-    if isinstance(record, list):
-        return lmap(
-            sync_image,
-            record
-        )
-
-    q = {'hash': record['hash']}
-    try:
-        db().images.update_one(q, {'$set': record}, upsert=True)
-        i = db().images.find_one({'hash': record['hash']})
-        return i
-    except ValueError:
-        pass
-
-    return None

utils/Sync/sync/models/Album.py

@@ -14,7 +14,7 @@ from sync.data import Provider, list_images
 from sync.models import Model
 from sync.models.Image import Image
 
-from utils.fn import swap_ext, first, map_cases, constant, combine, iterate_iter_over_fns
+from utils.fn import swap_ext, first, map_cases, constant, combine
 from utils.hash import hash_str
 from utils.io import read_yaml_md, parse_yaml_front_matter
 from utils.text.transform import url_encode_text

utils/Sync/sync/models/Document.py

@@ -1,16 +1,13 @@
 import os
 from tempfile import mkstemp
 
-import requests
-
 import settings
 from db import collection
 from sync.data import scan_subdirs, request
 from sync.models import Model
 from sync.models.Image import Image
-from utils.fn import last_good, lmap
+from utils.fn import last_good
 from utils.hash import hash_str
-from utils.image import create_image
 from utils.image.resize import image_magick_pdf_to_img
 from utils.text.transform import url_encode_text, url_encode_file
 

utils/Sync/sync/models/Page.py

@@ -34,13 +34,13 @@ class Page(Model):
         documents = [i for i in provider.scan('.') if provider.is_dir(i)]
         documents = [provider.type_filter(i, '.md') for i in documents]
         documents = [i[0] for i in documents if len(i)]
-        documents = [Page.read_path(provider, i) for i in documents]
+        documents = [Page.read(provider, i) for i in documents]
         documents = [i for i in documents if i]
 
         return documents
 
     @staticmethod
-    def read_path(provider, path):
+    def read(provider, path):
         params = {
             'file': os.path.basename(path),
             'folder': os.path.dirname(path),
@@ -89,17 +89,6 @@ class Page(Model):
 
         return not (self.hash == i['hash'])
 
-    async def save(self):
-        document = self.bake()
-
-        query = {'id': document['id']}
-        try:
-            self.store.update_one(query, {'$set': document}, upsert=True)
-            return self
-        except ValueError:
-            pass
-        return None
-
     async def setup_images(self, sizes, url_factory):
         folder = self.params['folder']
         images = []
@@ -142,15 +131,16 @@ class Page(Model):
 
     def __set_id(self):
         if 'id' in self.params:
-            self.id = self.params['id']
+            self.id = self.get_param('id')
         else:
-            self.id = url_encode_text(self.params['url'])
+            self.id = url_encode_text(self.get_param('url'))
 
     def __set_url(self):
         self.url = self.params['url']
 
     def __set_hash(self):
-        images = [os.path.join(self.params['folder'], i) for i in self.params['images']]
+        f = self.get_param('folder')
+        images = [os.path.join(f, i) for i in self.get_param('images')]
         files = sorted([self.file] + images)
         hashes = [self.provider.hash(i) for i in files]
 

utils/Sync/sync/models/product.py

@@ -1,8 +1,6 @@
 import settings
 from db import collection
 from sync.core.person import SyncPerson
-from sync.image import sync_image
-from utils.fn import key_mapper
 from utils.text.transform import interpolate_swift
 
 
@@ -17,9 +15,6 @@ def resolve_value(param, store):
     return store(i)
 
 
-unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
-
-
 def resolve_teacher(i):
     sync = SyncPerson(
         collection(settings.collection_collective),
@@ -32,13 +27,14 @@ def resolve_teacher(i):
 def create_product(doc):
     # author, image, title, awards, teacher, date
 
-    unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
+    # unpop_image = key_mapper('image', lambda i: sync_image(i)['_id'])
+    # unpop_teacher = key_mapper('teacher', lambda i: i['_id'])
 
     product = {**doc}
-    unpop_image(product)
+    # unpop_image(product)
 
-    product['teacher'] = resolve_teacher(doc['teacher'])
-    if (product['teacher']):
-        unpop_teacher(product)
+    # product['teacher'] = resolve_teacher(doc['teacher'])
+    # if (product['teacher']):
+        # unpop_teacher(product)
 
     return product