From b98a80954358b569608c84b2774b47dc7e4efbfe Mon Sep 17 00:00:00 2001 From: Roman Timashev Date: Sun, 18 Sep 2016 01:47:38 +0300 Subject: [PATCH] [sync] updated post events --- utils/Sync/sync.py | 6 ++++++ utils/Sync/sync/core/post.py | 12 ++++++------ utils/Sync/sync/data/__init__.py | 3 +++ utils/Sync/sync/data/fs.py | 2 ++ utils/Sync/sync/data/yandexdisk.py | 7 +++++++ utils/Sync/sync/post.py | 11 +---------- utils/Sync/utils/image/__init__.py | 17 +++++++---------- utils/Sync/utils/image/resize.py | 20 +++++++++----------- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/utils/Sync/sync.py b/utils/Sync/sync.py index 815d784f..b2ddaac0 100644 --- a/utils/Sync/sync.py +++ b/utils/Sync/sync.py @@ -1,9 +1,15 @@ +from subprocess import call + import settings from db import db from sync.data.fs import FSProvider from sync.data.yandexdisk import YDProvider +from sync.document import sync_documents from sync.post import sync_posts +from sync.schedule import sync_schedules from utils.fn import lprint_json +from utils.image import read_image +from utils.image import thumbnail def get_provider(provider_type, root): diff --git a/utils/Sync/sync/core/post.py b/utils/Sync/sync/core/post.py index 4de3776c..92051095 100644 --- a/utils/Sync/sync/core/post.py +++ b/utils/Sync/sync/core/post.py @@ -26,16 +26,17 @@ class SyncPost(Sync): return document def create_hash(self, document): - hash_images = lmap( + files = sorted(document['images']) + hashes = lmap( self.provider.hash, lmap( lambda i: os.path.join(document['folder'], i), - document['images'] + files ) ) document['hash'] = hash_str( - combine([hash_str(document)] + hash_images) + combine([hash_str(document)] + hashes) ) return document @@ -54,10 +55,9 @@ class SyncPost(Sync): images = [] for img in post_html.cssselect('img'): src = img.get('src') - img_path = os.path.abspath( - os.path.join(document['folder'], src) - ) + relative_image_path = os.path.join(document['folder'], src) + img_path = self.provider.get_local(relative_image_path) if os.path.exists(img_path): img_id = url_encode_text(os.path.splitext(src)[0]) url_fn = lambda size, ext: self.image_url_base.format( diff --git a/utils/Sync/sync/data/__init__.py b/utils/Sync/sync/data/__init__.py index 9815e3e8..e6022a53 100644 --- a/utils/Sync/sync/data/__init__.py +++ b/utils/Sync/sync/data/__init__.py @@ -38,6 +38,9 @@ class Provider: rel = os.path.relpath(path, self.root) return rel + def get_local(self, path): + return path + def is_dir(self, path): return False diff --git a/utils/Sync/sync/data/fs.py b/utils/Sync/sync/data/fs.py index 3ce94064..0f3ed756 100644 --- a/utils/Sync/sync/data/fs.py +++ b/utils/Sync/sync/data/fs.py @@ -44,3 +44,5 @@ class FSProvider(Provider): def is_dir(self, path): return os.path.isdir(self.get_abs(path)) + def get_local(self, path): + return self.get_abs(path) diff --git a/utils/Sync/sync/data/yandexdisk.py b/utils/Sync/sync/data/yandexdisk.py index bf391a87..ea8be6c4 100644 --- a/utils/Sync/sync/data/yandexdisk.py +++ b/utils/Sync/sync/data/yandexdisk.py @@ -3,6 +3,8 @@ import os import re import io +from tempfile import mkstemp + import requests from sync.data import Provider @@ -61,3 +63,8 @@ class YDProvider(Provider): def is_dir(self, path): res = self.api.get_content(self.get_abs(path)) return res.type == 'dir' + + def get_local(self, path): + ext = os.path.splitext(path)[1] + _, temp_in = mkstemp(suffix=ext) + return self.copy(path, temp_in) diff --git a/utils/Sync/sync/post.py b/utils/Sync/sync/post.py index 3d1fbf7e..0750090b 100644 --- a/utils/Sync/sync/post.py +++ b/utils/Sync/sync/post.py @@ -1,21 +1,12 @@ import os -from datetime import datetime -from glob import glob - -import lxml.html import settings from sync.data import list_images -from db import db from sync import create_date_and_title_from_folder_name, create_post_from_image_list, create_date, images_from_html, \ synced_images_ids, untouched from sync.core.post import SyncPost -from sync.image import sync_image -from utils.fn import combine, lmap, map_cases, first, lprint, key_mapper, lprint_json, lmapfn, constant -from utils.hash import hash_file, hash_str -from utils.image import create_image +from utils.fn import lmap, map_cases, first, key_mapper, lmapfn, constant from utils.io import read_yaml_md -from utils.text.transform import url_encode_text def get_manifest(provider, path): diff --git a/utils/Sync/utils/image/__init__.py b/utils/Sync/utils/image/__init__.py index e3c9ca35..a9d8fbd3 100644 --- a/utils/Sync/utils/image/__init__.py +++ b/utils/Sync/utils/image/__init__.py @@ -3,7 +3,7 @@ import os from PIL import Image from utils.hash import hash_file -from utils.image import resize +# from utils.image import resize from utils.image.resize import optimize, thumbnail, orient @@ -62,14 +62,11 @@ def create_image(file, sizes, url_fn, output_dir, skip_processing=False): def process_image(input_file, output_file, size): size_name, width, height = size - image = read_image(input_file) - image_width, image_height = image.size - if image_height > image_width: - width, height = height, width - if size_name == 'original': - thumb = optimize(input_file, output_file, quality=90) + return optimize(input_file, output_file, quality=90) else: - thumb = thumbnail(input_file, output_file, (width, height)) - - return thumb + image = read_image(input_file) + image_width, image_height = image.size + if image_height > image_width: + width, height = height, width + return thumbnail(input_file, output_file, (width, height)) diff --git a/utils/Sync/utils/image/resize.py b/utils/Sync/utils/image/resize.py index aec7147a..759440a9 100644 --- a/utils/Sync/utils/image/resize.py +++ b/utils/Sync/utils/image/resize.py @@ -10,13 +10,13 @@ import utils.image def thumbnail(src, dest, size, ext=None): - image_magick_resize(src, dest, size, quality=85) - return utils.image.read_image(dest) + # image_magick_resize(src, dest, size, quality=85) + # return utils.image.read_image(dest) - # image = read_image(src) - # image.thumbnail(size) - # image.save(override_extension(dest, ext)) - # return image + image = utils.image.read_image(src) + image.thumbnail(size) + image.save(override_extension(dest, ext)) + return image def optimize(src, dest, quality=85, ext=None): @@ -39,17 +39,15 @@ def crop(src, dest, size): crop_options['width'] = width crop_options['height'] = height - crop = SmartCrop() - result = crop.crop(image.copy(), crop_options) + crop_instance = SmartCrop() + result = crop_instance.crop(image.copy(), crop_options) box = (result['topCrop']['x'], result['topCrop']['y'], result['topCrop']['x'] + result['topCrop']['width'], result['topCrop']['y'] + result['topCrop']['height']) - i = utils.image\ - .read_image(src)\ - .crop(box) + i = utils.image.read_image(src).crop(box) i.thumbnail(size, Image.ANTIALIAS) i.save(dest)