sync: fixed & updated

This commit is contained in:
tmshv committed 2016-08-06 01:50:18 +03:00
1 parent 4a38abd13a
commit a27a0cddce
5 files changed
+75 -58

No files matched your search

+29 -1
View File
@@ -2,6 +2,7 @@ import os
from tempfile import mkstemp
from sync.core import Sync
from utils.fn import last_good, lmap
from utils.hash import hash_file, hash_str
from utils.image import create_image
from utils.image.resize import image_magick_pdf_to_img
@@ -24,6 +25,10 @@ class SyncDocument(Sync):
document['type'] = document_type(document)
document['preview'] = self.create_preview(file, sizes=self.sizes, preview_dir=self.dir_static_previews)
document['title'] = last_good([
document['title'],
get_pdf_title(self.provider, document['file'])
])
document['file'] = {
'name': filename,
'size': self.provider.size(file)
@@ -45,8 +50,13 @@ class SyncDocument(Sync):
return document
def create_hash(self, document):
hash_keys = ['id', 'url']
hash_doc = lmap(
lambda key: document[key],
hash_keys
)
document['hash'] = hash_str(
hash_str(document) + self.provider.hash(document['file'])
hash_str(hash_doc) + self.provider.hash(document['file'])
)
return document
@@ -77,3 +87,21 @@ def pdf_to_jpg(provider, pdf):
os.remove(temp_in)
return temp_out
def get_pdf_title(provider, file):
from PyPDF2 import PdfFileReader
from PyPDF2.generic import TextStringObject
from PyPDF2.generic import IndirectObject
pdf = PdfFileReader(provider.read(file))
info = pdf.getDocumentInfo()
if info:
if type(info.title) == TextStringObject:
return str(info.title)
if type(info.title_raw) == IndirectObject:
o = pdf.getObject(info.title_raw)
return str(o)
return None
+1 -1
View File
@@ -9,7 +9,7 @@ from utils.hash import hash_file
class FSProvider(Provider):
def open(self, path):
def read(self, path):
return open(self.get_abs(path), 'rb')
def scan(self, path):
+20 -53
View File
@@ -116,29 +116,29 @@ def get_documents(sync):
# documents = documents_from_yaml()
documents = documents_from_subdirs(sync.provider)
# SETUP TITLE BASED ON FILE NAME
documents = lmapfn(documents)(
lambda i: {
**i,
'title': os.path.splitext(os.path.basename(i['file']))[0]
}
documents = lmap(
lambda document: read_document(sync, document),
documents
)
# CHOICE BETTER TITLE
# documents = lmapfn(documents)(
# lambda i: {
# **i,
# 'title': until_none([i['title'], get_pdf_title(sync.provider, i['file'])])
# }
# )
documents = lmap(sync.create_id, documents)
documents = lmap(sync.create_hash, documents)
documents = lmap(sync.create_url, documents)
return documents
def read_document(sync, path):
name = os.path.basename(path)
document = {
'file': path,
'category': os.path.dirname(path),
'title': os.path.splitext(name)[0]
}
document = sync.create_id(document)
document = sync.create_url(document)
document = sync.create_hash(document)
return document
def static_path_fn(dir_path):
return lambda document: os.path.join(dir_path, os.path.basename(document['url']))
@@ -202,42 +202,9 @@ def documents_from_subdirs(provider):
))
documents = lmap(
lambda folder: lmap(
lambda path: {
# 'file': provider.get_rel(path),
'file': path,
'category': os.path.basename(folder)
},
provider.type_filter(folder, '.pdf')
),
lambda folder: provider.type_filter(folder, '.pdf'),
documents
)
documents = combine(documents)
return documents
def get_pdf_title(provider, file):
from PyPDF2 import PdfFileReader
from PyPDF2.generic import TextStringObject
from PyPDF2.generic import IndirectObject
pdf = PdfFileReader(provider.read(file))
info = pdf.getDocumentInfo()
if info:
if type(info.title) == TextStringObject:
return str(info.title)
if type(info.title_raw) == IndirectObject:
o = pdf.getObject(info.title_raw)
return str(o)
return None
def until_none(ls):
better = None
for i in ls:
if (i is not None) and i != '':
better = i
return better