tmshv/hudozka
1
0 Fork 0
Code Issues Pull Requests Actions Releases Wiki Activity

rm sync

Browse Source
This commit is contained in:
tmshv committed 2026-04-24 20:54:41 +03:00
1 parent 53a822f503
commit 41fbb72216
33 files changed
-2671
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

modules/sync/README.md
-71
View File
@@ -1,71 +0,0 @@
## Usage
Sync tool
```bash
<ENV> python3 sync.py
```
### Environment variables
**YANDEX_DISK_ACCESS_TOKEN**
For access to files using Yandex.Disk API
[How to obtain token for testing](https://tech.yandex.ru/oauth/doc/dg/tasks/get-oauth-token-docpage/)
Example:
```bash
YANDEX_DISK_ACCESS_TOKEN=<Your_Access_Token>
```
Authorize url. Replace with your app id
```
https://oauth.yandex.ru/authorize?response_type=token&client_id=<идентификатор приложения>
```
#### SYNC_LOCAL_PATH
Path to scan files from
Example:
```bash
SYNC_LOCAL_PATH=/Users/tmshv/Yandex.Disk
```
#### SYNC_STATIC
Path to local folder for compiled files (images, uploads)
Example:
```bash
SYNC_STATIC=~/Desktop/Hudozka Static
```
#### SYNC_PROVIDER
Type of file provider
- *fs* for local access
- *yd* for access using Yandex.Disk API (YANDEX_DISK_ACCESS_TOKEN is required)
Example:
```bash
SYNC_PROVIDER=fs
```
#### MONGO_URI
URI for MongoDB access
Example:
```bash
mongodb://localhost:27017/hudozka
```
#### SYNC_ENV
Environment like *NODE_ENV*
- *production* default value
- *development* don't update/delete documents from DB
modules/sync/config.yaml
-22
View File
@@ -1,22 +0,0 @@
---
database_uri: 'mongodb://localhost:27017/hudozka'
env: 'production'
yandex_disk_access_token: '<YANDEX_DISK_TOKEN>'
interval: 0
skip_unchanged: True
image_processing_enabled: True
upload_enabled: True
upload_auth:
login: '<LOGIN>'
password: '<PASSWORD>'
provider:
name: 'fs'
root: '<FS_ROOT>'
update_enabled: True
delete_enabled: True
modules/sync/db.py
-12
View File
@@ -1,12 +0,0 @@
from pymongo import MongoClient
import settings
def db():
client = MongoClient(settings.database_uri)
return client.hudozka
def collection(name):
return db()[name]
modules/sync/kazimir/CSVToken.py
-46
View File
@@ -1,46 +0,0 @@
import csv
import io
from kazimir.Token import FileToken, is_file
extensions = ['.csv']
class CSVToken(FileToken):
@staticmethod
def test(data: str):
return is_file(data, extensions)
def __init__(self, data) -> None:
super().__init__(data=data)
self.name = 'csv'
self.joinable = False
self.build = None
async def compile(self):
data = await self.build(self.data)
f = io.StringIO(data)
reader = csv.reader(f, delimiter=',', quotechar='|')
headers = next(reader, None)
row = lambda xs: '\n'.join([
f'<td>{x}</td>' for x in xs
])
head = '\n'.join([
f'<th>{x}</th>' for x in headers
])
body = '\n'.join([
f'<tr>{row(x)}</tr>' for x in reader
])
return f'''
<table class="MarkerTable">
<thead><tr>
{head}
</tr></thead>
<tbody>{body}</tbody>
</table>
'''.strip()
modules/sync/kazimir/DocumentToken.py
-17
View File
@@ -1,17 +0,0 @@
from kazimir.Token import parse_file, is_document, Token
class DocumentToken(Token):
@staticmethod
def test(data: str):
return is_document(data)
def __init__(self, data) -> None:
super().__init__(name='file', data=data)
self.joinable = False
self.build = None
async def get_data(self):
data = parse_file(self.data)
document = await self.build(data)
return document
modules/sync/kazimir/InstagramToken.py
-85
View File
@@ -1,85 +0,0 @@
from urllib.parse import urlparse
import re
import requests
from markdown.util import etree
from kazimir.Token import UrlToken, is_url
pattr = re.compile(r'https?://(www\.)?instagram\.com/p/(?P<media>[\w\d]+)/?(.*)$')
class InstagramToken(UrlToken):
@staticmethod
def test(data: str):
if not is_url(data):
return False
url = urlparse(data)
return url.netloc in ['www.instagram.com', 'instagram.com']
def __init__(self, data: str) -> None:
super().__init__(data=data)
self.name = 'instagram'
async def get_data(self):
m = pattr.match(self.data)
embed = None
if m:
m = m.group('media')
embed = await embed_instagram(m)
return {
'url': self.data,
'embed': embed,
}
async def embed_instagram(media):
"""
<blockquote class="instagram-media" data-instgrm-captioned data-instgrm-version="7" style=" background:#FFF; border:0; border-radius:3px; box-shadow:0 0 1px 0 rgba(0,0,0,0.5),0 1px 10px 0 rgba(0,0,0,0.15); margin: 1px; max-width:658px; padding:0; width:99.375%; width:-webkit-calc(100% - 2px); width:calc(100% - 2px);">
<div style="padding:8px;">
<div style=" background:#F8F8F8; line-height:0; margin-top:40px; padding:35.0% 0; text-align:center; width:100%;">
<div style=" background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsCAMAAAApWqozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAMUExURczMzPf399fX1+bm5mzY9AMAAADiSURBVDjLvZXbEsMgCES5/P8/t9FuRVCRmU73JWlzosgSIIZURCjo/ad+EQJJB4Hv8BFt+IDpQoCx1wjOSBFhh2XssxEIYn3ulI/6MNReE07UIWJEv8UEOWDS88LY97kqyTliJKKtuYBbruAyVh5wOHiXmpi5we58Ek028czwyuQdLKPG1Bkb4NnM+VeAnfHqn1k4+GPT6uGQcvu2h2OVuIf/gWUFyy8OWEpdyZSa3aVCqpVoVvzZZ2VTnn2wU8qzVjDDetO90GSy9mVLqtgYSy231MxrY6I2gGqjrTY0L8fxCxfCBbhWrsYYAAAAAElFTkSuQmCC); display:block; height:44px; margin:0 auto -44px; position:relative; top:-22px; width:44px;">
</div>
</div>
<p style=" margin:8px 0 0 0; padding:0 4px;">
<a href="https://www.instagram.com/p/BUelE04l1kU/" style=" color:#000; font-family:Arial,sans-serif; font-size:14px; font-style:normal; font-weight:normal; line-height:17px; text-decoration:none; word-wrap:break-word;" target="_blank">
Композиция. Второй год обучения. Преподаватель Гоголева О.Д. #shlb_hudozka #живопись #дети #детирисуют #гуашь #пословицыипоговорки #художка #шлиссельбург #shlisselburg
</a>
</p>
<p style=" color:#c9c8cd; font-family:Arial,sans-serif; font-size:14px; line-height:17px; margin-bottom:0; margin-top:8px; overflow:hidden; padding:8px 0 7px; text-align:center; text-overflow:ellipsis; white-space:nowrap;">
A post shared by Шлиссельбургская ДХШ (@hudozka) on
<time style=" font-family:Arial,sans-serif; font-size:14px; line-height:17px;" datetime="2017-05-24T14:05:19+00:00">
May 24, 2017 at 7:05am PDT
</time>
</p>
</div>
</blockquote>
"""
url = f'https://api.instagram.com/oembed?url=http://instagr.am/p/{media}&hidecaption=true&omitscript=true'
req = requests.get(url)
if req.status_code == 200:
html = req.json()['html']
html = fix_html(html)
return html
# i = etree.fromstring(html)
# obj = etree.Element('div')
# obj.set('class', "kazimir__embed")
# obj.append(i)
# return etree.tostring(obj).decode()
else:
raise Exception(
'Failed to compile Instagram {media}: Status Code: {status}'.format(
media=media, status=req.status_code)
)
def fix_html(html: str):
return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""')
modules/sync/kazimir/Marker.py
-83
View File
@@ -1,83 +0,0 @@
import lxml.html
from markdown import markdown
from kazimir.Token import SplitToken
class Marker:
def __init__(self) -> None:
super().__init__()
self.tokens = []
def add_token_factory(self, token):
self.tokens.append(token)
def parse(self, text: str):
tokens = text.split('\n')
tokens = self.markup_tokens(tokens)
tokens = join_tokens(tokens)
tokens = merge_tokens(tokens)
return tokens
def markup_tokens(self, tokens: list):
marked_tokens = []
for data in tokens:
t = self.create_token(data)
if t:
marked_tokens.append(t)
else:
raise Exception(f'Token not recognized for data {data}')
return marked_tokens
def create_token(self, data):
for f in self.tokens:
if f.token.test(data):
return f.create(data=data)
return None
def join_tokens(tokens):
is_split = lambda x: isinstance(x, SplitToken)
buffer_tag = None
buffer = []
ts = []
for token in tokens:
tag = token.name
changed = buffer_tag is not None and tag != buffer_tag
flush = is_split(token)
if changed:
flush = True
if not token.joinable:
flush = True
if flush:
if buffer:
ts.append(buffer)
if is_split(token):
buffer = []
buffer_tag = None
else:
buffer = [token]
buffer_tag = tag
else:
buffer_tag = tag
buffer.append(token)
if len(buffer):
ts.append(buffer)
return ts
def merge_tokens(tokens: list):
tokens_out = []
for ts in tokens:
token = ts[0]
for x in ts[1:]:
token = token.merge(x)
tokens_out.append(token)
return tokens_out
modules/sync/kazimir/Token.py
-191
View File
@@ -1,191 +0,0 @@
import os
import re
from markdown import markdown
from urllib.parse import urlparse
image_extensions = ['.jpg', '.jpeg', '.png']
document_extensions = ['.doc', '.docx', '.pdf', '.xls', '.xlsx']
class TokenFactory:
def __init__(self, token) -> None:
super().__init__()
self.token = token
def create(self, data):
return self.token(data=data)
class BuildTokenFactory(TokenFactory):
def __init__(self, token, build) -> None:
super().__init__(token)
self.build = build
def create(self, data):
x = self.token(data=data)
x.build = self.build
return x
class Token:
@staticmethod
def test(data: str):
return True
def __init__(self, name, data) -> None:
super().__init__()
self.name = name
self.data = data
self.joinable = True
async def get_data(self):
return self.data
def merge(self, token):
return self
def __repr__(self) -> str:
return f'[Token {self.name} {self.data}]'
class SplitToken(Token):
@staticmethod
def test(data: str):
data = data.strip()
return data == ''
def __init__(self, data) -> None:
super().__init__(name='split', data=None)
class TextToken(Token):
@staticmethod
def test(data: str):
return True
def __init__(self, data) -> None:
super().__init__(name='text', data=data)
def merge(self, token: Token):
data = self.data
data += '\n'
data += token.data
return TextToken(data.strip())
class HtmlToken(Token):
@staticmethod
def test(data: str):
"""
Result of compiling markdown to html is the same as source data meaning that the source is html
"""
html = markdown(data)
return html == data
def __init__(self, data: str) -> None:
super().__init__(name='html', data=data)
self.joinable = False
class FileToken(Token):
@staticmethod
def test(data: str):
return is_file(data, document_extensions + image_extensions)
def __init__(self, data) -> None:
super().__init__(name='file', data=data)
self.joinable = False
async def get_data(self):
return parse_file(self.data)
class ImageToken(Token):
@staticmethod
def test(data: str):
return is_image(data)
def __init__(self, data) -> None:
super().__init__(name='image', data=data)
self.joinable = False
self.build = None
async def get_data(self):
data = parse_file(self.data)
data = await self.build(data)
return data
class UrlToken(Token):
@staticmethod
def test(data: str):
return is_url(data)
def __init__(self, data) -> None:
super().__init__(name='url', data=data)
self.joinable = False
def markdown_to_html(text):
return markdown(text)
def is_url(sample: str):
url = urlparse(sample)
return url.scheme and url.netloc
def is_image(sample: str):
file = parse_file(sample)['file']
return is_file(file, image_extensions)
def is_document(sample: str):
file = parse_file(sample)['file']
return is_file(file, document_extensions)
def is_file(sample: str, extensions):
name, ext = os.path.splitext(sample)
ext = ext.lower()
return ext in extensions
def parse_file(sample: str):
"""
![Caption](file.jpg) -> {file, caption}
file.jpg Caption -> {file, caption}
:param sample:
:return:
"""
p = re.match(r'!\[(.*)\]\((.*)\)', sample)
if p:
caption = p.group(1)
file = p.group(2)
else:
sample = sample.split(' ')
file = sample[0]
caption = ' '.join(sample[1:])
return {
'file': file,
'caption': caption,
}
def parse_link(sample: str):
"""
![Alt](href) -> {href, alt}
:param sample:
:return:
"""
p = re.match(r'!\[(.*)\]\((.*)\)', sample)
if not p:
return None
url = p.group(2)
return {
'url': urlparse(url),
'alt': p.group(1),
}
modules/sync/kazimir/YoutubeToken.py
-21
View File
@@ -1,21 +0,0 @@
from urllib.parse import urlparse, parse_qs
from kazimir.Token import UrlToken, is_url
class YoutubeToken(UrlToken):
@staticmethod
def test(data: str):
if not is_url(data):
return False
url = urlparse(data)
return url.netloc in ['www.youtube.com', 'youtube.com']
def __init__(self, data: str) -> None:
super().__init__(data=data)
self.name = 'youtube'
async def get_data(self):
return {
'url': self.data
}
modules/sync/requirements.txt
-13
View File
@@ -1,13 +0,0 @@
cssselect==0.9.2
lxml==6.1.0
Markdown==3.8.1
Pillow==12.2.0
pymongo==4.6.3
PyPDF2==1.27.9
python-frontmatter==0.3.1
PyYAML==5.4
requests==2.33.0
six==1.10.0
smartcrop==0.1
toml==0.9.2
Unidecode==0.4.19
modules/sync/settings.py
-89
View File
@@ -1,89 +0,0 @@
import os
import yaml
import argparse
__author__ = 'Roman Timashev'
name = 'HudozkaSync'
_parser = argparse.ArgumentParser(description='Hudozka Sync Daemon')
_parser.add_argument('--config', dest='config', default='config.yaml', help='path to config.yaml')
_args = _parser.parse_args()
config = yaml.load(open(_args.config, 'r'))
def value(key, default=None):
return config[key] if key in config else default
def absolute(path, ensure=True):
a = os.path.expanduser(path)
if ensure:
os.makedirs(a, exist_ok=True)
return a
def abs_fn(root):
return lambda path: os.path.join(root, path)
def env(param, default=None):
if param in os.environ:
return os.environ[param]
return default
interval = value('interval', 0)
database_uri = value('database_uri') # env('MONGO_URI')
skip_unchanged = value('skip_unchanged')
image_processing_enabled = value('image_processing')
image_ext = '.jpg'
image_sizes = [
('original', None, None),
# ('big', 1500, 667),
# ('medium', 400, 300),
]
date_formats = [
'%Y',
'%m.%Y',
'%d.%m.%Y',
'%d.%m.%Y %H:%M',
'%d.%m.%Y %H:%M:%S'
]
collection_images = 'images'
collection_documents = 'documents'
collection_pages = 'pages'
hash_salt_pages = '6'
hash_salt_documents = '4'
hash_salt_images = '5'
provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs')
provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs')
f = abs_fn(provider_root)
dir_pages = './Pages'
document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}'
document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}'
image_url_base = 'https://static.shlisselburg.org/art/images/'
yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN')
auth_s3 = {
'region_name': 'ams3',
'endpoint_url': 'https://dir.ams3.digitaloceanspaces.com',
'aws_access_key_id': value('s3')['client_id'],
'aws_secret_access_key': value('s3')['client_secret'],
}
modules/sync/sync-smart.py
-50
View File
@@ -1,50 +0,0 @@
import settings
from sync.data.fs import FSProvider
from sync.data.yandexdisk import YDProvider
import asyncio
async def run_check(run_interval=0):
run_interval = 1
from yandexdisk import YDClient
client = YDClient(settings.yandex_disk_access_token)
async def need_to_update():
res = client.last_uploaded()
items = res['items']
paths = [i['path'] for i in items]
for i in paths:
if i.startswith('disk:/Hudozka/Site'):
return True
return False
while True:
status = await need_to_update()
if status:
print('DO SYNC')
# await run(0)
if run_interval == 0:
break
else:
await asyncio.sleep(run_interval)
def get_provider(provider_type, root):
if provider_type == 'fs':
return FSProvider(root)
elif provider_type == 'yd':
return YDProvider(root, settings.yandex_disk_access_token)
return None
if __name__ == '__main__':
interval = settings.interval
loop = asyncio.get_event_loop()
loop.run_until_complete(run_check(interval))
loop.close()
modules/sync/sync.py
-75
View File
@@ -1,75 +0,0 @@
import settings
from sync.core import Sync
from sync.data.fs import FSProvider
from sync.data.yandexdisk import YDProvider
import asyncio
import logging
from sync.models.Page import Page, PageController
def init_logger(name: str, file: str):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler(file)
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
# ch.setLevel(logging.ERROR)
ch.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
async def run_sync(sync: Sync, **kwargs):
await sync.run(**kwargs)
async def run(run_interval=0):
provider = get_provider(settings.provider_name, settings.provider_root)
controller = PageController(provider)
sync_pages = Sync(
ctrl=controller,
name='Page'
)
sync_pages.set_options(True, True, True)
while True:
await asyncio.wait([
run_sync(sync_pages),
])
if run_interval == 0:
break
else:
await asyncio.sleep(run_interval)
def get_provider(provider_type, root):
if provider_type == 'fs':
return FSProvider(root)
elif provider_type == 'yd':
return YDProvider(root, settings.yandex_disk_access_token)
return None
if __name__ == '__main__':
init_logger(settings.name, 'hudozka.log')
interval = settings.interval
loop = asyncio.get_event_loop()
loop.run_until_complete(run(interval))
loop.close()
modules/sync/sync/__init__.py
-147
View File
@@ -1,147 +0,0 @@
import os
import re
from datetime import datetime
import lxml.html
import kazimir
import settings
from kazimir.CSVToken import CSVToken
from kazimir.DocumentToken import DocumentToken
from kazimir.YoutubeToken import YoutubeToken
from kazimir.InstagramToken import InstagramToken
from kazimir.Marker import Marker
from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, HtmlToken
from sync.data import Provider
from sync.models import Model
from sync.models.Document import Document
from sync.models.Image import Image
def create_date(date_str, date_formats=None):
"""
2016
2016.10
2016.10.10
-> datetime
:param date_formats:
:param date_str:
:return:
"""
if not date_formats:
date_formats = [
'%Y',
'%Y.%m',
'%Y.%m.%d'
]
for date_format in date_formats:
try:
return datetime.strptime(date_str, date_format)
except:
continue
return None
async def untouched(items: [Model]) -> [Model]:
if settings.skip_unchanged:
return items
result = []
for item in items:
status = await item.is_changed()
if status:
result.append(item)
return result
# def is_equals(e, n):
# return (n is None) or ('hash' not in n) or (e.hash != n['hash'])
#
# doc = lambda i: {'id': i.id, 'hash': i.hash}
#
# stored_documents = [(i, store.read(doc(i))) for i in documents]
# filtered_documents = [i[0] for i in stored_documents if is_equals(*i)]
# return filtered_documents
async def create_post(provider: Provider, folder: str, md: str, sizes):
images = []
documents = []
def rel(file: str) -> str:
return os.path.join(folder, file)
async def build_document(data):
filepath = os.path.join(folder, data['file'])
document = await Document.new(provider, filepath, settings.image_sizes)
title = data['caption']
if title:
document.set_title(title)
await document.save()
if not document:
raise Exception(f'Failed to build Document {filepath}')
text = data['caption'] if data['caption'] else data['file']
url = f'/document/{document.id}'
image_url = document.preview.get_src()
return {
'url': url,
'slug': document.id,
'image_url': image_url,
'file_url': document.url,
'title': document.title,
'file_size': document.file_size,
'file_format': 'application/pdf',
}
async def build_image(data):
filename = data['file']
relative_path = os.path.join(folder, filename)
image_path = provider.get_local(relative_path)
if os.path.exists(image_path):
image = await Image.new(provider, relative_path, sizes)
if image:
images.append(image)
src = image.get_src()
return {
'src': src,
'alt': data['caption'],
'caption': data['caption'],
}
raise Exception('Fail to get Image', relative_path)
async def read_file(data):
filepath = data
filepath = rel(filepath)
return provider.read(filepath).read().decode('utf-8')
m = Marker()
m.add_token_factory(TokenFactory(SplitToken))
m.add_token_factory(TokenFactory(YoutubeToken))
m.add_token_factory(TokenFactory(InstagramToken))
m.add_token_factory(TokenFactory(UrlToken))
m.add_token_factory(BuildTokenFactory(ImageToken, build=build_image))
m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document))
m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file))
# m.add_token_factory(TokenFactory(FileToken))
m.add_token_factory(TokenFactory(HtmlToken))
m.add_token_factory(TokenFactory(TextToken))
tokens = m.parse(md)
encoded_tokens = []
for t in tokens:
data = await t.get_data()
encoded_tokens.append({
'token': t.name,
'data': data,
})
return images, documents, encoded_tokens
modules/sync/sync/core/__init__.py
-114
View File
@@ -1,114 +0,0 @@
import logging
import settings
from sync import untouched
class Sync:
def __init__(self, ctrl, name: str):
super().__init__()
self.validate_urls = True
self.__default_build_args = {
'sizes': settings.image_sizes,
}
self.__delete_query = lambda item: {'_id': item['_id']}
self.logger = logging.getLogger('%s.%s' % (settings.name, name))
self.ctrl = ctrl
self.filter_item_fn = lambda x: True
self.skip_unchanged = True
self.update_enabled = True
self.delete_enabled = True
def set_options(self, su: bool, u: bool, d: bool):
self.skip_unchanged = su
self.update_enabled = u
self.delete_enabled = d
return self
def set_item_filter(self, fn):
self.filter_item_fn = fn
return self
async def build(self, item):
args = self._get_build_args()
await item.build(**args)
return item
async def update(self, item):
await item.upload()
await item.save()
return item
def _get_build_args(self):
return {
**self.__default_build_args
}
def __items_to_delete_query(self, items):
query = {
'id': {'$nin': [doc.id for doc in items]},
}
return query
async def run(self):
"""
# Get scope files
# Validate these files. Raise an error
# calc_hashes
# make diff with previous run
# compile objects
# upload
:return:
"""
self.logger.info('Checking for update')
items = await self.ctrl.get_items()
self.logger.info('Found {} Item(s)'.format(len(items)))
all_items = items
if self.validate_urls:
self.check_urls_uniqueness(all_items)
# SKIP UNTOUCHED DOCUMENTS
if self.skip_unchanged:
items = await untouched(all_items)
# Optionally trim list of models by custom filter
items = [
x
for x in items
if self.filter_item_fn(x)
]
self.logger.info(f'Ready to process {len(items)} item(s)')
# BUILDING
for item in items:
self.logger.info(f'Build Item {item}')
await self.build(item)
# UPDATING
if self.update_enabled:
for item in items:
await self.update(item)
self.logger.info('Updated Item {}'.format(item))
# DELETING
if self.delete_enabled:
obsolete_items = await self.ctrl.find(self.__items_to_delete_query(all_items))
for item in obsolete_items:
self.logger.info('Deleting Item {}'.format(item['id']))
await self.ctrl.delete_item(self.__delete_query(item))
def check_urls_uniqueness(self, items):
urls = [x.url for x in items]
unique_urls = set(urls)
if len(urls) != len(unique_urls):
self.logger.error('URLs is not unique')
raise Exception('URLs is not unique')
modules/sync/sync/data/__init__.py
-78
View File
@@ -1,78 +0,0 @@
import os
from io import BytesIO
from utils.fn import combine
class Provider:
def __init__(self, root):
super().__init__()
self.root = root
def read(self, path):
return BytesIO(bytes())
def scan(self, path):
return []
def glob(self, pattr):
return []
def type_filter(self, path, ext):
return []
def copy(self, path, out):
pass
def hash(self, path):
return None
def size(self, path):
return 0
def exists(self, path):
return True
def get_abs(self, path):
a = os.path.join(self.root, path)
return os.path.normpath(a)
def get_rel(self, path):
rel = os.path.relpath(path, self.root)
return rel
def get_local(self, path):
return path
def is_dir(self, path):
return False
def scan_subdirs(provider, ext, path='.'):
"""
Creates flat list of files with specific ext
It iterates only one level of sub directories
:param provider: instance of Provider
:param ext: file extension like .jpg of .pdf for file filtering
:param path: optional path prefix
:return: list of files relative to provider.root
"""
items = provider.scan(path)
items = filter(
lambda i: provider.is_dir(i),
items
)
items = [provider.type_filter(folder, ext) for folder in items]
items = combine(items)
return items
def get_data(provider, path, transform):
data = provider.read(path)
return transform(data)
def list_images(provider, path):
image_types = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG']
images = [provider.type_filter(path, ext) for ext in image_types]
return combine(images)
modules/sync/sync/data/fs.py
-48
View File
@@ -1,48 +0,0 @@
import os
from glob import glob
from shutil import copyfile
from sync.data import Provider
from utils.hash import hash_file
class FSProvider(Provider):
def read(self, path):
return open(self.get_abs(path), 'rb')
def scan(self, path):
paths = [i.path for i in os.scandir(self.get_abs(path))]
return [self.get_rel(path) for path in paths]
def glob(self, pattr):
items = glob(self.get_abs(pattr))
return [self.get_rel(i) for i in items]
def type_filter(self, path, ext):
return list(filter(
lambda fn: os.path.splitext(fn)[1] == ext,
self.scan(path)
))
def hash(self, path):
return hash_file(self.get_abs(path))
def copy(self, path, out):
file = self.get_abs(path)
return copyfile(file, out)
def size(self, path):
file = self.get_abs(path)
return os.path.getsize(file)
def exists(self, path):
file = self.get_abs(path)
return os.path.exists(file)
def is_dir(self, path):
return os.path.isdir(self.get_abs(path))
def get_local(self, path):
return self.get_abs(path)
modules/sync/sync/data/request.py
-40
View File
@@ -1,40 +0,0 @@
import aiohttp
from boto3 import session
from mimetypes import MimeTypes
import logging
import settings
logger = logging.getLogger(settings.name + '.S3')
async def s3_put(obj, path):
"""
:param obj: uploads/file.pdf
:param path: to/to/local/file.pdf
:return:
"""
logger.info('Put {}'.format(obj))
s = session.Session()
client = s.client('s3', **settings.auth_s3)
mime = MimeTypes()
mime_type, enc = mime.guess_type(path)
# Upload a file to your Space
client.upload_file(path, 'hudozka', obj, ExtraArgs={
'ACL': 'public-read',
'ContentType': mime_type,
})
async def get(url):
async with aiohttp.ClientSession as aio_session:
res = aio_session.get(url)
return await res.text()
async def post(url, data, params={}, headers={}):
async with aiohttp.ClientSession() as aio_session:
async with aio_session.post(url, data=data, params=params, headers=headers) as resp:
return await resp.text()
modules/sync/sync/data/yandexdisk.py
-69
View File
@@ -1,69 +0,0 @@
import os
import re
import io
from tempfile import mkstemp
import requests
from sync.data import Provider
from yandexdisk import YDClient
class YDProvider(Provider):
def __init__(self, root, access_token):
super().__init__(root)
self.api = YDClient(access_token)
def read(self, path):
remote = self.api.get_download_link(self.get_abs(path))
url = remote['href']
req = requests.get(url)
data = req.content
return io.BytesIO(data)
def scan(self, path):
res = self.api.get_content(self.get_abs(path))
files = []
for item in res.children:
files.append(item.path)
files = map(
lambda fp: re.sub('disk:', '', fp),
files
)
return [self.get_rel(i) for i in files]
def glob(self, pattr):
return []
def type_filter(self, path, ext):
return list(filter(
lambda fn: os.path.splitext(fn)[1] == ext,
self.scan(path)
))
def hash(self, path):
res = self.api.get_content(self.get_abs(path))
return res.sha256
def copy(self, path, out):
data = self.read(path)
f = open(out, 'wb')
f.write(data.read())
f.close()
return out
def size(self, path):
res = self.api.get_content(self.get_abs(path))
return res['size']
def is_dir(self, path):
res = self.api.get_content(self.get_abs(path))
return res.type == 'dir'
def get_local(self, path):
ext = os.path.splitext(path)[1]
_, temp_in = mkstemp(suffix=ext)
return self.copy(path, temp_in)
modules/sync/sync/models/Document.py
-183
View File
@@ -1,183 +0,0 @@
import os
from tempfile import mkstemp
import settings
from db import collection
from sync.data import request
from sync.models import Model
from sync.models.DocumentPreviewImage import DocumentPreviewImage
from utils.hash import hash_str, md5
from utils.image import image_magick_pdf_to_img
store = collection(settings.collection_documents)
class Document(Model):
@staticmethod
async def find_one(query):
return store.find_one(query)
@staticmethod
async def find(query):
return store.find(query)
@staticmethod
async def delete(query):
return store.find_one_and_delete(query)
@staticmethod
def read_path(provider, path):
name = os.path.basename(path)
params = {
'title': os.path.splitext(name)[0]
}
return Document(
provider,
path,
params
)
@staticmethod
async def new(provider, file, sizes):
document = Document.read_path(provider, file)
changed = await document.is_changed()
if changed:
await document.build(sizes=sizes)
await document.upload()
await document.save()
else:
doc = await Document.find_one({'file': file})
document = Document(provider, file, doc)
document.ref = doc['_id']
await document.create_preview(sizes)
return document
def __init__(self, provider, file, params=None):
self.__hash_salt = settings.hash_salt_documents
self.__hash_keys = ['id', 'url']
self.__id_template: str = '{category}-{file}'
self.__url_template: str = 'https://static.shlisselburg.org/art/uploads/{file}'
self.__url_preview_template: str = settings.document_url_preview_template
self.title = None
self.file_name = None
self.file_size = None
self.preview = None
super().__init__(provider, store, file, params=params)
def init(self):
self.__set_id()
self.__set_url()
self.__set_hash()
filename = os.path.basename(self.file)
self.file_name = filename
self.file_size = self.provider.size(self.file)
if self.has_param('title'):
self.title = self.get_param('title')
def set_title(self, value: str):
self.title = value
async def build(self, **kwargs):
sizes = kwargs['sizes']
await self.create_preview(sizes)
pdf_title = await get_pdf_title(self.provider, self.file)
if pdf_title:
self.title = pdf_title
async def save(self):
document = self.bake()
query = {'id': document['id']}
try:
self.store.update_one(query, {'$set': document}, upsert=True)
doc = await Document.find_one(query)
self.ref = doc['_id']
return self
except ValueError:
pass
return None
async def upload(self):
filepath = self.provider.get_local(self.file)
name = os.path.basename(self.url)
await request.s3_put(f'uploads/{name}', filepath)
def bake(self):
return {
**self.params,
'id': self.id,
'file': self.file,
'hash': self.hash,
'url': self.url,
'title': self.title,
'preview': self.preview.ref,
'fileInfo': {
'name': self.file_name,
'size': self.file_size,
},
}
def __filename(self):
return os.path.basename(self.file)
def __set_id(self):
self.id = md5(self.file)
def __set_url(self):
s = os.path.splitext(self.file)
file = f'{self.id}{s[1]}'
self.url = self.__url_template.format(
file=file
)
def __set_hash(self):
doc = [getattr(self, key) for key in self.__hash_keys]
self.hash = hash_str(
self.__hash_salt + hash_str(doc) + self.provider.hash(self.file)
)
async def create_preview(self, sizes):
self.preview = await DocumentPreviewImage.new(self.provider, self.file, sizes)
def __str__(self):
return f'<Document file={self.file} id={self.id}>'
async def get_pdf_title(provider, file):
from PyPDF2 import PdfFileReader
from PyPDF2.generic import TextStringObject
from PyPDF2.generic import IndirectObject
def t(value):
value = str(value)
value = value.replace('\u0000', '')
if value == 'None':
return None
if value == '':
return None
return value
pdf = PdfFileReader(provider.read(file))
info = pdf.getDocumentInfo()
if info:
if type(info.title) == TextStringObject:
return t(info.title)
if type(info.title_raw) == IndirectObject:
o = pdf.getObject(info.title_raw)
return t(info.title)
return None
modules/sync/sync/models/DocumentPreviewImage.py
-64
View File
@@ -1,64 +0,0 @@
import os
from tempfile import mkstemp
from sync.models.Image import Image, default_url_factory
from utils.image import image_magick_pdf_to_img
class DocumentPreviewImage(Image):
@staticmethod
async def find_one(query):
return await Image.find_one(query)
@staticmethod
async def find(query):
return await Image.find(query)
@staticmethod
async def delete(query):
return await Image.delete(query)
@staticmethod
async def new(provider, file, sizes, **kwargs):
img = DocumentPreviewImage(provider, file, data=None)
changed = await img.is_changed()
if changed:
await img.build(sizes=sizes)
await img.upload()
await img.save()
else:
doc = await DocumentPreviewImage.find_one({'file': file})
img = DocumentPreviewImage(provider, file, doc['data'])
img.ref = doc['_id']
return img
def __init__(self, provider, file, data):
super().__init__(provider, file, data, url_factory=default_url_factory)
self.__hash_salt = ''
self.temp_image_path = None
async def is_changed(self):
i = await DocumentPreviewImage.find_one({'file': self.file})
return self._is_changed_hash(i)
async def build(self, sizes):
self.temp_image_path = await pdf_to_jpg(self.provider, self.file)
await super().build(sizes)
def _get_image_file(self):
return self.temp_image_path
def __get_hash(self):
return self.provider.hash(self.file)
async def pdf_to_jpg(provider, pdf):
_, temp_in = mkstemp(suffix='.pdf')
_, temp_out = mkstemp(suffix='.jpg')
abspdf = provider.copy(pdf, temp_in)
await image_magick_pdf_to_img(abspdf, temp_out)
os.remove(temp_in)
return temp_out
modules/sync/sync/models/Image.py
-244
View File
@@ -1,244 +0,0 @@
import logging
import os
import tempfile
import asyncio
from typing import Optional, List
import settings
from db import collection
from sync.data import request
from sync.models import Model
from utils.hash import md5, hash_str
from utils.image import image_magick_resize
from utils.text.transform import url_encode_text
logger = logging.getLogger(settings.name + '.Image')
store = collection(settings.collection_images)
def default_url_factory(file, size, ext):
url = settings.image_url_base + '{}-{}{}'
return url.format(md5(file), size, ext.lower())
class Image(Model):
@staticmethod
async def find_one(query):
document = store.find_one(query)
return document
@staticmethod
async def find(query):
return store.find(query)
@staticmethod
async def delete(query):
return store.find_one_and_delete(query)
@staticmethod
async def new(provider, file, sizes, url_factory=None):
url_factory = url_factory if url_factory else default_url_factory
img = Image(provider, file, data=None, url_factory=url_factory)
changed = await img.is_changed()
if changed:
await img.build(sizes=sizes)
await img.upload()
await img.save()
else:
doc = await Image.find_one({'file': file})
img = Image(provider, file, doc['data'], url_factory)
img.ref = doc['_id']
return img
@staticmethod
def read_path(provider, path):
return Image(
provider,
path,
data=None,
url_factory=default_url_factory
)
def __init__(self, provider, file, data, url_factory):
self.__hash_salt = settings.hash_salt_images
self.data = data
self.url_factory = url_factory
super().__init__(provider, store, file)
def init(self):
self.id = self.__get_id()
self.hash = self.__get_hash()
async def is_changed(self):
i = await Image.find_one({'file': self.file})
return self._is_changed_hash(i)
async def build(self, sizes):
logger.info(f'Building Image {self._get_abs(self.file)}')
img_in = self._get_image_file()
img_out = tempfile.mkdtemp()
images = await create_image(img_in, sizes, self.url_factory, img_out)
if not images:
raise Exception('Failed to create Image {}'.format(img_in))
data = {}
for i in images:
if i:
size_name = i['size']
data[size_name] = {
**i,
}
self.data = data
async def upload(self):
if self.data:
async def fn(i):
file = i['file']
filename = os.path.basename(file)
await request.s3_put(f'images/{filename}', file)
await asyncio.wait([fn(i) for i in self.data.values()])
async def save(self):
c = sync_image(self.bake())
self.ref = c['_id']
def bake(self):
return {
'id': self.id,
'file': self.file,
'hash': self.hash,
'data': self.data,
}
def get_size(self, size):
if size in self.data:
return self.data[size]
return None
def get_src(self):
return self.get_size('original')['url']
def _get_image_file(self):
return self._get_abs(self.file)
def __filename(self):
return os.path.basename(self.file)
def __get_id(self):
return md5(self.file)
def __get_hash(self):
file_hash = self.provider.hash(self.file)
return hash_str(self.__hash_salt + file_hash)
def __str__(self):
return '<Image hash={} file={}>'.format(self.hash, self.file)
def sync_image(record):
from db import db
q = {'file': record['file']}
try:
db().images.update_one(q, {'$set': record}, upsert=True)
i = db().images.find_one({'file': record['file']})
return i
except ValueError:
pass
return None
async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]:
"""
:param file: Image path to process
:param sizes: list of sizes to generate image (<size_name>, <width>, <height>)
:param url_fn: function (size_name, ext) for creating image url
:param output_dir: path to local folder for thumbs storing
:return:
"""
if not os.path.exists(file):
return None
_, ext = os.path.splitext(file)
images = []
if not settings.image_processing_enabled:
logger.warning(
f'Image processing disabled {file}. Size of image will be set to original'
)
async def fn(size):
size_name, width, height = size
image_url = url_fn(file, size_name, ext)
image_filename = os.path.basename(image_url)
local_image_path = os.path.join(output_dir, image_filename)
if settings.image_processing_enabled:
image = await process_image(file, local_image_path, size)
else:
image = await read_image(file)
if not image:
logger.warning(f'Failed to process image {file}')
return None
width, height = image.size
images.append({
'file': local_image_path,
'url': image_url,
'size': size_name,
'width': width,
'height': height
})
await asyncio.wait([fn(s) for s in sizes])
return images
async def process_image(input_file, output_file, size):
size_name, width, height = size
if size_name == 'original':
return await optimize(input_file, output_file, quality=90)
else:
image = await read_image(input_file)
if not image:
return None
image_width, image_height = image.size
if image_height > image_width:
width, height = height, width
return await thumbnail(input_file, output_file, (width, height), quality=90)
async def read_image(src):
import aiofiles
from PIL import Image
import io
async with aiofiles.open(src, mode='rb') as f:
image_data = await f.read()
image = Image.open(io.BytesIO(image_data))
return image
async def thumbnail(src, dest, size, quality):
await image_magick_resize(src, dest, size, quality)
return await read_image(dest)
async def optimize(src, dest, quality):
image = await read_image(src)
w, h = image.size
return await thumbnail(src, dest, (w, h), quality)
modules/sync/sync/models/Page.py
-209
View File
@@ -1,209 +0,0 @@
import os
import logging
from lxml import etree
import lxml.html
import settings
from db import collection
from sync.data import request
from sync.models import Model
from sync import create_date, create_post
from sync.models.Image import Image
from utils.hash import hash_str
from utils.front_matter import parse_yaml_front_matter
from utils.text.transform import url_encode_text
from markdown import markdown
logger = logging.getLogger(settings.name + '.Page')
store = collection(settings.collection_pages)
def extract_files(html) -> [str]:
images = html.cssselect('img')
images = [img.get('src') for img in images]
return images, []
def title_from_html(html):
titles = html.cssselect('h1')
if len(titles):
return titles[0].text
return ''
def find(store, item_id: str):
q = {'id': item_id}
try:
return store.find_one(q)
except ValueError:
pass
return None
class PageController:
def __init__(self, provider):
self.provider = provider
async def find(self, query):
return store.find(query)
async def delete_item(self, query):
return store.find_one_and_delete(query)
async def get_items(self):
scan_path = settings.dir_pages
items = []
for path in self.provider.scan(scan_path):
if not self.provider.is_dir(path):
continue
item = await self.create_item(path)
if not item:
continue
items.append(item)
return items
async def create_item(self, path: str):
"""
:param path: path to page manifest file. For example: /Pages/<Some_Page>/<Some_Page>.md
:return:
"""
dir_name = os.path.basename(path)
manifest_path = os.path.join(path, f'{dir_name}.md')
if not self.provider.exists(manifest_path):
logger.warning(f'Manifest not found on path {manifest_path}')
return None
manifest = await self.get_manifest(manifest_path)
if not manifest:
logger.warning(f'Cannot define manifest. Skip {manifest_path}')
return None
return Page(
self.provider,
manifest_path,
params=manifest,
)
async def get_manifest(self, path: str):
data = self.provider.read(path).read().decode('utf-8')
manifest = parse_yaml_front_matter(data)
if 'url' not in manifest:
logger.warning(f'URL not specified for Page {path}')
return None
if 'date' in manifest:
manifest['date'] = create_date(
manifest['date'], settings.date_formats)
if 'until' in manifest:
manifest['until'] = create_date(
manifest['until'], settings.date_formats)
if 'id' in manifest:
manifest['id'] = str(manifest['id'])
return {
**manifest,
'file': os.path.basename(path),
'folder': os.path.dirname(path),
}
class Page(Model):
def __init__(self, provider, file, params=None):
self.__hash_salt = settings.hash_salt_pages
self.__hash_keys = ['id', 'url']
self.images = []
self.documents = []
self.title = None
self.preview = None
self.tokens = None
self.__id_template: str = '{category}-{file}'
self.__url_template: str = settings.document_url_template
self.__url_preview_template: str = settings.document_url_preview_template
super().__init__(provider, store, file, params=params)
def init(self):
content = self.get_param('content')
html = markdown(content)
html_tree = lxml.html.fromstring(html)
images, documents = extract_files(html_tree)
self.title = title_from_html(html_tree)
self.params['images'] = images
self.params['documents'] = documents
self.__set_id()
self.__set_url()
self.__set_hash()
async def is_changed(self):
i = find(self.store, self.id)
if not i:
return True
if 'hash' not in i:
return True
return not (self.hash == i['hash'])
async def build(self, **kwargs):
sizes = kwargs['sizes']
markdown_post = self.get_param('content')
images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes)
self.images = images
self.documents = documents
self.tokens = tokens
if self.has_param('preview'):
preview = self.get_param('preview')
preview = self._get_relpath(preview)
self.preview = await Image.new(self.provider, preview, sizes)
def bake(self):
images = [x.ref for x in self.images]
documents = [x.ref for x in self.documents]
return {
**self.params,
'id': self.id,
'hash': self.hash,
'url': self.url,
'file': self.file,
'images': images,
'documents': documents,
'title': self.title,
'tokens': self.tokens,
'preview': self.preview.ref if self.preview else None,
}
def __filename(self):
return os.path.basename(self.file)
def __set_id(self):
if 'id' in self.params:
self.id = self.get_param('id')
else:
self.id = url_encode_text(self.get_param('url'))
def __set_url(self):
self.url = self.params['url']
def __set_hash(self):
files = []
files += self.get_param('images')
files += self.get_param('documents')
folder = self.get_param('folder')
files = [os.path.join(folder, x) for x in files]
files = sorted([self.file] + files)
hashes = [self.provider.hash(x) for x in files]
self.hash = hash_str(self.__hash_salt + ''.join(hashes))
def __str__(self):
return '<Page file={} id={}>'.format(self.file, self.id)
modules/sync/sync/models/__init__.py
-87
View File
@@ -1,87 +0,0 @@
import os
import settings
def find(store, item_id: str):
q = {'id': item_id}
try:
return store.find_one(q)
except ValueError:
pass
return None
class Model:
def __init__(self, provider, store, file, params=None):
super().__init__()
self.provider = provider
self.store = store
self.file = file
self.id = None
self.url = None
self.hash = None
self.ref = None # Data base reference
self.params = params if params else {}
self.init()
def init(self):
pass
async def upload(self, **kwargs):
pass
async def build(self, **kwargs):
pass
def bake(self):
return None
async def is_changed(self):
i = find(self.store, self.id)
return self._is_changed_hash(i)
def has_param(self, name):
if not self.params:
return False
return name in self.params
def get_param(self, name, default_value=None):
if not self.params:
return default_value
if not (name in self.params):
return default_value
return self.params[name]
def _is_changed_hash(self, item):
if not item:
return True
if 'hash' not in item:
return True
return not (self.hash == item['hash'])
def _filename(self):
return os.path.basename(self.file)
def _get_relpath(self, path):
folder = self.get_param('folder', self.file)
return os.path.join(folder, path)
def _get_abs(self, path):
return self.provider.get_abs(path)
async def save(self):
document = self.bake()
query = {'id': document['id']}
try:
self.store.update_one(query, {'$set': document}, upsert=True)
return self
except ValueError:
pass
return None
modules/sync/test/sync/models/test_model.py
-13
View File
@@ -1,13 +0,0 @@
import unittest
from sync.models import Model
class MyTestCase(unittest.TestCase):
def test_bake(self):
m = Model()
self.assertIsNone(m.bake())
if __name__ == '__main__':
unittest.main()
modules/sync/utils/fn.py
-45
View File
@@ -1,45 +0,0 @@
import os
from functools import reduce
def constant(value):
return lambda _: value
def combine(items):
if len(items) == 0:
return []
return reduce(
lambda total, i: total + i,
items
)
def compose(*fns):
return lambda i: reduce(
lambda out, fn: fn(out),
fns, i
)
first = lambda i: i[0] if len(i) > 0 else None
def swap_ext(extension: str):
return lambda i: os.path.splitext(i)[0] + extension
def map_cases(param, cases):
for predicate, fn in cases:
if predicate(param):
return fn(param)
return param
def until(condition_fn, iter_fn, items, default_value=None):
for i in items:
result = iter_fn(i)
if condition_fn(result):
return result
return default_value
modules/sync/utils/front_matter.py
-36
View File
@@ -1,36 +0,0 @@
import re
import yaml
FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE)
def parse_yaml_front_matter(data: str):
fm, content = split_yaml_front_matter(data)
fm = fm if fm else {}
return {
**fm,
'content': content,
}
def split_yaml_front_matter(text: str):
"""
Parse text with YAML frontmatter, return metadata and content.
Pass in optional metadata defaults as keyword args.
If frontmatter is not found, returns an empty metadata dictionary
and original text content.
"""
# split on the first two triple-dashes
try:
_, fm, content = FM_BOUNDARY.split(text, 2)
except ValueError:
# if we can't split, bail
return None, None
# parse yaml, now that we have frontmatter
fm = yaml.safe_load(fm)
if isinstance(fm, dict):
return fm, content.strip()
return None, None
modules/sync/utils/hash.py
-38
View File
@@ -1,38 +0,0 @@
import hashlib
import json
import bson.json_util
from utils.fn import map_cases
def hash_file(file):
with open(file, 'rb') as f:
data = f.read()
return hash_data(data)
def hash_data(data):
sha = hashlib.sha256(data)
return sha.hexdigest()
def hash_str(data):
to_json = lambda i: json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default)
data = map_cases(data, [
(lambda i: isinstance(i, dict), to_json),
(lambda i: isinstance(i, tuple), to_json),
(lambda i: isinstance(i, list), to_json),
(lambda i: isinstance(i, int), int),
(lambda i: isinstance(i, float), float),
])
if isinstance(data, dict):
data = to_json(data)
return hash_data(str(data).encode())
def md5(data: str) -> str:
i = hashlib.md5(data.encode('utf-8'))
return i.hexdigest()
modules/sync/utils/image.py
-35
View File
@@ -1,35 +0,0 @@
import asyncio
from subprocess import call
async def image_magick_resize(input_file, output_file, size, quality=85):
interlace = 'Plane'
s = '%dx%d' % size
cmd = [
'/usr/local/bin/convert',
'"%s"' % input_file,
'-strip',
'-interlace', interlace,
'-auto-orient',
'-resize', s,
'-quality', str(quality),
'"%s"' % output_file,
]
cmd = ' '.join(cmd)
process = await asyncio.create_subprocess_shell(cmd)
await process.wait()
async def image_magick_pdf_to_img(input_file, output_file):
call([
'/usr/local/bin/gs',
'-q',
'-sDEVICE=jpeg',
'-dLastPage=1',
'-o', output_file,
input_file,
])
modules/sync/utils/lxml_utils.py
-29
View File
@@ -1,29 +0,0 @@
import re
def get_following_tags(element, stop_tag_pattr):
stop_exp = re.compile(stop_tag_pattr)
following = element.xpath('./following-sibling::*')
tags = []
i = 0
while True:
try:
tag = following[i]
if stop_exp.match(tag.tag):
break
tags.append(tag)
except IndexError as tag:
break
i += 1
return tags
def get_text_following_by_tag(element, tag, contains, strip=True):
e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains))
if len(e) > 0:
if strip:
return e[0].strip()
else:
return e[0]
else:
return None
modules/sync/utils/text/split.py
-18
View File
@@ -1,18 +0,0 @@
from utils.fn import until
def split(data, separator=' '):
try:
s = data.index(separator)
return data[:s], data[s:]
except ValueError:
return None, None
def split_with(separators):
return lambda data: until(
lambda i: i[0] and i[1],
lambda s: split(data, s),
separators,
(None, None)
)
modules/sync/utils/text/transform.py
-94
View File
@@ -1,94 +0,0 @@
import re
from unidecode import unidecode
from utils.fn import compose
remove_pattr = re.compile('[«»<>#,!&"\']')
reduce_dash_patterns = [
(re.compile('[-]+'), '-'),
(re.compile('^-'), '')
]
def translit(text):
"""
RUSSIAN TEXT => ENGLISH TEXT
"""
return unidecode(text)
def interpolate(text, storage, key_fn=None):
if not key_fn:
key_fn = lambda i: i
key = key_fn(text)
return storage[key] if key in storage else text
def interpolate_swift(i):
return re.search("([\d\w-]+)", i).group(1)
def remove_punctuation(i):
return remove_pattr.sub('', i)
def reduce_dashes(i):
value = i
for reg_exp, replace in reduce_dash_patterns:
value = reg_exp.sub(replace, value)
return value
def space_to_dash(i):
return i.replace(' ', '-')
def short_dash(i):
"""
minus -> dash
tire -> dash
:param i:
:return:
"""
return i.replace('–', '-').replace('—', '-')
def dot_to_dash(i):
return i.replace('.', '-')
def slash_to_dash(i):
return i.replace('/', '-')
def remove_underscores(i):
return i.replace('_', '')
def text_lower(i):
return i.lower()
url_encode_file = compose(
translit,
space_to_dash,
short_dash,
remove_underscores,
remove_punctuation,
reduce_dashes,
text_lower,
)
url_encode_text = compose(
translit,
space_to_dash,
dot_to_dash,
short_dash,
slash_to_dash,
remove_underscores,
remove_punctuation,
reduce_dashes,
text_lower,
)
modules/sync/yandexdisk/__init__.py
-305
View File
@@ -1,305 +0,0 @@
import requests
class YDClient:
"""
Implementation of https://tech.yandex.ru/disk/poligon/
"""
base_url = 'https://cloud-api.yandex.net/v1/disk'
def __init__(self, access_token):
self.access_token = access_token
self.base_headers = dict(
Accept='application/json',
Authorization='OAuth %s' % self.access_token,
)
async def get_flat(self, limit=20):
"""
https://cloud-api.yandex.net/v1/disk/resources/files?
[ limit=<количество файлов в списке>]
[& media_type=<тип запрашиваемых файлов>]
[& offset=<смещение относительно начала списка>]
[& fields=<нужные ключи ответа>]
[& preview_size=<размер превью>]
[& preview_crop=<признак обрезки превью>]
:return:
"""
url = '{0}/resources/files'.format(self.base_url)
payload = dict(
limit=limit,
offset=0,
)
r = requests.get(url, headers=self.base_headers, params=payload)
self.check_code(r)
json_dict = r.json()
# return Directory(**json_dict)
return json_dict
def get_disk_metadata(self):
"""
:return: disk metadata
"""
url = self.base_url
r = requests.get(url, headers=self.base_headers)
self.check_code(r)
disk = r.json()
return Disk(
trash_size=disk['trash_size'],
total_space=disk['total_space'],
used_space=disk['used_space'],
system_folders=disk['system_folders']
)
def last_uploaded(self):
"""
https://cloud-api.yandex.net/v1/disk/resources/last-uploaded?
[ limit=<количество файлов в списке>]
[& media_type=<тип запрашиваемых файлов>]
[& fields=<нужные ключи ответа>]
[& preview_size=<размер превью>]
[& preview_crop=<признак обрезки превью>]
"""
url = '{0}/resources/last-uploaded'.format(self.base_url)
payload = dict(
limit=100,
)
r = requests.get(url, headers=self.base_headers, params=payload)
self.check_code(r)
json_dict = r.json()
# return Directory(**json_dict)
return json_dict
def get_content(self, path):
"""
:param path: path to folder
:return: content of folder
"""
url = '{0}/resources'.format(self.base_url)
payload = dict(
path=path,
limit=1000000
)
r = requests.get(url, headers=self.base_headers, params=payload)
self.check_code(r)
json_dict = r.json()
return Directory(**json_dict)
def create_folder(self, path_to_folder):
"""
:param path_to_folder: path to folder
:return: created folder
"""
url = '{0}/resources'.format(self.base_url)
payload = {'path': path_to_folder}
r = requests.put(url, headers=self.base_headers, params=payload)
self.check_code(r)
return self.get_content(path_to_folder)
def remove_folder_or_file(self, path):
"""
Remove folder or file
:param path: path to file or folder
"""
url = '{0}/resources'.format(self.base_url)
payload = {'path': path}
r = requests.delete(url, headers=self.base_headers, params=payload)
self.check_code(r)
def copy_folder_or_file(self, path_from, path_to):
"""
Copy folder or file
:param path_from: from path
:param path_to: to path
"""
url = '{0}/resources/copy'.format(self.base_url)
payload = {'path': path_to, 'from': path_from}
r = requests.post(url, headers=self.base_headers, params=payload)
self.check_code(r)
def get_download_link(self, path):
"""
:param path: path to file
:return: download link to file
"""
url = '{0}/resources/download'.format(self.base_url)
payload = {'path': path}
r = requests.get(url, headers=self.base_headers, params=payload)
self.check_code(r)
return r.json()
def get_published_elements(self):
"""
:return: published elements
"""
json_dict = self._get_dictionary_of_published_files()
elements = []
for item in json_dict['items']:
if item['type'] == 'dir':
elements.append(Directory(**item))
elif item['type'] == 'file':
elements.append(File(**item))
return elements
def get_public_link_to_folder_or_file(self, path):
"""
:param path: path
:return: public link to folder or file
"""
url = '{0}/resources/publish'.format(self.base_url)
payload = {'path': path}
r = requests.put(url, headers=self.base_headers, params=payload)
self.check_code(r)
files = self._get_dictionary_of_published_files()
for file in files['items']:
if str(file['path']).endswith(path):
return file['public_url']
return ''
def unpublish_folder_or_file(self, path):
"""
Unpublish folder of file
:param path: path to file or folder
"""
url = '{0}/resources/unpublish'.format(self.base_url)
payload = {'path': path}
r = requests.put(url, headers=self.base_headers, params=payload)
self.check_code(r)
def get_list_of_all_files(self):
"""
:return: List of all files
"""
url = '{0}/resources/files'.format(self.base_url)
r = requests.get(url, headers=self.base_headers)
self.check_code(r)
json_dict = r.json()
files = []
for item in json_dict['items']:
f = File(**item)
files.append(f)
return files
def move_folder_or_file(self, path_from, path_to):
"""
Move folder or file
:param path_from: path from
:param path_to: path to
"""
url = '{0}/resources/move'.format(self.base_url)
payload = {'path': path_to, 'from': path_from}
r = requests.post(url, headers=self.base_headers, params=payload)
self.check_code(r)
def upload_file(self, path_from, path_to):
"""
Upload file
:param path_from: path from
:param path_to: path to yandex disk
"""
url = '{0}/resources/upload'.format(self.base_url)
payload = {'path': path_to}
r = requests.get(url, headers=self.base_headers, params=payload)
self.check_code(r)
json_dict = r.json()
upload_link = json_dict["href"]
with open(path_from, 'rb') as fh:
files = {'file': fh}
r2 = requests.put(upload_link, headers=self.base_headers, files=files)
self.check_code(r2)
def upload_file_from_url(self, from_url, path_to):
"""
Upload file by URL
:param from_url: URL path from
:param path_to: path to yandex disk
"""
url = '{0}/resources/upload'.format(self.base_url)
payload = {'path': path_to, 'url': from_url}
r = requests.post(url, headers=self.base_headers, params=payload)
self.check_code(r)
def _get_dictionary_of_published_files(self):
url = '{0}/resources/public'.format(self.base_url)
r = requests.get(url, headers=self.base_headers)
self.check_code(r)
return r.json()
def check_code(self, req):
if not str(req.status_code).startswith('2'):
raise YDException(req.status_code, req.text)
class YDException(Exception):
code = None
def __init__(self, code, text):
super(YDException, self).__init__(text)
self.code = code
def __str__(self):
return "%d. %s" % (self.code, super(YDException, self).__str__())
class Directory:
def __init__(self, **kwargs):
self.children = []
for key in kwargs:
if key is not '_embedded':
setattr(self, key, kwargs[key])
if "_embedded" in kwargs:
for item in kwargs['_embedded']['items']:
if item['type'] == 'dir':
d = Directory(**item)
self.children.append(d)
if item['type'] == 'file':
f = File(**item)
self.children.append(f)
def get_children(self):
return self.children
class File:
def __init__(self, **kwargs):
for key in kwargs:
setattr(self, key, kwargs[key])
class Disk:
def __init__(self, trash_size, total_space, used_space, system_folders):
self.trash_size = trash_size
self.total_space = total_space
self.used_space = used_space
self.system_folders = system_folders