diff --git a/modules/sync/README.md b/modules/sync/README.md deleted file mode 100644 index c6d6b99e..00000000 --- a/modules/sync/README.md +++ /dev/null @@ -1,71 +0,0 @@ -## Usage - -Sync tool -```bash - python3 sync.py -``` - - -### Environment variables - -**YANDEX_DISK_ACCESS_TOKEN** - -For access to files using Yandex.Disk API - -[How to obtain token for testing](https://tech.yandex.ru/oauth/doc/dg/tasks/get-oauth-token-docpage/) - -Example: -```bash -YANDEX_DISK_ACCESS_TOKEN= -``` - -Authorize url. Replace with your app id -``` -https://oauth.yandex.ru/authorize?response_type=token&client_id=<идентификатор приложения> -``` - -#### SYNC_LOCAL_PATH - -Path to scan files from - -Example: -```bash -SYNC_LOCAL_PATH=/Users/tmshv/Yandex.Disk -``` - -#### SYNC_STATIC - -Path to local folder for compiled files (images, uploads) - -Example: -```bash -SYNC_STATIC=~/Desktop/Hudozka Static -``` - -#### SYNC_PROVIDER - -Type of file provider - -- *fs* for local access -- *yd* for access using Yandex.Disk API (YANDEX_DISK_ACCESS_TOKEN is required) - -Example: -```bash -SYNC_PROVIDER=fs -``` - -#### MONGO_URI - -URI for MongoDB access - -Example: - ```bash - mongodb://localhost:27017/hudozka - ``` - -#### SYNC_ENV - -Environment like *NODE_ENV* - -- *production* default value -- *development* don't update/delete documents from DB diff --git a/modules/sync/config.yaml b/modules/sync/config.yaml deleted file mode 100644 index c0ad1801..00000000 --- a/modules/sync/config.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -database_uri: 'mongodb://localhost:27017/hudozka' - -env: 'production' -yandex_disk_access_token: '' -interval: 0 - -skip_unchanged: True - -image_processing_enabled: True - -upload_enabled: True -upload_auth: - login: '' - password: '' - -provider: - name: 'fs' - root: '' - -update_enabled: True -delete_enabled: True diff --git a/modules/sync/db.py b/modules/sync/db.py deleted file mode 100644 index bfe91db3..00000000 --- a/modules/sync/db.py +++ /dev/null @@ -1,12 +0,0 @@ -from pymongo import MongoClient - -import settings - - -def db(): - client = MongoClient(settings.database_uri) - return client.hudozka - - -def collection(name): - return db()[name] diff --git a/modules/sync/kazimir/CSVToken.py b/modules/sync/kazimir/CSVToken.py deleted file mode 100644 index 8ffd0733..00000000 --- a/modules/sync/kazimir/CSVToken.py +++ /dev/null @@ -1,46 +0,0 @@ -import csv -import io - -from kazimir.Token import FileToken, is_file - -extensions = ['.csv'] - - -class CSVToken(FileToken): - @staticmethod - def test(data: str): - return is_file(data, extensions) - - def __init__(self, data) -> None: - super().__init__(data=data) - self.name = 'csv' - self.joinable = False - self.build = None - - async def compile(self): - data = await self.build(self.data) - - f = io.StringIO(data) - reader = csv.reader(f, delimiter=',', quotechar='|') - - headers = next(reader, None) - - row = lambda xs: '\n'.join([ - f'{x}' for x in xs - ]) - - head = '\n'.join([ - f'{x}' for x in headers - ]) - body = '\n'.join([ - f'{row(x)}' for x in reader - ]) - - return f''' - - - {head} - - {body} -
- '''.strip() diff --git a/modules/sync/kazimir/DocumentToken.py b/modules/sync/kazimir/DocumentToken.py deleted file mode 100644 index c9c97c25..00000000 --- a/modules/sync/kazimir/DocumentToken.py +++ /dev/null @@ -1,17 +0,0 @@ -from kazimir.Token import parse_file, is_document, Token - - -class DocumentToken(Token): - @staticmethod - def test(data: str): - return is_document(data) - - def __init__(self, data) -> None: - super().__init__(name='file', data=data) - self.joinable = False - self.build = None - - async def get_data(self): - data = parse_file(self.data) - document = await self.build(data) - return document diff --git a/modules/sync/kazimir/InstagramToken.py b/modules/sync/kazimir/InstagramToken.py deleted file mode 100644 index e3d3fc78..00000000 --- a/modules/sync/kazimir/InstagramToken.py +++ /dev/null @@ -1,85 +0,0 @@ -from urllib.parse import urlparse - -import re -import requests -from markdown.util import etree - -from kazimir.Token import UrlToken, is_url - -pattr = re.compile(r'https?://(www\.)?instagram\.com/p/(?P[\w\d]+)/?(.*)$') - - -class InstagramToken(UrlToken): - @staticmethod - def test(data: str): - if not is_url(data): - return False - - url = urlparse(data) - return url.netloc in ['www.instagram.com', 'instagram.com'] - - def __init__(self, data: str) -> None: - super().__init__(data=data) - self.name = 'instagram' - - async def get_data(self): - m = pattr.match(self.data) - embed = None - if m: - m = m.group('media') - embed = await embed_instagram(m) - - return { - 'url': self.data, - 'embed': embed, - } - - -async def embed_instagram(media): - """ -
- -
- """ - - url = f'https://api.instagram.com/oembed?url=http://instagr.am/p/{media}&hidecaption=true&omitscript=true' - req = requests.get(url) - if req.status_code == 200: - html = req.json()['html'] - html = fix_html(html) - - return html - # i = etree.fromstring(html) - - # obj = etree.Element('div') - # obj.set('class', "kazimir__embed") - # obj.append(i) - # return etree.tostring(obj).decode() - else: - raise Exception( - 'Failed to compile Instagram {media}: Status Code: {status}'.format( - media=media, status=req.status_code) - ) - - -def fix_html(html: str): - return html.replace('data-instgrm-captioned', 'data-instgrm-captioned=""') diff --git a/modules/sync/kazimir/Marker.py b/modules/sync/kazimir/Marker.py deleted file mode 100644 index ee013743..00000000 --- a/modules/sync/kazimir/Marker.py +++ /dev/null @@ -1,83 +0,0 @@ -import lxml.html -from markdown import markdown - -from kazimir.Token import SplitToken - - -class Marker: - def __init__(self) -> None: - super().__init__() - self.tokens = [] - - def add_token_factory(self, token): - self.tokens.append(token) - - def parse(self, text: str): - tokens = text.split('\n') - tokens = self.markup_tokens(tokens) - tokens = join_tokens(tokens) - tokens = merge_tokens(tokens) - - return tokens - - def markup_tokens(self, tokens: list): - marked_tokens = [] - - for data in tokens: - t = self.create_token(data) - if t: - marked_tokens.append(t) - else: - raise Exception(f'Token not recognized for data {data}') - return marked_tokens - - def create_token(self, data): - for f in self.tokens: - if f.token.test(data): - return f.create(data=data) - return None - - -def join_tokens(tokens): - is_split = lambda x: isinstance(x, SplitToken) - - buffer_tag = None - buffer = [] - ts = [] - for token in tokens: - tag = token.name - changed = buffer_tag is not None and tag != buffer_tag - - flush = is_split(token) - if changed: - flush = True - - if not token.joinable: - flush = True - - if flush: - if buffer: - ts.append(buffer) - if is_split(token): - buffer = [] - buffer_tag = None - else: - buffer = [token] - buffer_tag = tag - else: - buffer_tag = tag - buffer.append(token) - - if len(buffer): - ts.append(buffer) - return ts - - -def merge_tokens(tokens: list): - tokens_out = [] - for ts in tokens: - token = ts[0] - for x in ts[1:]: - token = token.merge(x) - tokens_out.append(token) - return tokens_out diff --git a/modules/sync/kazimir/Token.py b/modules/sync/kazimir/Token.py deleted file mode 100644 index 0ac9a9c6..00000000 --- a/modules/sync/kazimir/Token.py +++ /dev/null @@ -1,191 +0,0 @@ -import os -import re -from markdown import markdown -from urllib.parse import urlparse - -image_extensions = ['.jpg', '.jpeg', '.png'] -document_extensions = ['.doc', '.docx', '.pdf', '.xls', '.xlsx'] - - -class TokenFactory: - def __init__(self, token) -> None: - super().__init__() - self.token = token - - def create(self, data): - return self.token(data=data) - - -class BuildTokenFactory(TokenFactory): - def __init__(self, token, build) -> None: - super().__init__(token) - self.build = build - - def create(self, data): - x = self.token(data=data) - x.build = self.build - return x - - -class Token: - @staticmethod - def test(data: str): - return True - - def __init__(self, name, data) -> None: - super().__init__() - self.name = name - self.data = data - self.joinable = True - - async def get_data(self): - return self.data - - def merge(self, token): - return self - - def __repr__(self) -> str: - return f'[Token {self.name} {self.data}]' - - -class SplitToken(Token): - @staticmethod - def test(data: str): - data = data.strip() - return data == '' - - def __init__(self, data) -> None: - super().__init__(name='split', data=None) - - -class TextToken(Token): - @staticmethod - def test(data: str): - return True - - def __init__(self, data) -> None: - super().__init__(name='text', data=data) - - def merge(self, token: Token): - data = self.data - data += '\n' - data += token.data - return TextToken(data.strip()) - - -class HtmlToken(Token): - @staticmethod - def test(data: str): - """ - Result of compiling markdown to html is the same as source data meaning that the source is html - """ - html = markdown(data) - return html == data - - def __init__(self, data: str) -> None: - super().__init__(name='html', data=data) - self.joinable = False - - -class FileToken(Token): - @staticmethod - def test(data: str): - return is_file(data, document_extensions + image_extensions) - - def __init__(self, data) -> None: - super().__init__(name='file', data=data) - self.joinable = False - - async def get_data(self): - return parse_file(self.data) - - -class ImageToken(Token): - @staticmethod - def test(data: str): - return is_image(data) - - def __init__(self, data) -> None: - super().__init__(name='image', data=data) - self.joinable = False - self.build = None - - async def get_data(self): - data = parse_file(self.data) - data = await self.build(data) - return data - - -class UrlToken(Token): - @staticmethod - def test(data: str): - return is_url(data) - - def __init__(self, data) -> None: - super().__init__(name='url', data=data) - self.joinable = False - - -def markdown_to_html(text): - return markdown(text) - - -def is_url(sample: str): - url = urlparse(sample) - return url.scheme and url.netloc - - -def is_image(sample: str): - file = parse_file(sample)['file'] - return is_file(file, image_extensions) - - -def is_document(sample: str): - file = parse_file(sample)['file'] - return is_file(file, document_extensions) - - -def is_file(sample: str, extensions): - name, ext = os.path.splitext(sample) - ext = ext.lower() - return ext in extensions - - -def parse_file(sample: str): - """ - ![Caption](file.jpg) -> {file, caption} - file.jpg Caption -> {file, caption} - - :param sample: - :return: - """ - p = re.match(r'!\[(.*)\]\((.*)\)', sample) - if p: - caption = p.group(1) - file = p.group(2) - else: - sample = sample.split(' ') - file = sample[0] - caption = ' '.join(sample[1:]) - return { - 'file': file, - 'caption': caption, - } - - -def parse_link(sample: str): - """ - ![Alt](href) -> {href, alt} - - :param sample: - :return: - """ - p = re.match(r'!\[(.*)\]\((.*)\)', sample) - if not p: - return None - - url = p.group(2) - return { - 'url': urlparse(url), - 'alt': p.group(1), - } diff --git a/modules/sync/kazimir/YoutubeToken.py b/modules/sync/kazimir/YoutubeToken.py deleted file mode 100644 index 2157d957..00000000 --- a/modules/sync/kazimir/YoutubeToken.py +++ /dev/null @@ -1,21 +0,0 @@ -from urllib.parse import urlparse, parse_qs -from kazimir.Token import UrlToken, is_url - - -class YoutubeToken(UrlToken): - @staticmethod - def test(data: str): - if not is_url(data): - return False - - url = urlparse(data) - return url.netloc in ['www.youtube.com', 'youtube.com'] - - def __init__(self, data: str) -> None: - super().__init__(data=data) - self.name = 'youtube' - - async def get_data(self): - return { - 'url': self.data - } diff --git a/modules/sync/requirements.txt b/modules/sync/requirements.txt deleted file mode 100644 index 3e2ac7cc..00000000 --- a/modules/sync/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -cssselect==0.9.2 -lxml==6.1.0 -Markdown==3.8.1 -Pillow==12.2.0 -pymongo==4.6.3 -PyPDF2==1.27.9 -python-frontmatter==0.3.1 -PyYAML==5.4 -requests==2.33.0 -six==1.10.0 -smartcrop==0.1 -toml==0.9.2 -Unidecode==0.4.19 diff --git a/modules/sync/settings.py b/modules/sync/settings.py deleted file mode 100644 index d8935eef..00000000 --- a/modules/sync/settings.py +++ /dev/null @@ -1,89 +0,0 @@ -import os - -import yaml -import argparse - -__author__ = 'Roman Timashev' - -name = 'HudozkaSync' - -_parser = argparse.ArgumentParser(description='Hudozka Sync Daemon') -_parser.add_argument('--config', dest='config', default='config.yaml', help='path to config.yaml') -_args = _parser.parse_args() - -config = yaml.load(open(_args.config, 'r')) - - -def value(key, default=None): - return config[key] if key in config else default - - -def absolute(path, ensure=True): - a = os.path.expanduser(path) - if ensure: - os.makedirs(a, exist_ok=True) - return a - - -def abs_fn(root): - return lambda path: os.path.join(root, path) - - -def env(param, default=None): - if param in os.environ: - return os.environ[param] - return default - - -interval = value('interval', 0) - -database_uri = value('database_uri') # env('MONGO_URI') - -skip_unchanged = value('skip_unchanged') - -image_processing_enabled = value('image_processing') - -image_ext = '.jpg' -image_sizes = [ - ('original', None, None), - - # ('big', 1500, 667), - # ('medium', 400, 300), -] - -date_formats = [ - '%Y', - '%m.%Y', - '%d.%m.%Y', - '%d.%m.%Y %H:%M', - '%d.%m.%Y %H:%M:%S' -] - -collection_images = 'images' -collection_documents = 'documents' -collection_pages = 'pages' - -hash_salt_pages = '6' -hash_salt_documents = '4' -hash_salt_images = '5' - -provider_name = value('provider')['name'] # env('SYNC_PROVIDER', 'fs') -provider_root = value('provider')['root'] # env('SYNC_PROVIDER', 'fs') - -f = abs_fn(provider_root) - -dir_pages = './Pages' - -document_url_template = 'https://static.shlisselburg.org/art/uploads/{file}' -document_url_preview_template = 'https://static.shlisselburg.org/art/images/{id}-{size}{ext}' - -image_url_base = 'https://static.shlisselburg.org/art/images/' - -yandex_disk_access_token = value('yandex_disk_access_token') # env('YANDEX_DISK_ACCESS_TOKEN') - -auth_s3 = { - 'region_name': 'ams3', - 'endpoint_url': 'https://dir.ams3.digitaloceanspaces.com', - 'aws_access_key_id': value('s3')['client_id'], - 'aws_secret_access_key': value('s3')['client_secret'], -} diff --git a/modules/sync/sync-smart.py b/modules/sync/sync-smart.py deleted file mode 100644 index d7686198..00000000 --- a/modules/sync/sync-smart.py +++ /dev/null @@ -1,50 +0,0 @@ -import settings -from sync.data.fs import FSProvider -from sync.data.yandexdisk import YDProvider - -import asyncio - - -async def run_check(run_interval=0): - run_interval = 1 - - from yandexdisk import YDClient - client = YDClient(settings.yandex_disk_access_token) - - async def need_to_update(): - res = client.last_uploaded() - items = res['items'] - paths = [i['path'] for i in items] - - for i in paths: - if i.startswith('disk:/Hudozka/Site'): - return True - - return False - - while True: - status = await need_to_update() - if status: - print('DO SYNC') - # await run(0) - - if run_interval == 0: - break - else: - await asyncio.sleep(run_interval) - - -def get_provider(provider_type, root): - if provider_type == 'fs': - return FSProvider(root) - elif provider_type == 'yd': - return YDProvider(root, settings.yandex_disk_access_token) - return None - - -if __name__ == '__main__': - interval = settings.interval - - loop = asyncio.get_event_loop() - loop.run_until_complete(run_check(interval)) - loop.close() diff --git a/modules/sync/sync.py b/modules/sync/sync.py deleted file mode 100644 index d9c230fb..00000000 --- a/modules/sync/sync.py +++ /dev/null @@ -1,75 +0,0 @@ -import settings -from sync.core import Sync -from sync.data.fs import FSProvider -from sync.data.yandexdisk import YDProvider - -import asyncio -import logging - -from sync.models.Page import Page, PageController - - -def init_logger(name: str, file: str): - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - - # create file handler which logs even debug messages - fh = logging.FileHandler(file) - fh.setLevel(logging.DEBUG) - - # create console handler with a higher log level - ch = logging.StreamHandler() - # ch.setLevel(logging.ERROR) - ch.setLevel(logging.DEBUG) - - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - ch.setFormatter(formatter) - - # add the handlers to the logger - logger.addHandler(fh) - logger.addHandler(ch) - - -async def run_sync(sync: Sync, **kwargs): - await sync.run(**kwargs) - - -async def run(run_interval=0): - provider = get_provider(settings.provider_name, settings.provider_root) - controller = PageController(provider) - - sync_pages = Sync( - ctrl=controller, - name='Page' - ) - sync_pages.set_options(True, True, True) - - while True: - await asyncio.wait([ - run_sync(sync_pages), - ]) - - if run_interval == 0: - break - else: - await asyncio.sleep(run_interval) - - -def get_provider(provider_type, root): - if provider_type == 'fs': - return FSProvider(root) - elif provider_type == 'yd': - return YDProvider(root, settings.yandex_disk_access_token) - return None - - -if __name__ == '__main__': - init_logger(settings.name, 'hudozka.log') - - interval = settings.interval - - loop = asyncio.get_event_loop() - loop.run_until_complete(run(interval)) - loop.close() diff --git a/modules/sync/sync/__init__.py b/modules/sync/sync/__init__.py deleted file mode 100644 index 572ed51c..00000000 --- a/modules/sync/sync/__init__.py +++ /dev/null @@ -1,147 +0,0 @@ -import os -import re -from datetime import datetime - -import lxml.html -import kazimir -import settings -from kazimir.CSVToken import CSVToken -from kazimir.DocumentToken import DocumentToken -from kazimir.YoutubeToken import YoutubeToken -from kazimir.InstagramToken import InstagramToken -from kazimir.Marker import Marker -from kazimir.Token import TokenFactory, SplitToken, UrlToken, ImageToken, BuildTokenFactory, TextToken, HtmlToken -from sync.data import Provider -from sync.models import Model -from sync.models.Document import Document -from sync.models.Image import Image - - -def create_date(date_str, date_formats=None): - """ - 2016 - 2016.10 - 2016.10.10 - -> datetime - - :param date_formats: - :param date_str: - :return: - """ - if not date_formats: - date_formats = [ - '%Y', - '%Y.%m', - '%Y.%m.%d' - ] - - for date_format in date_formats: - try: - return datetime.strptime(date_str, date_format) - except: - continue - return None - - -async def untouched(items: [Model]) -> [Model]: - if settings.skip_unchanged: - return items - - result = [] - for item in items: - status = await item.is_changed() - if status: - result.append(item) - return result - - # def is_equals(e, n): - # return (n is None) or ('hash' not in n) or (e.hash != n['hash']) - # - # doc = lambda i: {'id': i.id, 'hash': i.hash} - # - # stored_documents = [(i, store.read(doc(i))) for i in documents] - # filtered_documents = [i[0] for i in stored_documents if is_equals(*i)] - - # return filtered_documents - - -async def create_post(provider: Provider, folder: str, md: str, sizes): - images = [] - documents = [] - - def rel(file: str) -> str: - return os.path.join(folder, file) - - async def build_document(data): - filepath = os.path.join(folder, data['file']) - document = await Document.new(provider, filepath, settings.image_sizes) - - title = data['caption'] - if title: - document.set_title(title) - await document.save() - - if not document: - raise Exception(f'Failed to build Document {filepath}') - - text = data['caption'] if data['caption'] else data['file'] - url = f'/document/{document.id}' - - image_url = document.preview.get_src() - - return { - 'url': url, - 'slug': document.id, - 'image_url': image_url, - 'file_url': document.url, - 'title': document.title, - 'file_size': document.file_size, - 'file_format': 'application/pdf', - } - - async def build_image(data): - filename = data['file'] - relative_path = os.path.join(folder, filename) - image_path = provider.get_local(relative_path) - if os.path.exists(image_path): - image = await Image.new(provider, relative_path, sizes) - if image: - images.append(image) - - src = image.get_src() - return { - 'src': src, - 'alt': data['caption'], - 'caption': data['caption'], - } - raise Exception('Fail to get Image', relative_path) - - async def read_file(data): - filepath = data - filepath = rel(filepath) - - return provider.read(filepath).read().decode('utf-8') - - m = Marker() - m.add_token_factory(TokenFactory(SplitToken)) - m.add_token_factory(TokenFactory(YoutubeToken)) - m.add_token_factory(TokenFactory(InstagramToken)) - m.add_token_factory(TokenFactory(UrlToken)) - m.add_token_factory(BuildTokenFactory(ImageToken, build=build_image)) - m.add_token_factory(BuildTokenFactory(DocumentToken, build=build_document)) - m.add_token_factory(BuildTokenFactory(CSVToken, build=read_file)) - # m.add_token_factory(TokenFactory(FileToken)) - m.add_token_factory(TokenFactory(HtmlToken)) - m.add_token_factory(TokenFactory(TextToken)) - - tokens = m.parse(md) - - encoded_tokens = [] - for t in tokens: - data = await t.get_data() - encoded_tokens.append({ - 'token': t.name, - 'data': data, - }) - - return images, documents, encoded_tokens diff --git a/modules/sync/sync/core/__init__.py b/modules/sync/sync/core/__init__.py deleted file mode 100644 index 4c8d3555..00000000 --- a/modules/sync/sync/core/__init__.py +++ /dev/null @@ -1,114 +0,0 @@ -import logging -import settings -from sync import untouched - - -class Sync: - def __init__(self, ctrl, name: str): - super().__init__() - - self.validate_urls = True - self.__default_build_args = { - 'sizes': settings.image_sizes, - } - self.__delete_query = lambda item: {'_id': item['_id']} - - self.logger = logging.getLogger('%s.%s' % (settings.name, name)) - - self.ctrl = ctrl - self.filter_item_fn = lambda x: True - - self.skip_unchanged = True - self.update_enabled = True - self.delete_enabled = True - - def set_options(self, su: bool, u: bool, d: bool): - self.skip_unchanged = su - self.update_enabled = u - self.delete_enabled = d - return self - - def set_item_filter(self, fn): - self.filter_item_fn = fn - return self - - async def build(self, item): - args = self._get_build_args() - await item.build(**args) - return item - - async def update(self, item): - await item.upload() - await item.save() - return item - - def _get_build_args(self): - return { - **self.__default_build_args - } - - def __items_to_delete_query(self, items): - query = { - 'id': {'$nin': [doc.id for doc in items]}, - } - - return query - - async def run(self): - """ - # Get scope files - # Validate these files. Raise an error - # calc_hashes - # make diff with previous run - # compile objects - # upload - - :return: - """ - self.logger.info('Checking for update') - - items = await self.ctrl.get_items() - self.logger.info('Found {} Item(s)'.format(len(items))) - all_items = items - - if self.validate_urls: - self.check_urls_uniqueness(all_items) - - # SKIP UNTOUCHED DOCUMENTS - if self.skip_unchanged: - items = await untouched(all_items) - - # Optionally trim list of models by custom filter - items = [ - x - for x in items - if self.filter_item_fn(x) - ] - - self.logger.info(f'Ready to process {len(items)} item(s)') - - # BUILDING - for item in items: - self.logger.info(f'Build Item {item}') - await self.build(item) - - # UPDATING - if self.update_enabled: - for item in items: - await self.update(item) - self.logger.info('Updated Item {}'.format(item)) - - # DELETING - if self.delete_enabled: - obsolete_items = await self.ctrl.find(self.__items_to_delete_query(all_items)) - for item in obsolete_items: - self.logger.info('Deleting Item {}'.format(item['id'])) - await self.ctrl.delete_item(self.__delete_query(item)) - - def check_urls_uniqueness(self, items): - urls = [x.url for x in items] - unique_urls = set(urls) - - if len(urls) != len(unique_urls): - self.logger.error('URLs is not unique') - raise Exception('URLs is not unique') diff --git a/modules/sync/sync/data/__init__.py b/modules/sync/sync/data/__init__.py deleted file mode 100644 index e7c46347..00000000 --- a/modules/sync/sync/data/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -import os -from io import BytesIO - -from utils.fn import combine - - -class Provider: - def __init__(self, root): - super().__init__() - self.root = root - - def read(self, path): - return BytesIO(bytes()) - - def scan(self, path): - return [] - - def glob(self, pattr): - return [] - - def type_filter(self, path, ext): - return [] - - def copy(self, path, out): - pass - - def hash(self, path): - return None - - def size(self, path): - return 0 - - def exists(self, path): - return True - - def get_abs(self, path): - a = os.path.join(self.root, path) - return os.path.normpath(a) - - def get_rel(self, path): - rel = os.path.relpath(path, self.root) - return rel - - def get_local(self, path): - return path - - def is_dir(self, path): - return False - - -def scan_subdirs(provider, ext, path='.'): - """ - Creates flat list of files with specific ext - It iterates only one level of sub directories - :param provider: instance of Provider - :param ext: file extension like .jpg of .pdf for file filtering - :param path: optional path prefix - :return: list of files relative to provider.root - """ - items = provider.scan(path) - items = filter( - lambda i: provider.is_dir(i), - items - ) - items = [provider.type_filter(folder, ext) for folder in items] - items = combine(items) - return items - - -def get_data(provider, path, transform): - data = provider.read(path) - return transform(data) - - -def list_images(provider, path): - image_types = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG'] - images = [provider.type_filter(path, ext) for ext in image_types] - return combine(images) diff --git a/modules/sync/sync/data/fs.py b/modules/sync/sync/data/fs.py deleted file mode 100644 index 2674110f..00000000 --- a/modules/sync/sync/data/fs.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from glob import glob - -from shutil import copyfile - -from sync.data import Provider -from utils.hash import hash_file - - -class FSProvider(Provider): - def read(self, path): - return open(self.get_abs(path), 'rb') - - def scan(self, path): - paths = [i.path for i in os.scandir(self.get_abs(path))] - - return [self.get_rel(path) for path in paths] - - def glob(self, pattr): - items = glob(self.get_abs(pattr)) - return [self.get_rel(i) for i in items] - - def type_filter(self, path, ext): - return list(filter( - lambda fn: os.path.splitext(fn)[1] == ext, - self.scan(path) - )) - - def hash(self, path): - return hash_file(self.get_abs(path)) - - def copy(self, path, out): - file = self.get_abs(path) - return copyfile(file, out) - - def size(self, path): - file = self.get_abs(path) - return os.path.getsize(file) - - def exists(self, path): - file = self.get_abs(path) - return os.path.exists(file) - - def is_dir(self, path): - return os.path.isdir(self.get_abs(path)) - - def get_local(self, path): - return self.get_abs(path) diff --git a/modules/sync/sync/data/request.py b/modules/sync/sync/data/request.py deleted file mode 100644 index 4b2cf544..00000000 --- a/modules/sync/sync/data/request.py +++ /dev/null @@ -1,40 +0,0 @@ -import aiohttp -from boto3 import session -from mimetypes import MimeTypes -import logging -import settings - -logger = logging.getLogger(settings.name + '.S3') - - -async def s3_put(obj, path): - """ - :param obj: uploads/file.pdf - :param path: to/to/local/file.pdf - :return: - """ - logger.info('Put {}'.format(obj)) - - s = session.Session() - client = s.client('s3', **settings.auth_s3) - - mime = MimeTypes() - mime_type, enc = mime.guess_type(path) - - # Upload a file to your Space - client.upload_file(path, 'hudozka', obj, ExtraArgs={ - 'ACL': 'public-read', - 'ContentType': mime_type, - }) - - -async def get(url): - async with aiohttp.ClientSession as aio_session: - res = aio_session.get(url) - return await res.text() - - -async def post(url, data, params={}, headers={}): - async with aiohttp.ClientSession() as aio_session: - async with aio_session.post(url, data=data, params=params, headers=headers) as resp: - return await resp.text() diff --git a/modules/sync/sync/data/yandexdisk.py b/modules/sync/sync/data/yandexdisk.py deleted file mode 100644 index f19678a4..00000000 --- a/modules/sync/sync/data/yandexdisk.py +++ /dev/null @@ -1,69 +0,0 @@ -import os - -import re - -import io -from tempfile import mkstemp - -import requests - -from sync.data import Provider -from yandexdisk import YDClient - - -class YDProvider(Provider): - def __init__(self, root, access_token): - super().__init__(root) - self.api = YDClient(access_token) - - def read(self, path): - remote = self.api.get_download_link(self.get_abs(path)) - url = remote['href'] - req = requests.get(url) - data = req.content - return io.BytesIO(data) - - def scan(self, path): - res = self.api.get_content(self.get_abs(path)) - files = [] - for item in res.children: - files.append(item.path) - - files = map( - lambda fp: re.sub('disk:', '', fp), - files - ) - return [self.get_rel(i) for i in files] - - def glob(self, pattr): - return [] - - def type_filter(self, path, ext): - return list(filter( - lambda fn: os.path.splitext(fn)[1] == ext, - self.scan(path) - )) - - def hash(self, path): - res = self.api.get_content(self.get_abs(path)) - return res.sha256 - - def copy(self, path, out): - data = self.read(path) - f = open(out, 'wb') - f.write(data.read()) - f.close() - return out - - def size(self, path): - res = self.api.get_content(self.get_abs(path)) - return res['size'] - - def is_dir(self, path): - res = self.api.get_content(self.get_abs(path)) - return res.type == 'dir' - - def get_local(self, path): - ext = os.path.splitext(path)[1] - _, temp_in = mkstemp(suffix=ext) - return self.copy(path, temp_in) diff --git a/modules/sync/sync/models/Document.py b/modules/sync/sync/models/Document.py deleted file mode 100644 index f32ff68e..00000000 --- a/modules/sync/sync/models/Document.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -from tempfile import mkstemp - -import settings -from db import collection -from sync.data import request -from sync.models import Model -from sync.models.DocumentPreviewImage import DocumentPreviewImage -from utils.hash import hash_str, md5 -from utils.image import image_magick_pdf_to_img - -store = collection(settings.collection_documents) - - -class Document(Model): - @staticmethod - async def find_one(query): - return store.find_one(query) - - @staticmethod - async def find(query): - return store.find(query) - - @staticmethod - async def delete(query): - return store.find_one_and_delete(query) - - @staticmethod - def read_path(provider, path): - name = os.path.basename(path) - params = { - 'title': os.path.splitext(name)[0] - } - - return Document( - provider, - path, - params - ) - - @staticmethod - async def new(provider, file, sizes): - document = Document.read_path(provider, file) - changed = await document.is_changed() - if changed: - await document.build(sizes=sizes) - await document.upload() - await document.save() - else: - doc = await Document.find_one({'file': file}) - document = Document(provider, file, doc) - document.ref = doc['_id'] - await document.create_preview(sizes) - - return document - - def __init__(self, provider, file, params=None): - self.__hash_salt = settings.hash_salt_documents - self.__hash_keys = ['id', 'url'] - - self.__id_template: str = '{category}-{file}' - self.__url_template: str = 'https://static.shlisselburg.org/art/uploads/{file}' - self.__url_preview_template: str = settings.document_url_preview_template - - self.title = None - self.file_name = None - self.file_size = None - self.preview = None - - super().__init__(provider, store, file, params=params) - - def init(self): - self.__set_id() - self.__set_url() - self.__set_hash() - - filename = os.path.basename(self.file) - self.file_name = filename - self.file_size = self.provider.size(self.file) - - if self.has_param('title'): - self.title = self.get_param('title') - - def set_title(self, value: str): - self.title = value - - async def build(self, **kwargs): - sizes = kwargs['sizes'] - await self.create_preview(sizes) - - pdf_title = await get_pdf_title(self.provider, self.file) - if pdf_title: - self.title = pdf_title - - async def save(self): - document = self.bake() - - query = {'id': document['id']} - try: - self.store.update_one(query, {'$set': document}, upsert=True) - - doc = await Document.find_one(query) - self.ref = doc['_id'] - - return self - except ValueError: - pass - return None - - async def upload(self): - filepath = self.provider.get_local(self.file) - name = os.path.basename(self.url) - - await request.s3_put(f'uploads/{name}', filepath) - - def bake(self): - return { - **self.params, - 'id': self.id, - 'file': self.file, - 'hash': self.hash, - 'url': self.url, - 'title': self.title, - 'preview': self.preview.ref, - 'fileInfo': { - 'name': self.file_name, - 'size': self.file_size, - }, - } - - def __filename(self): - return os.path.basename(self.file) - - def __set_id(self): - self.id = md5(self.file) - - def __set_url(self): - s = os.path.splitext(self.file) - file = f'{self.id}{s[1]}' - - self.url = self.__url_template.format( - file=file - ) - - def __set_hash(self): - doc = [getattr(self, key) for key in self.__hash_keys] - - self.hash = hash_str( - self.__hash_salt + hash_str(doc) + self.provider.hash(self.file) - ) - - async def create_preview(self, sizes): - self.preview = await DocumentPreviewImage.new(self.provider, self.file, sizes) - - def __str__(self): - return f'' - - -async def get_pdf_title(provider, file): - from PyPDF2 import PdfFileReader - from PyPDF2.generic import TextStringObject - from PyPDF2.generic import IndirectObject - - def t(value): - value = str(value) - value = value.replace('\u0000', '') - if value == 'None': - return None - if value == '': - return None - return value - - pdf = PdfFileReader(provider.read(file)) - info = pdf.getDocumentInfo() - - if info: - if type(info.title) == TextStringObject: - return t(info.title) - - if type(info.title_raw) == IndirectObject: - o = pdf.getObject(info.title_raw) - return t(info.title) - return None diff --git a/modules/sync/sync/models/DocumentPreviewImage.py b/modules/sync/sync/models/DocumentPreviewImage.py deleted file mode 100644 index 731c8fbe..00000000 --- a/modules/sync/sync/models/DocumentPreviewImage.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -from tempfile import mkstemp - -from sync.models.Image import Image, default_url_factory -from utils.image import image_magick_pdf_to_img - - -class DocumentPreviewImage(Image): - @staticmethod - async def find_one(query): - return await Image.find_one(query) - - @staticmethod - async def find(query): - return await Image.find(query) - - @staticmethod - async def delete(query): - return await Image.delete(query) - - @staticmethod - async def new(provider, file, sizes, **kwargs): - img = DocumentPreviewImage(provider, file, data=None) - changed = await img.is_changed() - if changed: - await img.build(sizes=sizes) - await img.upload() - await img.save() - else: - doc = await DocumentPreviewImage.find_one({'file': file}) - img = DocumentPreviewImage(provider, file, doc['data']) - img.ref = doc['_id'] - - return img - - def __init__(self, provider, file, data): - super().__init__(provider, file, data, url_factory=default_url_factory) - self.__hash_salt = '' - self.temp_image_path = None - - async def is_changed(self): - i = await DocumentPreviewImage.find_one({'file': self.file}) - return self._is_changed_hash(i) - - async def build(self, sizes): - self.temp_image_path = await pdf_to_jpg(self.provider, self.file) - await super().build(sizes) - - def _get_image_file(self): - return self.temp_image_path - - def __get_hash(self): - return self.provider.hash(self.file) - - -async def pdf_to_jpg(provider, pdf): - _, temp_in = mkstemp(suffix='.pdf') - _, temp_out = mkstemp(suffix='.jpg') - - abspdf = provider.copy(pdf, temp_in) - await image_magick_pdf_to_img(abspdf, temp_out) - - os.remove(temp_in) - return temp_out diff --git a/modules/sync/sync/models/Image.py b/modules/sync/sync/models/Image.py deleted file mode 100644 index f9d35f7c..00000000 --- a/modules/sync/sync/models/Image.py +++ /dev/null @@ -1,244 +0,0 @@ -import logging -import os -import tempfile -import asyncio - -from typing import Optional, List - -import settings -from db import collection -from sync.data import request -from sync.models import Model -from utils.hash import md5, hash_str -from utils.image import image_magick_resize -from utils.text.transform import url_encode_text - -logger = logging.getLogger(settings.name + '.Image') - -store = collection(settings.collection_images) - - -def default_url_factory(file, size, ext): - url = settings.image_url_base + '{}-{}{}' - return url.format(md5(file), size, ext.lower()) - - -class Image(Model): - @staticmethod - async def find_one(query): - document = store.find_one(query) - return document - - @staticmethod - async def find(query): - return store.find(query) - - @staticmethod - async def delete(query): - return store.find_one_and_delete(query) - - @staticmethod - async def new(provider, file, sizes, url_factory=None): - url_factory = url_factory if url_factory else default_url_factory - - img = Image(provider, file, data=None, url_factory=url_factory) - changed = await img.is_changed() - if changed: - await img.build(sizes=sizes) - await img.upload() - await img.save() - else: - doc = await Image.find_one({'file': file}) - img = Image(provider, file, doc['data'], url_factory) - img.ref = doc['_id'] - - return img - - @staticmethod - def read_path(provider, path): - return Image( - provider, - path, - data=None, - url_factory=default_url_factory - ) - - def __init__(self, provider, file, data, url_factory): - self.__hash_salt = settings.hash_salt_images - - self.data = data - self.url_factory = url_factory - - super().__init__(provider, store, file) - - def init(self): - self.id = self.__get_id() - self.hash = self.__get_hash() - - async def is_changed(self): - i = await Image.find_one({'file': self.file}) - return self._is_changed_hash(i) - - async def build(self, sizes): - logger.info(f'Building Image {self._get_abs(self.file)}') - - img_in = self._get_image_file() - img_out = tempfile.mkdtemp() - - images = await create_image(img_in, sizes, self.url_factory, img_out) - if not images: - raise Exception('Failed to create Image {}'.format(img_in)) - data = {} - for i in images: - if i: - size_name = i['size'] - data[size_name] = { - **i, - } - self.data = data - - async def upload(self): - if self.data: - async def fn(i): - file = i['file'] - filename = os.path.basename(file) - - await request.s3_put(f'images/{filename}', file) - - await asyncio.wait([fn(i) for i in self.data.values()]) - - async def save(self): - c = sync_image(self.bake()) - self.ref = c['_id'] - - def bake(self): - return { - 'id': self.id, - 'file': self.file, - 'hash': self.hash, - 'data': self.data, - } - - def get_size(self, size): - if size in self.data: - return self.data[size] - return None - - def get_src(self): - return self.get_size('original')['url'] - - def _get_image_file(self): - return self._get_abs(self.file) - - def __filename(self): - return os.path.basename(self.file) - - def __get_id(self): - return md5(self.file) - - def __get_hash(self): - file_hash = self.provider.hash(self.file) - return hash_str(self.__hash_salt + file_hash) - - def __str__(self): - return ''.format(self.hash, self.file) - - -def sync_image(record): - from db import db - - q = {'file': record['file']} - try: - db().images.update_one(q, {'$set': record}, upsert=True) - i = db().images.find_one({'file': record['file']}) - return i - except ValueError: - pass - - return None - - -async def create_image(file: str, sizes: [()], url_fn, output_dir: str) -> Optional[List[dict]]: - """ - - :param file: Image path to process - :param sizes: list of sizes to generate image (, , ) - :param url_fn: function (size_name, ext) for creating image url - :param output_dir: path to local folder for thumbs storing - :return: - """ - if not os.path.exists(file): - return None - - _, ext = os.path.splitext(file) - images = [] - - if not settings.image_processing_enabled: - logger.warning( - f'Image processing disabled {file}. Size of image will be set to original' - ) - - async def fn(size): - size_name, width, height = size - - image_url = url_fn(file, size_name, ext) - image_filename = os.path.basename(image_url) - local_image_path = os.path.join(output_dir, image_filename) - - if settings.image_processing_enabled: - image = await process_image(file, local_image_path, size) - else: - image = await read_image(file) - if not image: - logger.warning(f'Failed to process image {file}') - return None - width, height = image.size - - images.append({ - 'file': local_image_path, - 'url': image_url, - 'size': size_name, - 'width': width, - 'height': height - }) - - await asyncio.wait([fn(s) for s in sizes]) - - return images - - -async def process_image(input_file, output_file, size): - size_name, width, height = size - - if size_name == 'original': - return await optimize(input_file, output_file, quality=90) - else: - image = await read_image(input_file) - if not image: - return None - image_width, image_height = image.size - if image_height > image_width: - width, height = height, width - return await thumbnail(input_file, output_file, (width, height), quality=90) - - -async def read_image(src): - import aiofiles - from PIL import Image - import io - - async with aiofiles.open(src, mode='rb') as f: - image_data = await f.read() - image = Image.open(io.BytesIO(image_data)) - return image - - -async def thumbnail(src, dest, size, quality): - await image_magick_resize(src, dest, size, quality) - return await read_image(dest) - - -async def optimize(src, dest, quality): - image = await read_image(src) - w, h = image.size - return await thumbnail(src, dest, (w, h), quality) diff --git a/modules/sync/sync/models/Page.py b/modules/sync/sync/models/Page.py deleted file mode 100644 index aec6a836..00000000 --- a/modules/sync/sync/models/Page.py +++ /dev/null @@ -1,209 +0,0 @@ -import os -import logging -from lxml import etree -import lxml.html -import settings -from db import collection -from sync.data import request -from sync.models import Model -from sync import create_date, create_post -from sync.models.Image import Image -from utils.hash import hash_str -from utils.front_matter import parse_yaml_front_matter -from utils.text.transform import url_encode_text -from markdown import markdown - - -logger = logging.getLogger(settings.name + '.Page') -store = collection(settings.collection_pages) - - -def extract_files(html) -> [str]: - images = html.cssselect('img') - images = [img.get('src') for img in images] - - return images, [] - - -def title_from_html(html): - titles = html.cssselect('h1') - if len(titles): - return titles[0].text - return '' - - -def find(store, item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None - - -class PageController: - def __init__(self, provider): - self.provider = provider - - async def find(self, query): - return store.find(query) - - async def delete_item(self, query): - return store.find_one_and_delete(query) - - async def get_items(self): - scan_path = settings.dir_pages - items = [] - for path in self.provider.scan(scan_path): - if not self.provider.is_dir(path): - continue - item = await self.create_item(path) - if not item: - continue - items.append(item) - return items - - async def create_item(self, path: str): - """ - :param path: path to page manifest file. For example: /Pages//.md - :return: - """ - dir_name = os.path.basename(path) - manifest_path = os.path.join(path, f'{dir_name}.md') - if not self.provider.exists(manifest_path): - logger.warning(f'Manifest not found on path {manifest_path}') - return None - - manifest = await self.get_manifest(manifest_path) - if not manifest: - logger.warning(f'Cannot define manifest. Skip {manifest_path}') - return None - - return Page( - self.provider, - manifest_path, - params=manifest, - ) - - async def get_manifest(self, path: str): - data = self.provider.read(path).read().decode('utf-8') - manifest = parse_yaml_front_matter(data) - - if 'url' not in manifest: - logger.warning(f'URL not specified for Page {path}') - return None - - if 'date' in manifest: - manifest['date'] = create_date( - manifest['date'], settings.date_formats) - - if 'until' in manifest: - manifest['until'] = create_date( - manifest['until'], settings.date_formats) - - if 'id' in manifest: - manifest['id'] = str(manifest['id']) - - return { - **manifest, - 'file': os.path.basename(path), - 'folder': os.path.dirname(path), - } - - -class Page(Model): - def __init__(self, provider, file, params=None): - self.__hash_salt = settings.hash_salt_pages - self.__hash_keys = ['id', 'url'] - self.images = [] - self.documents = [] - self.title = None - self.preview = None - self.tokens = None - - self.__id_template: str = '{category}-{file}' - self.__url_template: str = settings.document_url_template - self.__url_preview_template: str = settings.document_url_preview_template - - super().__init__(provider, store, file, params=params) - - def init(self): - content = self.get_param('content') - html = markdown(content) - html_tree = lxml.html.fromstring(html) - images, documents = extract_files(html_tree) - self.title = title_from_html(html_tree) - - self.params['images'] = images - self.params['documents'] = documents - - self.__set_id() - self.__set_url() - self.__set_hash() - - async def is_changed(self): - i = find(self.store, self.id) - if not i: - return True - - if 'hash' not in i: - return True - - return not (self.hash == i['hash']) - - async def build(self, **kwargs): - sizes = kwargs['sizes'] - - markdown_post = self.get_param('content') - images, documents, tokens = await create_post(self.provider, self.get_param('folder'), markdown_post, sizes) - self.images = images - self.documents = documents - self.tokens = tokens - - if self.has_param('preview'): - preview = self.get_param('preview') - preview = self._get_relpath(preview) - self.preview = await Image.new(self.provider, preview, sizes) - - def bake(self): - images = [x.ref for x in self.images] - documents = [x.ref for x in self.documents] - - return { - **self.params, - 'id': self.id, - 'hash': self.hash, - 'url': self.url, - 'file': self.file, - 'images': images, - 'documents': documents, - 'title': self.title, - 'tokens': self.tokens, - 'preview': self.preview.ref if self.preview else None, - } - - def __filename(self): - return os.path.basename(self.file) - - def __set_id(self): - if 'id' in self.params: - self.id = self.get_param('id') - else: - self.id = url_encode_text(self.get_param('url')) - - def __set_url(self): - self.url = self.params['url'] - - def __set_hash(self): - files = [] - files += self.get_param('images') - files += self.get_param('documents') - folder = self.get_param('folder') - files = [os.path.join(folder, x) for x in files] - files = sorted([self.file] + files) - - hashes = [self.provider.hash(x) for x in files] - self.hash = hash_str(self.__hash_salt + ''.join(hashes)) - - def __str__(self): - return ''.format(self.file, self.id) diff --git a/modules/sync/sync/models/__init__.py b/modules/sync/sync/models/__init__.py deleted file mode 100644 index 0f5ec7ee..00000000 --- a/modules/sync/sync/models/__init__.py +++ /dev/null @@ -1,87 +0,0 @@ -import os - -import settings - - -def find(store, item_id: str): - q = {'id': item_id} - try: - return store.find_one(q) - except ValueError: - pass - return None - - -class Model: - def __init__(self, provider, store, file, params=None): - super().__init__() - - self.provider = provider - self.store = store - self.file = file - - self.id = None - self.url = None - self.hash = None - self.ref = None # Data base reference - self.params = params if params else {} - - self.init() - - def init(self): - pass - - async def upload(self, **kwargs): - pass - - async def build(self, **kwargs): - pass - - def bake(self): - return None - - async def is_changed(self): - i = find(self.store, self.id) - return self._is_changed_hash(i) - - def has_param(self, name): - if not self.params: - return False - return name in self.params - - def get_param(self, name, default_value=None): - if not self.params: - return default_value - if not (name in self.params): - return default_value - return self.params[name] - - def _is_changed_hash(self, item): - if not item: - return True - - if 'hash' not in item: - return True - - return not (self.hash == item['hash']) - - def _filename(self): - return os.path.basename(self.file) - - def _get_relpath(self, path): - folder = self.get_param('folder', self.file) - return os.path.join(folder, path) - - def _get_abs(self, path): - return self.provider.get_abs(path) - - async def save(self): - document = self.bake() - - query = {'id': document['id']} - try: - self.store.update_one(query, {'$set': document}, upsert=True) - return self - except ValueError: - pass - return None diff --git a/modules/sync/test/sync/models/test_model.py b/modules/sync/test/sync/models/test_model.py deleted file mode 100644 index 650342ea..00000000 --- a/modules/sync/test/sync/models/test_model.py +++ /dev/null @@ -1,13 +0,0 @@ -import unittest - -from sync.models import Model - - -class MyTestCase(unittest.TestCase): - def test_bake(self): - m = Model() - self.assertIsNone(m.bake()) - - -if __name__ == '__main__': - unittest.main() diff --git a/modules/sync/utils/fn.py b/modules/sync/utils/fn.py deleted file mode 100644 index e24bd04d..00000000 --- a/modules/sync/utils/fn.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -from functools import reduce - - -def constant(value): - return lambda _: value - - -def combine(items): - if len(items) == 0: - return [] - - return reduce( - lambda total, i: total + i, - items - ) - - -def compose(*fns): - return lambda i: reduce( - lambda out, fn: fn(out), - fns, i - ) - - -first = lambda i: i[0] if len(i) > 0 else None - - -def swap_ext(extension: str): - return lambda i: os.path.splitext(i)[0] + extension - - -def map_cases(param, cases): - for predicate, fn in cases: - if predicate(param): - return fn(param) - return param - - -def until(condition_fn, iter_fn, items, default_value=None): - for i in items: - result = iter_fn(i) - if condition_fn(result): - return result - return default_value diff --git a/modules/sync/utils/front_matter.py b/modules/sync/utils/front_matter.py deleted file mode 100644 index 6d3977bf..00000000 --- a/modules/sync/utils/front_matter.py +++ /dev/null @@ -1,36 +0,0 @@ -import re -import yaml - -FM_BOUNDARY = re.compile(r'^-{3,}', re.MULTILINE) - - -def parse_yaml_front_matter(data: str): - fm, content = split_yaml_front_matter(data) - fm = fm if fm else {} - return { - **fm, - 'content': content, - } - - -def split_yaml_front_matter(text: str): - """ - Parse text with YAML frontmatter, return metadata and content. - Pass in optional metadata defaults as keyword args. - - If frontmatter is not found, returns an empty metadata dictionary - and original text content. - """ - - # split on the first two triple-dashes - try: - _, fm, content = FM_BOUNDARY.split(text, 2) - except ValueError: - # if we can't split, bail - return None, None - - # parse yaml, now that we have frontmatter - fm = yaml.safe_load(fm) - if isinstance(fm, dict): - return fm, content.strip() - return None, None diff --git a/modules/sync/utils/hash.py b/modules/sync/utils/hash.py deleted file mode 100644 index 010f3cce..00000000 --- a/modules/sync/utils/hash.py +++ /dev/null @@ -1,38 +0,0 @@ -import hashlib -import json - -import bson.json_util - -from utils.fn import map_cases - - -def hash_file(file): - with open(file, 'rb') as f: - data = f.read() - return hash_data(data) - - -def hash_data(data): - sha = hashlib.sha256(data) - return sha.hexdigest() - - -def hash_str(data): - to_json = lambda i: json.dumps(i, sort_keys=True, ensure_ascii=False, default=bson.json_util.default) - - data = map_cases(data, [ - (lambda i: isinstance(i, dict), to_json), - (lambda i: isinstance(i, tuple), to_json), - (lambda i: isinstance(i, list), to_json), - (lambda i: isinstance(i, int), int), - (lambda i: isinstance(i, float), float), - ]) - if isinstance(data, dict): - data = to_json(data) - - return hash_data(str(data).encode()) - - -def md5(data: str) -> str: - i = hashlib.md5(data.encode('utf-8')) - return i.hexdigest() diff --git a/modules/sync/utils/image.py b/modules/sync/utils/image.py deleted file mode 100644 index 4dd90db5..00000000 --- a/modules/sync/utils/image.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio -from subprocess import call - - -async def image_magick_resize(input_file, output_file, size, quality=85): - interlace = 'Plane' - s = '%dx%d' % size - - cmd = [ - '/usr/local/bin/convert', - '"%s"' % input_file, - - '-strip', - '-interlace', interlace, - '-auto-orient', - '-resize', s, - '-quality', str(quality), - - '"%s"' % output_file, - ] - - cmd = ' '.join(cmd) - process = await asyncio.create_subprocess_shell(cmd) - await process.wait() - - -async def image_magick_pdf_to_img(input_file, output_file): - call([ - '/usr/local/bin/gs', - '-q', - '-sDEVICE=jpeg', - '-dLastPage=1', - '-o', output_file, - input_file, - ]) diff --git a/modules/sync/utils/lxml_utils.py b/modules/sync/utils/lxml_utils.py deleted file mode 100644 index 4972ced0..00000000 --- a/modules/sync/utils/lxml_utils.py +++ /dev/null @@ -1,29 +0,0 @@ -import re - - -def get_following_tags(element, stop_tag_pattr): - stop_exp = re.compile(stop_tag_pattr) - following = element.xpath('./following-sibling::*') - tags = [] - i = 0 - while True: - try: - tag = following[i] - if stop_exp.match(tag.tag): - break - tags.append(tag) - except IndexError as tag: - break - i += 1 - return tags - - -def get_text_following_by_tag(element, tag, contains, strip=True): - e = element.xpath('.//%s[contains(text(), "%s")]/following-sibling::text()' % (tag, contains)) - if len(e) > 0: - if strip: - return e[0].strip() - else: - return e[0] - else: - return None \ No newline at end of file diff --git a/modules/sync/utils/text/split.py b/modules/sync/utils/text/split.py deleted file mode 100644 index b4f567b8..00000000 --- a/modules/sync/utils/text/split.py +++ /dev/null @@ -1,18 +0,0 @@ -from utils.fn import until - - -def split(data, separator=' '): - try: - s = data.index(separator) - return data[:s], data[s:] - except ValueError: - return None, None - - -def split_with(separators): - return lambda data: until( - lambda i: i[0] and i[1], - lambda s: split(data, s), - separators, - (None, None) - ) diff --git a/modules/sync/utils/text/transform.py b/modules/sync/utils/text/transform.py deleted file mode 100644 index 4380a687..00000000 --- a/modules/sync/utils/text/transform.py +++ /dev/null @@ -1,94 +0,0 @@ -import re - -from unidecode import unidecode - -from utils.fn import compose - -remove_pattr = re.compile('[«»<>#,!&"\']') -reduce_dash_patterns = [ - (re.compile('[-]+'), '-'), - (re.compile('^-'), '') -] - - -def translit(text): - """ - RUSSIAN TEXT => ENGLISH TEXT - """ - return unidecode(text) - - -def interpolate(text, storage, key_fn=None): - if not key_fn: - key_fn = lambda i: i - - key = key_fn(text) - return storage[key] if key in storage else text - - -def interpolate_swift(i): - return re.search("([\d\w-]+)", i).group(1) - - -def remove_punctuation(i): - return remove_pattr.sub('', i) - - -def reduce_dashes(i): - value = i - for reg_exp, replace in reduce_dash_patterns: - value = reg_exp.sub(replace, value) - return value - - -def space_to_dash(i): - return i.replace(' ', '-') - - -def short_dash(i): - """ - minus -> dash - tire -> dash - :param i: - :return: - """ - return i.replace('–', '-').replace('—', '-') - - -def dot_to_dash(i): - return i.replace('.', '-') - - -def slash_to_dash(i): - return i.replace('/', '-') - - -def remove_underscores(i): - return i.replace('_', '') - - -def text_lower(i): - return i.lower() - - -url_encode_file = compose( - translit, - space_to_dash, - short_dash, - remove_underscores, - remove_punctuation, - reduce_dashes, - text_lower, -) - -url_encode_text = compose( - translit, - space_to_dash, - dot_to_dash, - short_dash, - slash_to_dash, - remove_underscores, - remove_punctuation, - reduce_dashes, - text_lower, -) diff --git a/modules/sync/yandexdisk/__init__.py b/modules/sync/yandexdisk/__init__.py deleted file mode 100644 index 4157de7a..00000000 --- a/modules/sync/yandexdisk/__init__.py +++ /dev/null @@ -1,305 +0,0 @@ -import requests - - -class YDClient: - """ - Implementation of https://tech.yandex.ru/disk/poligon/ - """ - - base_url = 'https://cloud-api.yandex.net/v1/disk' - - def __init__(self, access_token): - self.access_token = access_token - - self.base_headers = dict( - Accept='application/json', - Authorization='OAuth %s' % self.access_token, - ) - - async def get_flat(self, limit=20): - """ - https://cloud-api.yandex.net/v1/disk/resources/files? - [ limit=<количество файлов в списке>] - [& media_type=<тип запрашиваемых файлов>] - [& offset=<смещение относительно начала списка>] - [& fields=<нужные ключи ответа>] - [& preview_size=<размер превью>] - [& preview_crop=<признак обрезки превью>] - :return: - """ - - url = '{0}/resources/files'.format(self.base_url) - payload = dict( - limit=limit, - offset=0, - ) - r = requests.get(url, headers=self.base_headers, params=payload) - self.check_code(r) - - json_dict = r.json() - # return Directory(**json_dict) - return json_dict - - def get_disk_metadata(self): - """ - :return: disk metadata - """ - url = self.base_url - r = requests.get(url, headers=self.base_headers) - self.check_code(r) - disk = r.json() - return Disk( - trash_size=disk['trash_size'], - total_space=disk['total_space'], - used_space=disk['used_space'], - system_folders=disk['system_folders'] - ) - - def last_uploaded(self): - """ - https://cloud-api.yandex.net/v1/disk/resources/last-uploaded? - [ limit=<количество файлов в списке>] - [& media_type=<тип запрашиваемых файлов>] - [& fields=<нужные ключи ответа>] - [& preview_size=<размер превью>] - [& preview_crop=<признак обрезки превью>] - """ - url = '{0}/resources/last-uploaded'.format(self.base_url) - payload = dict( - limit=100, - ) - r = requests.get(url, headers=self.base_headers, params=payload) - self.check_code(r) - - json_dict = r.json() - # return Directory(**json_dict) - return json_dict - - def get_content(self, path): - """ - :param path: path to folder - :return: content of folder - """ - url = '{0}/resources'.format(self.base_url) - - payload = dict( - path=path, - limit=1000000 - ) - r = requests.get(url, headers=self.base_headers, params=payload) - self.check_code(r) - - json_dict = r.json() - return Directory(**json_dict) - - def create_folder(self, path_to_folder): - """ - :param path_to_folder: path to folder - :return: created folder - """ - url = '{0}/resources'.format(self.base_url) - - payload = {'path': path_to_folder} - r = requests.put(url, headers=self.base_headers, params=payload) - self.check_code(r) - - return self.get_content(path_to_folder) - - def remove_folder_or_file(self, path): - """ - Remove folder or file - :param path: path to file or folder - """ - url = '{0}/resources'.format(self.base_url) - - payload = {'path': path} - r = requests.delete(url, headers=self.base_headers, params=payload) - self.check_code(r) - - def copy_folder_or_file(self, path_from, path_to): - """ - Copy folder or file - :param path_from: from path - :param path_to: to path - """ - url = '{0}/resources/copy'.format(self.base_url) - - payload = {'path': path_to, 'from': path_from} - r = requests.post(url, headers=self.base_headers, params=payload) - self.check_code(r) - - def get_download_link(self, path): - """ - :param path: path to file - :return: download link to file - """ - url = '{0}/resources/download'.format(self.base_url) - - payload = {'path': path} - r = requests.get(url, headers=self.base_headers, params=payload) - self.check_code(r) - - return r.json() - - def get_published_elements(self): - """ - :return: published elements - """ - json_dict = self._get_dictionary_of_published_files() - - elements = [] - for item in json_dict['items']: - if item['type'] == 'dir': - elements.append(Directory(**item)) - elif item['type'] == 'file': - elements.append(File(**item)) - return elements - - def get_public_link_to_folder_or_file(self, path): - """ - :param path: path - :return: public link to folder or file - """ - url = '{0}/resources/publish'.format(self.base_url) - - payload = {'path': path} - r = requests.put(url, headers=self.base_headers, params=payload) - self.check_code(r) - - files = self._get_dictionary_of_published_files() - for file in files['items']: - if str(file['path']).endswith(path): - return file['public_url'] - return '' - - def unpublish_folder_or_file(self, path): - """ - Unpublish folder of file - :param path: path to file or folder - """ - url = '{0}/resources/unpublish'.format(self.base_url) - - payload = {'path': path} - r = requests.put(url, headers=self.base_headers, params=payload) - self.check_code(r) - - def get_list_of_all_files(self): - """ - :return: List of all files - """ - url = '{0}/resources/files'.format(self.base_url) - - r = requests.get(url, headers=self.base_headers) - self.check_code(r) - - json_dict = r.json() - - files = [] - for item in json_dict['items']: - f = File(**item) - files.append(f) - return files - - def move_folder_or_file(self, path_from, path_to): - """ - Move folder or file - :param path_from: path from - :param path_to: path to - """ - url = '{0}/resources/move'.format(self.base_url) - - payload = {'path': path_to, 'from': path_from} - r = requests.post(url, headers=self.base_headers, params=payload) - self.check_code(r) - - def upload_file(self, path_from, path_to): - """ - Upload file - :param path_from: path from - :param path_to: path to yandex disk - """ - url = '{0}/resources/upload'.format(self.base_url) - - payload = {'path': path_to} - r = requests.get(url, headers=self.base_headers, params=payload) - self.check_code(r) - - json_dict = r.json() - upload_link = json_dict["href"] - - with open(path_from, 'rb') as fh: - files = {'file': fh} - - r2 = requests.put(upload_link, headers=self.base_headers, files=files) - self.check_code(r2) - - def upload_file_from_url(self, from_url, path_to): - """ - Upload file by URL - :param from_url: URL path from - :param path_to: path to yandex disk - """ - url = '{0}/resources/upload'.format(self.base_url) - - payload = {'path': path_to, 'url': from_url} - r = requests.post(url, headers=self.base_headers, params=payload) - self.check_code(r) - - def _get_dictionary_of_published_files(self): - url = '{0}/resources/public'.format(self.base_url) - - r = requests.get(url, headers=self.base_headers) - self.check_code(r) - - return r.json() - - def check_code(self, req): - if not str(req.status_code).startswith('2'): - raise YDException(req.status_code, req.text) - - -class YDException(Exception): - code = None - - def __init__(self, code, text): - super(YDException, self).__init__(text) - self.code = code - - def __str__(self): - return "%d. %s" % (self.code, super(YDException, self).__str__()) - - -class Directory: - def __init__(self, **kwargs): - self.children = [] - - for key in kwargs: - if key is not '_embedded': - setattr(self, key, kwargs[key]) - - if "_embedded" in kwargs: - for item in kwargs['_embedded']['items']: - if item['type'] == 'dir': - d = Directory(**item) - self.children.append(d) - - if item['type'] == 'file': - f = File(**item) - self.children.append(f) - - def get_children(self): - return self.children - - -class File: - def __init__(self, **kwargs): - for key in kwargs: - setattr(self, key, kwargs[key]) - - -class Disk: - def __init__(self, trash_size, total_space, used_space, system_folders): - self.trash_size = trash_size - self.total_space = total_space - self.used_space = used_space - self.system_folders = system_folders