diff --git a/API.md b/API.md index 43c6dd9f..b060e92c 100644 --- a/API.md +++ b/API.md @@ -63,6 +63,8 @@ - [Listing snapshots](#listing-snapshots) - Global info - [Getting global info](#getting-global-info) + - File uploads + - [Uploading temporary file](#uploading-temporary-file) 3. [Resources](#resources) @@ -105,16 +107,28 @@ application/json`. An exception to this rule are requests that upload files. ## File uploads -Requests that upload files must use `multipart/form-data` encoding. JSON -metadata must then be included as field of name `metadata`, whereas files must -be included as separate fields with names specific to each request type. +Requests that upload files must use `multipart/form-data` encoding. Any request +that bundles user files, must send the request data (which is JSON) as an +additional file with the special name of `metadata` (whereas the actual files +must have names specific to the API that is being used.) Alternatively, the server can download the files from the Internet on client's behalf. In that case, the request doesn't need to be specially encoded in any -way. The files, however, should be passed as regular fields appended with `Url` -suffix. For example, to download a file named `content` from -`http://example.com/file.jpg`, the client should pass -`{"contentUrl":"http://example.com/file.jpg"}` as part of the message body. +way. The files, however, should be passed as regular fields appended with a +`Url` suffix. For example, to use `http://example.com/file.jpg` in an API that +accepts a file named `content`, the client should pass +`{"contentUrl":"http://example.com/file.jpg"}` as a part of the JSON message +body. + +Finally, in some cases the user might want to reuse one file between the +requests to save the bandwidth (for example, reverse search + consecutive +upload). In this case one should use [temporary file +uploads](#uploading-temporary-file), and pass the tokens returned by the API as +regular fields appended with a `Token` suffix. For example, to use previously +uploaded data, which was given token `deadbeef`, in an API that accepts a file +named `content`, the client should pass `{"contentToken":"deadbeef"}` as part +of the JSON message body. If the file with the particular token doesn't exist +or it has expired, the server will show an error. ## Error handling @@ -1593,6 +1607,35 @@ data. exception of privilege array keys being converted to lower camel case to match the API convention. +## Uploading temporary file + +- **Request** + + `POST /uploads` + +- **Files** + + - `content` - the content of the file to upload. Note that in this + particular API, one can't use token-based uploads. + +- **Output** + + ```json5 + { + "token": + } + ``` + +- **Errors** + + - privileges are too low + +- **Description** + + Puts a file in temporary storage and assigns it a token that can be used in + other requests. The files uploaded that way are deleted after a short while + so clients shouldn't use it as a free upload service. + # Resources diff --git a/config.yaml.dist b/config.yaml.dist index 90c48752..a38393a0 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -119,3 +119,5 @@ privileges: 'comments:score': regular 'snapshots:list': power + + 'uploads:create': regular diff --git a/server/szurubooru/api/__init__.py b/server/szurubooru/api/__init__.py index 308b86bf..2a2d5af7 100644 --- a/server/szurubooru/api/__init__.py +++ b/server/szurubooru/api/__init__.py @@ -6,3 +6,4 @@ import szurubooru.api.tag_category_api import szurubooru.api.comment_api import szurubooru.api.password_reset_api import szurubooru.api.snapshot_api +import szurubooru.api.upload_api diff --git a/server/szurubooru/api/upload_api.py b/server/szurubooru/api/upload_api.py new file mode 100644 index 00000000..eaf2880b --- /dev/null +++ b/server/szurubooru/api/upload_api.py @@ -0,0 +1,10 @@ +from szurubooru.rest import routes +from szurubooru.func import auth, file_uploads + + +@routes.post('/uploads/?') +def create_temporary_file(ctx, _params=None): + auth.verify_privilege(ctx.user, 'uploads:create') + content = ctx.get_file('content', required=True, allow_tokens=False) + token = file_uploads.save(content) + return {'token': token} diff --git a/server/szurubooru/errors.py b/server/szurubooru/errors.py index 33bdfce1..f7edf85d 100644 --- a/server/szurubooru/errors.py +++ b/server/szurubooru/errors.py @@ -36,6 +36,10 @@ class MissingRequiredFileError(ValidationError): pass +class MissingOrExpiredRequiredFileError(MissingRequiredFileError): + pass + + class MissingRequiredParameterError(ValidationError): pass diff --git a/server/szurubooru/facade.py b/server/szurubooru/facade.py index 5211ca81..e29aae13 100644 --- a/server/szurubooru/facade.py +++ b/server/szurubooru/facade.py @@ -1,11 +1,13 @@ ''' Exports create_app. ''' import os +import time import logging +import threading import coloredlogs import sqlalchemy.orm.exc from szurubooru import config, errors, rest -from szurubooru.func import posts +from szurubooru.func import posts, file_uploads # pylint: disable=unused-import from szurubooru import api, middleware @@ -79,6 +81,15 @@ def validate_config(): raise errors.ConfigError('Database is not configured') +def purge_old_uploads(): + while True: + try: + file_uploads.purge_old_uploads() + except Exception as ex: + logging.exception(ex) + time.sleep(60 * 5) + + def create_app(): ''' Create a WSGI compatible App object. ''' validate_config() @@ -88,6 +99,9 @@ def create_app(): if config.config['show_sql']: logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + purge_thread = threading.Thread(target=purge_old_uploads) + purge_thread.daemon = True + purge_thread.start() posts.populate_reverse_search() rest.errors.handle(errors.AuthError, _on_auth_error) diff --git a/server/szurubooru/func/file_uploads.py b/server/szurubooru/func/file_uploads.py new file mode 100644 index 00000000..95698e36 --- /dev/null +++ b/server/szurubooru/func/file_uploads.py @@ -0,0 +1,29 @@ +import datetime +from szurubooru.func import files, util + + +MAX_MINUTES = 60 + + +def _get_path(checksum): + return 'temporary-uploads/%s.dat' % checksum + + +def purge_old_uploads(): + now = datetime.datetime.now() + for file in files.scan('temporary-uploads'): + file_time = datetime.datetime.fromtimestamp(file.stat().st_ctime) + if now - file_time > datetime.timedelta(minutes=MAX_MINUTES): + files.delete('temporary-uploads/%s' % file.name) + + +def get(checksum): + return files.get('temporary-uploads/%s.dat' % checksum) + + +def save(content): + checksum = util.get_sha1(content) + path = _get_path(checksum) + if not files.has(path): + files.save(path, content) + return checksum diff --git a/server/szurubooru/func/files.py b/server/szurubooru/func/files.py index 48340450..3ca87776 100644 --- a/server/szurubooru/func/files.py +++ b/server/szurubooru/func/files.py @@ -16,6 +16,12 @@ def has(path): return os.path.exists(_get_full_path(path)) +def scan(path): + if has(path): + return os.scandir(_get_full_path(path)) + return [] + + def move(source_path, target_path): return os.rename(_get_full_path(source_path), _get_full_path(target_path)) diff --git a/server/szurubooru/rest/context.py b/server/szurubooru/rest/context.py index 6f74a4ca..081064ed 100644 --- a/server/szurubooru/rest/context.py +++ b/server/szurubooru/rest/context.py @@ -1,5 +1,5 @@ from szurubooru import errors -from szurubooru.func import net +from szurubooru.func import net, file_uploads def _lower_first(source): @@ -43,18 +43,26 @@ class Context: def get_header(self, name): return self._headers.get(name, None) - def has_file(self, name): - return name in self._files or name + 'Url' in self._params + def has_file(self, name, allow_tokens=True): + return (name in self._files + or name + 'Url' in self._params + or (allow_tokens and name + 'Token' in self._params)) - def get_file(self, name, required=False): + def get_file(self, name, required=False, allow_tokens=True): + ret = None if name in self._files: - return self._files[name] - if name + 'Url' in self._params: - return net.download(self._params[name + 'Url']) - if not required: - return None - raise errors.MissingRequiredFileError( - 'Required file %r is missing.' % name) + ret = self._files[name] + elif name + 'Url' in self._params: + ret = net.download(self._params[name + 'Url']) + elif allow_tokens and name + 'Token' in self._params: + ret = file_uploads.get(self._params[name + 'Token']) + if required and not ret: + raise errors.MissingOrExpiredRequiredFileError( + 'Required file %r is missing or has expired.' % name) + if required and not ret: + raise errors.MissingRequiredFileError( + 'Required file %r is missing.' % name) + return ret def has_param(self, name): return name in self._params