server/uploads: add file upload api

API.md

@@ -63,6 +63,8 @@
         - [Listing snapshots](#listing-snapshots)
     - Global info
         - [Getting global info](#getting-global-info)
+    - File uploads
+        - [Uploading temporary file](#uploading-temporary-file)
 
 3. [Resources](#resources)
 
@@ -105,16 +107,28 @@ application/json`. An exception to this rule are requests that upload files.
 
 ## File uploads
 
-Requests that upload files must use `multipart/form-data` encoding. JSON
+Requests that upload files must use `multipart/form-data` encoding. Any request
-metadata must then be included as field of name `metadata`, whereas files must
+that bundles user files, must send the request data (which is JSON) as an
-be included as separate fields with names specific to each request type.
+additional file with the special name of `metadata` (whereas the actual files
+must have names specific to the API that is being used.)
 
 Alternatively, the server can download the files from the Internet on client's
 behalf. In that case, the request doesn't need to be specially encoded in any
-way. The files, however, should be passed as regular fields appended with `Url`
+way. The files, however, should be passed as regular fields appended with a
-suffix. For example, to download a file named `content` from
+`Url` suffix. For example, to use `http://example.com/file.jpg` in an API that
-`http://example.com/file.jpg`, the client should pass
+accepts a file named `content`, the client should pass
-`{"contentUrl":"http://example.com/file.jpg"}` as part of the message body.
+`{"contentUrl":"http://example.com/file.jpg"}` as a part of the JSON message
+body.
+
+Finally, in some cases the user might want to reuse one file between the
+requests to save the bandwidth (for example, reverse search + consecutive
+upload). In this case one should use [temporary file
+uploads](#uploading-temporary-file), and pass the tokens returned by the API as
+regular fields appended with a `Token` suffix. For example, to use previously
+uploaded data, which was given token `deadbeef`, in an API that accepts a file
+named `content`, the client should pass `{"contentToken":"deadbeef"}` as part
+of the JSON message body. If the file with the particular token doesn't exist
+or it has expired, the server will show an error.
 
 ## Error handling
 
@@ -1593,6 +1607,35 @@ data.
     exception of privilege array keys being converted to lower camel case to
     match the API convention.
 
+## Uploading temporary file
+
+- **Request**
+
+    `POST /uploads`
+
+- **Files**
+
+    - `content` - the content of the file to upload. Note that in this
+      particular API, one can't use token-based uploads.
+
+- **Output**
+
+    ```json5
+    {
+        "token": <token>
+    }
+    ```
+
+- **Errors**
+
+    - privileges are too low
+
+- **Description**
+
+    Puts a file in temporary storage and assigns it a token that can be used in
+    other requests. The files uploaded that way are deleted after a short while
+    so clients shouldn't use it as a free upload service.
+
 
 
 # Resources

config.yaml.dist

@@ -119,3 +119,5 @@ privileges:
     'comments:score':               regular
 
     'snapshots:list':               power
+
+    'uploads:create':               regular

server/szurubooru/api/__init__.py

@@ -6,3 +6,4 @@ import szurubooru.api.tag_category_api
 import szurubooru.api.comment_api
 import szurubooru.api.password_reset_api
 import szurubooru.api.snapshot_api
+import szurubooru.api.upload_api

server/szurubooru/api/upload_api.py

@@ -0,0 +1,10 @@
+from szurubooru.rest import routes
+from szurubooru.func import auth, file_uploads
+
+
+@routes.post('/uploads/?')
+def create_temporary_file(ctx, _params=None):
+    auth.verify_privilege(ctx.user, 'uploads:create')
+    content = ctx.get_file('content', required=True, allow_tokens=False)
+    token = file_uploads.save(content)
+    return {'token': token}

server/szurubooru/errors.py

@@ -36,6 +36,10 @@ class MissingRequiredFileError(ValidationError):
     pass
 
 
+class MissingOrExpiredRequiredFileError(MissingRequiredFileError):
+    pass
+
+
 class MissingRequiredParameterError(ValidationError):
     pass
 

server/szurubooru/facade.py

@@ -1,11 +1,13 @@
 ''' Exports create_app. '''
 
 import os
+import time
 import logging
+import threading
 import coloredlogs
 import sqlalchemy.orm.exc
 from szurubooru import config, errors, rest
-from szurubooru.func import posts
+from szurubooru.func import posts, file_uploads
 # pylint: disable=unused-import
 from szurubooru import api, middleware
 
@@ -79,6 +81,15 @@ def validate_config():
         raise errors.ConfigError('Database is not configured')
 
 
+def purge_old_uploads():
+    while True:
+        try:
+            file_uploads.purge_old_uploads()
+        except Exception as ex:
+            logging.exception(ex)
+        time.sleep(60 * 5)
+
+
 def create_app():
     ''' Create a WSGI compatible App object. '''
     validate_config()
@@ -88,6 +99,9 @@ def create_app():
     if config.config['show_sql']:
         logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
 
+    purge_thread = threading.Thread(target=purge_old_uploads)
+    purge_thread.daemon = True
+    purge_thread.start()
     posts.populate_reverse_search()
 
     rest.errors.handle(errors.AuthError, _on_auth_error)

server/szurubooru/func/file_uploads.py

@@ -0,0 +1,29 @@
+import datetime
+from szurubooru.func import files, util
+
+
+MAX_MINUTES = 60
+
+
+def _get_path(checksum):
+    return 'temporary-uploads/%s.dat' % checksum
+
+
+def purge_old_uploads():
+    now = datetime.datetime.now()
+    for file in files.scan('temporary-uploads'):
+        file_time = datetime.datetime.fromtimestamp(file.stat().st_ctime)
+        if now - file_time > datetime.timedelta(minutes=MAX_MINUTES):
+            files.delete('temporary-uploads/%s' % file.name)
+
+
+def get(checksum):
+    return files.get('temporary-uploads/%s.dat' % checksum)
+
+
+def save(content):
+    checksum = util.get_sha1(content)
+    path = _get_path(checksum)
+    if not files.has(path):
+        files.save(path, content)
+    return checksum

server/szurubooru/func/files.py

@@ -16,6 +16,12 @@ def has(path):
     return os.path.exists(_get_full_path(path))
 
 
+def scan(path):
+    if has(path):
+        return os.scandir(_get_full_path(path))
+    return []
+
+
 def move(source_path, target_path):
     return os.rename(_get_full_path(source_path), _get_full_path(target_path))
 

server/szurubooru/rest/context.py

@@ -1,5 +1,5 @@
 from szurubooru import errors
-from szurubooru.func import net
+from szurubooru.func import net, file_uploads
 
 
 def _lower_first(source):
@@ -43,18 +43,26 @@ class Context:
     def get_header(self, name):
         return self._headers.get(name, None)
 
-    def has_file(self, name):
+    def has_file(self, name, allow_tokens=True):
-        return name in self._files or name + 'Url' in self._params
+        return (name in self._files
+            or name + 'Url' in self._params
+            or (allow_tokens and name + 'Token' in self._params))
 
-    def get_file(self, name, required=False):
+    def get_file(self, name, required=False, allow_tokens=True):
+        ret = None
         if name in self._files:
-            return self._files[name]
+            ret = self._files[name]
-        if name + 'Url' in self._params:
+        elif name + 'Url' in self._params:
-            return net.download(self._params[name + 'Url'])
+            ret = net.download(self._params[name + 'Url'])
-        if not required:
+        elif allow_tokens and name + 'Token' in self._params:
-            return None
+            ret = file_uploads.get(self._params[name + 'Token'])
+            if required and not ret:
+                raise errors.MissingOrExpiredRequiredFileError(
+                    'Required file %r is missing or has expired.' % name)
+        if required and not ret:
             raise errors.MissingRequiredFileError(
                 'Required file %r is missing.' % name)
+        return ret
 
     def has_param(self, name):
         return name in self._params