From 0709d739df748a44edf2bfe32694da5860385253 Mon Sep 17 00:00:00 2001 From: pbf <3612022+pbf@users.noreply.github.com> Date: Fri, 21 Jul 2023 23:15:48 +0200 Subject: [PATCH] server+client: add ability to configure allowed file types --- client/js/api.js | 4 ++ client/js/views/post_upload_view.js | 5 +- server/config.yaml.dist | 12 ++++ server/szurubooru/api/info_api.py | 7 +- server/szurubooru/func/mime.py | 64 +++++++++---------- server/szurubooru/func/posts.py | 6 +- server/szurubooru/tests/api/test_info.py | 2 + .../tests/api/test_post_creating.py | 4 ++ server/szurubooru/tests/func/test_posts.py | 48 ++++++++++++++ 9 files changed, 116 insertions(+), 36 deletions(-) diff --git a/client/js/api.js b/client/js/api.js index 5bde6d81..b4f40316 100644 --- a/client/js/api.js +++ b/client/js/api.js @@ -100,6 +100,10 @@ class Api extends events.EventTarget { return remoteConfig.contactEmail; } + getAllowedExtensions() { + return remoteConfig.allowedExtensions; + } + canSendMails() { return !!remoteConfig.canSendMails; } diff --git a/client/js/views/post_upload_view.js b/client/js/views/post_upload_view.js index 4ef4c1ad..22ad8941 100644 --- a/client/js/views/post_upload_view.js +++ b/client/js/views/post_upload_view.js @@ -161,11 +161,14 @@ class PostUploadView extends events.EventTarget { return this._uploadables.findIndex((u2) => u.key === u2.key); }; + let allowedExtensions = api.getAllowedExtensions().map( + function(e) {return "." + e} + ); this._contentFileDropper = new FileDropperControl( this._contentInputNode, { extraText: - "Allowed extensions: .jpg, .png, .gif, .webm, .mp4, .swf, .avif, .heif, .heic", + "Allowed extensions: " + allowedExtensions.join(", "), allowUrls: true, allowMultiple: true, lock: false, diff --git a/server/config.yaml.dist b/server/config.yaml.dist index 193aac3a..382521f2 100644 --- a/server/config.yaml.dist +++ b/server/config.yaml.dist @@ -29,6 +29,18 @@ convert: to_webm: false to_mp4: false +# specify which MIME types are allowed +allowed_mime_types: + - image/jpeg + - image/png + - image/gif + - video/webm + - video/mp4 + - application/x-shockwave-flash + - image/avif + - image/heif + - image/heic + # allow posts to be uploaded even if some image processing errors occur allow_broken_uploads: false diff --git a/server/szurubooru/api/info_api.py b/server/szurubooru/api/info_api.py index 757b09cf..11a78ea0 100644 --- a/server/szurubooru/api/info_api.py +++ b/server/szurubooru/api/info_api.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta from typing import Dict, Optional from szurubooru import config, rest -from szurubooru.func import auth, posts, users, util +from szurubooru.func import auth, mime, posts, users, util _cache_time = None # type: Optional[datetime] _cache_result = None # type: Optional[int] @@ -49,6 +49,11 @@ def get_info(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Response: "privileges": util.snake_case_to_lower_camel_case_keys( config.config["privileges"] ), + "allowedExtensions": [ + mime.MIME_EXTENSIONS_MAP[i] + for i in config.config["allowed_mime_types"] + if i in mime.MIME_EXTENSIONS_MAP + ], }, } if auth.has_privilege(ctx.user, "posts:view:featured"): diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 8fae5679..3f838f49 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -1,6 +1,33 @@ import re +from collections import ChainMap from typing import Optional +MIME_TYPES_MAP = { + "image": { + "image/gif": "gif", + "image/jpeg": "jpg", + "image/png": "png", + "image/webp": "webp", + "image/bmp": "bmp", + "image/avif": "avif", + "image/heif": "heif", + "image/heic": "heic", + }, + "video": { + "application/ogg": None, + "video/mp4": "mp4", + "video/quicktime": "mov", + "video/webm": "webm", + }, + "flash": { + "application/x-shockwave-flash": "swf" + }, + "other": { + "application/octet-stream": "dat", + }, +} +MIME_EXTENSIONS_MAP = ChainMap(*MIME_TYPES_MAP.values()) + def get_mime_type(content: bytes) -> str: if not content: @@ -46,48 +73,19 @@ def get_mime_type(content: bytes) -> str: def get_extension(mime_type: str) -> Optional[str]: - extension_map = { - "application/x-shockwave-flash": "swf", - "image/gif": "gif", - "image/jpeg": "jpg", - "image/png": "png", - "image/webp": "webp", - "image/bmp": "bmp", - "image/avif": "avif", - "image/heif": "heif", - "image/heic": "heic", - "video/mp4": "mp4", - "video/quicktime": "mov", - "video/webm": "webm", - "application/octet-stream": "dat", - } - return extension_map.get((mime_type or "").strip().lower(), None) + return MIME_EXTENSIONS_MAP.get((mime_type or "").strip().lower(), None) def is_flash(mime_type: str) -> bool: - return mime_type.lower() == "application/x-shockwave-flash" + return mime_type.lower() in MIME_TYPES_MAP["flash"] def is_video(mime_type: str) -> bool: - return mime_type.lower() in ( - "application/ogg", - "video/mp4", - "video/quicktime", - "video/webm", - ) + return mime_type.lower() in MIME_TYPES_MAP["video"] def is_image(mime_type: str) -> bool: - return mime_type.lower() in ( - "image/jpeg", - "image/png", - "image/gif", - "image/webp", - "image/bmp", - "image/avif", - "image/heif", - "image/heic", - ) + return mime_type.lower() in MIME_TYPES_MAP["image"] def is_animated_gif(content: bytes) -> bool: diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index be2259cf..bdcc21cb 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -611,7 +611,11 @@ def update_post_content(post: model.Post, content: Optional[bytes]) -> None: update_signature = False post.mime_type = mime.get_mime_type(content) - if mime.is_flash(post.mime_type): + if post.mime_type not in config.config["allowed_mime_types"]: + raise InvalidPostContentError( + "File type not allowed: %r" % post.mime_type + ) + elif mime.is_flash(post.mime_type): post.type = model.Post.TYPE_FLASH elif mime.is_image(post.mime_type): update_signature = True diff --git a/server/szurubooru/tests/api/test_info.py b/server/szurubooru/tests/api/test_info.py index 37099e8d..bf7ea02f 100644 --- a/server/szurubooru/tests/api/test_info.py +++ b/server/szurubooru/tests/api/test_info.py @@ -34,6 +34,7 @@ def test_info_api( "smtp": { "host": "example.com", }, + "allowed_mime_types": ["application/octet-stream"], } ) db.session.add_all([post_factory(), post_factory()]) @@ -54,6 +55,7 @@ def test_info_api( "posts:view:featured": "regular", }, "canSendMails": True, + "allowedExtensions": ["dat"], } with fake_datetime("2016-01-01 13:00"): diff --git a/server/szurubooru/tests/api/test_post_creating.py b/server/szurubooru/tests/api/test_post_creating.py index a1ad4de7..4c479fa5 100644 --- a/server/szurubooru/tests/api/test_post_creating.py +++ b/server/szurubooru/tests/api/test_post_creating.py @@ -343,6 +343,10 @@ def test_errors_not_spending_ids( "uploads:use_downloader": model.User.RANK_POWER, }, "secret": "test", + "allowed_mime_types": [ + "image/png", + "image/jpeg", + ], } ) auth_user = user_factory(rank=model.User.RANK_REGULAR) diff --git a/server/szurubooru/tests/func/test_posts.py b/server/szurubooru/tests/func/test_posts.py index fa1b3bb6..a29e86d9 100644 --- a/server/szurubooru/tests/func/test_posts.py +++ b/server/szurubooru/tests/func/test_posts.py @@ -489,6 +489,18 @@ def test_update_post_content_for_new_post( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": [ + "image/png", + "image/jpeg", + "image/gif", + "image/bmp", + "image/avif", + "image/heic", + "image/heif", + "video/webm", + "video/mp4", + "application/x-shockwave-flash", + ], } ) output_file_path = "{}/data/posts/{}".format(tmpdir, output_file_name) @@ -526,6 +538,7 @@ def test_update_post_content_to_existing_content( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": ["image/png"], } ) post = post_factory() @@ -553,6 +566,7 @@ def test_update_post_content_with_broken_content( }, "secret": "test", "allow_broken_uploads": allow_broken_uploads, + "allowed_mime_types": ["image/png"], } ) post = post_factory() @@ -576,6 +590,7 @@ def test_update_post_content_with_invalid_content( config_injector( { "allow_broken_uploads": True, + "allowed_mime_types": ["application/octet-stream"], } ) post = model.Post() @@ -583,6 +598,29 @@ def test_update_post_content_with_invalid_content( posts.update_post_content(post, input_content) +def test_update_post_content_with_unallowed_mime_type( + tmpdir, config_injector, post_factory, read_asset +): + config_injector( + { + "data_dir": str(tmpdir.mkdir("data")), + "thumbnails": { + "post_width": 300, + "post_height": 300, + }, + "secret": "test", + "allow_broken_uploads": False, + "allowed_mime_types": [], + } + ) + post = post_factory() + db.session.add(post) + db.session.flush() + content = read_asset("png.png") + with pytest.raises(posts.InvalidPostContentError): + posts.update_post_content(post, content) + + @pytest.mark.parametrize("is_existing", (True, False)) def test_update_post_thumbnail_to_new_one( tmpdir, config_injector, read_asset, post_factory, is_existing @@ -596,6 +634,7 @@ def test_update_post_thumbnail_to_new_one( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": ["image/png"], } ) post = post_factory(id=1) @@ -637,6 +676,7 @@ def test_update_post_thumbnail_to_default( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": ["image/png"], } ) post = post_factory(id=1) @@ -677,6 +717,7 @@ def test_update_post_thumbnail_with_broken_thumbnail( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": ["image/png"], } ) post = post_factory(id=1) @@ -721,6 +762,7 @@ def test_update_post_content_leaving_custom_thumbnail( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": ["image/png"], } ) post = post_factory(id=1) @@ -754,6 +796,11 @@ def test_update_post_content_convert_heif_to_png_when_processing( }, "secret": "test", "allow_broken_uploads": False, + "allowed_mime_types": [ + "image/avif", + "image/heic", + "image/heif", + ], } ) post = post_factory(id=1) @@ -1176,6 +1223,7 @@ def test_merge_posts_replaces_content( "post_height": 300, }, "secret": "test", + "allowed_mime_types": ["image/png"], } ) source_post = post_factory(id=1)