Merge 0709d739df into 376f687c38

The minimum version requirements are rough guesses, in practice any decently modern docker installation should work.

client/js/api.js

@@ -100,6 +100,10 @@ class Api extends events.EventTarget {
         return remoteConfig.contactEmail;
     }
 
+    getAllowedExtensions() {
+    	return remoteConfig.allowedExtensions;
+    }
+
     canSendMails() {
         return !!remoteConfig.canSendMails;
     }

client/js/views/post_upload_view.js

@@ -161,11 +161,14 @@ class PostUploadView extends events.EventTarget {
             return this._uploadables.findIndex((u2) => u.key === u2.key);
         };
 
+				let allowedExtensions = api.getAllowedExtensions().map(
+					function(e) {return "." + e}
+				);
         this._contentFileDropper = new FileDropperControl(
             this._contentInputNode,
             {
                 extraText:
-                    "Allowed extensions: .jpg, .png, .gif, .webm, .mp4, .swf, .avif, .heif, .heic",
+                    "Allowed extensions: " + allowedExtensions.join(", "),
                 allowUrls: true,
                 allowMultiple: true,
                 lock: false,

doc/API.md

@@ -789,7 +789,7 @@ data.
     | `fav-time`           | alias of `fav-date`                                                                                                                     |
     | `feature-date`       | featured at given date                                                                                                                  |
     | `feature-time`       | alias of `feature-time`                                                                                                                 |
-    | `safety`             | having given safety. `<value>` can be either `safe`, `sketchy` (or `questionable`) or `unsafe`.                                         |
+    | `safety`             | having given safety. `<value>` can be either `safe`, `sketchy` or `unsafe`.                                         |
     | `rating`             | alias of `safety`                                                                                                                       |
 
     **Sort style tokens**

doc/INSTALL.md

@@ -1,5 +1,5 @@
-This assumes that you have Docker (version 17.05 or greater)
+This assumes that you have Docker (version 19.03 or greater)
-and Docker Compose (version 1.6.0 or greater) already installed.
+and the Docker Compose CLI (version 1.27.0 or greater) already installed.
 
 ### Prepare things
 
@@ -38,7 +38,7 @@ and Docker Compose (version 1.6.0 or greater) already installed.
 
     This pulls the latest containers from docker.io:
     ```console
-    user@host:szuru$ docker-compose pull
+    user@host:szuru$ docker compose pull
     ```
 
     If you have modified the application's source and would like to manually
@@ -49,17 +49,17 @@ and Docker Compose (version 1.6.0 or greater) already installed.
 
     For first run, it is recommended to start the database separately:
     ```console
-    user@host:szuru$ docker-compose up -d sql
+    user@host:szuru$ docker compose up -d sql
     ```
 
     To start all containers:
     ```console
-    user@host:szuru$ docker-compose up -d
+    user@host:szuru$ docker compose up -d
     ```
 
     To view/monitor the application logs:
     ```console
-    user@host:szuru$ docker-compose logs -f
+    user@host:szuru$ docker compose logs -f
     # (CTRL+C to exit)
     ```
 
@@ -84,13 +84,13 @@ and Docker Compose (version 1.6.0 or greater) already installed.
 2. Build the containers:
 
     ```console
-    user@host:szuru$ docker-compose build
+    user@host:szuru$ docker compose build
     ```
 
     That will attempt to build both containers, but you can specify `client`
     or `server` to make it build only one.
 
-    If `docker-compose build` spits out:
+    If `docker compose build` spits out:
 
     ```
     ERROR: Service 'server' failed to build: failed to parse platform : "" is an invalid component of "": platform specifier component must match "^[A-Za-z0-9_-]+$": invalid argument
@@ -102,7 +102,7 @@ and Docker Compose (version 1.6.0 or greater) already installed.
     user@host:szuru$ export DOCKER_BUILDKIT=1; export COMPOSE_DOCKER_CLI_BUILD=1
     ```
 
-    ...and run `docker-compose build` again.
+    ...and run `docker compose build` again.
 
 *Note: If your changes are not taking effect in your builds, consider building
 with `--no-cache`.*
@@ -117,7 +117,7 @@ with `--no-cache`.*
     run from docker:
 
     ```console
-    user@host:szuru$ docker-compose run server ./szuru-admin --help
+    user@host:szuru$ docker compose run server ./szuru-admin --help
     ```
 
     will give you a breakdown on all available commands.

docker-compose.yml

@@ -1,9 +1,7 @@
 ## Example Docker Compose configuration
 ##
-## Use this as a template to set up docker-compose, or as guide to set up other
+## Use this as a template to set up docker compose, or as guide to set up other
 ## orchestration services
-version: '2'
-
 services:
 
   server:

server/config.yaml.dist

@@ -29,6 +29,18 @@ convert:
      to_webm: false
      to_mp4: false
 
+# specify which MIME types are allowed
+allowed_mime_types:
+   - image/jpeg
+   - image/png
+   - image/gif
+   - video/webm
+   - video/mp4
+   - application/x-shockwave-flash
+   - image/avif
+   - image/heif
+   - image/heic
+
 # allow posts to be uploaded even if some image processing errors occur
 allow_broken_uploads: false
 

server/szurubooru/api/info_api.py

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 from typing import Dict, Optional
 
 from szurubooru import config, rest
-from szurubooru.func import auth, posts, users, util
+from szurubooru.func import auth, mime, posts, users, util
 
 _cache_time = None  # type: Optional[datetime]
 _cache_result = None  # type: Optional[int]
@@ -49,6 +49,11 @@ def get_info(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Response:
             "privileges": util.snake_case_to_lower_camel_case_keys(
                 config.config["privileges"]
             ),
+            "allowedExtensions": [
+                mime.MIME_EXTENSIONS_MAP[i]
+                for i in config.config["allowed_mime_types"]
+                if i in mime.MIME_EXTENSIONS_MAP
+            ],
         },
     }
     if auth.has_privilege(ctx.user, "posts:view:featured"):

server/szurubooru/func/mime.py

@@ -1,6 +1,33 @@
 import re
+from collections import ChainMap
 from typing import Optional
 
+MIME_TYPES_MAP = {
+    "image": {
+        "image/gif": "gif",
+        "image/jpeg": "jpg",
+        "image/png": "png",
+        "image/webp": "webp",
+        "image/bmp": "bmp",
+        "image/avif": "avif",
+        "image/heif": "heif",
+        "image/heic": "heic",
+    },
+    "video": {
+        "application/ogg": None,
+        "video/mp4": "mp4",
+        "video/quicktime": "mov",
+        "video/webm": "webm",
+    },
+    "flash": {
+        "application/x-shockwave-flash": "swf"
+    },
+    "other": {
+        "application/octet-stream": "dat",
+    },
+}
+MIME_EXTENSIONS_MAP = ChainMap(*MIME_TYPES_MAP.values())
+
 
 def get_mime_type(content: bytes) -> str:
     if not content:
@@ -46,48 +73,19 @@ def get_mime_type(content: bytes) -> str:
 
 
 def get_extension(mime_type: str) -> Optional[str]:
-    extension_map = {
+    return MIME_EXTENSIONS_MAP.get((mime_type or "").strip().lower(), None)
-        "application/x-shockwave-flash": "swf",
-        "image/gif": "gif",
-        "image/jpeg": "jpg",
-        "image/png": "png",
-        "image/webp": "webp",
-        "image/bmp": "bmp",
-        "image/avif": "avif",
-        "image/heif": "heif",
-        "image/heic": "heic",
-        "video/mp4": "mp4",
-        "video/quicktime": "mov",
-        "video/webm": "webm",
-        "application/octet-stream": "dat",
-    }
-    return extension_map.get((mime_type or "").strip().lower(), None)
 
 
 def is_flash(mime_type: str) -> bool:
-    return mime_type.lower() == "application/x-shockwave-flash"
+    return mime_type.lower() in MIME_TYPES_MAP["flash"]
 
 
 def is_video(mime_type: str) -> bool:
-    return mime_type.lower() in (
+    return mime_type.lower() in MIME_TYPES_MAP["video"]
-        "application/ogg",
-        "video/mp4",
-        "video/quicktime",
-        "video/webm",
-    )
 
 
 def is_image(mime_type: str) -> bool:
-    return mime_type.lower() in (
+    return mime_type.lower() in MIME_TYPES_MAP["image"]
-        "image/jpeg",
-        "image/png",
-        "image/gif",
-        "image/webp",
-        "image/bmp",
-        "image/avif",
-        "image/heif",
-        "image/heic",
-    )
 
 
 def is_animated_gif(content: bytes) -> bool:

server/szurubooru/func/posts.py

@@ -611,7 +611,11 @@ def update_post_content(post: model.Post, content: Optional[bytes]) -> None:
 
     update_signature = False
     post.mime_type = mime.get_mime_type(content)
-    if mime.is_flash(post.mime_type):
+    if post.mime_type not in config.config["allowed_mime_types"]:
+        raise InvalidPostContentError(
+            "File type not allowed: %r" % post.mime_type
+        )
+    elif mime.is_flash(post.mime_type):
         post.type = model.Post.TYPE_FLASH
     elif mime.is_image(post.mime_type):
         update_signature = True

server/szurubooru/tests/api/test_info.py

@@ -34,6 +34,7 @@ def test_info_api(
             "smtp": {
                 "host": "example.com",
             },
+            "allowed_mime_types": ["application/octet-stream"],
         }
     )
     db.session.add_all([post_factory(), post_factory()])
@@ -54,6 +55,7 @@ def test_info_api(
             "posts:view:featured": "regular",
         },
         "canSendMails": True,
+        "allowedExtensions": ["dat"],
     }
 
     with fake_datetime("2016-01-01 13:00"):

server/szurubooru/tests/api/test_post_creating.py

@@ -343,6 +343,10 @@ def test_errors_not_spending_ids(
                 "uploads:use_downloader": model.User.RANK_POWER,
             },
             "secret": "test",
+            "allowed_mime_types": [
+                "image/png",
+                "image/jpeg",
+            ],
         }
     )
     auth_user = user_factory(rank=model.User.RANK_REGULAR)

server/szurubooru/tests/func/test_posts.py

@@ -489,6 +489,18 @@ def test_update_post_content_for_new_post(
                 },
                 "secret": "test",
                 "allow_broken_uploads": False,
+                "allowed_mime_types": [
+                    "image/png",
+                    "image/jpeg",
+                    "image/gif",
+                    "image/bmp",
+                    "image/avif",
+                    "image/heic",
+                    "image/heif",
+                    "video/webm",
+                    "video/mp4",
+                    "application/x-shockwave-flash",
+                ],
             }
         )
         output_file_path = "{}/data/posts/{}".format(tmpdir, output_file_name)
@@ -526,6 +538,7 @@ def test_update_post_content_to_existing_content(
             },
             "secret": "test",
             "allow_broken_uploads": False,
+            "allowed_mime_types": ["image/png"],
         }
     )
     post = post_factory()
@@ -553,6 +566,7 @@ def test_update_post_content_with_broken_content(
             },
             "secret": "test",
             "allow_broken_uploads": allow_broken_uploads,
+            "allowed_mime_types": ["image/png"],
         }
     )
     post = post_factory()
@@ -576,6 +590,7 @@ def test_update_post_content_with_invalid_content(
     config_injector(
         {
             "allow_broken_uploads": True,
+            "allowed_mime_types": ["application/octet-stream"],
         }
     )
     post = model.Post()
@@ -583,6 +598,29 @@ def test_update_post_content_with_invalid_content(
         posts.update_post_content(post, input_content)
 
 
+def test_update_post_content_with_unallowed_mime_type(
+        tmpdir, config_injector, post_factory, read_asset
+):
+    config_injector(
+        {
+            "data_dir": str(tmpdir.mkdir("data")),
+            "thumbnails": {
+                "post_width": 300,
+                "post_height": 300,
+            },
+            "secret": "test",
+            "allow_broken_uploads": False,
+            "allowed_mime_types": [],
+        }
+    )
+    post = post_factory()
+    db.session.add(post)
+    db.session.flush()
+    content = read_asset("png.png")
+    with pytest.raises(posts.InvalidPostContentError):
+        posts.update_post_content(post, content)
+
+
 @pytest.mark.parametrize("is_existing", (True, False))
 def test_update_post_thumbnail_to_new_one(
     tmpdir, config_injector, read_asset, post_factory, is_existing
@@ -596,6 +634,7 @@ def test_update_post_thumbnail_to_new_one(
             },
             "secret": "test",
             "allow_broken_uploads": False,
+            "allowed_mime_types": ["image/png"],
         }
     )
     post = post_factory(id=1)
@@ -637,6 +676,7 @@ def test_update_post_thumbnail_to_default(
             },
             "secret": "test",
             "allow_broken_uploads": False,
+            "allowed_mime_types": ["image/png"],
         }
     )
     post = post_factory(id=1)
@@ -677,6 +717,7 @@ def test_update_post_thumbnail_with_broken_thumbnail(
             },
             "secret": "test",
             "allow_broken_uploads": False,
+            "allowed_mime_types": ["image/png"],
         }
     )
     post = post_factory(id=1)
@@ -721,6 +762,7 @@ def test_update_post_content_leaving_custom_thumbnail(
             },
             "secret": "test",
             "allow_broken_uploads": False,
+            "allowed_mime_types": ["image/png"],
         }
     )
     post = post_factory(id=1)
@@ -754,6 +796,11 @@ def test_update_post_content_convert_heif_to_png_when_processing(
             },
             "secret": "test",
             "allow_broken_uploads": False,
+            "allowed_mime_types": [
+                "image/avif",
+                "image/heic",
+                "image/heif",
+            ],
         }
     )
     post = post_factory(id=1)
@@ -1176,6 +1223,7 @@ def test_merge_posts_replaces_content(
                 "post_height": 300,
             },
             "secret": "test",
+            "allowed_mime_types": ["image/png"],
         }
     )
     source_post = post_factory(id=1)