From 4f57f49ebe5395a4c3a8e20c84c5d881c9f98dfc Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Wed, 15 Sep 2021 16:06:48 -0400 Subject: [PATCH 01/48] client+server: migrate to GitHub actions --- .github/workflows/build-containers.yml | 93 ++++++++++++++++++++++++++ .github/workflows/run-unit-tests.yml | 16 +++++ client/hooks/build | 16 ----- client/hooks/post_push | 19 ------ server/hooks/build | 7 -- server/hooks/post_push | 19 ------ server/hooks/test | 8 --- 7 files changed, 109 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/build-containers.yml create mode 100644 .github/workflows/run-unit-tests.yml delete mode 100755 client/hooks/build delete mode 100755 client/hooks/post_push delete mode 100755 server/hooks/build delete mode 100755 server/hooks/post_push delete mode 100755 server/hooks/test diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml new file mode 100644 index 00000000..41baea71 --- /dev/null +++ b/.github/workflows/build-containers.yml @@ -0,0 +1,93 @@ +name: build-containers +on: [push] +jobs: + build-client: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Determine metadata + run: | + CLOSEST_VER="$(git describe --tags --abbrev=0 $GITHUB_SHA)" + CLOSEST_MAJOR_VER="$(echo ${CLOSEST_VER} | cut -d'.' -f1)" + CLOSEST_MINOR_VER="$(echo ${CLOSEST_VER} | cut -d'.' -f2)" + SHORT_COMMIT=$(echo $GITHUB_SHA | cut -c1-8) + BUILD_INFO="v${CLOSEST_VER}-${SHORT_COMMIT}" + BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + + echo "major_tag=${CLOSEST_MAJOR_VER}" >> $GITHUB_ENV + echo "minor_tag=${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}" >> $GITHUB_ENV + echo "build_info=${BUILD_INFO}" >> $GITHUB_ENV + echo "build_date=${BUILD_DATE}" >> $GITHUB_ENV + + echo "Build Info: ${BUILD_INFO}" + echo "Build Date: ${BUILD_DATE}" + + - name: Build container + run: > + docker build + --build-arg BUILD_INFO=${{ env.build_info }} + --build-arg BUILD_DATE=${{ env.build_date }} + --build-arg SOURCE_COMMIT=$GITHUB_SHA + --build-arg DOCKER_REPO=szurubooru/client + -t "szurubooru/client:latest" + -t "szurubooru/client:${{ env.major_tag }}" + -t "szurubooru/client:${{ env.minor_tag }}" + ./client + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Push containers + run: docker push -a szurubooru/client + + build-server: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Determine metadata + run: | + CLOSEST_VER="$(git describe --tags --abbrev=0 $GITHUB_SHA)" + CLOSEST_MAJOR_VER="$(echo ${CLOSEST_VER} | cut -d'.' -f1)" + CLOSEST_MINOR_VER="$(echo ${CLOSEST_VER} | cut -d'.' -f2)" + SHORT_COMMIT=$(echo $GITHUB_SHA | cut -c1-8) + BUILD_INFO="v${CLOSEST_VER}-${SHORT_COMMIT}" + BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + + echo "major_tag=${CLOSEST_MAJOR_VER}" >> $GITHUB_ENV + echo "minor_tag=${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}" >> $GITHUB_ENV + echo "build_info=${BUILD_INFO}" >> $GITHUB_ENV + echo "build_date=${BUILD_DATE}" >> $GITHUB_ENV + + echo "Build Info: ${BUILD_INFO}" + echo "Build Date: ${BUILD_DATE}" + + - name: Build container + run: > + docker build + --build-arg BUILD_DATE=${{ env.build_date }} + --build-arg SOURCE_COMMIT=$GITHUB_SHA + --build-arg DOCKER_REPO=szurubooru/server + -t "szurubooru/server:latest" + -t "szurubooru/server:${{ env.major_tag }}" + -t "szurubooru/server:${{ env.minor_tag }}" + ./server + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Push containers + run: docker push -a szurubooru/server diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml new file mode 100644 index 00000000..87ab4e85 --- /dev/null +++ b/.github/workflows/run-unit-tests.yml @@ -0,0 +1,16 @@ +name: run-unit-tests +on: [pull_request] +jobs: + test-server: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Build test container + run: | + TAG=$(docker build --target testing -q ./server) + echo "image_tag=${TAG}" >> $GITHUB_ENV + + - name: Run unit tests + run: docker run --rm -t ${{ env.image_tag }} --color=no szurubooru/ diff --git a/client/hooks/build b/client/hooks/build deleted file mode 100755 index 46443f40..00000000 --- a/client/hooks/build +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -CLOSEST_VER=$(git describe --tags --abbrev=0 ${SOURCE_COMMIT}) -if git describe --exact-match --abbrev=0 ${SOURCE_COMMIT} 2> /dev/null; then - BUILD_INFO="v${CLOSEST_VER}" -else - BUILD_INFO="v${CLOSEST_VER}-edge-$(git rev-parse --short ${SOURCE_COMMIT})" -fi - -echo "Using BUILD_INFO=${BUILD_INFO}" -docker build \ - --build-arg BUILD_INFO=${BUILD_INFO} \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg SOURCE_COMMIT \ - --build-arg DOCKER_REPO \ - -f $DOCKERFILE_PATH -t $IMAGE_NAME . diff --git a/client/hooks/post_push b/client/hooks/post_push deleted file mode 100755 index 1b1e0ad9..00000000 --- a/client/hooks/post_push +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -add_tag() { - echo "Also tagging image as ${DOCKER_REPO}:${1}" - docker tag $IMAGE_NAME $DOCKER_REPO:$1 - docker push $DOCKER_REPO:$1 -} - -CLOSEST_VER=$(git describe --tags --abbrev=0) -CLOSEST_MAJOR_VER=$(echo ${CLOSEST_VER} | cut -d'.' -f1) -CLOSEST_MINOR_VER=$(echo ${CLOSEST_VER} | cut -d'.' -f2) - -add_tag "${CLOSEST_MAJOR_VER}-edge" -add_tag "${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}-edge" - -if git describe --exact-match --abbrev=0 2> /dev/null; then - add_tag "${CLOSEST_MAJOR_VER}" - add_tag "${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}" -fi diff --git a/server/hooks/build b/server/hooks/build deleted file mode 100755 index b5e914b2..00000000 --- a/server/hooks/build +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -docker build \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg SOURCE_COMMIT \ - --build-arg DOCKER_REPO \ - -f $DOCKERFILE_PATH -t $IMAGE_NAME . diff --git a/server/hooks/post_push b/server/hooks/post_push deleted file mode 100755 index 1b1e0ad9..00000000 --- a/server/hooks/post_push +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -add_tag() { - echo "Also tagging image as ${DOCKER_REPO}:${1}" - docker tag $IMAGE_NAME $DOCKER_REPO:$1 - docker push $DOCKER_REPO:$1 -} - -CLOSEST_VER=$(git describe --tags --abbrev=0) -CLOSEST_MAJOR_VER=$(echo ${CLOSEST_VER} | cut -d'.' -f1) -CLOSEST_MINOR_VER=$(echo ${CLOSEST_VER} | cut -d'.' -f2) - -add_tag "${CLOSEST_MAJOR_VER}-edge" -add_tag "${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}-edge" - -if git describe --exact-match --abbrev=0 2> /dev/null; then - add_tag "${CLOSEST_MAJOR_VER}" - add_tag "${CLOSEST_MAJOR_VER}.${CLOSEST_MINOR_VER}" -fi diff --git a/server/hooks/test b/server/hooks/test deleted file mode 100755 index b3251864..00000000 --- a/server/hooks/test +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e - -docker run --rm \ - -t $(docker build --target testing -q .) \ - --color=no szurubooru/ - -exit $? From c64983002ed727cb9692e5959ab3e9ed17564d68 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Thu, 16 Sep 2021 11:54:03 -0400 Subject: [PATCH 02/48] client+server/docker: build ARM images for Docker Hub --- .github/workflows/build-containers.yml | 44 ++++++++++++++++---------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 41baea71..f808f657 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -4,6 +4,12 @@ jobs: build-client: runs-on: ubuntu-latest steps: + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Checkout uses: actions/checkout@v2 with: @@ -26,9 +32,17 @@ jobs: echo "Build Info: ${BUILD_INFO}" echo "Build Date: ${BUILD_DATE}" + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Build container run: > - docker build + docker buildx build --push + --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 --build-arg BUILD_INFO=${{ env.build_info }} --build-arg BUILD_DATE=${{ env.build_date }} --build-arg SOURCE_COMMIT=$GITHUB_SHA @@ -38,18 +52,15 @@ jobs: -t "szurubooru/client:${{ env.minor_tag }}" ./client + build-server: + runs-on: ubuntu-latest + steps: - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push containers - run: docker push -a szurubooru/client - - build-server: - runs-on: ubuntu-latest - steps: - name: Checkout uses: actions/checkout@v2 with: @@ -72,9 +83,17 @@ jobs: echo "Build Info: ${BUILD_INFO}" echo "Build Date: ${BUILD_DATE}" + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Build container run: > - docker build + docker buildx build --push + --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 --build-arg BUILD_DATE=${{ env.build_date }} --build-arg SOURCE_COMMIT=$GITHUB_SHA --build-arg DOCKER_REPO=szurubooru/server @@ -82,12 +101,3 @@ jobs: -t "szurubooru/server:${{ env.major_tag }}" -t "szurubooru/server:${{ env.minor_tag }}" ./server - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Push containers - run: docker push -a szurubooru/server From c3b81371d83c3daf31aeeb821d4865e232cd6b0d Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Sun, 19 Sep 2021 12:03:32 -0400 Subject: [PATCH 03/48] client+server/docker: fix ARM build platform issue --- .github/workflows/build-containers.yml | 4 ++-- client/Dockerfile | 4 ++-- server/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index f808f657..518bf5bc 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -42,7 +42,7 @@ jobs: - name: Build container run: > docker buildx build --push - --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --build-arg BUILD_INFO=${{ env.build_info }} --build-arg BUILD_DATE=${{ env.build_date }} --build-arg SOURCE_COMMIT=$GITHUB_SHA @@ -93,7 +93,7 @@ jobs: - name: Build container run: > docker buildx build --push - --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + --platform linux/amd64,linux/arm/v7,linux/arm64/v8 --build-arg BUILD_DATE=${{ env.build_date }} --build-arg SOURCE_COMMIT=$GITHUB_SHA --build-arg DOCKER_REPO=szurubooru/server diff --git a/client/Dockerfile b/client/Dockerfile index 2aeaf3b5..3ab0016f 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -1,4 +1,4 @@ -FROM node:lts as builder +FROM --platform=$BUILDPLATFORM node:lts as builder WORKDIR /opt/app COPY package.json package-lock.json ./ @@ -12,7 +12,7 @@ ARG CLIENT_BUILD_ARGS="" RUN BASE_URL="__BASEURL__" node build.js --gzip ${CLIENT_BUILD_ARGS} -FROM scratch as approot +FROM --platform=$BUILDPLATFORM scratch as approot COPY docker-start.sh / diff --git a/server/Dockerfile b/server/Dockerfile index 4beec1cf..205c8e4c 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -37,7 +37,7 @@ COPY ./ /opt/app/ RUN rm -rf /opt/app/szurubooru/tests -FROM prereqs as testing +FROM --platform=$BUILDPLATFORM prereqs as testing WORKDIR /opt/app RUN apk --no-cache add \ From ad9d3599bccd825fa0f17e0b4334dfcf01e877ef Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Wed, 22 Sep 2021 22:08:07 -0400 Subject: [PATCH 04/48] server/net: return more useful error messages --- server/szurubooru/func/net.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py index 9dff3c45..3f085a0e 100644 --- a/server/szurubooru/func/net.py +++ b/server/szurubooru/func/net.py @@ -42,10 +42,17 @@ def download(url: str, use_video_downloader: bool = False) -> bytes: while (chunk := handle.read(_dl_chunk_size)) : length_tally += len(chunk) if length_tally > config.config["max_dl_filesize"]: - raise DownloadTooLargeError(url) + raise DownloadTooLargeError( + "Download target exceeds maximum. (%d)" + % (config.config["max_dl_filesize"]), + extra_fields={"URL": url}, + ) content_buffer += chunk except urllib.error.HTTPError as ex: - raise DownloadError(url) from ex + raise DownloadError( + "Download target returned HTTP %d. (%s)" % (ex.code, ex.reason), + extra_fields={"URL": url}, + ) from ex if ( youtube_dl_error @@ -69,7 +76,8 @@ def _get_youtube_dl_content_url(url: str) -> str: ) except subprocess.CalledProcessError: raise errors.ThirdPartyError( - "Could not extract content location from %s" % (url) + "Could not extract content location from URL.", + extra_fields={"URL": url}, ) from None From d08308440713063502979a40c04c641ddef7dde7 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Thu, 23 Sep 2021 12:24:56 -0400 Subject: [PATCH 05/48] server/tests: use transactional db for faster unit tests * `test_modify_saves_non_empty_diffs` needs non-transactional db, so moved to seperate file * Replaced incompatable usage of `db.session.rollback()` with parametrerized function calls * xfail conditionals for search removed, as we can no longer get current driver with binds * Also remove usage of deprecated `pytest.yield_fixture` --- .../szurubooru/tests/api/test_tag_updating.py | 14 +---- server/szurubooru/tests/conftest.py | 16 ++++- .../szurubooru/tests/func/test_snapshots.py | 42 +------------ .../test_snapshots_transactional_isolation.py | 59 +++++++++++++++++++ .../tests/func/test_tag_categories.py | 15 +++-- server/szurubooru/tests/func/test_tags.py | 16 +++-- .../search/configs/test_pool_search_config.py | 2 - .../search/configs/test_tag_search_config.py | 2 - 8 files changed, 91 insertions(+), 75 deletions(-) create mode 100644 server/szurubooru/tests/func/test_snapshots_transactional_isolation.py diff --git a/server/szurubooru/tests/api/test_tag_updating.py b/server/szurubooru/tests/api/test_tag_updating.py index 729734d9..be5f4858 100644 --- a/server/szurubooru/tests/api/test_tag_updating.py +++ b/server/szurubooru/tests/api/test_tag_updating.py @@ -145,8 +145,9 @@ def test_trying_to_update_without_privileges( ) +@pytest.mark.parametrize("type", ["suggestions", "implications"]) def test_trying_to_create_tags_without_privileges( - config_injector, context_factory, tag_factory, user_factory + config_injector, context_factory, tag_factory, user_factory, type ): tag = tag_factory(names=["tag"]) db.session.add(tag) @@ -165,16 +166,7 @@ def test_trying_to_create_tags_without_privileges( with pytest.raises(errors.AuthError): api.tag_api.update_tag( context_factory( - params={"suggestions": ["tag1", "tag2"], "version": 1}, - user=user_factory(rank=model.User.RANK_REGULAR), - ), - {"tag_name": "tag"}, - ) - db.session.rollback() - with pytest.raises(errors.AuthError): - api.tag_api.update_tag( - context_factory( - params={"implications": ["tag1", "tag2"], "version": 1}, + params={type: ["tag1", "tag2"], "version": 1}, user=user_factory(rank=model.User.RANK_REGULAR), ), {"tag_name": "tag"}, diff --git a/server/szurubooru/tests/conftest.py b/server/szurubooru/tests/conftest.py index e7811fe1..280987ca 100644 --- a/server/szurubooru/tests/conftest.py +++ b/server/szurubooru/tests/conftest.py @@ -43,14 +43,26 @@ def query_logger(pytestconfig): logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -@pytest.yield_fixture(scope="function", autouse=True) -def session(query_logger, postgresql_db): +@pytest.fixture(scope="function", autouse=True) +def session(query_logger, transacted_postgresql_db): + db.session = transacted_postgresql_db.session + transacted_postgresql_db.create_table(*model.Base.metadata.sorted_tables) + try: + yield transacted_postgresql_db.session + finally: + transacted_postgresql_db.reset_db() + + +@pytest.fixture(scope="function") +def nontransacted_session(query_logger, postgresql_db): + old_db_session = db.session db.session = postgresql_db.session postgresql_db.create_table(*model.Base.metadata.sorted_tables) try: yield postgresql_db.session finally: postgresql_db.reset_db() + db.session = old_db_session @pytest.fixture diff --git a/server/szurubooru/tests/func/test_snapshots.py b/server/szurubooru/tests/func/test_snapshots.py index da935307..dc68ff05 100644 --- a/server/szurubooru/tests/func/test_snapshots.py +++ b/server/szurubooru/tests/func/test_snapshots.py @@ -1,7 +1,7 @@ from datetime import datetime from unittest.mock import patch -import pytest +import pytest # noqa: F401 from szurubooru import db, model from szurubooru.func import snapshots, users @@ -144,46 +144,6 @@ def test_create(tag_factory, user_factory): assert results[0].data == "mocked" -def test_modify_saves_non_empty_diffs(post_factory, user_factory): - if "sqlite" in db.session.get_bind().driver: - pytest.xfail( - "SQLite doesn't support transaction isolation, " - "which is required to retrieve original entity" - ) - post = post_factory() - post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="old")] - user = user_factory() - db.session.add_all([post, user]) - db.session.commit() - post.source = "new source" - post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")] - db.session.flush() - with patch("szurubooru.func.snapshots._post_to_webhooks"): - snapshots.modify(post, user) - db.session.flush() - results = db.session.query(model.Snapshot).all() - assert len(results) == 1 - assert results[0].data == { - "type": "object change", - "value": { - "source": { - "type": "primitive change", - "old-value": None, - "new-value": "new source", - }, - "notes": { - "type": "list change", - "removed": [ - {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"} - ], - "added": [ - {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"} - ], - }, - }, - } - - def test_modify_doesnt_save_empty_diffs(tag_factory, user_factory): tag = tag_factory(names=["dummy"]) user = user_factory() diff --git a/server/szurubooru/tests/func/test_snapshots_transactional_isolation.py b/server/szurubooru/tests/func/test_snapshots_transactional_isolation.py new file mode 100644 index 00000000..b98cea7a --- /dev/null +++ b/server/szurubooru/tests/func/test_snapshots_transactional_isolation.py @@ -0,0 +1,59 @@ +from unittest.mock import patch + +import pytest + +from szurubooru import db, model +from szurubooru.func import snapshots + + +@pytest.fixture(autouse=True) +def session(query_logger, postgresql_db): + """ + Override db session for this specific test section only + """ + db.session = postgresql_db.session + postgresql_db.create_table(*model.Base.metadata.sorted_tables) + try: + yield postgresql_db.session + finally: + postgresql_db.reset_db() + + +def test_modify_saves_non_empty_diffs(post_factory, user_factory): + if "sqlite" in db.session.get_bind().driver: + pytest.xfail( + "SQLite doesn't support transaction isolation, " + "which is required to retrieve original entity" + ) + post = post_factory() + post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="old")] + user = user_factory() + db.session.add_all([post, user]) + db.session.commit() + post.source = "new source" + post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")] + db.session.flush() + with patch("szurubooru.func.snapshots._post_to_webhooks"): + snapshots.modify(post, user) + db.session.flush() + results = db.session.query(model.Snapshot).all() + assert len(results) == 1 + assert results[0].data == { + "type": "object change", + "value": { + "source": { + "type": "primitive change", + "old-value": None, + "new-value": "new source", + }, + "notes": { + "type": "list change", + "removed": [ + {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"} + ], + "added": [ + {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"} + ], + }, + }, + } diff --git a/server/szurubooru/tests/func/test_tag_categories.py b/server/szurubooru/tests/func/test_tag_categories.py index 11300cf4..9e649a34 100644 --- a/server/szurubooru/tests/func/test_tag_categories.py +++ b/server/szurubooru/tests/func/test_tag_categories.py @@ -107,17 +107,16 @@ def test_update_category_name_reusing_other_name( tag_categories.update_category_name(category, "NAME") +@pytest.mark.parametrize("name", ["name", "NAME"]) def test_update_category_name_reusing_own_name( - config_injector, tag_category_factory + config_injector, tag_category_factory, name ): config_injector({"tag_category_name_regex": ".*"}) - for name in ["name", "NAME"]: - category = tag_category_factory(name="name") - db.session.add(category) - db.session.flush() - tag_categories.update_category_name(category, name) - assert category.name == name - db.session.rollback() + category = tag_category_factory(name="name") + db.session.add(category) + db.session.flush() + tag_categories.update_category_name(category, name) + assert category.name == name def test_update_category_color_with_empty_string(tag_category_factory): diff --git a/server/szurubooru/tests/func/test_tags.py b/server/szurubooru/tests/func/test_tags.py index ac8963c7..60df1220 100644 --- a/server/szurubooru/tests/func/test_tags.py +++ b/server/szurubooru/tests/func/test_tags.py @@ -513,15 +513,14 @@ def test_update_tag_names_trying_to_use_taken_name( tags.update_tag_names(tag, ["A"]) -def test_update_tag_names_reusing_own_name(config_injector, tag_factory): +@pytest.mark.parametrize("name", list("aA")) +def test_update_tag_names_reusing_own_name(config_injector, tag_factory, name): config_injector({"tag_name_regex": "^[a-zA-Z]*$"}) - for name in list("aA"): - tag = tag_factory(names=["a"]) - db.session.add(tag) - db.session.flush() - tags.update_tag_names(tag, [name]) - assert [tag_name.name for tag_name in tag.names] == [name] - db.session.rollback() + tag = tag_factory(names=["a"]) + db.session.add(tag) + db.session.flush() + tags.update_tag_names(tag, [name]) + assert [tag_name.name for tag_name in tag.names] == [name] def test_update_tag_names_changing_primary_name(config_injector, tag_factory): @@ -533,7 +532,6 @@ def test_update_tag_names_changing_primary_name(config_injector, tag_factory): db.session.flush() db.session.refresh(tag) assert [tag_name.name for tag_name in tag.names] == ["b", "a"] - db.session.rollback() @pytest.mark.parametrize("attempt", ["name", "NAME", "alias", "ALIAS"]) diff --git a/server/szurubooru/tests/search/configs/test_pool_search_config.py b/server/szurubooru/tests/search/configs/test_pool_search_config.py index 202635c6..1103ec40 100644 --- a/server/szurubooru/tests/search/configs/test_pool_search_config.py +++ b/server/szurubooru/tests/search/configs/test_pool_search_config.py @@ -136,8 +136,6 @@ def test_escaping( ) db.session.flush() - if db_driver and db.session.get_bind().driver != db_driver: - pytest.xfail() if expected_pool_names is None: with pytest.raises(errors.SearchError): executor.execute(input, offset=0, limit=100) diff --git a/server/szurubooru/tests/search/configs/test_tag_search_config.py b/server/szurubooru/tests/search/configs/test_tag_search_config.py index 8175b73c..9fe9a80e 100644 --- a/server/szurubooru/tests/search/configs/test_tag_search_config.py +++ b/server/szurubooru/tests/search/configs/test_tag_search_config.py @@ -134,8 +134,6 @@ def test_escaping(executor, tag_factory, input, expected_tag_names, db_driver): ) db.session.flush() - if db_driver and db.session.get_bind().driver != db_driver: - pytest.xfail() if expected_tag_names is None: with pytest.raises(errors.SearchError): executor.execute(input, offset=0, limit=100) From d699979d359949789be27b7927b00cacefef1f21 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Thu, 23 Sep 2021 12:38:08 -0400 Subject: [PATCH 06/48] client+server: cleanup GitHub actions workflow names Also run unit test action on push --- .github/workflows/build-containers.yml | 4 +++- .github/workflows/run-unit-tests.yml | 24 ++++++++++++++++++------ .pre-commit-config.yaml | 11 ----------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 518bf5bc..3dda368f 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -1,7 +1,8 @@ -name: build-containers +name: Build Docker containers on: [push] jobs: build-client: + name: Build and push client/ Docker container runs-on: ubuntu-latest steps: - name: Login to Docker Hub @@ -53,6 +54,7 @@ jobs: ./client build-server: + name: Build and push server/ Docker container runs-on: ubuntu-latest steps: - name: Login to Docker Hub diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 87ab4e85..9ea14ed5 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -1,16 +1,28 @@ -name: run-unit-tests -on: [pull_request] +name: Run unit tests +on: [push, pull_request] jobs: test-server: + name: Run pytest for server/ runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Build test container - run: | - TAG=$(docker build --target testing -q ./server) - echo "image_tag=${TAG}" >> $GITHUB_ENV + run: > + docker buildx build --load + --platform linux/amd64 --target testing + -t test_container + ./server - name: Run unit tests - run: docker run --rm -t ${{ env.image_tag }} --color=no szurubooru/ + run: > + docker run --rm -t test_container + --color=no + --cov-report=term-missing:skip-covered + --cov=szurubooru + szurubooru/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5053b6e0..c793e753 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -83,17 +83,6 @@ repos: files: server/szurubooru/ exclude: server/szurubooru/migrations/ pass_filenames: false - stages: [push] - - - id: pytest-cov - name: pytest - entry: bash -c 'docker run --rm -t $(docker build --target testing -q server/) --cov-report=term-missing:skip-covered --cov=szurubooru szurubooru/' - language: system - types: [python] - files: server/szurubooru/ - exclude: server/szurubooru/migrations/ - pass_filenames: false - verbose: true stages: [manual] fail_fast: true From a5fbaae4b3fdd51486dc92199d8dad3b89ece5a7 Mon Sep 17 00:00:00 2001 From: skybldev Date: Sun, 28 Nov 2021 10:07:04 -0500 Subject: [PATCH 07/48] updated build files - is no longer valid as per https://github.com/npm/cli/wiki/Support-Policy#long-term-support-lts - updated pre-commit config to use latest repos --- .pre-commit-config.yaml | 23 ++++++++++++----------- client/Dockerfile | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c793e753..c2e4d53e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,29 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.0.1 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: mixed-line-ending + - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.9 + rev: v1.1.10 hooks: - id: remove-tabs - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.11b1 hooks: - id: black files: 'server/' types: [python] - language_version: python3.8 + language_version: python3.9 -- repo: https://github.com/timothycrosley/isort - rev: '5.4.2' +- repo: https://github.com/PyCQA/isort + rev: '5.10.1' hooks: - id: isort files: 'server/' @@ -31,8 +32,8 @@ repos: additional_dependencies: - toml -- repo: https://github.com/prettier/prettier - rev: '2.1.1' +- repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.5.0 hooks: - id: prettier files: client/js/ @@ -40,7 +41,7 @@ repos: args: ['--config', 'client/.prettierrc.yml'] - repo: https://github.com/pre-commit/mirrors-eslint - rev: v7.8.0 + rev: v8.3.0 hooks: - id: eslint files: client/js/ @@ -48,8 +49,8 @@ repos: additional_dependencies: - eslint-config-prettier -- repo: https://gitlab.com/pycqa/flake8 - rev: '3.8.3' +- repo: https://gitlab.com/PyCQA/flake8 + rev: '4.0.1' hooks: - id: flake8 files: server/szurubooru/ diff --git a/client/Dockerfile b/client/Dockerfile index 3ab0016f..e105e845 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -2,7 +2,7 @@ FROM --platform=$BUILDPLATFORM node:lts as builder WORKDIR /opt/app COPY package.json package-lock.json ./ -RUN npm install -g npm@lts +RUN npm install -g npm RUN npm install COPY . ./ From 98c0941c97e1019f526e04a2b37419ff5bfc6381 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 29 Nov 2021 18:09:56 -0500 Subject: [PATCH 08/48] client/docker: Do not pin LTS version of Node See: https://github.com/npm/cli/wiki/Support-Policy#long-term-support-lts --- client/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/client/Dockerfile b/client/Dockerfile index 3ab0016f..ea5151fa 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -2,7 +2,6 @@ FROM --platform=$BUILDPLATFORM node:lts as builder WORKDIR /opt/app COPY package.json package-lock.json ./ -RUN npm install -g npm@lts RUN npm install COPY . ./ From 9b3123a8150faee660b4311d9d8c2c85a83b32b5 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 29 Nov 2021 18:39:34 -0500 Subject: [PATCH 09/48] server: fix python docstring formatting --- server/szurubooru/facade.py | 2 +- server/szurubooru/func/auth.py | 8 ++++---- server/szurubooru/func/net.py | 2 +- server/szurubooru/func/util.py | 6 +++--- server/szurubooru/middleware/authenticator.py | 6 +++--- server/szurubooru/rest/app.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/szurubooru/facade.py b/server/szurubooru/facade.py index a7e48449..4c8084f6 100644 --- a/server/szurubooru/facade.py +++ b/server/szurubooru/facade.py @@ -135,7 +135,7 @@ _live_migrations = ( def create_app() -> Callable[[Any, Any], Any]: - """ Create a WSGI compatible App object. """ + """Create a WSGI compatible App object.""" validate_config() coloredlogs.install(fmt="[%(asctime)-15s] %(name)s %(message)s") if config.config["debug"]: diff --git a/server/szurubooru/func/auth.py b/server/szurubooru/func/auth.py index d0137756..17d25f70 100644 --- a/server/szurubooru/func/auth.py +++ b/server/szurubooru/func/auth.py @@ -25,7 +25,7 @@ RANK_MAP = OrderedDict( def get_password_hash(salt: str, password: str) -> Tuple[str, int]: - """ Retrieve argon2id password hash. """ + """Retrieve argon2id password hash.""" return ( pwhash.argon2id.str( (config.config["secret"] + salt + password).encode("utf8") @@ -37,7 +37,7 @@ def get_password_hash(salt: str, password: str) -> Tuple[str, int]: def get_sha256_legacy_password_hash( salt: str, password: str ) -> Tuple[str, int]: - """ Retrieve old-style sha256 password hash. """ + """Retrieve old-style sha256 password hash.""" digest = hashlib.sha256() digest.update(config.config["secret"].encode("utf8")) digest.update(salt.encode("utf8")) @@ -46,7 +46,7 @@ def get_sha256_legacy_password_hash( def get_sha1_legacy_password_hash(salt: str, password: str) -> Tuple[str, int]: - """ Retrieve old-style sha1 password hash. """ + """Retrieve old-style sha1 password hash.""" digest = hashlib.sha1() digest.update(b"1A2/$_4xVa") digest.update(salt.encode("utf8")) @@ -125,7 +125,7 @@ def verify_privilege(user: model.User, privilege_name: str) -> None: def generate_authentication_token(user: model.User) -> str: - """ Generate nonguessable challenge (e.g. links in password reminder). """ + """Generate nonguessable challenge (e.g. links in password reminder).""" assert user digest = hashlib.md5() digest.update(config.config["secret"].encode("utf8")) diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py index 3f085a0e..c53a62eb 100644 --- a/server/szurubooru/func/net.py +++ b/server/szurubooru/func/net.py @@ -39,7 +39,7 @@ def download(url: str, use_video_downloader: bool = False) -> bytes: length_tally = 0 try: with urllib.request.urlopen(request) as handle: - while (chunk := handle.read(_dl_chunk_size)) : + while chunk := handle.read(_dl_chunk_size): length_tally += len(chunk) if length_tally > config.config["max_dl_filesize"]: raise DownloadTooLargeError( diff --git a/server/szurubooru/func/util.py b/server/szurubooru/func/util.py index f8391365..453e1213 100644 --- a/server/szurubooru/func/util.py +++ b/server/szurubooru/func/util.py @@ -83,12 +83,12 @@ def flip(source: Dict[Any, Any]) -> Dict[Any, Any]: def is_valid_email(email: Optional[str]) -> bool: - """ Return whether given email address is valid or empty. """ + """Return whether given email address is valid or empty.""" return not email or re.match(r"^[^@]*@[^@]*\.[^@]*$", email) is not None class dotdict(dict): - """ dot.notation access to dictionary attributes. """ + """dot.notation access to dictionary attributes.""" def __getattr__(self, attr: str) -> Any: return self.get(attr) @@ -98,7 +98,7 @@ class dotdict(dict): def parse_time_range(value: str) -> Tuple[datetime, datetime]: - """ Return tuple containing min/max time for given text representation. """ + """Return tuple containing min/max time for given text representation.""" one_day = timedelta(days=1) one_second = timedelta(seconds=1) almost_one_day = one_day - one_second diff --git a/server/szurubooru/middleware/authenticator.py b/server/szurubooru/middleware/authenticator.py index e73b235e..436543b1 100644 --- a/server/szurubooru/middleware/authenticator.py +++ b/server/szurubooru/middleware/authenticator.py @@ -7,7 +7,7 @@ from szurubooru.rest.errors import HttpBadRequest def _authenticate_basic_auth(username: str, password: str) -> model.User: - """ Try to authenticate user. Throw AuthError for invalid users. """ + """Try to authenticate user. Throw AuthError for invalid users.""" user = users.get_user_by_name(username) if not auth.is_valid_password(user, password): raise errors.AuthError("Invalid password.") @@ -17,7 +17,7 @@ def _authenticate_basic_auth(username: str, password: str) -> model.User: def _authenticate_token( username: str, token: str ) -> Tuple[model.User, model.UserToken]: - """ Try to authenticate user. Throw AuthError for invalid users. """ + """Try to authenticate user. Throw AuthError for invalid users.""" user = users.get_user_by_name(username) user_token = user_tokens.get_by_user_and_token(user, token) if not auth.is_valid_token(user_token): @@ -72,7 +72,7 @@ def _get_user(ctx: rest.Context, bump_login: bool) -> Optional[model.User]: def process_request(ctx: rest.Context) -> None: - """ Bind the user to request. Update last login time if needed. """ + """Bind the user to request. Update last login time if needed.""" bump_login = ctx.get_param_as_bool("bump-login", default=False) auth_user = _get_user(ctx, bump_login) if auth_user: diff --git a/server/szurubooru/rest/app.py b/server/szurubooru/rest/app.py index a6f10fbc..c098bd04 100644 --- a/server/szurubooru/rest/app.py +++ b/server/szurubooru/rest/app.py @@ -11,7 +11,7 @@ from szurubooru.rest import context, errors, middleware, routes def _json_serializer(obj: Any) -> str: - """ JSON serializer for objects not serializable by default JSON code """ + """JSON serializer for objects not serializable by default JSON code""" if isinstance(obj, datetime): serial = obj.isoformat("T") + "Z" return serial From 9f95e9eb90258f77518f7f5d3d7421333ba28e34 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 29 Nov 2021 18:44:20 -0500 Subject: [PATCH 10/48] client: linting --- client/js/controllers/user_controller.js | 5 ++--- client/js/controls/expander_control.js | 5 ++--- client/js/controls/post_edit_sidebar_control.js | 5 ++--- client/js/controls/post_notes_overlay_control.js | 5 ++--- client/js/util/markdown.js | 8 ++------ client/js/views/post_main_view.js | 5 ++--- client/js/views/post_upload_view.js | 4 ++-- client/js/views/posts_header_view.js | 5 ++--- client/js/views/user_tokens_view.js | 10 ++++------ 9 files changed, 20 insertions(+), 32 deletions(-) diff --git a/client/js/controllers/user_controller.js b/client/js/controllers/user_controller.js index 326736b5..068d329e 100644 --- a/client/js/controllers/user_controller.js +++ b/client/js/controllers/user_controller.js @@ -31,9 +31,8 @@ class UserController { userTokenPromise = UserToken.get(userName).then( (userTokens) => { return userTokens.map((token) => { - token.isCurrentAuthToken = api.isCurrentAuthToken( - token - ); + token.isCurrentAuthToken = + api.isCurrentAuthToken(token); return token; }); }, diff --git a/client/js/controls/expander_control.js b/client/js/controls/expander_control.js index 11ad3ef5..ffb0e904 100644 --- a/client/js/controls/expander_control.js +++ b/client/js/controls/expander_control.js @@ -45,9 +45,8 @@ class ExpanderControl { // eslint-disable-next-line accessor-pairs set title(newTitle) { if (this._expanderNode) { - this._expanderNode.querySelector( - "header span" - ).textContent = newTitle; + this._expanderNode.querySelector("header span").textContent = + newTitle; } } diff --git a/client/js/controls/post_edit_sidebar_control.js b/client/js/controls/post_edit_sidebar_control.js index b8ad9dab..eabb98ae 100644 --- a/client/js/controls/post_edit_sidebar_control.js +++ b/client/js/controls/post_edit_sidebar_control.js @@ -203,9 +203,8 @@ class PostEditSidebarControl extends events.EventTarget { ); if (this._formNode) { - const inputNodes = this._formNode.querySelectorAll( - "input, textarea" - ); + const inputNodes = + this._formNode.querySelectorAll("input, textarea"); for (let node of inputNodes) { node.addEventListener("change", (e) => this.dispatchEvent(new CustomEvent("change")) diff --git a/client/js/controls/post_notes_overlay_control.js b/client/js/controls/post_notes_overlay_control.js index 030f7f28..e9aad045 100644 --- a/client/js/controls/post_notes_overlay_control.js +++ b/client/js/controls/post_notes_overlay_control.js @@ -727,9 +727,8 @@ class PostNotesOverlayControl extends events.EventTarget { } _showNoteText(note) { - this._textNode.querySelector( - ".wrapper" - ).innerHTML = misc.formatMarkdown(note.text); + this._textNode.querySelector(".wrapper").innerHTML = + misc.formatMarkdown(note.text); this._textNode.style.display = "block"; const bodyRect = document.body.getBoundingClientRect(); const noteRect = this._textNode.getBoundingClientRect(); diff --git a/client/js/util/markdown.js b/client/js/util/markdown.js index 22cdae50..7bd4d584 100644 --- a/client/js/util/markdown.js +++ b/client/js/util/markdown.js @@ -136,12 +136,8 @@ function createRenderer() { const renderer = new marked.Renderer(); renderer.image = (href, title, alt) => { - let [ - _, - url, - width, - height, - ] = /^(.+?)(?:\s=\s*(\d*)\s*x\s*(\d*)\s*)?$/.exec(href); + let [_, url, width, height] = + /^(.+?)(?:\s=\s*(\d*)\s*x\s*(\d*)\s*)?$/.exec(href); let res = '' + sanitize(alt); if (width) { res += ' Date: Mon, 29 Nov 2021 20:06:20 -0500 Subject: [PATCH 11/48] client/upload: restore option to pause upload chain on error --- client/css/post-upload.styl | 5 +++++ client/html/post_upload.tpl | 12 ++++++++++-- client/js/controllers/post_upload_controller.js | 17 +++++++++++++---- client/js/views/post_upload_view.js | 10 +++++++++- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/client/css/post-upload.styl b/client/css/post-upload.styl index 38362937..ea79fcac 100644 --- a/client/css/post-upload.styl +++ b/client/css/post-upload.styl @@ -14,9 +14,11 @@ $cancel-button-color = tomato &.inactive input[type=submit], &.inactive .skip-duplicates &.inactive .always-upload-similar + &.inactive .pause-remain-on-error &.uploading input[type=submit], &.uploading .skip-duplicates, &.uploading .always-upload-similar + &.uploading .pause-remain-on-error &:not(.uploading) .cancel display: none @@ -44,6 +46,9 @@ $cancel-button-color = tomato .always-upload-similar margin-left: 1em + .pause-remain-on-error + margin-left: 1em + form>.messages margin-top: 1em diff --git a/client/html/post_upload.tpl b/client/html/post_upload.tpl index 6374fe8c..3c1b2388 100644 --- a/client/html/post_upload.tpl +++ b/client/html/post_upload.tpl @@ -7,7 +7,7 @@ <%= ctx.makeCheckbox({ - text: 'Skip duplicates', + text: 'Skip duplicate', name: 'skip-duplicates', checked: false, }) %> @@ -15,12 +15,20 @@ <%= ctx.makeCheckbox({ - text: 'Always upload similar', + text: 'Force upload similar', name: 'always-upload-similar', checked: false, }) %> + + <%= ctx.makeCheckbox({ + text: 'Pause on error', + name: 'pause-remain-on-error', + checked: true, + }) %> + + diff --git a/client/js/controllers/post_upload_controller.js b/client/js/controllers/post_upload_controller.js index a54baec7..720a116c 100644 --- a/client/js/controllers/post_upload_controller.js +++ b/client/js/controllers/post_upload_controller.js @@ -90,21 +90,30 @@ class PostUploadController { uploadable ); } + if (e.detail.pauseRemainOnError) { + return Promise.reject(); + } }) ), Promise.resolve() ) .then(() => { if (anyFailures) { - this._view.showError(genericErrorMessage); - this._view.enableForm(); - } else { + return Promise.reject(); + } + }) + .then( + () => { this._view.clearMessages(); misc.disableExitConfirmation(); const ctx = router.show(uri.formatClientLink("posts")); ctx.controller.showSuccess("Posts uploaded."); + }, + (error) => { + this._view.showError(genericErrorMessage); + this._view.enableForm(); } - }); + ); } _uploadSinglePost(uploadable, skipDuplicates, alwaysUploadSimilar) { diff --git a/client/js/views/post_upload_view.js b/client/js/views/post_upload_view.js index 4f0f0bf1..fc98a19e 100644 --- a/client/js/views/post_upload_view.js +++ b/client/js/views/post_upload_view.js @@ -285,7 +285,7 @@ class PostUploadView extends events.EventTarget { for (let uploadable of this._uploadables) { this._updateUploadableFromDom(uploadable); } - this._submitButtonNode.value = "Resume upload"; + this._submitButtonNode.value = "Resume"; this._emit("submit"); } @@ -362,6 +362,8 @@ class PostUploadView extends events.EventTarget { skipDuplicates: this._skipDuplicatesCheckboxNode.checked, alwaysUploadSimilar: this._alwaysUploadSimilarCheckboxNode.checked, + pauseRemainOnError: + this._pauseRemainOnErrorCheckboxNode.checked, }, }) ); @@ -431,6 +433,12 @@ class PostUploadView extends events.EventTarget { ); } + get _pauseRemainOnErrorCheckboxNode() { + return this._hostNode.querySelector( + "form [name=pause-remain-on-error]" + ); + } + get _submitButtonNode() { return this._hostNode.querySelector("form [type=submit]"); } From a14ead1842ed2058a6baf64e85ceb3b294e01ae9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jan 2022 01:05:54 +0000 Subject: [PATCH 12/48] build(deps): bump marked from 0.7.0 to 4.0.10 in /client Bumps [marked](https://github.com/markedjs/marked) from 0.7.0 to 4.0.10. - [Release notes](https://github.com/markedjs/marked/releases) - [Changelog](https://github.com/markedjs/marked/blob/master/.releaserc.json) - [Commits](https://github.com/markedjs/marked/compare/v0.7.0...v4.0.10) --- updated-dependencies: - dependency-name: marked dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 18 +++++++++--------- client/package.json | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 14f94096..bd9c668a 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -10,7 +10,7 @@ "font-awesome": "^4.7.0", "ios-inner-height": "^1.0.3", "js-cookie": "^2.2.0", - "marked": "^0.7.0", + "marked": "^4.0.10", "mousetrap": "^1.6.2", "nprogress": "^0.2.0", "superagent": "^3.8.3" @@ -2997,14 +2997,14 @@ "dev": true }, "node_modules/marked": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.7.0.tgz", - "integrity": "sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.0.10.tgz", + "integrity": "sha512-+QvuFj0nGgO970fySghXGmuw+Fd0gD2x3+MqCWLIPf5oxdv1Ka6b2q+z9RP01P/IaKPMEramy+7cNy/Lw8c3hw==", "bin": { - "marked": "bin/marked" + "marked": "bin/marked.js" }, "engines": { - "node": ">=0.10.0" + "node": ">= 12" } }, "node_modules/md5.js": { @@ -7280,9 +7280,9 @@ "dev": true }, "marked": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.7.0.tgz", - "integrity": "sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==" + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.0.10.tgz", + "integrity": "sha512-+QvuFj0nGgO970fySghXGmuw+Fd0gD2x3+MqCWLIPf5oxdv1Ka6b2q+z9RP01P/IaKPMEramy+7cNy/Lw8c3hw==" }, "md5.js": { "version": "1.3.4", diff --git a/client/package.json b/client/package.json index a83a9cb4..9d0005c4 100644 --- a/client/package.json +++ b/client/package.json @@ -10,7 +10,7 @@ "font-awesome": "^4.7.0", "ios-inner-height": "^1.0.3", "js-cookie": "^2.2.0", - "marked": "^0.7.0", + "marked": "^4.0.10", "mousetrap": "^1.6.2", "nprogress": "^0.2.0", "superagent": "^3.8.3" From 106dcc41356fdf7671a3b2a98a5fe3023c0e7af9 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Sun, 16 Jan 2022 11:07:46 -0500 Subject: [PATCH 13/48] server/func/images: Do not pass file content to ffmpeg stdin --- server/szurubooru/func/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/szurubooru/func/images.py b/server/szurubooru/func/images.py index de41222f..4d4011fe 100644 --- a/server/szurubooru/func/images.py +++ b/server/szurubooru/func/images.py @@ -277,10 +277,10 @@ class Image: proc = subprocess.Popen( cli, stdout=subprocess.PIPE, - stdin=subprocess.PIPE, + stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, ) - out, err = proc.communicate(input=self.content) + out, err = proc.communicate() if proc.returncode != 0: logger.warning( "Failed to execute ffmpeg command (cli=%r, err=%r)", From d5a6609f754b62536a1d884b0d488adab3ee218e Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Wed, 6 Oct 2021 14:59:17 +0200 Subject: [PATCH 14/48] client: remove URL rewriting from the markdown handler --- client/js/util/markdown.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/client/js/util/markdown.js b/client/js/util/markdown.js index 7bd4d584..6d821348 100644 --- a/client/js/util/markdown.js +++ b/client/js/util/markdown.js @@ -65,17 +65,6 @@ class TagPermalinkFixWrapper extends BaseMarkdownWrapper { // post, user and tags permalinks class EntityPermalinkWrapper extends BaseMarkdownWrapper { preprocess(text) { - // URL-based permalinks - text = text.replace(new RegExp("\\b/post/(\\d+)/?\\b", "g"), "@$1"); - text = text.replace( - new RegExp("\\b/tag/([a-zA-Z0-9_-]+?)/?", "g"), - "#$1" - ); - text = text.replace( - new RegExp("\\b/user/([a-zA-Z0-9_-]+?)/?", "g"), - "+$1" - ); - text = text.replace( /(^|^\(|(?:[^\]])\(|[\s<>\[\]\)])([+#@][a-zA-Z0-9_-]+)/g, "$1[$2]($2)" From e2419a30badd27a429bb4e514640fb8aa85ed47f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jan 2022 14:29:29 +0000 Subject: [PATCH 15/48] build(deps): bump cached-path-relative from 1.0.2 to 1.1.0 in /client Bumps [cached-path-relative](https://github.com/ashaffer/cached-path-relative) from 1.0.2 to 1.1.0. - [Release notes](https://github.com/ashaffer/cached-path-relative/releases) - [Commits](https://github.com/ashaffer/cached-path-relative/commits) --- updated-dependencies: - dependency-name: cached-path-relative dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 14f94096..f123052e 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1506,9 +1506,9 @@ "dev": true }, "node_modules/cached-path-relative": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.0.2.tgz", - "integrity": "sha512-5r2GqsoEb4qMTTN9J+WzXfjov+hjxT+j3u5K+kIVNIwAd99DLCJE9pBIMP1qVeybV6JiijL385Oz0DcYxfbOIg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.1.0.tgz", + "integrity": "sha512-WF0LihfemtesFcJgO7xfOoOcnWzY/QHR4qeDqV44jPU3HTI54+LnfXK3SA27AVVGCdZFgjjFFaqUA9Jx7dMJZA==", "dev": true }, "node_modules/call-bind": { @@ -3983,12 +3983,6 @@ "minimist": "^1.1.0" } }, - "node_modules/subarg/node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true - }, "node_modules/superagent": { "version": "3.8.3", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.3.tgz", @@ -6053,9 +6047,9 @@ "dev": true }, "cached-path-relative": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.0.2.tgz", - "integrity": "sha512-5r2GqsoEb4qMTTN9J+WzXfjov+hjxT+j3u5K+kIVNIwAd99DLCJE9pBIMP1qVeybV6JiijL385Oz0DcYxfbOIg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.1.0.tgz", + "integrity": "sha512-WF0LihfemtesFcJgO7xfOoOcnWzY/QHR4qeDqV44jPU3HTI54+LnfXK3SA27AVVGCdZFgjjFFaqUA9Jx7dMJZA==", "dev": true }, "call-bind": { @@ -8116,14 +8110,6 @@ "dev": true, "requires": { "minimist": "^1.1.0" - }, - "dependencies": { - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true - } } }, "superagent": { From a22485afda0a37ac1074f5bfea10828c8fef3151 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 7 Feb 2022 12:51:25 -0500 Subject: [PATCH 16/48] server/func/images: upgrade to heif-image-plugin --- server/Dockerfile | 2 +- server/requirements.txt | 18 +++++++++--------- server/szurubooru/func/image_hash.py | 5 +---- server/szurubooru/func/images.py | 2 ++ 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 205c8e4c..aac0a652 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -28,9 +28,9 @@ RUN apk --no-cache add \ && pip3 install --no-cache-dir --disable-pip-version-check \ alembic \ "coloredlogs==5.0" \ + heif-image-plugin \ youtube_dl \ pillow-avif-plugin \ - pyheif-pillow-opener \ && apk --no-cache del py3-pip COPY ./ /opt/app/ diff --git a/server/requirements.txt b/server/requirements.txt index 2a09b24b..6b032d35 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,14 +1,14 @@ alembic>=0.8.5 -pyyaml>=3.11 -psycopg2-binary>=2.6.1 -SQLAlchemy>=1.0.12, <1.4 -coloredlogs==5.0 certifi>=2017.11.5 +coloredlogs==5.0 +heif-image-plugin>=0.3.2 numpy>=1.8.2 -pillow>=4.3.0 -pynacl>=1.2.1 -pytz>=2018.3 -pyRFC3339>=1.0 pillow-avif-plugin>=1.1.0 -pyheif-pillow-opener>=0.1.0 +pillow>=4.3.0 +psycopg2-binary>=2.6.1 +pynacl>=1.2.1 +pyRFC3339>=1.0 +pytz>=2018.3 +pyyaml>=3.11 +SQLAlchemy>=1.0.12, <1.4 youtube_dl diff --git a/server/szurubooru/func/image_hash.py b/server/szurubooru/func/image_hash.py index 05b27a42..76d5a846 100644 --- a/server/szurubooru/func/image_hash.py +++ b/server/szurubooru/func/image_hash.py @@ -4,16 +4,13 @@ from datetime import datetime from io import BytesIO from typing import Any, Callable, List, Optional, Set, Tuple +import HeifImagePlugin import numpy as np import pillow_avif -import pyheif from PIL import Image -from pyheif_pillow_opener import register_heif_opener from szurubooru import config, errors -register_heif_opener() - logger = logging.getLogger(__name__) # Math based on paper from H. Chi Wong, Marshall Bern and David Goldberg diff --git a/server/szurubooru/func/images.py b/server/szurubooru/func/images.py index 4d4011fe..e135d182 100644 --- a/server/szurubooru/func/images.py +++ b/server/szurubooru/func/images.py @@ -7,6 +7,8 @@ import subprocess from io import BytesIO from typing import List +import HeifImagePlugin +import pillow_avif from PIL import Image as PILImage from szurubooru import errors From 6de0a742570058365753266d6b8f43b354de9b11 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 7 Feb 2022 16:44:56 -0500 Subject: [PATCH 17/48] server/config: fix deprecated database string format --- server/szurubooru/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/szurubooru/config.py b/server/szurubooru/config.py index 1515a54f..8f87642f 100644 --- a/server/szurubooru/config.py +++ b/server/szurubooru/config.py @@ -33,7 +33,7 @@ def _docker_config() -> Dict: "show_sql": int(os.getenv("LOG_SQL", 0)), "data_url": os.getenv("DATA_URL", "data/"), "data_dir": "/data/", - "database": "postgres://%(user)s:%(pass)s@%(host)s:%(port)d/%(db)s" + "database": "postgresql://%(user)s:%(pass)s@%(host)s:%(port)d/%(db)s" % { "user": os.getenv("POSTGRES_USER"), "pass": os.getenv("POSTGRES_PASSWORD"), From 8ad9457b2423a6db7ed8fe98533276952ce74a17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Feb 2022 15:00:00 +0000 Subject: [PATCH 18/48] build(deps): bump path-parse from 1.0.6 to 1.0.7 in /client Bumps [path-parse](https://github.com/jbgutierrez/path-parse) from 1.0.6 to 1.0.7. - [Release notes](https://github.com/jbgutierrez/path-parse/releases) - [Commits](https://github.com/jbgutierrez/path-parse/commits/v1.0.7) --- updated-dependencies: - dependency-name: path-parse dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 14f94096..f0ffb342 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -3385,9 +3385,9 @@ } }, "node_modules/path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "node_modules/path-platform": { @@ -7607,9 +7607,9 @@ "dev": true }, "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "path-platform": { From 82541536afd95ebbbb72b1983caae77b67eb8b2d Mon Sep 17 00:00:00 2001 From: noirscape Date: Sat, 12 Feb 2022 22:16:13 +0100 Subject: [PATCH 19/48] Make waitress thread count configurable. This should fix most scaling problems without needing to start more server instances. By default, waitress maintains at most 4 threads. This works fine if the database is small (sub 100k posts) but causes a large Task queue depth to occur if the database is larger. Letting users increase the amount of threads means that one server instance is able to handle more requests without locking up the rest of the site. This adds a new environment variable to .env, THREADS, which can be used to configure the amount of threads to start and is by default set to 4 (the default amount used by waitress). --- doc/example.env | 6 ++++++ docker-compose.yml | 1 + server/Dockerfile | 3 +++ server/docker-start.sh | 4 ++-- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/example.env b/doc/example.env index 59e1e859..303a25e6 100644 --- a/doc/example.env +++ b/doc/example.env @@ -10,6 +10,12 @@ BUILD_INFO=latest # otherwise the port specified here will be publicly accessible PORT=8080 +# How many waitress threads to start +# 4 is the default amount of threads. If you experience performance +# degradation with a large number of posts, increasing this may +# improve performance, since waitress is most likely clogging up with Tasks. +THREADS=4 + # URL base to run szurubooru under # See "Additional Features" section in INSTALL.md BASE_URL=/ diff --git a/docker-compose.yml b/docker-compose.yml index 1da23bd6..38e08b97 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: #POSTGRES_DB: defaults to same as POSTGRES_USER #POSTGRES_PORT: 5432 #LOG_SQL: 0 (1 for verbose SQL logs) + THREADS: volumes: - "${MOUNT_DATA}:/data" - "./server/config.yaml:/opt/app/config.yaml" diff --git a/server/Dockerfile b/server/Dockerfile index aac0a652..a13e2301 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -83,6 +83,9 @@ ARG PORT=6666 ENV PORT=${PORT} EXPOSE ${PORT} +ARG THREADS=4 +ENV THREADS=${THREADS} + VOLUME ["/data/"] ARG DOCKER_REPO diff --git a/server/docker-start.sh b/server/docker-start.sh index 34a0e498..eebef1c7 100755 --- a/server/docker-start.sh +++ b/server/docker-start.sh @@ -4,5 +4,5 @@ cd /opt/app alembic upgrade head -echo "Starting szurubooru API on port ${PORT}" -exec waitress-serve-3 --port ${PORT} szurubooru.facade:app +echo "Starting szurubooru API on port ${PORT} - Running on ${THREADS} threads" +exec waitress-serve-3 --port ${PORT} --threads ${THREADS} szurubooru.facade:app From 514b8467816a16d1e3388b73c0646e6178b7a17f Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Wed, 16 Feb 2022 09:09:21 -0500 Subject: [PATCH 20/48] client/js/markdown: fix processing of inline markdown --- client/.eslintrc.yml | 1 - client/js/util/markdown.js | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client/.eslintrc.yml b/client/.eslintrc.yml index 6c28a2fe..a9d703ff 100644 --- a/client/.eslintrc.yml +++ b/client/.eslintrc.yml @@ -10,4 +10,3 @@ ignorePatterns: - build.js parserOptions: ecmaVersion: 11 - sourceType: module diff --git a/client/js/util/markdown.js b/client/js/util/markdown.js index ab31a7f0..e71e3263 100644 --- a/client/js/util/markdown.js +++ b/client/js/util/markdown.js @@ -1,6 +1,6 @@ "use strict"; -import { marked } from "marked"; +const marked = require("marked"); const DOMPurify = require("dompurify"); class BaseMarkdownWrapper { @@ -159,7 +159,7 @@ function formatMarkdown(text) { for (let wrapper of wrappers) { text = wrapper.preprocess(text); } - text = marked(text, options); + text = marked.parse(text, options); wrappers.reverse(); for (let wrapper of wrappers) { text = wrapper.postprocess(text); @@ -185,7 +185,7 @@ function formatInlineMarkdown(text) { for (let wrapper of wrappers) { text = wrapper.preprocess(text); } - text = marked.inlineLexer(text, [], options); + text = marked.parseInline(text, options); wrappers.reverse(); for (let wrapper of wrappers) { text = wrapper.postprocess(text); From 929071ea1a57dfd8ba48d487b4465418b633a04c Mon Sep 17 00:00:00 2001 From: Maksymilian Babarowski Date: Mon, 21 Mar 2022 14:24:40 +0100 Subject: [PATCH 21/48] doc: fix external link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a86ef795..93070557 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Szurubooru is an image board engine inspired by services such as Danbooru, Gelbooru and Moebooru dedicated for small and medium communities. Its name [has its roots in Polish language and has onomatopeic meaning of scraping or -scrubbing](http://sjp.pwn.pl/sjp/;2527372). It is pronounced as *shoorubooru*. +scrubbing](https://sjp.pwn.pl/sjp/;2527372). It is pronounced as *shoorubooru*. ## Features From 79d0efc25b42d5c750743bd9c0d6c0d18df8e147 Mon Sep 17 00:00:00 2001 From: Skybbles Date: Mon, 28 Mar 2022 16:17:56 -0400 Subject: [PATCH 22/48] doc: added BuildKit flags fix to INSTALL.md Added this because recently, there have been more problems with `docker-compose build` where it errors: ERROR: Service 'server' failed to build: failed to parse platform : "" is an invalid component of "": platform specifier component must match "^[A-Za-z0-9_-]+$": invalid argument Recent Docker versions have switched to using `buildx` (BuildKit) to build containers, but that needs to be enabled, either in `daemon.json` or through an environment variable. But since we are using Docker Compose, it doesn't pass it to Docker; so the environment variable needs to be set. At least that's what I've heard and figured out sweat_smile My explanation might be very wrong - but it works :) --- doc/INSTALL.md | 90 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/doc/INSTALL.md b/doc/INSTALL.md index d978e4a8..ca0212bf 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -34,33 +34,79 @@ and Docker Compose (version 1.6.0 or greater) already installed. Read the comments to guide you. Note that `.env` should be in the root directory of this repository. -### Running the Application +4. Pull the containers: -Download containers: -```console -user@host:szuru$ docker-compose pull -``` + This pulls the latest containers from docker.io: + ```console + user@host:szuru$ docker-compose pull + ``` -For first run, it is recommended to start the database separately: -```console -user@host:szuru$ docker-compose up -d sql -``` + If you have modified the application's source and would like to manually + build it, follow the instructions in [**Building**](#Building) instead, + then read here once you're done. -To start all containers: -```console -user@host:szuru$ docker-compose up -d -``` +5. Run it! -To view/monitor the application logs: -```console -user@host:szuru$ docker-compose logs -f -# (CTRL+C to exit) -``` + For first run, it is recommended to start the database separately: + ```console + user@host:szuru$ docker-compose up -d sql + ``` + + To start all containers: + ```console + user@host:szuru$ docker-compose up -d + ``` + + To view/monitor the application logs: + ```console + user@host:szuru$ docker-compose logs -f + # (CTRL+C to exit) + ``` + +### Building + +1. Edit `docker-compose.yml` to tell Docker to build instead of pull containers: + + ```diff yaml + ... + server: + - image: szurubooru/server:latest + + build: server + ... + client: + - image: szurubooru/client:latest + + build: client + ... + ``` + + You can choose to build either one from source. + +2. Build the containers: + + ```console + user@host:szuru$ docker-compose build + ``` + + That will attempt to build both containers, but you can specify `client` + or `server` to make it build only one. + + If `docker-compose build` spits out: + + ``` + ERROR: Service 'server' failed to build: failed to parse platform : "" is an invalid component of "": platform specifier component must match "^[A-Za-z0-9_-]+$": invalid argument + ``` + + ...you will need to export Docker BuildKit flags: + + ```console + user@host:szuru$ export DOCKER_BUILDKIT=1; export COMPOSE_DOCKER_CLI_BUILD=1 + ``` + + ...and run `docker-compose build` again. + +*Note: If your changes are not taking effect in your builds, consider building +with `--no-cache`.* -To stop all containers: -```console -user@host:szuru$ docker-compose down -``` ### Additional Features From 6088e89ea1258ddd7686370355ca2ce4a026f759 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Wed, 30 Mar 2022 23:04:16 -0400 Subject: [PATCH 23/48] server/szuru-admin: Add thumbnail regeneration script Closes #467 --- .pre-commit-config.yaml | 2 +- server/szuru-admin | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2e4d53e..7b550ca1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: remove-tabs - repo: https://github.com/psf/black - rev: 21.11b1 + rev: '22.3.0' hooks: - id: black files: 'server/' diff --git a/server/szuru-admin b/server/szuru-admin index 004a7517..08ba1827 100755 --- a/server/szuru-admin +++ b/server/szuru-admin @@ -91,6 +91,15 @@ def reset_filenames() -> None: rename_in_dir("posts/custom-thumbnails/") +def regenerate_thumbnails() -> None: + for post in db.session.query(model.Post).all(): + print("Generating tumbnail for post %d ..." % post.post_id, end="\r") + try: + postfuncs.generate_post_thumbnail(post) + except Exception: + pass + + def main() -> None: parser_top = ArgumentParser( description="Collection of CLI commands for an administrator to use", @@ -114,6 +123,12 @@ def main() -> None: help="reset and rename the content and thumbnail " "filenames in case of a lost/changed secret key", ) + parser.add_argument( + "--regenerate-thumbnails", + action="store_true", + help="regenerate the thumbnails for posts if the " + "thumbnail files are missing", + ) command = parser_top.parse_args() try: @@ -123,6 +138,8 @@ def main() -> None: check_audio() elif command.reset_filenames: reset_filenames() + elif command.regenerate_thumbnails: + regenerate_thumbnails() except errors.BaseError as e: print(e, file=stderr) From e746f09911cbe1c94055296310a96ab057daecdb Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Thu, 31 Mar 2022 18:43:37 -0400 Subject: [PATCH 24/48] server: fix build error due to broken pip requirements Pinned pyheif to v0.6.1 --- server/Dockerfile | 20 ++++++++++---------- server/requirements.txt | 3 ++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index a13e2301..487f1923 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -7,8 +7,13 @@ WORKDIR /opt/app RUN apk --no-cache add \ python3 \ python3-dev \ - ffmpeg \ py3-pip \ + build-base \ + libheif \ + libheif-dev \ + libavif \ + libavif-dev \ + ffmpeg \ # from requirements.txt: py3-yaml \ py3-psycopg2 \ @@ -19,18 +24,13 @@ RUN apk --no-cache add \ py3-pynacl \ py3-tz \ py3-pyrfc3339 \ - build-base \ - && apk --no-cache add \ - libheif \ - libavif \ - libheif-dev \ - libavif-dev \ && pip3 install --no-cache-dir --disable-pip-version-check \ - alembic \ + "alembic>=0.8.5" \ "coloredlogs==5.0" \ - heif-image-plugin \ + "pyheif==0.6.1" \ + "heif-image-plugin>=0.3.2" \ youtube_dl \ - pillow-avif-plugin \ + "pillow-avif-plugin>=1.1.0" \ && apk --no-cache del py3-pip COPY ./ /opt/app/ diff --git a/server/requirements.txt b/server/requirements.txt index 6b032d35..16b29fff 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,11 +1,12 @@ alembic>=0.8.5 certifi>=2017.11.5 coloredlogs==5.0 -heif-image-plugin>=0.3.2 +heif-image-plugin==0.3.2 numpy>=1.8.2 pillow-avif-plugin>=1.1.0 pillow>=4.3.0 psycopg2-binary>=2.6.1 +pyheif==0.6.1 pynacl>=1.2.1 pyRFC3339>=1.0 pytz>=2018.3 From 7e5d48b6e85c651283176b503e2d84f207c72926 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 22:45:47 +0000 Subject: [PATCH 25/48] build(deps): bump minimist from 1.2.5 to 1.2.6 in /client Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 76497611..2e213de6 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -3108,9 +3108,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "node_modules/mkdirp": { @@ -7358,9 +7358,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "mkdirp": { From 1b9ce79f4e79745d7992eef14e3ab4104bd74fe4 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Thu, 31 Mar 2022 18:54:08 -0400 Subject: [PATCH 26/48] client+server: only trigger autobuild on master branch pushes --- .github/workflows/build-containers.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 3dda368f..c688a5f9 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -1,5 +1,8 @@ name: Build Docker containers -on: [push] +on: + push: + branches: + - master jobs: build-client: name: Build and push client/ Docker container From 70f2164dc6c5d9950bb4279eb01ce5c7c33e80a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 22:45:47 +0000 Subject: [PATCH 27/48] build(deps): bump minimist from 1.2.5 to 1.2.6 in /client Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 76497611..2e213de6 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -3108,9 +3108,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "node_modules/mkdirp": { @@ -7358,9 +7358,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "mkdirp": { From 6075ae9326b95fa8c863ca80abca040c6e0d6561 Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Mon, 2 May 2022 12:57:35 +0200 Subject: [PATCH 28/48] all: add .gitattributes This forces shell scripts to always have LF line endings. By default Windows uses CRLF which breaks the docker build, because docker-start.sh doesn't have the correct line endings. Adding this file should fix that. --- .gitattributes | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..48e3403f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Shell scripts require LF +*.sh text eol=lf From 6c3b50d287a8b8ba67d2bb4cf3cffe25bcd3b4e6 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 8 Jun 2022 20:06:09 -0300 Subject: [PATCH 29/48] doc: add GET /post//around to API.md --- doc/API.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/API.md b/doc/API.md index 3d280fd1..63a50a2f 100644 --- a/doc/API.md +++ b/doc/API.md @@ -37,6 +37,7 @@ - [Creating post](#creating-post) - [Updating post](#updating-post) - [Getting post](#getting-post) + - [Getting around post](#getting-around-post) - [Deleting post](#deleting-post) - [Merging posts](#merging-posts) - [Rating post](#rating-post) @@ -951,6 +952,29 @@ data. Retrieves information about an existing post. +## Getting around post +- **Request** + + `GET /post//around` + +- **Output** + + ```json5 + { + "prev": , + "next": + } + ``` + +- **Errors** + + - the post does not exist + - privileges are too low + +- **Description** + + Retrieves information about posts that are before or after an existing post. + ## Deleting post - **Request** From 5b43c5bebd28fc735e524854b00066b8b019007f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jun 2022 01:45:34 +0000 Subject: [PATCH 30/48] build(deps): bump jpeg-js from 0.4.0 to 0.4.4 in /client Bumps [jpeg-js](https://github.com/eugeneware/jpeg-js) from 0.4.0 to 0.4.4. - [Release notes](https://github.com/eugeneware/jpeg-js/releases) - [Commits](https://github.com/eugeneware/jpeg-js/compare/v0.4.0...v0.4.4) --- updated-dependencies: - dependency-name: jpeg-js dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 2e213de6..daf29055 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -2859,9 +2859,9 @@ "dev": true }, "node_modules/jpeg-js": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.0.tgz", - "integrity": "sha512-960VHmtN1vTpasX/1LupLohdP5odwAT7oK/VSm6mW0M58LbrBnowLAPWAZhWGhDAGjzbMnPXZxzB/QYgBwkN0w==", + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", "dev": true }, "node_modules/js-cookie": { @@ -7152,9 +7152,9 @@ } }, "jpeg-js": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.0.tgz", - "integrity": "sha512-960VHmtN1vTpasX/1LupLohdP5odwAT7oK/VSm6mW0M58LbrBnowLAPWAZhWGhDAGjzbMnPXZxzB/QYgBwkN0w==", + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", "dev": true }, "js-cookie": { From 42bb364dd0a9b3d8510c64a30d0dbbf913b936dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 21:38:21 +0000 Subject: [PATCH 31/48] build(deps): bump shell-quote from 1.6.1 to 1.7.3 in /client Bumps [shell-quote](https://github.com/substack/node-shell-quote) from 1.6.1 to 1.7.3. - [Release notes](https://github.com/substack/node-shell-quote/releases) - [Changelog](https://github.com/substack/node-shell-quote/blob/master/CHANGELOG.md) - [Commits](https://github.com/substack/node-shell-quote/compare/1.6.1...1.7.3) --- updated-dependencies: - dependency-name: shell-quote dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 64 +++++----------------------------------- 1 file changed, 8 insertions(+), 56 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 2e213de6..a32dd4b8 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -476,24 +476,6 @@ "node": ">= 8" } }, - "node_modules/array-filter": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/array-filter/-/array-filter-0.0.1.tgz", - "integrity": "sha1-fajPLiZijtcygDWB/SH2fKzS7uw=", - "dev": true - }, - "node_modules/array-map": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/array-map/-/array-map-0.0.0.tgz", - "integrity": "sha1-iKK6tz0c97zVwbEYoAP2b2ZfpmI=", - "dev": true - }, - "node_modules/array-reduce": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/array-reduce/-/array-reduce-0.0.0.tgz", - "integrity": "sha1-FziZ0//Rx9k4PkR5Ul2+J4yrXys=", - "dev": true - }, "node_modules/asn1.js": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz", @@ -3770,16 +3752,10 @@ } }, "node_modules/shell-quote": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.6.1.tgz", - "integrity": "sha1-9HgZSczkAmlxJ0MOo7PFR29IF2c=", - "dev": true, - "dependencies": { - "array-filter": "~0.0.0", - "array-map": "~0.0.0", - "array-reduce": "~0.0.0", - "jsonify": "~0.0.0" - } + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.3.tgz", + "integrity": "sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==", + "dev": true }, "node_modules/simple-concat": { "version": "1.0.0", @@ -5041,24 +5017,6 @@ "picomatch": "^2.0.4" } }, - "array-filter": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/array-filter/-/array-filter-0.0.1.tgz", - "integrity": "sha1-fajPLiZijtcygDWB/SH2fKzS7uw=", - "dev": true - }, - "array-map": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/array-map/-/array-map-0.0.0.tgz", - "integrity": "sha1-iKK6tz0c97zVwbEYoAP2b2ZfpmI=", - "dev": true - }, - "array-reduce": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/array-reduce/-/array-reduce-0.0.0.tgz", - "integrity": "sha1-FziZ0//Rx9k4PkR5Ul2+J4yrXys=", - "dev": true - }, "asn1.js": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz", @@ -7930,16 +7888,10 @@ } }, "shell-quote": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.6.1.tgz", - "integrity": "sha1-9HgZSczkAmlxJ0MOo7PFR29IF2c=", - "dev": true, - "requires": { - "array-filter": "~0.0.0", - "array-map": "~0.0.0", - "array-reduce": "~0.0.0", - "jsonify": "~0.0.0" - } + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.3.tgz", + "integrity": "sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==", + "dev": true }, "simple-concat": { "version": "1.0.0", From da71c672ddb9dbe2e3c0d8120ca9d0b8d7ec57a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Jul 2022 01:23:35 +0000 Subject: [PATCH 32/48] build(deps-dev): bump terser from 3.7.7 to 4.8.1 in /client Bumps [terser](https://github.com/terser/terser) from 3.7.7 to 4.8.1. - [Release notes](https://github.com/terser/terser/releases) - [Changelog](https://github.com/terser/terser/blob/master/CHANGELOG.md) - [Commits](https://github.com/terser/terser/commits) --- updated-dependencies: - dependency-name: terser dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 50 ++++++++++++++++++++-------------------- client/package.json | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 2e213de6..c7241a9f 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -28,7 +28,7 @@ "jimp": "^0.13.0", "pretty-error": "^3.0.3", "stylus": "^0.54.8", - "terser": "^3.7.7", + "terser": "^4.8.1", "underscore": "^1.12.1", "watchify": "^4.0.0", "ws": "^7.4.6" @@ -4022,26 +4022,26 @@ } }, "node_modules/terser": { - "version": "3.7.7", - "resolved": "https://registry.npmjs.org/terser/-/terser-3.7.7.tgz", - "integrity": "sha512-RRLIxE7S52vSOI9cEbOaisgBd2y6MNgfg2ihUkidsFnuP1eDmZ79+lBWbyvgfFTAc/r8nSjL0k3cpZDDIYiYiA==", + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.1.tgz", + "integrity": "sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw==", "dev": true, "dependencies": { - "commander": "~2.14.1", + "commander": "^2.20.0", "source-map": "~0.6.1", - "source-map-support": "~0.5.6" + "source-map-support": "~0.5.12" }, "bin": { - "terser": "bin/uglifyjs" + "terser": "bin/terser" }, "engines": { - "node": ">=0.8.0" + "node": ">=6.0.0" } }, "node_modules/terser/node_modules/commander": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz", - "integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, "node_modules/terser/node_modules/source-map": { @@ -4054,9 +4054,9 @@ } }, "node_modules/terser/node_modules/source-map-support": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.6.tgz", - "integrity": "sha512-N4KXEz7jcKqPf2b2vZF11lQIz9W5ZMuUcIOGj243lduidkf2fjkVKJS9vNxVWn3u/uxX38AcE8U9nnH9FPcq+g==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", "dev": true, "dependencies": { "buffer-from": "^1.0.0", @@ -8145,20 +8145,20 @@ } }, "terser": { - "version": "3.7.7", - "resolved": "https://registry.npmjs.org/terser/-/terser-3.7.7.tgz", - "integrity": "sha512-RRLIxE7S52vSOI9cEbOaisgBd2y6MNgfg2ihUkidsFnuP1eDmZ79+lBWbyvgfFTAc/r8nSjL0k3cpZDDIYiYiA==", + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.1.tgz", + "integrity": "sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw==", "dev": true, "requires": { - "commander": "~2.14.1", + "commander": "^2.20.0", "source-map": "~0.6.1", - "source-map-support": "~0.5.6" + "source-map-support": "~0.5.12" }, "dependencies": { "commander": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz", - "integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, "source-map": { @@ -8168,9 +8168,9 @@ "dev": true }, "source-map-support": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.6.tgz", - "integrity": "sha512-N4KXEz7jcKqPf2b2vZF11lQIz9W5ZMuUcIOGj243lduidkf2fjkVKJS9vNxVWn3u/uxX38AcE8U9nnH9FPcq+g==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", "dev": true, "requires": { "buffer-from": "^1.0.0", diff --git a/client/package.json b/client/package.json index 9d0005c4..1935b814 100644 --- a/client/package.json +++ b/client/package.json @@ -28,7 +28,7 @@ "jimp": "^0.13.0", "pretty-error": "^3.0.3", "stylus": "^0.54.8", - "terser": "^3.7.7", + "terser": "^4.8.1", "underscore": "^1.12.1", "watchify": "^4.0.0", "ws": "^7.4.6" From 8088ff3bbe101907c2cea3341bc4437285b3d12d Mon Sep 17 00:00:00 2001 From: w1kl4s Date: Sat, 10 Sep 2022 08:44:16 +0200 Subject: [PATCH 33/48] support ftypiso6 file signature --- server/szurubooru/func/mime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 3be43f77..ee10b9db 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -36,7 +36,7 @@ def get_mime_type(content: bytes) -> str: if content[0:4] == b"\x1A\x45\xDF\xA3": return "video/webm" - if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypmp42", b"ftypM4V "): + if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypiso6", b"ftypmp42", b"ftypM4V "): return "video/mp4" return "application/octet-stream" From 86f50ec74203cf851dfa7006efd1e3d0f081a1fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Dec 2022 15:29:20 +0000 Subject: [PATCH 34/48] build(deps): bump decode-uri-component from 0.2.0 to 0.2.2 in /client Bumps [decode-uri-component](https://github.com/SamVerschueren/decode-uri-component) from 0.2.0 to 0.2.2. - [Release notes](https://github.com/SamVerschueren/decode-uri-component/releases) - [Commits](https://github.com/SamVerschueren/decode-uri-component/compare/v0.2.0...v0.2.2) --- updated-dependencies: - dependency-name: decode-uri-component dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 2e213de6..e663fd9d 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1850,9 +1850,9 @@ } }, "node_modules/decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true, "engines": { "node": ">=0.10" @@ -6357,9 +6357,9 @@ } }, "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true }, "define-properties": { From e950fe7ea5b3dd9c8c9d0945f20d7b5ef213987d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Dec 2022 17:36:52 +0000 Subject: [PATCH 35/48] build(deps): bump qs from 6.5.2 to 6.11.0 in /client Bumps [qs](https://github.com/ljharb/qs) from 6.5.2 to 6.11.0. - [Release notes](https://github.com/ljharb/qs/releases) - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.5.2...v6.11.0) --- updated-dependencies: - dependency-name: qs dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 64 ++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 2e213de6..b01fb053 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1515,7 +1515,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -2254,8 +2253,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/get-assigned-identifiers": { "version": "1.2.0", @@ -2267,7 +2265,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -2351,7 +2348,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -2384,7 +2380,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -3224,7 +3219,6 @@ "version": "1.10.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -3507,11 +3501,17 @@ "dev": true }, "node_modules/qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, "engines": { "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/querystring": { @@ -3781,6 +3781,19 @@ "jsonify": "~0.0.0" } }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/simple-concat": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", @@ -6056,7 +6069,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -6691,8 +6703,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "get-assigned-identifiers": { "version": "1.2.0", @@ -6704,7 +6715,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", - "dev": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -6772,7 +6782,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -6795,8 +6804,7 @@ "has-symbols": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", - "dev": true + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" }, "hash-base": { "version": "3.0.4", @@ -7460,8 +7468,7 @@ "object-inspect": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", - "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==", - "dev": true + "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==" }, "object-keys": { "version": "1.1.1", @@ -7699,9 +7706,12 @@ "dev": true }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } }, "querystring": { "version": "0.2.0", @@ -7941,6 +7951,16 @@ "jsonify": "~0.0.0" } }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "simple-concat": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", From e3062b1c77b52ff9fad23a279844b946d6491417 Mon Sep 17 00:00:00 2001 From: Neo <50623835+neobooru@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:44:31 +0100 Subject: [PATCH 36/48] client: add bulk delete feature (#459) This introduces a new privilege 'posts:bulk-edit:delete' which by default is given to power users. --- client/css/post-list-view.styl | 37 ++++++++++++++- client/html/posts_header.tpl | 7 +++ client/html/posts_page.tpl | 4 ++ client/js/controllers/post_list_controller.js | 46 +++++++++++++++++++ client/js/views/posts_header_view.js | 45 ++++++++++++++++++ client/js/views/posts_page_view.js | 35 ++++++++++++++ server/config.yaml.dist | 1 + 7 files changed, 174 insertions(+), 1 deletion(-) diff --git a/client/css/post-list-view.styl b/client/css/post-list-view.styl index 0272ee15..7f6aa80c 100644 --- a/client/css/post-list-view.styl +++ b/client/css/post-list-view.styl @@ -114,6 +114,29 @@ &[data-disabled] background: rgba(200, 200, 200, 0.7) + .delete-flipper + display: inline-block + padding: 0.5em + box-sizing: border-box + border: 0 + &:after + display: inline-block + width: 1em + height: 1em + text-align: center + line-height: 1em + font-size: 2.2em + &.delete + background: rgba(255, 0, 0, 0.7) + &:after + color: white + font-family: FontAwesome; + content: "\f1f8"; // fa-trash + &:not(.delete) + background: rgba(200, 200, 200, 0.7) + &:after + color: white + content: '-' .thumbnail width: 100% @@ -215,7 +238,19 @@ .append @media (max-width: 1000px) margin-left: 0 - + .bulk-edit-delete + &.opened + .start + @media (max-width: 1000px) + margin-left: 0 + &:not(.opened) + .start + display: none + .append.open + @media (max-width: 1000px) + margin-left: 0 + .start + margin-left: 1em .safety margin-right: 0.25em &.safety-safe diff --git a/client/html/posts_header.tpl b/client/html/posts_header.tpl index e0ba0eae..d1422d2c 100644 --- a/client/html/posts_header.tpl +++ b/client/html/posts_header.tpl @@ -28,4 +28,11 @@ %>Stop editing safety<% %><% %><% } %><% + %><% if (ctx.canBulkDelete) { %><% + %>
<% + %>Mass delete<% + %><% + %>Stop deleting<% + %>
<% + %><% } %><% %> diff --git a/client/html/posts_page.tpl b/client/html/posts_page.tpl index 78362787..52011ad1 100644 --- a/client/html/posts_page.tpl +++ b/client/html/posts_page.tpl @@ -50,6 +50,10 @@ <% } %> <% } %> + <% if (ctx.canBulkDelete && ctx.parameters && ctx.parameters.delete) { %> + + + <% } %> <% } %> diff --git a/client/js/controllers/post_list_controller.js b/client/js/controllers/post_list_controller.js index 526d8f54..ec3e13c3 100644 --- a/client/js/controllers/post_list_controller.js +++ b/client/js/controllers/post_list_controller.js @@ -44,6 +44,7 @@ class PostListController { enableSafety: api.safetyEnabled(), canBulkEditTags: api.hasPrivilege("posts:bulk-edit:tags"), canBulkEditSafety: api.hasPrivilege("posts:bulk-edit:safety"), + canBulkDelete: api.hasPrivilege("posts:bulk-edit:delete"), bulkEdit: { tags: this._bulkEditTags, }, @@ -52,6 +53,14 @@ class PostListController { this._evtNavigate(e) ); + this._headerView._bulkDeleteEditor.addEventListener( + "deleteSelectedPosts", + (e) => { + this._evtDeleteSelectedPosts(e); + } + ); + + this._postsMarkedForDeletion = []; this._syncPageController(); } @@ -91,6 +100,38 @@ class PostListController { e.detail.post.save().catch((error) => window.alert(error.message)); } + _evtMarkForDeletion(e) { + const postId = e.detail; + + // Add or remove post from delete list + if (e.detail.delete) { + this._postsMarkedForDeletion.push(e.detail.post); + } else { + this._postsMarkedForDeletion = this._postsMarkedForDeletion.filter( + (x) => x.id != e.detail.post.id + ); + } + } + + _evtDeleteSelectedPosts(e) { + if (this._postsMarkedForDeletion.length == 0) return; + + if ( + confirm( + `Are you sure you want to delete ${this._postsMarkedForDeletion.length} posts?` + ) + ) { + Promise.all( + this._postsMarkedForDeletion.map((post) => post.delete()) + ) + .catch((error) => window.alert(error.message)) + .then(() => { + this._postsMarkedForDeletion = []; + this._headerView._navigate(); + }); + } + } + _syncPageController() { this._pageController.run({ parameters: this._ctx.parameters, @@ -117,8 +158,10 @@ class PostListController { canBulkEditSafety: api.hasPrivilege( "posts:bulk-edit:safety" ), + canBulkDelete: api.hasPrivilege("posts:bulk-edit:delete"), bulkEdit: { tags: this._bulkEditTags, + markedForDeletion: this._postsMarkedForDeletion, }, postFlow: settings.get().postFlow, }); @@ -128,6 +171,9 @@ class PostListController { view.addEventListener("changeSafety", (e) => this._evtChangeSafety(e) ); + view.addEventListener("markForDeletion", (e) => + this._evtMarkForDeletion(e) + ); return view; }, }); diff --git a/client/js/views/posts_header_view.js b/client/js/views/posts_header_view.js index f64060d6..38a4aa98 100644 --- a/client/js/views/posts_header_view.js +++ b/client/js/views/posts_header_view.js @@ -141,6 +141,34 @@ class BulkTagEditor extends BulkEditor { } } +class BulkDeleteEditor extends BulkEditor { + constructor(hostNode) { + super(hostNode); + this._hostNode.addEventListener("submit", (e) => + this._evtFormSubmit(e) + ); + } + + _evtFormSubmit(e) { + e.preventDefault(); + this.dispatchEvent( + new CustomEvent("deleteSelectedPosts", { detail: {} }) + ); + } + + _evtOpenLinkClick(e) { + e.preventDefault(); + this.toggleOpen(true); + this.dispatchEvent(new CustomEvent("open", { detail: {} })); + } + + _evtCloseLinkClick(e) { + e.preventDefault(); + this.toggleOpen(false); + this.dispatchEvent(new CustomEvent("close", { detail: {} })); + } +} + class PostsHeaderView extends events.EventTarget { constructor(ctx) { super(); @@ -186,6 +214,13 @@ class PostsHeaderView extends events.EventTarget { this._bulkEditors.push(this._bulkSafetyEditor); } + if (this._bulkEditDeleteNode) { + this._bulkDeleteEditor = new BulkDeleteEditor( + this._bulkEditDeleteNode + ); + this._bulkEditors.push(this._bulkDeleteEditor); + } + for (let editor of this._bulkEditors) { editor.addEventListener("submit", (e) => { this._navigate(); @@ -204,6 +239,8 @@ class PostsHeaderView extends events.EventTarget { this._openBulkEditor(this._bulkTagEditor); } else if (ctx.parameters.safety && this._bulkSafetyEditor) { this._openBulkEditor(this._bulkSafetyEditor); + } else if (ctx.parameters.delete && this._bulkDeleteEditor) { + this._openBulkEditor(this._bulkDeleteEditor); } } @@ -227,6 +264,10 @@ class PostsHeaderView extends events.EventTarget { return this._hostNode.querySelector(".bulk-edit-safety"); } + get _bulkEditDeleteNode() { + return this._hostNode.querySelector(".bulk-edit-delete"); + } + _openBulkEditor(editor) { editor.toggleOpen(true); this._hideBulkEditorsExcept(editor); @@ -293,6 +334,10 @@ class PostsHeaderView extends events.EventTarget { this._bulkSafetyEditor && this._bulkSafetyEditor.opened ? "1" : null; + parameters.delete = + this._bulkDeleteEditor && this._bulkDeleteEditor.opened + ? "1" + : null; this.dispatchEvent( new CustomEvent("navigate", { detail: { parameters: parameters } }) ); diff --git a/client/js/views/posts_page_view.js b/client/js/views/posts_page_view.js index ba07a63a..c4b19882 100644 --- a/client/js/views/posts_page_view.js +++ b/client/js/views/posts_page_view.js @@ -39,6 +39,13 @@ class PostsPageView extends events.EventTarget { ); } } + + const deleteFlipperNode = this._getDeleteFlipperNode(listItemNode); + if (deleteFlipperNode) { + deleteFlipperNode.addEventListener("click", (e) => + this._evtBulkToggleDeleteClick(e, post) + ); + } } this._syncBulkEditorsHighlights(); @@ -56,6 +63,10 @@ class PostsPageView extends events.EventTarget { return listItemNode.querySelector(".safety-flipper"); } + _getDeleteFlipperNode(listItemNode) { + return listItemNode.querySelector(".delete-flipper"); + } + _evtPostChange(e) { const listItemNode = this._postIdToListItemNode[e.detail.post.id]; for (let node of listItemNode.querySelectorAll("[data-disabled]")) { @@ -99,6 +110,20 @@ class PostsPageView extends events.EventTarget { ); } + _evtBulkToggleDeleteClick(e, post) { + e.preventDefault(); + const linkNode = e.target; + linkNode.classList.toggle("delete"); + this.dispatchEvent( + new CustomEvent("markForDeletion", { + detail: { + post, + delete: linkNode.classList.contains("delete"), + }, + }) + ); + } + _syncBulkEditorsHighlights() { for (let listItemNode of this._listItemNodes) { const postId = listItemNode.getAttribute("data-post-id"); @@ -123,6 +148,16 @@ class PostsPageView extends events.EventTarget { ); } } + + const deleteFlipperNode = this._getDeleteFlipperNode(listItemNode); + if (deleteFlipperNode) { + deleteFlipperNode.classList.toggle( + "delete", + this._ctx.bulkEdit.markedForDeletion.some( + (x) => x.id == postId + ) + ); + } } } } diff --git a/server/config.yaml.dist b/server/config.yaml.dist index bc4e3630..193aac3a 100644 --- a/server/config.yaml.dist +++ b/server/config.yaml.dist @@ -115,6 +115,7 @@ privileges: 'posts:favorite': regular 'posts:bulk-edit:tags': power 'posts:bulk-edit:safety': power + 'posts:bulk-edit:delete': power 'tags:create': regular 'tags:edit:names': power From 75635bbc434562dcbcd528c68b776548fd0b54fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 20:36:57 +0000 Subject: [PATCH 37/48] build(deps): bump cookiejar from 2.1.2 to 2.1.4 in /client Bumps [cookiejar](https://github.com/bmeck/node-cookiejar) from 2.1.2 to 2.1.4. - [Release notes](https://github.com/bmeck/node-cookiejar/releases) - [Commits](https://github.com/bmeck/node-cookiejar/commits) --- updated-dependencies: - dependency-name: cookiejar dependency-type: indirect ... Signed-off-by: dependabot[bot] --- client/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 2e213de6..16903280 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1682,9 +1682,9 @@ "dev": true }, "node_modules/cookiejar": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz", - "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==" + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz", + "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==" }, "node_modules/core-js": { "version": "2.5.7", @@ -6205,9 +6205,9 @@ "dev": true }, "cookiejar": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz", - "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==" + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz", + "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==" }, "core-js": { "version": "2.5.7", From d7d2a151a8d60a8bde87a8245da299cd6489552f Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Tue, 24 Jan 2023 22:19:24 +0100 Subject: [PATCH 38/48] client: workaround for #545, but not a fix --- client/js/controllers/post_list_controller.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/client/js/controllers/post_list_controller.js b/client/js/controllers/post_list_controller.js index ec3e13c3..fdb7b844 100644 --- a/client/js/controllers/post_list_controller.js +++ b/client/js/controllers/post_list_controller.js @@ -53,12 +53,14 @@ class PostListController { this._evtNavigate(e) ); - this._headerView._bulkDeleteEditor.addEventListener( - "deleteSelectedPosts", - (e) => { - this._evtDeleteSelectedPosts(e); - } - ); + if (this._headerView._bulkDeleteEditor) { + this._headerView._bulkDeleteEditor.addEventListener( + "deleteSelectedPosts", + (e) => { + this._evtDeleteSelectedPosts(e); + } + ); + } this._postsMarkedForDeletion = []; this._syncPageController(); From 196f92593ca8c5d91c21d9678b62e9964bf850d6 Mon Sep 17 00:00:00 2001 From: SediSocks Date: Mon, 13 Mar 2023 19:53:02 +0000 Subject: [PATCH 39/48] fix flow view on webkit browsers --- client/css/core-general.styl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/css/core-general.styl b/client/css/core-general.styl index d25c5f6e..7e5883aa 100644 --- a/client/css/core-general.styl +++ b/client/css/core-general.styl @@ -300,10 +300,10 @@ a .access-key background-size: 20px 20px img opacity: 0 - width: 100% + width: auto height: 100% video - width: 100% + width: auto height: 100% .flexbox-dummy From da3b4790ad4a4ec0ef8097452a93b7c36477b033 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Sat, 4 Feb 2023 19:08:31 -0500 Subject: [PATCH 40/48] server+client: bump versions in pre-commit --- .pre-commit-config.yaml | 44 ++++++++--------------------------------- client/package.json | 3 ++- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b550ca1..75a55d62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -10,12 +10,12 @@ repos: - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.10 + rev: v1.4.2 hooks: - id: remove-tabs - repo: https://github.com/psf/black - rev: '22.3.0' + rev: '23.1.0' hooks: - id: black files: 'server/' @@ -23,7 +23,7 @@ repos: language_version: python3.9 - repo: https://github.com/PyCQA/isort - rev: '5.10.1' + rev: '5.12.0' hooks: - id: isort files: 'server/' @@ -33,7 +33,7 @@ repos: - toml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.0 + rev: v2.7.1 hooks: - id: prettier files: client/js/ @@ -41,7 +41,7 @@ repos: args: ['--config', 'client/.prettierrc.yml'] - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.3.0 + rev: v8.33.0 hooks: - id: eslint files: client/js/ @@ -49,8 +49,8 @@ repos: additional_dependencies: - eslint-config-prettier -- repo: https://gitlab.com/PyCQA/flake8 - rev: '4.0.1' +- repo: https://github.com/PyCQA/flake8 + rev: '6.0.0' hooks: - id: flake8 files: server/szurubooru/ @@ -58,33 +58,5 @@ repos: - flake8-print args: ['--config=server/.flake8'] -- repo: local - hooks: - - id: docker-build-client - name: Docker - build client - entry: bash -c 'docker build client/' - language: system - types: [file] - files: client/ - pass_filenames: false - - - id: docker-build-server - name: Docker - build server - entry: bash -c 'docker build server/' - language: system - types: [file] - files: server/ - pass_filenames: false - - - id: pytest - name: pytest - entry: bash -c 'docker run --rm -t $(docker build --target testing -q server/) szurubooru/' - language: system - types: [python] - files: server/szurubooru/ - exclude: server/szurubooru/migrations/ - pass_filenames: false - stages: [manual] - fail_fast: true exclude: LICENSE.md diff --git a/client/package.json b/client/package.json index 9d0005c4..46de6cbe 100644 --- a/client/package.json +++ b/client/package.json @@ -3,7 +3,8 @@ "private": true, "scripts": { "build": "node build.js", - "watch": "node build.js --watch" + "watch": "node build.js --watch", + "build-container": "docker build -t szurubooru/client:dev ." }, "dependencies": { "dompurify": "^2.0.17", From 244a0f0b6c190a50946a6fed37c9768f151f0d29 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Sun, 5 Feb 2023 12:25:30 -0500 Subject: [PATCH 41/48] server/test: skip network tests by default --- server/szurubooru/tests/func/test_net.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/server/szurubooru/tests/func/test_net.py b/server/szurubooru/tests/func/test_net.py index c5b4c73e..be2f3c93 100644 --- a/server/szurubooru/tests/func/test_net.py +++ b/server/szurubooru/tests/func/test_net.py @@ -1,3 +1,5 @@ +import os + import pytest from szurubooru import errors @@ -16,6 +18,9 @@ def inject_config(tmpdir, config_injector): ) +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) def test_download(): url = "http://info.cern.ch/hypertext/WWW/TheProject.html" @@ -62,6 +67,9 @@ def test_download(): assert actual_content == expected_content +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) @pytest.mark.parametrize( "url", [ @@ -74,6 +82,9 @@ def test_too_large_download(url): net.download(url, use_video_downloader=True) +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) @pytest.mark.parametrize( "url,expected_sha1", [ @@ -96,6 +107,9 @@ def test_content_download(url, expected_sha1): assert get_sha1(actual_content) == expected_sha1 +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) def test_bad_content_downlaod(): url = "http://info.cern.ch/hypertext/WWW/TheProject.html" with pytest.raises(errors.ThirdPartyError): @@ -108,11 +122,13 @@ def test_no_webhooks(config_injector): assert len(res) == 0 +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) @pytest.mark.parametrize( "webhook,status_code", [ ("https://postman-echo.com/post", 200), - ("http://localhost/", 400), ("https://postman-echo.com/get", 400), ], ) @@ -121,6 +137,9 @@ def test_single_webhook(config_injector, webhook, status_code): assert ret == status_code +@pytest.mark.skipif( + "TEST_NET" not in os.environ, reason="Network tests skipped by default." +) def test_multiple_webhooks(config_injector): config_injector( { From 8a03015349c3d9642c4d83472dbe76b072dd05a1 Mon Sep 17 00:00:00 2001 From: skybldev Date: Fri, 5 Aug 2022 21:31:27 -0400 Subject: [PATCH 42/48] client+server: added quicktime upload support --- client/html/post_merge_side.tpl | 1 + client/html/post_readonly_sidebar.tpl | 1 + client/js/views/post_upload_view.js | 2 ++ server/szurubooru/func/mime.py | 6 +++++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/client/html/post_merge_side.tpl b/client/html/post_merge_side.tpl index fe705027..a08070f0 100644 --- a/client/html/post_merge_side.tpl +++ b/client/html/post_merge_side.tpl @@ -42,6 +42,7 @@ 'image/heic': 'HEIC', 'video/webm': 'WEBM', 'video/mp4': 'MPEG-4', + 'video/quicktime': 'MOV', 'application/x-shockwave-flash': 'SWF', }[ctx.post.mimeType] + ' (' + diff --git a/client/html/post_readonly_sidebar.tpl b/client/html/post_readonly_sidebar.tpl index 4f18624f..0f93ae3b 100644 --- a/client/html/post_readonly_sidebar.tpl +++ b/client/html/post_readonly_sidebar.tpl @@ -15,6 +15,7 @@ 'image/heic': 'HEIC', 'video/webm': 'WEBM', 'video/mp4': 'MPEG-4', + 'video/quicktime': 'MOV', 'application/x-shockwave-flash': 'SWF', }[ctx.post.mimeType] %> diff --git a/client/js/views/post_upload_view.js b/client/js/views/post_upload_view.js index fc98a19e..4ef4c1ad 100644 --- a/client/js/views/post_upload_view.js +++ b/client/js/views/post_upload_view.js @@ -22,6 +22,7 @@ function _mimeTypeToPostType(mimeType) { "image/heic": "image", "video/mp4": "video", "video/webm": "video", + "video/quicktime": "video", }[mimeType] || "unknown" ); } @@ -120,6 +121,7 @@ class Url extends Uploadable { heif: "image/heif", heic: "image/heic", mp4: "video/mp4", + mov: "video/quicktime", webm: "video/webm", }; for (let extension of Object.keys(mime)) { diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 3be43f77..4aa9c3dc 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -39,6 +39,9 @@ def get_mime_type(content: bytes) -> str: if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypmp42", b"ftypM4V "): return "video/mp4" + if content[4:12] in (b"ftypqt "): + return "video/quicktime" + return "application/octet-stream" @@ -54,6 +57,7 @@ def get_extension(mime_type: str) -> Optional[str]: "image/heif": "heif", "image/heic": "heic", "video/mp4": "mp4", + "video/quicktime": "mov", "video/webm": "webm", "application/octet-stream": "dat", } @@ -65,7 +69,7 @@ def is_flash(mime_type: str) -> bool: def is_video(mime_type: str) -> bool: - return mime_type.lower() in ("application/ogg", "video/mp4", "video/webm") + return mime_type.lower() in ("application/ogg", "video/mp4", "video/quicktime", "video/webm") def is_image(mime_type: str) -> bool: From 42524503b9fb5a73c69402c435e8d7aee3e036fa Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 17 Apr 2023 11:58:13 -0400 Subject: [PATCH 43/48] client/tests: add unit tests for quicktime videos --- server/szurubooru/func/mime.py | 9 +++++++-- server/szurubooru/tests/assets/mov.mov | Bin 0 -> 844 bytes server/szurubooru/tests/func/test_mime.py | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 server/szurubooru/tests/assets/mov.mov diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 4aa9c3dc..f01af0da 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -39,7 +39,7 @@ def get_mime_type(content: bytes) -> str: if content[4:12] in (b"ftypisom", b"ftypiso5", b"ftypmp42", b"ftypM4V "): return "video/mp4" - if content[4:12] in (b"ftypqt "): + if content[4:12] == b"ftypqt ": return "video/quicktime" return "application/octet-stream" @@ -69,7 +69,12 @@ def is_flash(mime_type: str) -> bool: def is_video(mime_type: str) -> bool: - return mime_type.lower() in ("application/ogg", "video/mp4", "video/quicktime", "video/webm") + return mime_type.lower() in ( + "application/ogg", + "video/mp4", + "video/quicktime", + "video/webm", + ) def is_image(mime_type: str) -> bool: diff --git a/server/szurubooru/tests/assets/mov.mov b/server/szurubooru/tests/assets/mov.mov new file mode 100644 index 0000000000000000000000000000000000000000..911ee8525be3b88429a94b9dc25585c089d4a5ce GIT binary patch literal 844 zcmZ`$u}&L75S_y$Si(q7Vj(~-E|M!Nwj97pC=ixnAqp%d1yoU#jrS~DzQxY%atJCz z1qFXWLl=P{Kj8cUA3>Ko6?t=eXB|WtYdo{>&Agl0C89#d91g!5*Cm=F8!3CxlO7Q* zDJcw*b4~dfgxmbrPbcd?e-mZq)nE{!!dGEeBBm^Jk3*RpG^EhS2M~`)qDH#*-+T(w z9h*S(L47v;=$n~TS)x`>VKz;egm`jpFrJN_sN_kbaVmPg#^#Z6ht-XGoj*$U1Ta_0 zr@PkxV^nLEuJi*=Ro4$fd!sQ$l-mU`*b;lv_fX_qsNP-&Y#~&UXnM>yiI9ZQe&D+k z6(qAwVZ=m-O48~SW#F+M8*Am_KIG1OZNB-y?zGX#<8jSc4OjV5*`n`nZni$!g|Ji> zzb!EAY*B?x|MS1_@BrqErzKx0Wk#b>3vJVg8}18+Xl}&*3`hlvKX|&EvsYH!acO*h zO24S-6g~jD+7@BEzEWMOS8LDF;qXo8he8h#c^dWwZK4N6nqbj`6tQcYL0D3o!~(b@ oOVP`i2i6|M7}u5HulJ=9h>Mr{AfUOOF$U_?*Q(3!s&8wxzmBzPhX4Qo literal 0 HcmV?d00001 diff --git a/server/szurubooru/tests/func/test_mime.py b/server/szurubooru/tests/func/test_mime.py index b33746b4..551ba7c3 100644 --- a/server/szurubooru/tests/func/test_mime.py +++ b/server/szurubooru/tests/func/test_mime.py @@ -7,6 +7,7 @@ from szurubooru.func import mime "input_path,expected_mime_type", [ ("mp4.mp4", "video/mp4"), + ("mov.mov", "video/quicktime"), ("webm.webm", "video/webm"), ("flash.swf", "application/x-shockwave-flash"), ("png.png", "image/png"), @@ -35,6 +36,7 @@ def test_get_mime_type_for_empty_file(): [ ("video/mp4", "mp4"), ("video/webm", "webm"), + ("video/quicktime", "mov"), ("application/x-shockwave-flash", "swf"), ("image/png", "png"), ("image/jpeg", "jpg"), @@ -70,6 +72,8 @@ def test_is_flash(input_mime_type, expected_state): ("VIDEO/WEBM", True), ("video/mp4", True), ("VIDEO/MP4", True), + ("video/quicktime", True), + ("VIDEO/QUICKTIME", True), ("video/anything_else", False), ("application/ogg", True), ("not a video", False), From 782f0690318d21e9e09ac88984afc59a1812b888 Mon Sep 17 00:00:00 2001 From: Shyam Sunder Date: Mon, 17 Apr 2023 19:50:40 -0400 Subject: [PATCH 44/48] client/upload: fix thumbnail width in post uploads Fixes regression caused by 648121d7 --- client/css/post-upload.styl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/css/post-upload.styl b/client/css/post-upload.styl index ea79fcac..cb6b0067 100644 --- a/client/css/post-upload.styl +++ b/client/css/post-upload.styl @@ -62,6 +62,14 @@ $cancel-button-color = tomato margin: 0 0 1.2em 0 padding-left: 13em + img + width: 100% + height: 100% + + video + width: 100% + height: 100% + &>.thumbnail-wrapper float: left width: 12em From ffdf115714f06f7c28b3898ccd04415ac0037fb6 Mon Sep 17 00:00:00 2001 From: Yochyo Date: Mon, 20 Mar 2023 15:47:56 +0100 Subject: [PATCH 45/48] docs (api): change micro post attribute name to id --- doc/API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/API.md b/doc/API.md index 63a50a2f..c8b88fef 100644 --- a/doc/API.md +++ b/doc/API.md @@ -2491,7 +2491,7 @@ One file together with its metadata posted to the site. ## Micro post **Description** -A [post resource](#post) stripped down to `name` and `thumbnailUrl` fields. +A [post resource](#post) stripped down to `id` and `thumbnailUrl` fields. ## Note **Description** From c2fdc2d0707640c4a7b21e76244645240f6a9c51 Mon Sep 17 00:00:00 2001 From: Yochyo Date: Mon, 3 Apr 2023 20:27:35 +0200 Subject: [PATCH 46/48] docs (tag categories): order is required when creating tag category --- doc/API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/API.md b/doc/API.md index c8b88fef..f588c89d 100644 --- a/doc/API.md +++ b/doc/API.md @@ -323,7 +323,7 @@ data. { "name": , "color": , - "order": // optional + "order": } ``` From 4806bbe0eda3a6c2f76800200a60d7f19971150c Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Thu, 7 Oct 2021 16:28:06 +0200 Subject: [PATCH 47/48] server: post category filter --- .../search/configs/post_search_config.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index ddc003b7..8d4672d4 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -122,6 +122,34 @@ def _pool_filter( )(query, criterion, negated) +def _category_filter( + query: SaQuery, criterion: Optional[criteria.BaseCriterion], negated: bool +) -> SaQuery: + assert criterion + + # Step 1. find the id for the category + q1 = db.session.query(model.TagCategory.tag_category_id).filter( + model.TagCategory.name == criterion.value + ) + + # Step 2. find the tags with that category + q2 = db.session.query(model.Tag.tag_id).filter( + model.Tag.category_id.in_(q1) + ) + + # Step 3. find all posts that have at least one of those tags + q3 = db.session.query(model.PostTag.post_id).filter( + model.PostTag.tag_id.in_(q2) + ) + + # Step 4. profit + expr = model.Post.post_id.in_(q3) + if negated: + expr = ~expr + + return query.filter(expr) + + class PostSearchConfig(BaseSearchConfig): def __init__(self) -> None: self.user = None # type: Optional[model.User] @@ -349,6 +377,7 @@ class PostSearchConfig(BaseSearchConfig): ), ), (["pool"], _pool_filter), + (["category"], _category_filter), ] ) From 7a82e9d5813d8b88e2f49ebdabbf19957b2f393a Mon Sep 17 00:00:00 2001 From: neobooru <50623835+neobooru@users.noreply.github.com> Date: Mon, 26 Jun 2023 20:32:41 +0200 Subject: [PATCH 48/48] tests/server: post category filter --- .../search/configs/test_post_search_config.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/server/szurubooru/tests/search/configs/test_post_search_config.py b/server/szurubooru/tests/search/configs/test_post_search_config.py index 4fb8191a..b86fa273 100644 --- a/server/szurubooru/tests/search/configs/test_post_search_config.py +++ b/server/szurubooru/tests/search/configs/test_post_search_config.py @@ -863,3 +863,55 @@ def test_tumbleweed( db.session.flush() verify_unpaged("special:tumbleweed", [4]) verify_unpaged("-special:tumbleweed", [1, 2, 3]) + + +@pytest.mark.parametrize( + "input,expected_post_ids", + [ + ("category:cat1", [1, 2, 3]), + ("category:cat2", [3, 4]), + ], +) +def test_search_by_tag_category( + verify_unpaged, + post_factory, + tag_factory, + tag_category_factory, + input, + expected_post_ids, +): + cat1 = tag_category_factory(name="cat1") + cat2 = tag_category_factory(name="cat2") + tag1 = tag_factory(names=["t1"], category=cat1) + tag2 = tag_factory(names=["t2"], category=cat1) + tag3 = tag_factory(names=["t3"], category=cat2) + + post1 = post_factory(id=1) + post1.tags.append(tag1) + + post2 = post_factory(id=2) + post2.tags.append(tag2) + + post3 = post_factory(id=3) + post3.tags.append(tag1) + post3.tags.append(tag3) + + post4 = post_factory(id=4) + post4.tags.append(tag3) + + post5 = post_factory(id=5) + + db.session.add_all( + [ + tag1, + tag2, + tag3, + post1, + post2, + post3, + post4, + post5, + ] + ) + db.session.flush() + verify_unpaged(input, expected_post_ids)