ulysia/szurubooru
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into file-last-modifed-time

Browse source
This commit is contained in:
Luxray5474 2020-08-24 08:48:25 -04:00
commit 22f2b21450
21 changed files with 242 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.pre-commit-config.yaml
View file

@ -75,6 +75,7 @@ repos:
files: server/szurubooru/
exclude: server/szurubooru/migrations/
pass_filenames: false
stages: [push]
- id: pytest-cov
name: pytest

2
client/css/post-list-view.styl
View file

@ -182,7 +182,7 @@
.hint
display: none
input[name=tag]
width: 12em
width: 24em
@media (max-width: 1000px)
display: block
width: 100%

7
client/js/controls/auto_complete_control.js
View file

@ -62,13 +62,16 @@ class AutoCompleteControl {
let prefix = "";
let suffix = this._sourceInputNode.value.substring(start);
let middle = this._sourceInputNode.value.substring(0, start);
const index = middle.lastIndexOf(" ");
const spaceIndex = middle.lastIndexOf(" ");
const commaIndex = middle.lastIndexOf(",");
const index = spaceIndex < commaIndex ? commaIndex : spaceIndex;
const delimiter = spaceIndex < commaIndex ? "" : " ";
if (index !== -1) {
prefix = this._sourceInputNode.value.substring(0, index + 1);
middle = this._sourceInputNode.value.substring(index + 1);
}
this._sourceInputNode.value =
prefix + result.toString() + " " + suffix.trimLeft();
prefix + result.toString() + delimiter + suffix.trimLeft();
if (!addSpace) {
this._sourceInputNode.value = this._sourceInputNode.value.trim();
}

27
client/js/views/posts_header_view.js
View file

@ -6,6 +6,7 @@ const keyboard = require("../util/keyboard.js");
const misc = require("../util/misc.js");
const search = require("../util/search.js");
const views = require("../util/views.js");
const TagList = require("../models/tag_list.js");
const TagAutoCompleteControl = require("../controls/tag_auto_complete_control.js");
const template = views.getTemplate("posts-header");
@ -74,11 +75,27 @@ class BulkTagEditor extends BulkEditor {
this._autoCompleteControl = new TagAutoCompleteControl(
this._inputNode,
{
confirm: (tag) =>
this._autoCompleteControl.replaceSelectedText(
tag.names[0],
false
),
confirm: (tag) => {
let tag_list = new TagList();
tag_list
.addByName(tag.names[0], true)
.then(
() => {
return tag_list
.map((s) => s.names[0])
.join(" ");
},
(err) => {
return tag.names[0];
}
)
.then((tag_str) => {
this._autoCompleteControl.replaceSelectedText(
tag_str,
false
);
});
},
}
);
this._hostNode.addEventListener("submit", (e) =>

6
client/package-lock.json generated
View file

@ -1761,9 +1761,9 @@
"dev": true
},
"elliptic": {
"version": "6.4.0",
"resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.0.tgz",
"integrity": "sha1-ysmvh2LIWDYYcAPI3+GT5eLq5d8=",
"version": "6.5.3",
"resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
"integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
"dev": true,
"requires": {
"bn.js": "^4.4.0",

10
doc/API.md
View file

@ -13,6 +13,7 @@
- [Error handling](#error-handling)
- [Field selecting](#field-selecting)
- [Versioning](#versioning)
- [Webhooks](#webhooks)
2. [API reference](#api-reference)
@ -276,6 +277,15 @@ reject the request as well, in which case the client is encouraged to notify
the user about the situation.
## Webhooks
System administrators can choose to configure webhooks to track events.
Webhook URIs can be configured in `config.yaml` (See `config.yaml.dist` for
example). Upon any event, the API will send a `POST` request to the listed
URIs with a [snapshot resource](#snapshot) generated with anonymous user
privileges as the message body, in JSON format.
# API reference
Depending on the deployment, the URLs might be relative to some base path such

28
doc/developer-utils/create-alembic-migration.sh Executable file
View file

@ -0,0 +1,28 @@
#!/bin/sh
# Helper script to create an alembic migration file via Docker
if [ $# -lt 1 ]; then
echo "Need to pass a name for your migration file" > /dev/stderr
exit 1
fi
# Create a dummy container
WORKDIR="$(git rev-parse --show-toplevel)/server"
IMAGE=$(docker build -q "${WORKDIR}")
CONTAINER=$(docker run -d ${IMAGE} tail -f /dev/null)
# Create the migration script
docker exec -i \
-e PYTHONPATH='/opt/app' \
-e POSTGRES_HOST='x' \
-e POSTGRES_USER='x' \
-e POSTGRES_PASSWORD='x' \
${CONTAINER} alembic revision -m "$1"
# Copy the file over from the container
docker cp ${CONTAINER}:/opt/app/szurubooru/migrations/versions/ \
"${WORKDIR}/szurubooru/migrations/"
# Destroy the dummy container
docker rm -f ${CONTAINER} > /dev/null

11
server/config.yaml.dist
View file

@ -39,8 +39,9 @@ smtp:
user: # example: bot
pass: # example: groovy123
from: # example: noreply@example.com
# if host is left empty the password reset feature will be disabled, in which case it is
# recommended to fill contactEmail so that users know who to contact when they want to reset their password
# if host is left empty the password reset feature will be disabled,
# in which case it is recommended to fill contactEmail so that users
# know who to contact when they want to reset their password
contact_email: # example: bob@example.com. Meant for manual password reset procedures
@ -58,6 +59,12 @@ pool_category_name_regex: ^[^\s%+#/]+$
password_regex: '^.{5,}$'
user_name_regex: '^[a-zA-Z0-9_-]{1,32}$'
# webhooks to call when events occur (such as post/tag/user/etc. changes)
# the listed urls will be called with a HTTP POST request with a payload
# containing a snapshot resource as JSON. See doc/API.md for details
webhooks:
# - https://api.example.com/webhooks/
default_rank: regular
privileges:

5
server/szurubooru/facade.py
View file

@ -94,6 +94,11 @@ def validate_config() -> None:
if not config.config["database"]:
raise errors.ConfigError("Database is not configured")
if config.config["webhooks"] and not isinstance(
config.config["webhooks"], list
):
raise errors.ConfigError("Webhooks must be provided as a list of URLs")
if config.config["smtp"]["host"]:
if not config.config["smtp"]["port"]:
raise errors.ConfigError("SMTP host is set but port is not set")

24
server/szurubooru/func/net.py
View file

@ -1,7 +1,10 @@
import json
import logging
import os
import urllib.error
import urllib.request
from tempfile import NamedTemporaryFile
from typing import Any, Dict, List
from youtube_dl import YoutubeDL
from youtube_dl.utils import YoutubeDLError
@ -58,3 +61,24 @@ def _youtube_dl_wrapper(url: str) -> bytes:
raise errors.ThirdPartyError(
"Error downloading video %s (file could not be saved)" % (url)
)
def post_to_webhooks(payload: Dict[str, Any]) -> List[int]:
return_list = []
for webhook in config.config["webhooks"] or []:
req = urllib.request.Request(webhook)
req.data = json.dumps(
payload, default=lambda x: x.isoformat("T") + "Z",
).encode("utf-8")
req.add_header("Content-Type", "application/json")
try:
res = urllib.request.urlopen(req)
if not 200 <= res.status <= 299:
logger.warning(
f"Webhook {webhook} returned {res.status} {res.reason}"
)
return_list.append(res.status)
except urllib.error.URLError as e:
logger.error(f"Unable to call webhook {webhook}: {str(e)}")
return_list.append(400)
return return_list

13
server/szurubooru/func/snapshots.py
View file

@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, Optional
import sqlalchemy as sa
from szurubooru import db, model
from szurubooru.func import diff, users
from szurubooru.func import diff, net, users
def get_tag_category_snapshot(category: model.TagCategory) -> Dict[str, Any]:
@ -93,6 +93,13 @@ def serialize_snapshot(
}
def _post_to_webhooks(snapshot: model.Snapshot) -> None:
webhook_user = model.User()
webhook_user.name = None
webhook_user.rank = "anonymous"
net.post_to_webhooks(serialize_snapshot(snapshot, webhook_user))
def _create(
operation: str, entity: model.Base, auth_user: Optional[model.User]
) -> model.Snapshot:
@ -116,6 +123,7 @@ def create(entity: model.Base, auth_user: Optional[model.User]) -> None:
snapshot_factory = _snapshot_factories[snapshot.resource_type]
snapshot.data = snapshot_factory(entity)
db.session.add(snapshot)
_post_to_webhooks(snapshot)
def modify(entity: model.Base, auth_user: Optional[model.User]) -> None:
@ -147,6 +155,7 @@ def modify(entity: model.Base, auth_user: Optional[model.User]) -> None:
if not snapshot.data:
return
db.session.add(snapshot)
_post_to_webhooks(snapshot)
def delete(entity: model.Base, auth_user: Optional[model.User]) -> None:
@ -155,6 +164,7 @@ def delete(entity: model.Base, auth_user: Optional[model.User]) -> None:
snapshot_factory = _snapshot_factories[snapshot.resource_type]
snapshot.data = snapshot_factory(entity)
db.session.add(snapshot)
_post_to_webhooks(snapshot)
def merge(
@ -174,3 +184,4 @@ def merge(
) = model.util.get_resource_info(target_entity)
snapshot.data = [resource_type, resource_name]
db.session.add(snapshot)
_post_to_webhooks(snapshot)

2
server/szurubooru/tests/api/test_pool_category_updating.py
View file

@ -67,7 +67,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.pool_categories.serialize_category"), patch(
"szurubooru.func.pool_categories.update_category_name"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
api.pool_category_api.update_pool_category(
context_factory(
params={**params, **{"version": 1}},

2
server/szurubooru/tests/api/test_pool_creating.py
View file

@ -70,7 +70,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.pools.create_pool"), patch(
"szurubooru.func.pools.serialize_pool"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
pools.create_pool.return_value = pool_factory()
api.pool_api.create_pool(
context_factory(

23
server/szurubooru/tests/api/test_pool_deleting.py
View file

@ -34,17 +34,18 @@ def test_deleting_used(
pool.posts.append(post)
db.session.add_all([pool, post])
db.session.commit()
api.pool_api.delete_pool(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"pool_id": 1},
)
db.session.refresh(post)
assert db.session.query(model.Pool).count() == 0
assert db.session.query(model.PoolPost).count() == 0
assert post.pools == []
with patch("szurubooru.func.snapshots._post_to_webhooks"):
api.pool_api.delete_pool(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"pool_id": 1},
)
db.session.refresh(post)
assert db.session.query(model.Pool).count() == 0
assert db.session.query(model.PoolPost).count() == 0
assert post.pools == []
def test_trying_to_delete_non_existing(user_factory, context_factory):

12
server/szurubooru/tests/api/test_post_creating.py
View file

@ -143,7 +143,9 @@ def test_anonymous_uploads(
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.posts.create_post"
), patch("szurubooru.func.posts.update_post_source"):
), patch("szurubooru.func.posts.update_post_source"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
config_injector(
{
"privileges": {
@ -181,6 +183,8 @@ def test_creating_from_url_saves_source(
"szurubooru.func.posts.serialize_post"
), patch("szurubooru.func.posts.create_post"), patch(
"szurubooru.func.posts.update_post_source"
), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
config_injector(
{
@ -223,6 +227,8 @@ def test_creating_from_url_with_source_specified(
"szurubooru.func.posts.serialize_post"
), patch("szurubooru.func.posts.create_post"), patch(
"szurubooru.func.posts.update_post_source"
), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
config_injector(
{
@ -334,7 +340,7 @@ def test_errors_not_spending_ids(
# successful request
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.posts.update_post_tags"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
posts.serialize_post.side_effect = lambda post, *_, **__: post.post_id
post1_id = api.post_api.create_post(
context_factory(
@ -357,7 +363,7 @@ def test_errors_not_spending_ids(
# successful request
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.posts.update_post_tags"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
posts.serialize_post.side_effect = lambda post, *_, **__: post.post_id
post2_id = api.post_api.create_post(
context_factory(

8
server/szurubooru/tests/api/test_post_featuring.py
View file

@ -55,7 +55,9 @@ def test_trying_to_feature_the_same_post_twice(
):
db.session.add(post_factory(id=1))
db.session.commit()
with patch("szurubooru.func.posts.serialize_post"):
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
api.post_api.set_featured_post(
context_factory(
params={"id": 1},
@ -80,7 +82,9 @@ def test_featuring_one_post_after_another(
assert posts.try_get_featured_post() is None
assert not posts.get_post_by_id(1).is_featured
assert not posts.get_post_by_id(2).is_featured
with patch("szurubooru.func.posts.serialize_post"):
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
with fake_datetime("1997"):
api.post_api.set_featured_post(
context_factory(

2
server/szurubooru/tests/api/test_tag_category_updating.py
View file

@ -65,7 +65,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.tag_categories.serialize_category"), patch(
"szurubooru.func.tag_categories.update_category_name"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
api.tag_category_api.update_tag_category(
context_factory(
params={**params, **{"version": 1}},

2
server/szurubooru/tests/api/test_tag_creating.py
View file

@ -71,7 +71,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.tags.create_tag"), patch(
"szurubooru.func.tags.serialize_tag"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
tags.create_tag.return_value = tag_factory()
api.tag_api.create_tag(
context_factory(

21
server/szurubooru/tests/api/test_tag_deleting.py
View file

@ -34,16 +34,17 @@ def test_deleting_used(
post.tags.append(tag)
db.session.add_all([tag, post])
db.session.commit()
api.tag_api.delete_tag(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"tag_name": "tag"},
)
db.session.refresh(post)
assert db.session.query(model.Tag).count() == 0
assert post.tags == []
with patch("szurubooru.func.snapshots._post_to_webhooks"):
api.tag_api.delete_tag(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"tag_name": "tag"},
)
db.session.refresh(post)
assert db.session.query(model.Tag).count() == 0
assert post.tags == []
def test_trying_to_delete_non_existing(user_factory, context_factory):

45
server/szurubooru/tests/func/test_net.py
View file

@ -1,3 +1,6 @@
from datetime import datetime
from unittest.mock import patch
import pytest
from szurubooru import errors
@ -99,3 +102,45 @@ def test_video_download(url, expected_sha1):
def test_failed_video_download(url):
with pytest.raises(errors.ThirdPartyError):
net.download(url, use_video_downloader=True)
def test_no_webhooks(config_injector):
config_injector({"webhooks": []})
res = net.post_to_webhooks(None)
assert len(res) == 0
@pytest.mark.parametrize(
"webhook,status_code",
[
("https://postman-echo.com/post", 200),
("http://localhost/", 400),
("https://postman-echo.com/get", 400),
],
)
def test_single_webhook(config_injector, webhook, status_code):
config_injector({"webhooks": [webhook]})
res = net.post_to_webhooks({"test_arg": "test_value"})
assert len(res) == 1
assert res[0] == status_code
def test_multiple_webhooks(config_injector):
config_injector(
{
"webhooks": [
"https://postman-echo.com/post",
"https://postman-echo.com/post",
]
}
)
res = net.post_to_webhooks({"test_arg": "test_value"})
assert len(res) == 2
assert res[0] == 200
assert res[1] == 200
def test_malformed_webhooks(config_injector):
config_injector({"webhooks": ["malformed_url"]})
with pytest.raises(ValueError):
net.post_to_webhooks({"test_arg": "test_value"})

64
server/szurubooru/tests/func/test_snapshots.py
View file

@ -132,7 +132,9 @@ def test_create(tag_factory, user_factory):
tag = tag_factory(names=["dummy"])
db.session.add(tag)
db.session.flush()
with patch("szurubooru.func.snapshots.get_tag_snapshot"):
with patch("szurubooru.func.snapshots.get_tag_snapshot"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
snapshots.get_tag_snapshot.return_value = "mocked"
snapshots.create(tag, user_factory())
db.session.flush()
@ -156,29 +158,30 @@ def test_modify_saves_non_empty_diffs(post_factory, user_factory):
post.source = "new source"
post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")]
db.session.flush()
snapshots.modify(post, user)
db.session.flush()
results = db.session.query(model.Snapshot).all()
assert len(results) == 1
assert results[0].data == {
"type": "object change",
"value": {
"source": {
"type": "primitive change",
"old-value": None,
"new-value": "new source",
with patch("szurubooru.func.snapshots._post_to_webhooks"):
snapshots.modify(post, user)
db.session.flush()
results = db.session.query(model.Snapshot).all()
assert len(results) == 1
assert results[0].data == {
"type": "object change",
"value": {
"source": {
"type": "primitive change",
"old-value": None,
"new-value": "new source",
},
"notes": {
"type": "list change",
"removed": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"}
],
"added": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"}
],
},
},
"notes": {
"type": "list change",
"removed": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"}
],
"added": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"}
],
},
},
}
}
def test_modify_doesnt_save_empty_diffs(tag_factory, user_factory):
@ -195,7 +198,9 @@ def test_delete(tag_factory, user_factory):
tag = tag_factory(names=["dummy"])
db.session.add(tag)
db.session.flush()
with patch("szurubooru.func.snapshots.get_tag_snapshot"):
with patch("szurubooru.func.snapshots.get_tag_snapshot"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
snapshots.get_tag_snapshot.return_value = "mocked"
snapshots.delete(tag, user_factory())
db.session.flush()
@ -210,8 +215,9 @@ def test_merge(tag_factory, user_factory):
target_tag = tag_factory(names=["target"])
db.session.add_all([source_tag, target_tag])
db.session.flush()
snapshots.merge(source_tag, target_tag, user_factory())
db.session.flush()
result = db.session.query(model.Snapshot).one()
assert result.operation == model.Snapshot.OPERATION_MERGED
assert result.data == ["tag", "target"]
with patch("szurubooru.func.snapshots._post_to_webhooks"):
snapshots.merge(source_tag, target_tag, user_factory())
db.session.flush()
result = db.session.query(model.Snapshot).one()
assert result.operation == model.Snapshot.OPERATION_MERGED
assert result.data == ["tag", "target"]