ulysia/szurubooru
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server: API support for webhooks

Browse source 
Closes #339
This commit is contained in:
Shyam Sunder 2020-08-13 19:14:14 -04:00
parent b74492974d
commit 4595f9a2aa
15 changed files with 182 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
doc/API.md
View file

@ -13,6 +13,7 @@
- [Error handling](#error-handling)
- [Field selecting](#field-selecting)
- [Versioning](#versioning)
- [Webhooks](#webhooks)
2. [API reference](#api-reference)
@ -276,6 +277,15 @@ reject the request as well, in which case the client is encouraged to notify
the user about the situation.
## Webhooks
System administrators can choose to configure webhooks to track events.
Webhook URIs can be configured in `config.yaml` (See `config.yaml.dist` for
example). Upon any event, the API will send a `POST` request to the listed
URIs with a [snapshot resource](#snapshot) generated with anonymous user
privileges as the message body, in JSON format.
# API reference
Depending on the deployment, the URLs might be relative to some base path such

11
server/config.yaml.dist
View file

@ -39,8 +39,9 @@ smtp:
user: # example: bot
pass: # example: groovy123
from: # example: noreply@example.com
# if host is left empty the password reset feature will be disabled, in which case it is
# recommended to fill contactEmail so that users know who to contact when they want to reset their password
# if host is left empty the password reset feature will be disabled,
# in which case it is recommended to fill contactEmail so that users
# know who to contact when they want to reset their password
contact_email: # example: bob@example.com. Meant for manual password reset procedures
@ -58,6 +59,12 @@ pool_category_name_regex: ^[^\s%+#/]+$
password_regex: '^.{5,}$'
user_name_regex: '^[a-zA-Z0-9_-]{1,32}$'
# webhooks to call when events occur (such as post/tag/user/etc. changes)
# the listed urls will be called with a HTTP POST request with a payload
# containing a snapshot resource as JSON. See doc/API.md for details
webhooks:
# - https://api.example.com/webhooks/
default_rank: regular
privileges:

5
server/szurubooru/facade.py
View file

@ -94,6 +94,11 @@ def validate_config() -> None:
if not config.config["database"]:
raise errors.ConfigError("Database is not configured")
if config.config["webhooks"] and not isinstance(
config.config["webhooks"], list
):
raise errors.ConfigError("Webhooks must be provided as a list of URLs")
if config.config["smtp"]["host"]:
if not config.config["smtp"]["port"]:
raise errors.ConfigError("SMTP host is set but port is not set")

24
server/szurubooru/func/net.py
View file

@ -1,7 +1,10 @@
import json
import logging
import os
import urllib.error
import urllib.request
from tempfile import NamedTemporaryFile
from typing import Any, Dict, List
from youtube_dl import YoutubeDL
from youtube_dl.utils import YoutubeDLError
@ -58,3 +61,24 @@ def _youtube_dl_wrapper(url: str) -> bytes:
raise errors.ThirdPartyError(
"Error downloading video %s (file could not be saved)" % (url)
)
def post_to_webhooks(payload: Dict[str, Any]) -> List[int]:
return_list = []
for webhook in config.config["webhooks"] or []:
req = urllib.request.Request(webhook)
req.data = json.dumps(
payload, default=lambda x: x.isoformat("T") + "Z",
).encode("utf-8")
req.add_header("Content-Type", "application/json")
try:
res = urllib.request.urlopen(req)
if not 200 <= res.status <= 299:
logger.warning(
f"Webhook {webhook} returned {res.status} {res.reason}"
)
return_list.append(res.status)
except urllib.error.URLError as e:
logger.error(f"Unable to call webhook {webhook}: {str(e)}")
return_list.append(400)
return return_list

13
server/szurubooru/func/snapshots.py
View file

@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, Optional
import sqlalchemy as sa
from szurubooru import db, model
from szurubooru.func import diff, users
from szurubooru.func import diff, net, users
def get_tag_category_snapshot(category: model.TagCategory) -> Dict[str, Any]:
@ -93,6 +93,13 @@ def serialize_snapshot(
}
def _post_to_webhooks(snapshot: model.Snapshot) -> None:
webhook_user = model.User()
webhook_user.name = None
webhook_user.rank = "anonymous"
net.post_to_webhooks(serialize_snapshot(snapshot, webhook_user))
def _create(
operation: str, entity: model.Base, auth_user: Optional[model.User]
) -> model.Snapshot:
@ -116,6 +123,7 @@ def create(entity: model.Base, auth_user: Optional[model.User]) -> None:
snapshot_factory = _snapshot_factories[snapshot.resource_type]
snapshot.data = snapshot_factory(entity)
db.session.add(snapshot)
_post_to_webhooks(snapshot)
def modify(entity: model.Base, auth_user: Optional[model.User]) -> None:
@ -147,6 +155,7 @@ def modify(entity: model.Base, auth_user: Optional[model.User]) -> None:
if not snapshot.data:
return
db.session.add(snapshot)
_post_to_webhooks(snapshot)
def delete(entity: model.Base, auth_user: Optional[model.User]) -> None:
@ -155,6 +164,7 @@ def delete(entity: model.Base, auth_user: Optional[model.User]) -> None:
snapshot_factory = _snapshot_factories[snapshot.resource_type]
snapshot.data = snapshot_factory(entity)
db.session.add(snapshot)
_post_to_webhooks(snapshot)
def merge(
@ -174,3 +184,4 @@ def merge(
) = model.util.get_resource_info(target_entity)
snapshot.data = [resource_type, resource_name]
db.session.add(snapshot)
_post_to_webhooks(snapshot)

2
server/szurubooru/tests/api/test_pool_category_updating.py
View file

@ -67,7 +67,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.pool_categories.serialize_category"), patch(
"szurubooru.func.pool_categories.update_category_name"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
api.pool_category_api.update_pool_category(
context_factory(
params={**params, **{"version": 1}},

2
server/szurubooru/tests/api/test_pool_creating.py
View file

@ -70,7 +70,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.pools.create_pool"), patch(
"szurubooru.func.pools.serialize_pool"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
pools.create_pool.return_value = pool_factory()
api.pool_api.create_pool(
context_factory(

23
server/szurubooru/tests/api/test_pool_deleting.py
View file

@ -34,17 +34,18 @@ def test_deleting_used(
pool.posts.append(post)
db.session.add_all([pool, post])
db.session.commit()
api.pool_api.delete_pool(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"pool_id": 1},
)
db.session.refresh(post)
assert db.session.query(model.Pool).count() == 0
assert db.session.query(model.PoolPost).count() == 0
assert post.pools == []
with patch("szurubooru.func.snapshots._post_to_webhooks"):
api.pool_api.delete_pool(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"pool_id": 1},
)
db.session.refresh(post)
assert db.session.query(model.Pool).count() == 0
assert db.session.query(model.PoolPost).count() == 0
assert post.pools == []
def test_trying_to_delete_non_existing(user_factory, context_factory):

12
server/szurubooru/tests/api/test_post_creating.py
View file

@ -143,7 +143,9 @@ def test_anonymous_uploads(
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.posts.create_post"
), patch("szurubooru.func.posts.update_post_source"):
), patch("szurubooru.func.posts.update_post_source"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
config_injector(
{
"privileges": {
@ -181,6 +183,8 @@ def test_creating_from_url_saves_source(
"szurubooru.func.posts.serialize_post"
), patch("szurubooru.func.posts.create_post"), patch(
"szurubooru.func.posts.update_post_source"
), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
config_injector(
{
@ -223,6 +227,8 @@ def test_creating_from_url_with_source_specified(
"szurubooru.func.posts.serialize_post"
), patch("szurubooru.func.posts.create_post"), patch(
"szurubooru.func.posts.update_post_source"
), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
config_injector(
{
@ -334,7 +340,7 @@ def test_errors_not_spending_ids(
# successful request
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.posts.update_post_tags"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
posts.serialize_post.side_effect = lambda post, *_, **__: post.post_id
post1_id = api.post_api.create_post(
context_factory(
@ -357,7 +363,7 @@ def test_errors_not_spending_ids(
# successful request
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.posts.update_post_tags"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
posts.serialize_post.side_effect = lambda post, *_, **__: post.post_id
post2_id = api.post_api.create_post(
context_factory(

8
server/szurubooru/tests/api/test_post_featuring.py
View file

@ -55,7 +55,9 @@ def test_trying_to_feature_the_same_post_twice(
):
db.session.add(post_factory(id=1))
db.session.commit()
with patch("szurubooru.func.posts.serialize_post"):
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
api.post_api.set_featured_post(
context_factory(
params={"id": 1},
@ -80,7 +82,9 @@ def test_featuring_one_post_after_another(
assert posts.try_get_featured_post() is None
assert not posts.get_post_by_id(1).is_featured
assert not posts.get_post_by_id(2).is_featured
with patch("szurubooru.func.posts.serialize_post"):
with patch("szurubooru.func.posts.serialize_post"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
with fake_datetime("1997"):
api.post_api.set_featured_post(
context_factory(

2
server/szurubooru/tests/api/test_tag_category_updating.py
View file

@ -65,7 +65,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.tag_categories.serialize_category"), patch(
"szurubooru.func.tag_categories.update_category_name"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
api.tag_category_api.update_tag_category(
context_factory(
params={**params, **{"version": 1}},

2
server/szurubooru/tests/api/test_tag_creating.py
View file

@ -71,7 +71,7 @@ def test_omitting_optional_field(
del params[field]
with patch("szurubooru.func.tags.create_tag"), patch(
"szurubooru.func.tags.serialize_tag"
):
), patch("szurubooru.func.snapshots._post_to_webhooks"):
tags.create_tag.return_value = tag_factory()
api.tag_api.create_tag(
context_factory(

21
server/szurubooru/tests/api/test_tag_deleting.py
View file

@ -34,16 +34,17 @@ def test_deleting_used(
post.tags.append(tag)
db.session.add_all([tag, post])
db.session.commit()
api.tag_api.delete_tag(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"tag_name": "tag"},
)
db.session.refresh(post)
assert db.session.query(model.Tag).count() == 0
assert post.tags == []
with patch("szurubooru.func.snapshots._post_to_webhooks"):
api.tag_api.delete_tag(
context_factory(
params={"version": 1},
user=user_factory(rank=model.User.RANK_REGULAR),
),
{"tag_name": "tag"},
)
db.session.refresh(post)
assert db.session.query(model.Tag).count() == 0
assert post.tags == []
def test_trying_to_delete_non_existing(user_factory, context_factory):

45
server/szurubooru/tests/func/test_net.py
View file

@ -1,3 +1,6 @@
from datetime import datetime
from unittest.mock import patch
import pytest
from szurubooru import errors
@ -99,3 +102,45 @@ def test_video_download(url, expected_sha1):
def test_failed_video_download(url):
with pytest.raises(errors.ThirdPartyError):
net.download(url, use_video_downloader=True)
def test_no_webhooks(config_injector):
config_injector({"webhooks": []})
res = net.post_to_webhooks(None)
assert len(res) == 0
@pytest.mark.parametrize(
"webhook,status_code",
[
("https://postman-echo.com/post", 200),
("http://localhost/", 400),
("https://postman-echo.com/get", 400),
],
)
def test_single_webhook(config_injector, webhook, status_code):
config_injector({"webhooks": [webhook]})
res = net.post_to_webhooks({"test_arg": "test_value"})
assert len(res) == 1
assert res[0] == status_code
def test_multiple_webhooks(config_injector):
config_injector(
{
"webhooks": [
"https://postman-echo.com/post",
"https://postman-echo.com/post",
]
}
)
res = net.post_to_webhooks({"test_arg": "test_value"})
assert len(res) == 2
assert res[0] == 200
assert res[1] == 200
def test_malformed_webhooks(config_injector):
config_injector({"webhooks": ["malformed_url"]})
with pytest.raises(ValueError):
net.post_to_webhooks({"test_arg": "test_value"})

64
server/szurubooru/tests/func/test_snapshots.py
View file

@ -132,7 +132,9 @@ def test_create(tag_factory, user_factory):
tag = tag_factory(names=["dummy"])
db.session.add(tag)
db.session.flush()
with patch("szurubooru.func.snapshots.get_tag_snapshot"):
with patch("szurubooru.func.snapshots.get_tag_snapshot"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
snapshots.get_tag_snapshot.return_value = "mocked"
snapshots.create(tag, user_factory())
db.session.flush()
@ -156,29 +158,30 @@ def test_modify_saves_non_empty_diffs(post_factory, user_factory):
post.source = "new source"
post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")]
db.session.flush()
snapshots.modify(post, user)
db.session.flush()
results = db.session.query(model.Snapshot).all()
assert len(results) == 1
assert results[0].data == {
"type": "object change",
"value": {
"source": {
"type": "primitive change",
"old-value": None,
"new-value": "new source",
with patch("szurubooru.func.snapshots._post_to_webhooks"):
snapshots.modify(post, user)
db.session.flush()
results = db.session.query(model.Snapshot).all()
assert len(results) == 1
assert results[0].data == {
"type": "object change",
"value": {
"source": {
"type": "primitive change",
"old-value": None,
"new-value": "new source",
},
"notes": {
"type": "list change",
"removed": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"}
],
"added": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"}
],
},
},
"notes": {
"type": "list change",
"removed": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"}
],
"added": [
{"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"}
],
},
},
}
}
def test_modify_doesnt_save_empty_diffs(tag_factory, user_factory):
@ -195,7 +198,9 @@ def test_delete(tag_factory, user_factory):
tag = tag_factory(names=["dummy"])
db.session.add(tag)
db.session.flush()
with patch("szurubooru.func.snapshots.get_tag_snapshot"):
with patch("szurubooru.func.snapshots.get_tag_snapshot"), patch(
"szurubooru.func.snapshots._post_to_webhooks"
):
snapshots.get_tag_snapshot.return_value = "mocked"
snapshots.delete(tag, user_factory())
db.session.flush()
@ -210,8 +215,9 @@ def test_merge(tag_factory, user_factory):
target_tag = tag_factory(names=["target"])
db.session.add_all([source_tag, target_tag])
db.session.flush()
snapshots.merge(source_tag, target_tag, user_factory())
db.session.flush()
result = db.session.query(model.Snapshot).one()
assert result.operation == model.Snapshot.OPERATION_MERGED
assert result.data == ["tag", "target"]
with patch("szurubooru.func.snapshots._post_to_webhooks"):
snapshots.merge(source_tag, target_tag, user_factory())
db.session.flush()
result = db.session.query(model.Snapshot).one()
assert result.operation == model.Snapshot.OPERATION_MERGED
assert result.data == ["tag", "target"]