ulysia/szurubooru
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

client+server: implement HTML caching and URL prefix determination

Browse source 
* Uses 'X-Forwarded-Prefix' header to determine base URL if not
  explicitly defined
* Authentication ignored for HTML generation to improve caching
* Also add logging of 'User-Agent' for HTML requests
This commit is contained in:
Shyam Sunder 2021-09-24 20:04:56 -04:00
parent d5a223652e
commit 636498ad38
8 changed files with 108 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
client/docker-start.sh
View file

@ -1,5 +1,9 @@
#!/usr/bin/dumb-init /bin/sh #!/usr/bin/dumb-init /bin/sh
# Create cache directory
mkdir -p /tmp/nginx-cache
chmod a+rwx /tmp/nginx-cache
# Integrate environment variables # Integrate environment variables
sed -i "s|__BACKEND__|${BACKEND_HOST}|" \ sed -i "s|__BACKEND__|${BACKEND_HOST}|" \
/etc/nginx/nginx.conf /etc/nginx/nginx.conf

34
client/nginx.conf.docker
View file

@ -19,6 +19,9 @@ http {
server_tokens off; server_tokens off;
keepalive_timeout 65; keepalive_timeout 65;
proxy_cache_path /tmp/nginx-cache
levels=1:2 keys_zone=spa_cache:1m max_size=50m inactive=60m use_temp_path=off;
upstream backend { upstream backend {
server __BACKEND__:6666; server __BACKEND__:6666;
} }
@ -50,6 +53,14 @@ http {
gzip_proxied expired no-cache no-store private auth; gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain application/json; gzip_types text/plain application/json;
if ($http_x_forwarded_host = '') {
set $http_x_forwarded_host $host;
}
if ($http_x_forwarded_proto = '') {
set $http_x_forwarded_proto 'http';
}
if ($request_uri ~* "/api/(.*)") { if ($request_uri ~* "/api/(.*)") {
proxy_pass http://backend/$1; proxy_pass http://backend/$1;
} }
@ -85,6 +96,29 @@ http {
location / { location / {
tcp_nodelay on; tcp_nodelay on;
proxy_cache spa_cache;
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
proxy_cache_background_update on;
proxy_cache_lock on;
gzip on;
gzip_comp_level 3;
gzip_min_length 20;
gzip_proxied any;
gzip_types text/plain application/json;
if ( $http_accept ~ "application/json" ) {
return 406 "API requests should be sent to the /api prefix";
}
if ($http_x_forwarded_host = '') {
set $http_x_forwarded_host $host;
}
if ($http_x_forwarded_proto = '') {
set $http_x_forwarded_proto 'http';
}
proxy_pass http://backend; proxy_pass http://backend;
error_page 500 502 503 504 @badproxy; error_page 500 502 503 504 @badproxy;

9
server/szurubooru/api/info_api.py
View file

@ -74,20 +74,17 @@ def generate_manifest(
"name": config.config["name"], "name": config.config["name"],
"icons": [ "icons": [
{ {
# TODO: Host Header and Proxy Prefix "src": f"{ctx.url_prefix}/img/android-chrome-192x192.png",
"src": "img/android-chrome-192x192.png",
"type": "image/png", "type": "image/png",
"sizes": "192x192", "sizes": "192x192",
}, },
{ {
# TODO: Host Header and Proxy Prefix "src": f"{ctx.url_prefix}/img/android-chrome-512x512.png",
"src": "img/android-chrome-512x512.png",
"type": "image/png", "type": "image/png",
"sizes": "512x512", "sizes": "512x512",
}, },
], ],
# TODO: Host Header and Proxy Prefix "start_url": f"{ctx.url_prefix}/",
"start_url": "/",
"theme_color": "#24aadd", "theme_color": "#24aadd",
"background_color": "#ffffff", "background_color": "#ffffff",
"display": "standalone", "display": "standalone",

31
server/szurubooru/api/opengraph_api.py
View file

@ -62,7 +62,7 @@ _apple_touch_startup_images = {
def _get_html_template( def _get_html_template(
meta_tags: Dict = {}, title: str = config.config["name"] meta_tags: Dict = {}, title: str = config.config["name"], prefix: str = ""
) -> Doc: ) -> Doc:
doc = Doc() doc = Doc()
doc.asis("<!DOCTYPE html>") doc.asis("<!DOCTYPE html>")
@ -73,18 +73,19 @@ def _get_html_template(
doc.stag("meta", name=name, content=content) doc.stag("meta", name=name, content=content)
with doc.tag("title"): with doc.tag("title"):
doc.text(title) doc.text(title)
# TODO: Host Header and Proxy Prefix doc.stag("base", href=f"{prefix}/")
doc.stag("base", href="/") doc.stag(
doc.stag("link", rel="manifest", href="/manifest.json") "link", rel="manifest", href=f"{prefix}/api/manifest.json"
)
doc.stag( doc.stag(
"link", "link",
href="css/app.min.css", href=f"{prefix}/css/app.min.css",
rel="stylesheet", rel="stylesheet",
type="text/css", type="text/css",
) )
doc.stag( doc.stag(
"link", "link",
href="css/vendor.min.css", href=f"{prefix}/css/vendor.min.css",
rel="stylesheet", rel="stylesheet",
type="text/css", type="text/css",
) )
@ -92,19 +93,19 @@ def _get_html_template(
"link", "link",
rel="shortcut icon", rel="shortcut icon",
type="image/png", type="image/png",
href="img/favicon.png", href=f"{prefix}/img/favicon.png",
) )
doc.stag( doc.stag(
"link", "link",
rel="apple-touch-icon", rel="apple-touch-icon",
sizes="180x180", sizes="180x180",
href="img/apple-touch-icon.png", href=f"{prefix}/img/apple-touch-icon.png",
) )
for res, media in _apple_touch_startup_images.items(): for res, media in _apple_touch_startup_images.items():
doc.stag( doc.stag(
"link", "link",
rel="apple-touch-startup-image", rel="apple-touch-startup-image",
href=f"img/apple-touch-startup-image-{res}.png", href=f"{prefix}/img/apple-touch-startup-image-{res}.png",
media=" and ".join( media=" and ".join(
f"({k}: {v})" for k, v in media.items() f"({k}: {v})" for k, v in media.items()
), ),
@ -148,21 +149,25 @@ def get_post_html(
except posts.InvalidPostIdError: except posts.InvalidPostIdError:
# Return the default template and let the browser JS handle the 404 # Return the default template and let the browser JS handle the 404
return _get_html_template() return _get_html_template()
metadata = { metadata = {
"og:site_name": config.config["name"], "og:site_name": config.config["name"],
"og:type": "image", "og:type": "image",
"og:title": title, "og:title": title,
# TODO: Host Header and Proxy Prefix "og:url": f"{ctx.url_prefix}/post/{params['post_id']}",
"og:url": f"{config.config['domain'] or ''}/post/{params['post_id']}",
} }
# Note: ctx.user will always be the anonymous user
if auth.has_privilege(ctx.user, "posts:view"): if auth.has_privilege(ctx.user, "posts:view"):
# TODO: Host Header and Proxy Prefix # TODO: Host Header and Proxy Prefix
metadata["og:image"] = posts.get_post_content_url(post) metadata["og:image"] = posts.get_post_content_url(post)
return _get_html_template(meta_tags=metadata, title=title)
return _get_html_template(
meta_tags=metadata, title=title, prefix=ctx.url_prefix
)
@rest.routes.get("/.*", accept="text/html") @rest.routes.get("/.*", accept="text/html")
def default_route( def default_route(
ctx: rest.Context, _params: Dict[str, str] = {} ctx: rest.Context, _params: Dict[str, str] = {}
) -> rest.Response: ) -> rest.Response:
return _get_html_template() return _get_html_template(prefix=ctx.url_prefix)

1
server/szurubooru/middleware/authenticator.py
View file

@ -73,6 +73,7 @@ def _get_user(ctx: rest.Context, bump_login: bool) -> Optional[model.User]:
def process_request(ctx: rest.Context) -> None: def process_request(ctx: rest.Context) -> None:
""" Bind the user to request. Update last login time if needed. """ """ Bind the user to request. Update last login time if needed. """
if ctx.accept == "application/json":
bump_login = ctx.get_param_as_bool("bump-login", default=False) bump_login = ctx.get_param_as_bool("bump-login", default=False)
auth_user = _get_user(ctx, bump_login) auth_user = _get_user(ctx, bump_login)
if auth_user: if auth_user:

8
server/szurubooru/middleware/request_logger.py
View file

@ -13,6 +13,7 @@ def process_request(_ctx: rest.Context) -> None:
@middleware.post_hook @middleware.post_hook
def process_response(ctx: rest.Context) -> None: def process_response(ctx: rest.Context) -> None:
if ctx.accept == "application/json":
logger.info( logger.info(
"%s %s (user=%s, queries=%d)", "%s %s (user=%s, queries=%d)",
ctx.method, ctx.method,
@ -20,3 +21,10 @@ def process_response(ctx: rest.Context) -> None:
ctx.user.name, ctx.user.name,
db.get_query_count(), db.get_query_count(),
) )
elif ctx.accept == "text/html":
logger.info(
"HTML %s (user-agent='%s' queries=%d)",
ctx.url,
ctx.get_header("User-Agent"),
db.get_query_count(),
)

43
server/szurubooru/rest/app.py
View file

@ -5,7 +5,7 @@ import urllib.parse
from datetime import datetime from datetime import datetime
from typing import Any, Callable, Dict, Tuple from typing import Any, Callable, Dict, Tuple
from szurubooru import db from szurubooru import config, db
from szurubooru.func import util from szurubooru.func import util
from szurubooru.rest import context, errors, middleware, routes from szurubooru.rest import context, errors, middleware, routes
@ -40,6 +40,23 @@ def _create_context(env: Dict[str, Any]) -> context.Context:
path = "/" + env["PATH_INFO"].lstrip("/") path = "/" + env["PATH_INFO"].lstrip("/")
path = path.encode("latin-1").decode("utf-8") # PEP-3333 path = path.encode("latin-1").decode("utf-8") # PEP-3333
headers = _get_headers(env) headers = _get_headers(env)
_raw_accept = headers.get("Accept", "text/html")
if "application/json" in _raw_accept:
accept = "application/json"
elif "text/html" in _raw_accept:
accept = "text/html"
else:
raise errors.HttpNotAcceptable(
"ValidationError",
"This API only supports the following response types: "
"application/json, text/html",
)
if config.config["domain"]:
url_prefix = config.config["domain"].rstrip("/")
else:
url_prefix = headers.get("X-Forwarded-Prefix", "")
files = {} files = {}
params = dict(urllib.parse.parse_qsl(env.get("QUERY_STRING", ""))) params = dict(urllib.parse.parse_qsl(env.get("QUERY_STRING", "")))
@ -70,27 +87,17 @@ def _create_context(env: Dict[str, Any]) -> context.Context:
"was incorrect or was not encoded as UTF-8.", "was incorrect or was not encoded as UTF-8.",
) )
return context.Context(env, method, path, headers, params, files) return context.Context(
env, method, path, headers, accept, url_prefix, params, files
)
def application( def application(
env: Dict[str, Any], start_response: Callable[[str, Any], Any] env: Dict[str, Any], start_response: Callable[[str, Any], Any]
) -> Tuple[bytes]: ) -> Tuple[bytes]:
accept = None
try: try:
ctx = _create_context(env) ctx = _create_context(env)
if "application/json" in ctx.get_header("Accept"): for url, allowed_methods in routes.routes[ctx.accept].items():
accept = "application/json"
elif "text/html" in (ctx.get_header("Accept") or "text/html"):
accept = "text/html"
else:
raise errors.HttpNotAcceptable(
"ValidationError",
"This API only supports the following response types: "
", ".join(routes.routes.keys()),
)
for url, allowed_methods in routes.routes[accept].items():
match = re.fullmatch(url, ctx.url) match = re.fullmatch(url, ctx.url)
if match: if match:
if ctx.method not in allowed_methods: if ctx.method not in allowed_methods:
@ -122,9 +129,9 @@ def application(
finally: finally:
db.session.remove() db.session.remove()
start_response("200", [("content-type", accept)]) start_response("200", [("content-type", ctx.accept)])
return ( return (
_serialize_response_body(response, accept).encode("utf-8"), _serialize_response_body(response, ctx.accept).encode("utf-8"),
) )
except Exception as ex: except Exception as ex:
@ -136,7 +143,7 @@ def application(
except errors.BaseHttpError as ex: except errors.BaseHttpError as ex:
start_response( start_response(
"%d %s" % (ex.code, ex.reason), "%d %s" % (ex.code, ex.reason),
[("content-type", accept or "application/json")], [("content-type", "application/json")],
) )
blob = { blob = {
"name": ex.name, "name": ex.name,

4
server/szurubooru/rest/context.py
View file

@ -15,12 +15,16 @@ class Context:
method: str, method: str,
url: str, url: str,
headers: Dict[str, str] = None, headers: Dict[str, str] = None,
accept: str = None,
url_prefix: str = None,
params: Request = None, params: Request = None,
files: Dict[str, bytes] = None, files: Dict[str, bytes] = None,
) -> None: ) -> None:
self.env = env self.env = env
self.method = method self.method = method
self.url = url self.url = url
self.accept = accept
self.url_prefix = url_prefix
self._headers = headers or {} self._headers = headers or {}
self._params = params or {} self._params = params or {}
self._files = files or {} self._files = files or {}