From 81645864ecaa2a43a55c333779e3e4ede4cbf404 Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 02:38:40 -0700 Subject: [PATCH 01/35] Support sorting post search results by pool post order --- .../search/configs/post_search_config.py | 37 +++++++++++++---- server/szurubooru/search/configs/util.py | 1 + server/szurubooru/search/executor.py | 16 +++++--- server/szurubooru/search/typing.py | 2 +- .../search/configs/test_post_search_config.py | 40 +++++++++++++++++++ 5 files changed, 81 insertions(+), 15 deletions(-) diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index ddc003b7..8e5fe71a 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, Callable, Union import sqlalchemy as sa @@ -114,17 +114,30 @@ def _pool_filter( query: SaQuery, criterion: Optional[criteria.BaseCriterion], negated: bool ) -> SaQuery: assert criterion - return search_util.create_subquery_filter( - model.Post.post_id, - model.PoolPost.post_id, - model.PoolPost.pool_id, - search_util.create_num_filter, - )(query, criterion, negated) + from szurubooru.search.configs import util as search_util + subquery = db.session.query(model.PoolPost.post_id.label("foreign_id")) + subquery = subquery.options(sa.orm.lazyload("*")) + subquery = search_util.create_num_filter(model.PoolPost.pool_id)(subquery, criterion, False) + subquery = subquery.subquery("t") + expression = model.Post.post_id.in_(subquery) + if negated: + expression = ~expression + return query.filter(expression) + + +def _pool_sort( + query: SaQuery, pool_id: Optional[int] +) -> SaQuery: + if pool_id is None: + return query + return query.join(model.PoolPost, sa.and_(model.PoolPost.post_id == model.Post.post_id, model.PoolPost.pool_id == pool_id)) \ + .order_by(model.PoolPost.order.desc()) class PostSearchConfig(BaseSearchConfig): def __init__(self) -> None: self.user = None # type: Optional[model.User] + self.pool_id = None # type: Optional[int] def on_search_query_parsed(self, search_query: SearchQuery) -> SaQuery: new_special_tokens = [] @@ -149,6 +162,10 @@ class PostSearchConfig(BaseSearchConfig): else: new_special_tokens.append(token) search_query.special_tokens = new_special_tokens + self.pool_id = None + for token in search_query.named_tokens: + if token.name == "pool" and isinstance(token.criterion, criteria.PlainCriterion): + self.pool_id = token.criterion.value def create_around_query(self) -> SaQuery: return db.session.query(model.Post).options(sa.orm.lazyload("*")) @@ -353,7 +370,7 @@ class PostSearchConfig(BaseSearchConfig): ) @property - def sort_columns(self) -> Dict[str, Tuple[SaColumn, str]]: + def sort_columns(self) -> Dict[str, Union[Tuple[SaColumn, str], Callable[[SaQuery], None]]]: return util.unalias_dict( [ ( @@ -415,6 +432,10 @@ class PostSearchConfig(BaseSearchConfig): ["feature-date", "feature-time"], (model.Post.last_feature_time, self.SORT_DESC), ), + ( + ["pool"], + lambda subquery: _pool_sort(subquery, self.pool_id) + ) ] ) diff --git a/server/szurubooru/search/configs/util.py b/server/szurubooru/search/configs/util.py index 58e6ebe5..fd40b43d 100644 --- a/server/szurubooru/search/configs/util.py +++ b/server/szurubooru/search/configs/util.py @@ -205,6 +205,7 @@ def create_subquery_filter( filter_column: SaColumn, filter_factory: SaColumn, subquery_decorator: Callable[[SaQuery], None] = None, + order: SaQuery = None, ) -> Filter: filter_func = filter_factory(filter_column) diff --git a/server/szurubooru/search/executor.py b/server/szurubooru/search/executor.py index a5ef9625..beb17f8a 100644 --- a/server/szurubooru/search/executor.py +++ b/server/szurubooru/search/executor.py @@ -181,14 +181,18 @@ class Executor: _format_dict_keys(self.config.sort_columns), ) ) - column, default_order = self.config.sort_columns[ + entry = self.config.sort_columns[ sort_token.name ] - order = _get_order(sort_token.order, default_order) - if order == sort_token.SORT_ASC: - db_query = db_query.order_by(column.asc()) - elif order == sort_token.SORT_DESC: - db_query = db_query.order_by(column.desc()) + if callable(entry): + db_query = entry(db_query) + else: + column, default_order = entry + order = _get_order(sort_token.order, default_order) + if order == sort_token.SORT_ASC: + db_query = db_query.order_by(column.asc()) + elif order == sort_token.SORT_DESC: + db_query = db_query.order_by(column.desc()) db_query = self.config.finalize_query(db_query) return db_query diff --git a/server/szurubooru/search/typing.py b/server/szurubooru/search/typing.py index 686c2cb6..011f7eae 100644 --- a/server/szurubooru/search/typing.py +++ b/server/szurubooru/search/typing.py @@ -1,4 +1,4 @@ -from typing import Any, Callable +from typing import Any, Callable, Union SaColumn = Any SaQuery = Any diff --git a/server/szurubooru/tests/search/configs/test_post_search_config.py b/server/szurubooru/tests/search/configs/test_post_search_config.py index 4fb8191a..0f5336e1 100644 --- a/server/szurubooru/tests/search/configs/test_post_search_config.py +++ b/server/szurubooru/tests/search/configs/test_post_search_config.py @@ -725,6 +725,7 @@ def test_filter_by_feature_date( "sort:fav-time", "sort:feature-date", "sort:feature-time", + "sort:pool", ], ) def test_sort_tokens(verify_unpaged, post_factory, input): @@ -863,3 +864,42 @@ def test_tumbleweed( db.session.flush() verify_unpaged("special:tumbleweed", [4]) verify_unpaged("-special:tumbleweed", [1, 2, 3]) + + +def test_sort_pool( + post_factory, pool_factory, pool_category_factory, verify_unpaged +): + post1 = post_factory(id=1) + post2 = post_factory(id=2) + post3 = post_factory(id=3) + post4 = post_factory(id=4) + pool1 = pool_factory( + id=1, + names=["pool1"], + description="desc", + category=pool_category_factory("test-cat1"), + ) + pool1.posts = [post1, post4, post3] + pool2 = pool_factory( + id=2, + names=["pool2"], + description="desc", + category=pool_category_factory("test-cat2"), + ) + pool2.posts = [post3, post4, post2] + db.session.add_all( + [ + post1, + post2, + post3, + post4, + pool1, + pool2 + ] + ) + db.session.flush() + verify_unpaged("pool:1 sort:pool", [1, 4, 3]) + verify_unpaged("pool:2 sort:pool", [3, 4, 2]) + verify_unpaged("pool:1 pool:2 sort:pool", [4, 3]) + verify_unpaged("pool:2 pool:1 sort:pool", [3, 4]) + verify_unpaged("sort:pool", [1, 2, 3, 4]) From 6fd48dcb5f0b3636847f281773cf13c1f6c49345 Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 02:41:26 -0700 Subject: [PATCH 02/35] Sort by pool by default from pool details page --- client/html/pool_delete.tpl | 2 +- client/html/pool_summary.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/html/pool_delete.tpl b/client/html/pool_delete.tpl index 1ef7fb53..f279e0df 100644 --- a/client/html/pool_delete.tpl +++ b/client/html/pool_delete.tpl @@ -1,6 +1,6 @@
-

This pool has '><%- ctx.pool.postCount %> post(s).

+

This pool has '><%- ctx.pool.postCount %> post(s).

  • diff --git a/client/html/pool_summary.tpl b/client/html/pool_summary.tpl index 8f4e27d0..b20672e4 100644 --- a/client/html/pool_summary.tpl +++ b/client/html/pool_summary.tpl @@ -18,6 +18,6 @@
    <%= ctx.makeMarkdown(ctx.pool.description || 'This pool has no description yet.') %> -

    This pool has '><%- ctx.pool.postCount %> post(s).

    +

    This pool has '><%- ctx.pool.postCount %> post(s).

From 47377395a2143fb0617265464799c905651fd9ca Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 05:04:39 -0700 Subject: [PATCH 03/35] Thumbnail view in pool list --- client/css/pool-list-view.styl | 109 +++++++++++------- client/html/pools_page.tpl | 59 +++------- client/js/controllers/pool_list_controller.js | 10 +- client/package.json | 2 +- 4 files changed, 94 insertions(+), 86 deletions(-) diff --git a/client/css/pool-list-view.styl b/client/css/pool-list-view.styl index b7ac15ed..272ac931 100644 --- a/client/css/pool-list-view.styl +++ b/client/css/pool-list-view.styl @@ -1,47 +1,60 @@ @import colors .pool-list - table - width: 100% - border-spacing: 0 - text-align: left - line-height: 1.3em - tr:hover td - background: $top-navigation-color - th, td - padding: 0.1em 0.5em - th - white-space: nowrap - background: $top-navigation-color - .names - width: 84% - .post-count - text-align: center - width: 8% - .creation-time - text-align: center - width: 8% - white-space: pre - ul - list-style-type: none - margin: 0 - padding: 0 - display: inline - li - padding: 0 - display: inline - &:not(:last-child):after - content: ', ' - @media (max-width: 800px) - .posts - display: none + ul + list-style-type: none + margin: 0 + padding: 0 + display: flex + align-content: flex-end + flex-wrap: wrap + margin: 0 -0.25em -.darktheme .pool-list - table - tr:hover td - background: $top-navigation-color-darktheme - th - background: $top-navigation-color-darktheme + li + position: relative + flex-grow: 1 + margin: 0 0.25em 0.5em 0.25em + display: inline-block + text-align: left + min-width: 10em + width: 12vw + &:not(.flexbox-dummy) + min-height: 7.5em + height: 9vw + + .thumbnail-wrapper + display: inline-block + width: 100% + height: 100% + line-height: 80% + font-size: 80% + color: white + outline-offset: -3px + box-shadow: 0 0 0 1px rgba(0,0,0,0.2) + + .thumbnail + width: 100% + height: 100% + outline-offset: -3px + &:not(.empty) + background-position: 50% 30% + + .pool-name + color: black + font-size: 1em + text-align: center + a + width: 100% + display: inline-block + + &:hover + background: $post-thumbnail-border-color + .thumbnail + opacity: .9 + + &:hover a, a:active, a:focus + .thumbnail + outline: 4px solid $main-color !important .pool-list-header label @@ -61,3 +74,19 @@ .darktheme .pool-list-header .append color: $inactive-link-color-darktheme + +.post-flow + ul + li + min-width: inherit + width: inherit + &:not(.flexbox-dummy) + height: 14vw + .thumbnail + outline-offset: -1px + .thumbnail-wrapper.no-tags + .thumbnail + outline: 2px solid $post-thumbnail-no-tags-border-color + &:hover a, a:active, a:focus + .thumbnail + outline: 2px solid $main-color !important diff --git a/client/html/pools_page.tpl b/client/html/pools_page.tpl index 0d811808..48530ea3 100644 --- a/client/html/pools_page.tpl +++ b/client/html/pools_page.tpl @@ -1,48 +1,19 @@ -
+<% if (ctx.postFlow) { %>
<% } else { %>
<% } %> <% if (ctx.response.results.length) { %> - - - - - - - - <% for (let pool of ctx.response.results) { %> - - - - - - <% } %> - -
- <% if (ctx.parameters.query == 'sort:name' || !ctx.parameters.query) { %> - '>Pool name(s) - <% } else { %> - '>Pool name(s) + - <% if (ctx.parameters.query == 'sort:post-count') { %> - '>Post count - <% } else { %> - '>Post count - <% } %> - - <% if (ctx.parameters.query == 'sort:creation-time') { %> - '>Created on - <% } else { %> - '>Created on - <% } %> -
-
    - <% for (let name of pool.names) { %> -
  • <%= ctx.makePoolLink(pool.id, false, false, pool, name) %>
    • - <% } %> -
- 		- '><%- pool.postCount %> - - <%= ctx.makeRelativeTime(pool.creationTime) %> -
+ +
+ <%= ctx.makePoolLink(pool.id, false, false, pool, name) %> +
+ + <% } %> + <%= ctx.makeFlexboxAlign() %> + <% } %>
diff --git a/client/js/controllers/pool_list_controller.js b/client/js/controllers/pool_list_controller.js index 91d655c5..67608684 100644 --- a/client/js/controllers/pool_list_controller.js +++ b/client/js/controllers/pool_list_controller.js @@ -2,6 +2,7 @@ const router = require("../router.js"); const api = require("../api.js"); +const settings = require("../models/settings.js"); const uri = require("../util/uri.js"); const PoolList = require("../models/pool_list.js"); const topNavigation = require("../models/top_navigation.js"); @@ -42,7 +43,9 @@ class PoolListController { }); this._headerView.addEventListener( "submit", - (e) => this._evtSubmit(e), + (e) => this._evtSubmit(e) + ); + this._headerView.addEventListener( "navigate", (e) => this._evtNavigate(e) ); @@ -106,6 +109,11 @@ class PoolListController { ); }, pageRenderer: (pageCtx) => { + Object.assign(pageCtx, { + canViewPosts: api.hasPrivilege("posts:view"), + canViewPools: api.hasPrivilege("pools:view"), + postFlow: settings.get().postFlow, + }); return new PoolsPageView(pageCtx); }, }); diff --git a/client/package.json b/client/package.json index 6c816432..991700ec 100644 --- a/client/package.json +++ b/client/package.json @@ -3,7 +3,7 @@ "private": true, "scripts": { "build": "node build.js", - "watch": "c1=\"\";while :;do c2=$(find html js css img -type f -and -not -iname '*autogen*'|sort|xargs cat|md5sum);[[ $c1 != $c2 ]]&&npm run build -- --debug --no-vendor-js;c1=$c2;sleep 1;done" + "watch": "c1=\"\";while :;do c2=$(find html js css img -type f -and -not -iname '*autogen*'|sort|xargs cat|md5sum);[[ $c1 != $c2 ]]&&npm run build -- --debug --no-vendor-js --no-web-app-files --no-binary-assets;c1=$c2;sleep 1;done" }, "dependencies": { "dompurify": "^2.0.17", From 161a3939c907a87d4adad8af722eb7414998a8f6 Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 16:15:39 -0700 Subject: [PATCH 04/35] Add new sort tokens for pools; update API doc --- doc/API.md | 91 ++++++++++--------- .../search/configs/pool_search_config.py | 4 +- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/doc/API.md b/doc/API.md index 3d280fd1..804f60a5 100644 --- a/doc/API.md +++ b/doc/API.md @@ -793,38 +793,39 @@ data. **Sort style tokens** - | `` | Description | - | ---------------- | ------------------------------------------------ | - | `random` | as random as it can get | - | `id` | highest to lowest post number | - | `score` | highest scored | - | `tag-count` | with most tags | - | `comment-count` | most commented first | - | `fav-count` | loved by most | - | `note-count` | with most annotations | - | `relation-count` | with most relations | - | `feature-count` | most often featured | - | `file-size` | largest files first | - | `image-width` | widest images first | - | `image-height` | tallest images first | - | `image-area` | largest images first | - | `width` | alias of `image-width` | - | `height` | alias of `image-height` | - | `area` | alias of `image-area` | - | `creation-date` | newest to oldest (pretty much same as id) | - | `creation-time` | alias of `creation-date` | - | `date` | alias of `creation-date` | - | `time` | alias of `creation-date` | - | `last-edit-date` | like creation-date, only looks at last edit time | - | `last-edit-time` | alias of `last-edit-date` | - | `edit-date` | alias of `last-edit-date` | - | `edit-time` | alias of `last-edit-date` | - | `comment-date` | recently commented by anyone | - | `comment-time` | alias of `comment-date` | - | `fav-date` | recently added to favorites by anyone | - | `fav-time` | alias of `fav-date` | - | `feature-date` | recently featured | - | `feature-time` | alias of `feature-time` | + | `` | Description | + | ---------------- | ------------------------------------------------ | + | `random` | as random as it can get | + | `id` | highest to lowest post number | + | `score` | highest scored | + | `tag-count` | with most tags | + | `comment-count` | most commented first | + | `fav-count` | loved by most | + | `note-count` | with most annotations | + | `relation-count` | with most relations | + | `feature-count` | most often featured | + | `file-size` | largest files first | + | `image-width` | widest images first | + | `image-height` | tallest images first | + | `image-area` | largest images first | + | `width` | alias of `image-width` | + | `height` | alias of `image-height` | + | `area` | alias of `image-area` | + | `creation-date` | newest to oldest (pretty much same as id) | + | `creation-time` | alias of `creation-date` | + | `date` | alias of `creation-date` | + | `time` | alias of `creation-date` | + | `last-edit-date` | like creation-date, only looks at last edit time | + | `last-edit-time` | alias of `last-edit-date` | + | `edit-date` | alias of `last-edit-date` | + | `edit-time` | alias of `last-edit-date` | + | `comment-date` | recently commented by anyone | + | `comment-time` | alias of `comment-date` | + | `fav-date` | recently added to favorites by anyone | + | `fav-time` | alias of `fav-date` | + | `feature-date` | recently featured | + | `feature-time` | alias of `feature-time` | + | `pool` | post order of the pool referenced by the `pool:` named token in the same search query | **Special tokens** @@ -1333,6 +1334,7 @@ data. | `` | Description | | ------------------- | ----------------------------------------- | + | `id` | having given pool number | | `name` | having given name (accepts wildcards) | | `category` | having given category (accepts wildcards) | | `creation-date` | created at given date | @@ -1345,18 +1347,19 @@ data. **Sort style tokens** - | `` | Description | - | ------------------- | ---------------------------- | - | `random` | as random as it can get | - | `name` | A to Z | - | `category` | category (A to Z) | - | `creation-date` | recently created first | - | `creation-time` | alias of `creation-date` | - | `last-edit-date` | recently edited first | - | `last-edit-time` | alias of `creation-time` | - | `edit-date` | alias of `creation-time` | - | `edit-time` | alias of `creation-time` | - | `post-count` | used in most posts first | + | `` | Description | + | ------------------- | ---------------------------- | + | `random` | as random as it can get | + | `id` | highest to lowest pool number | + | `name` | A to Z | + | `category` | category (A to Z) | + | `creation-date` | recently created first | + | `creation-time` | alias of `creation-date` | + | `last-edit-date` | recently edited first | + | `last-edit-time` | alias of `creation-time` | + | `edit-date` | alias of `creation-time` | + | `edit-time` | alias of `creation-time` | + | `post-count` | used in most posts first | **Special tokens** diff --git a/server/szurubooru/search/configs/pool_search_config.py b/server/szurubooru/search/configs/pool_search_config.py index 88b30a6e..30b4d4ea 100644 --- a/server/szurubooru/search/configs/pool_search_config.py +++ b/server/szurubooru/search/configs/pool_search_config.py @@ -30,7 +30,7 @@ class PoolSearchConfig(BaseSearchConfig): raise NotImplementedError() def finalize_query(self, query: SaQuery) -> SaQuery: - return query.order_by(model.Pool.first_name.asc()) + return query.order_by(model.Pool.pool_id.desc()) @property def anonymous_filter(self) -> Filter: @@ -45,6 +45,7 @@ class PoolSearchConfig(BaseSearchConfig): def named_filters(self) -> Dict[str, Filter]: return util.unalias_dict( [ + (["id"], search_util.create_num_filter(model.Pool.pool_id)), ( ["name"], search_util.create_subquery_filter( @@ -91,6 +92,7 @@ class PoolSearchConfig(BaseSearchConfig): ["random"], (sa.sql.expression.func.random(), self.SORT_NONE), ), + (["id"], (model.Pool.pool_id, self.SORT_DESC)), (["name"], (model.Pool.first_name, self.SORT_ASC)), (["category"], (model.PoolCategory.name, self.SORT_ASC)), ( From eee9b70b0e91bd824bbeb9235a63f48a1ae5257e Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 16:15:59 -0700 Subject: [PATCH 05/35] Stacked thumbnail appearance for pool list page --- client/css/pool-list-view.styl | 22 ++++++++++++++++++++-- client/html/pools_page.tpl | 4 ++-- client/js/util/views.js | 24 +++++++++++++++++++++--- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/client/css/pool-list-view.styl b/client/css/pool-list-view.styl index 272ac931..37ec877c 100644 --- a/client/css/pool-list-view.styl +++ b/client/css/pool-list-view.styl @@ -3,7 +3,6 @@ .pool-list ul list-style-type: none - margin: 0 padding: 0 display: flex align-content: flex-end @@ -13,7 +12,7 @@ li position: relative flex-grow: 1 - margin: 0 0.25em 0.5em 0.25em + margin: 2em 1.5em 2em 1.2em; display: inline-block text-align: left min-width: 10em @@ -38,6 +37,23 @@ outline-offset: -3px &:not(.empty) background-position: 50% 30% + position: absolute + display: inline-block + + .thumbnail-1 + right: -0px; + top: -0px; + z-index: 30; + + .thumbnail-2 + right: -10px; + top: -10px; + z-index: 20; + + .thumbnail-3 + right: -20px; + top: -20px; + z-index: 10; .pool-name color: black @@ -80,9 +96,11 @@ li min-width: inherit width: inherit + margin: 0 0.25em 0.5em 0.25em &:not(.flexbox-dummy) height: 14vw .thumbnail + position: static outline-offset: -1px .thumbnail-wrapper.no-tags .thumbnail diff --git a/client/html/pools_page.tpl b/client/html/pools_page.tpl index 48530ea3..c935785f 100644 --- a/client/html/pools_page.tpl +++ b/client/html/pools_page.tpl @@ -4,8 +4,8 @@ <% for (let pool of ctx.response.results) { %>
  • - <% if (ctx.canViewPosts && pool.posts.length > 0) { %> - <%= ctx.makeThumbnail(pool.posts.at(0).thumbnailUrl) %> + <% if (ctx.canViewPosts) { %> + <%= ctx.makePoolThumbnails(pool.posts, ctx.postFlow) %> <% } %>
    diff --git a/client/js/util/views.js b/client/js/util/views.js index 38c98a13..f9470375 100644 --- a/client/js/util/views.js +++ b/client/js/util/views.js @@ -40,12 +40,12 @@ function makeRelativeTime(time) { ); } -function makeThumbnail(url) { +function makeThumbnail(url, klass) { return makeElement( "span", url ? { - class: "thumbnail", + class: klass || "thumbnail", style: `background-image: url(\'${url}\')`, } : { class: "thumbnail empty" }, @@ -53,6 +53,23 @@ function makeThumbnail(url) { ); } +function makePoolThumbnails(posts, postFlow) { + if (posts.length == 0) { + return makeThumbnail(null); + } + if (postFlow) { + return makeThumbnail(posts.at(0).thumbnailUrl); + } + + let s = ""; + + for (let i = 0; i < Math.min(3, posts.length); i++) { + s += makeThumbnail(posts.at(i).thumbnailUrl, "thumbnail thumbnail-" + (i+1)); + } + + return s; +} + function makeRadio(options) { _imbueId(options); return makeElement( @@ -254,7 +271,7 @@ function makePoolLink(id, includeHash, includeCount, pool, name) { misc.escapeHtml(text) ) : makeElement( - "span", + "div", { class: misc.makeCssName(category, "pool") }, misc.escapeHtml(text) ); @@ -436,6 +453,7 @@ function getTemplate(templatePath) { makeFileSize: makeFileSize, makeMarkdown: makeMarkdown, makeThumbnail: makeThumbnail, + makePoolThumbnails: makePoolThumbnails, makeRadio: makeRadio, makeCheckbox: makeCheckbox, makeSelect: makeSelect, From 676a5ff97cbdd2dbcf47331a76d11bb5f272ec5b Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 16:42:30 -0700 Subject: [PATCH 06/35] Retrieve surrounding pool posts in pool search query --- .../search/configs/base_search_config.py | 17 +++++++++++ .../search/configs/post_search_config.py | 30 +++++++++++++++++++ server/szurubooru/search/executor.py | 13 +------- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/server/szurubooru/search/configs/base_search_config.py b/server/szurubooru/search/configs/base_search_config.py index d60f3617..34b93811 100644 --- a/server/szurubooru/search/configs/base_search_config.py +++ b/server/szurubooru/search/configs/base_search_config.py @@ -1,5 +1,7 @@ from typing import Callable, Dict, Optional, Tuple +import sqlalchemy as sa + from szurubooru.search import criteria, tokens from szurubooru.search.query import SearchQuery from szurubooru.search.typing import SaColumn, SaQuery @@ -24,6 +26,21 @@ class BaseSearchConfig: def create_around_query(self) -> SaQuery: raise NotImplementedError() + def create_around_filter_queries(self, filter_query: SaQuery, entity_id: int) -> Tuple[SaQuery, SaQuery]: + prev_filter_query = ( + filter_query.filter(self.id_column > entity_id) + .order_by(None) + .order_by(sa.func.abs(self.id_column - entity_id).asc()) + .limit(1) + ) + next_filter_query = ( + filter_query.filter(self.id_column < entity_id) + .order_by(None) + .order_by(sa.func.abs(self.id_column - entity_id).asc()) + .limit(1) + ) + return (prev_filter_query, next_filter_query) + def finalize_query(self, query: SaQuery) -> SaQuery: return query diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index 8e5fe71a..7fb7d82f 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -134,6 +134,31 @@ def _pool_sort( .order_by(model.PoolPost.order.desc()) +def _posts_around_pool(filter_query: SaQuery, post_id: int, pool_id: int) -> Tuple[SaQuery, SaQuery]: + this_order = db.session.query(model.PoolPost) \ + .filter(model.PoolPost.post_id == post_id) \ + .filter(model.PoolPost.pool_id == pool_id) \ + .one().order + + filter_query = db.session.query(model.Post) \ + .join(model.PoolPost, model.PoolPost.pool_id == pool_id) \ + .filter(model.PoolPost.post_id == model.Post.post_id) + + prev_filter_query = ( + filter_query.filter(model.PoolPost.order > this_order) + .order_by(None) + .order_by(sa.func.abs(model.PoolPost.order - this_order).asc()) + .limit(1) + ) + next_filter_query = ( + filter_query.filter(model.PoolPost.order < this_order) + .order_by(None) + .order_by(sa.func.abs(model.PoolPost.order - this_order).asc()) + .limit(1) + ) + return (prev_filter_query, next_filter_query) + + class PostSearchConfig(BaseSearchConfig): def __init__(self) -> None: self.user = None # type: Optional[model.User] @@ -170,6 +195,11 @@ class PostSearchConfig(BaseSearchConfig): def create_around_query(self) -> SaQuery: return db.session.query(model.Post).options(sa.orm.lazyload("*")) + def create_around_filter_queries(self, filter_query: SaQuery, entity_id: int) -> Tuple[SaQuery, SaQuery]: + if self.pool_id is not None: + return _posts_around_pool(filter_query, entity_id, self.pool_id) + return super(PostSearchConfig, self).create_around_filter_queries(filter_query, entity_id) + def create_filter_query(self, disable_eager_loads: bool) -> SaQuery: strategy = ( sa.orm.lazyload if disable_eager_loads else sa.orm.subqueryload diff --git a/server/szurubooru/search/executor.py b/server/szurubooru/search/executor.py index beb17f8a..6b39ff4a 100644 --- a/server/szurubooru/search/executor.py +++ b/server/szurubooru/search/executor.py @@ -47,18 +47,7 @@ class Executor: filter_query = self._prepare_db_query( filter_query, search_query, False ) - prev_filter_query = ( - filter_query.filter(self.config.id_column > entity_id) - .order_by(None) - .order_by(sa.func.abs(self.config.id_column - entity_id).asc()) - .limit(1) - ) - next_filter_query = ( - filter_query.filter(self.config.id_column < entity_id) - .order_by(None) - .order_by(sa.func.abs(self.config.id_column - entity_id).asc()) - .limit(1) - ) + prev_filter_query, next_filter_query = self.config.create_around_filter_queries(filter_query, entity_id) return ( prev_filter_query.one_or_none(), next_filter_query.one_or_none(), From 748f0e16eb301d0c7e8b539a182ce5076279149c Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 18:12:41 -0700 Subject: [PATCH 07/35] temp --- client/css/pool-navigator-control.styl | 9 ++++ client/css/pool-navigator-list-control.styl | 9 ++++ client/html/pool_navigator.tpl | 2 + client/html/pool_navigator_list.tpl | 4 ++ client/html/post_main.tpl | 4 ++ client/js/controllers/post_main_controller.js | 2 + client/js/controls/pool_navigator_control.js | 26 ++++++++++ .../controls/pool_navigator_list_control.js | 50 +++++++++++++++++++ client/js/views/post_main_view.js | 16 ++++++ 9 files changed, 122 insertions(+) create mode 100644 client/css/pool-navigator-control.styl create mode 100644 client/css/pool-navigator-list-control.styl create mode 100644 client/html/pool_navigator.tpl create mode 100644 client/html/pool_navigator_list.tpl create mode 100644 client/js/controls/pool_navigator_control.js create mode 100644 client/js/controls/pool_navigator_list_control.js diff --git a/client/css/pool-navigator-control.styl b/client/css/pool-navigator-control.styl new file mode 100644 index 00000000..aaf6fac2 --- /dev/null +++ b/client/css/pool-navigator-control.styl @@ -0,0 +1,9 @@ +@import colors +$pool-navigator-header-background-color = $top-navigation-color +$pool-navigator-header-background-color-darktheme = $top-navigation-color-darktheme + +.pool-navigator-container + padding: 0 0 0 60px + +.darktheme .pool-navigator-container + background: $pool-navigator-header-background-color-darktheme diff --git a/client/css/pool-navigator-list-control.styl b/client/css/pool-navigator-list-control.styl new file mode 100644 index 00000000..080ad01a --- /dev/null +++ b/client/css/pool-navigator-list-control.styl @@ -0,0 +1,9 @@ +.pool-navigators>ul + list-style-type: none + margin: 0 + padding: 0 + + >li + margin-bottom: 1em + &:last-child + margin-bottom: 0 diff --git a/client/html/pool_navigator.tpl b/client/html/pool_navigator.tpl new file mode 100644 index 00000000..e71c4194 --- /dev/null +++ b/client/html/pool_navigator.tpl @@ -0,0 +1,2 @@ +
    +
    diff --git a/client/html/pool_navigator_list.tpl b/client/html/pool_navigator_list.tpl new file mode 100644 index 00000000..0ea2e7e5 --- /dev/null +++ b/client/html/pool_navigator_list.tpl @@ -0,0 +1,4 @@ +
    +
      +
    +
    diff --git a/client/html/post_main.tpl b/client/html/post_main.tpl index 54c57333..b2e1e6f5 100644 --- a/client/html/post_main.tpl +++ b/client/html/post_main.tpl @@ -54,6 +54,10 @@
    + <% if (ctx.canListPools && ctx.canViewPools) { %> +
    + <% } %> + <% if (ctx.canListComments) { %>
    <% } %> diff --git a/client/js/controllers/post_main_controller.js b/client/js/controllers/post_main_controller.js index 95cfdb52..871d98dd 100644 --- a/client/js/controllers/post_main_controller.js +++ b/client/js/controllers/post_main_controller.js @@ -56,6 +56,8 @@ class PostMainController extends BasePostController { canFeaturePosts: api.hasPrivilege("posts:feature"), canListComments: api.hasPrivilege("comments:list"), canCreateComments: api.hasPrivilege("comments:create"), + canListPools: api.hasPrivilege("pools:list"), + canViewPools: api.hasPrivilege("pools:view"), parameters: parameters, }); diff --git a/client/js/controls/pool_navigator_control.js b/client/js/controls/pool_navigator_control.js new file mode 100644 index 00000000..afaa45ee --- /dev/null +++ b/client/js/controls/pool_navigator_control.js @@ -0,0 +1,26 @@ +"use strict"; + +const api = require("../api.js"); +const misc = require("../util/misc.js"); +const events = require("../events.js"); +const views = require("../util/views.js"); + +const template = views.getTemplate("pool-navigator"); + +class PoolNavigatorControl extends events.EventTarget { + constructor(hostNode, pool) { + super(); + this._hostNode = hostNode; + this._pool = pool; + } + + // get _formNode() { + // return this._hostNode.querySelector("form"); + // } + + // get _scoreContainerNode() { + // return this._hostNode.querySelector(".score-container"); + // } +} + +module.exports = PoolNavigatorControl; diff --git a/client/js/controls/pool_navigator_list_control.js b/client/js/controls/pool_navigator_list_control.js new file mode 100644 index 00000000..88c52d32 --- /dev/null +++ b/client/js/controls/pool_navigator_list_control.js @@ -0,0 +1,50 @@ +"use strict"; + +const events = require("../events.js"); +const views = require("../util/views.js"); +const PoolNavigatorControl = require("../controls/pool_navigator_control.js"); + +const template = views.getTemplate("pool-navigator-list"); + +class PoolNavigatorListControl extends events.EventTarget { + constructor(hostNode, a) { + super(); + this._hostNode = hostNode; + + const poolList = []; + for (let pool of poolList) { + this._installPoolNavigatorNode(pool); + } + } + + get _poolNavigatorListNode() { + return this._hostNode.querySelector("ul"); + } + + _installPoolNavigatorNode(pool) { + const poolListItemNode = document.createElement("li"); + const poolControl = new PoolNavigatorControl( + pool + ); + // events.proxyEvent(commentControl, this, "submit"); + // events.proxyEvent(commentControl, this, "score"); + // events.proxyEvent(commentControl, this, "delete"); + // this._commentIdToNode[comment.id] = commentListItemNode; + this._poolNavigatorListNode.appendChild(poolListItemNode); + } + + _uninstallCommentNode(pool) { + const poolListItemNode = this._commentIdToNode[pool.id]; + poolListItemNode.parentNode.removeChild(poolListItemNode); + } + + // _evtAdd(e) { + // this._installPoolNode(e.detail.comment); + // } + + // _evtRemove(e) { + // this._uninstallPoolNode(e.detail.comment); + // } +} + +module.exports = PoolNavigatorListControl; diff --git a/client/js/views/post_main_view.js b/client/js/views/post_main_view.js index c38a9337..5c031243 100644 --- a/client/js/views/post_main_view.js +++ b/client/js/views/post_main_view.js @@ -12,6 +12,7 @@ const PostReadonlySidebarControl = require("../controls/post_readonly_sidebar_co const PostEditSidebarControl = require("../controls/post_edit_sidebar_control.js"); const CommentControl = require("../controls/comment_control.js"); const CommentListControl = require("../controls/comment_list_control.js"); +const PoolNavigatorListControl = require("../controls/pool_navigator_list_control.js"); const template = views.getTemplate("post-main"); @@ -58,6 +59,7 @@ class PostMainView { this._installSidebar(ctx); this._installCommentForm(); this._installComments(ctx.post.comments); + this._installPoolNavigators(ctx); const showPreviousImage = () => { if (ctx.prevPostId) { @@ -138,6 +140,20 @@ class PostMainView { } } + _installPoolNavigators(ctx) { + const poolNavigatorsContainerNode = document.querySelector( + "#content-holder .poolnavigators-container" + ); + if (!poolNavigatorsContainerNode) { + return; + } + + this.poolNavigatorsControl = new PoolNavigatorListControl( + poolNavigatorsContainerNode, + null + ); + } + _installCommentForm() { const commentFormContainer = document.querySelector( "#content-holder .comment-form-container" From 8e8b15a1d8d8a938d953b1831826d1a1accbf869 Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 22:08:11 -0700 Subject: [PATCH 08/35] Route for getting previous/next posts in pool --- client/js/controllers/post_main_controller.js | 13 ++- .../controls/pool_navigator_list_control.js | 2 +- client/js/models/post_list.js | 9 +++ server/requirements.txt | 1 + server/szurubooru/api/post_api.py | 13 +++ server/szurubooru/func/posts.py | 79 ++++++++++++++++++- server/szurubooru/migrations/env.py | 4 + server/szurubooru/migrations/functions.py | 46 +++++++++++ ...8dc7_add_get_pool_posts_around_function.py | 33 ++++++++ 9 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 server/szurubooru/migrations/functions.py create mode 100644 server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py diff --git a/client/js/controllers/post_main_controller.js b/client/js/controllers/post_main_controller.js index 871d98dd..a55acb9d 100644 --- a/client/js/controllers/post_main_controller.js +++ b/client/js/controllers/post_main_controller.js @@ -16,6 +16,14 @@ class PostMainController extends BasePostController { constructor(ctx, editMode) { super(ctx); + let poolPostsAround = Promise.resolve({results: [], activePool: null}) + if (api.hasPrivilege("pools.list") && api.hasPrivilege("pools.view")) { + poolPostsAround = PostList.getPoolPostsAround( + ctxt.parameters.id, + parameters ? parameters.query : null + ); + } + let parameters = ctx.parameters; Promise.all([ Post.get(ctx.parameters.id), @@ -23,9 +31,10 @@ class PostMainController extends BasePostController { ctx.parameters.id, parameters ? parameters.query : null ), + poolPostsAround ]).then( (responses) => { - const [post, aroundResponse] = responses; + const [post, aroundResponse, poolPostsAroundResponse] = responses; // remove junk from query, but save it into history so that it can // be still accessed after history navigation / page refresh @@ -44,6 +53,8 @@ class PostMainController extends BasePostController { this._post = post; this._view = new PostMainView({ post: post, + poolPostsAround: poolPostsAroundResponse.results, + activePool: poolPostsAroundResponse.activePool, editMode: editMode, prevPostId: aroundResponse.prev ? aroundResponse.prev.id diff --git a/client/js/controls/pool_navigator_list_control.js b/client/js/controls/pool_navigator_list_control.js index 88c52d32..dd6a09b7 100644 --- a/client/js/controls/pool_navigator_list_control.js +++ b/client/js/controls/pool_navigator_list_control.js @@ -7,7 +7,7 @@ const PoolNavigatorControl = require("../controls/pool_navigator_control.js"); const template = views.getTemplate("pool-navigator-list"); class PoolNavigatorListControl extends events.EventTarget { - constructor(hostNode, a) { + constructor(hostNode, pools) { super(); this._hostNode = hostNode; diff --git a/client/js/models/post_list.js b/client/js/models/post_list.js index 8c2c9d4e..884dc5f0 100644 --- a/client/js/models/post_list.js +++ b/client/js/models/post_list.js @@ -16,6 +16,15 @@ class PostList extends AbstractList { ); } + static getPoolPostsAround(id, searchQuery) { + return api.get( + uri.formatApiLink("post", id, "pool-posts-around", { + query: PostList._decorateSearchQuery(searchQuery || ""), + fields: "id", + }) + ); + } + static search(text, offset, limit, fields) { return api .get( diff --git a/server/requirements.txt b/server/requirements.txt index 043c4f12..7df6a6d5 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -9,4 +9,5 @@ pillow>=4.3.0 pynacl>=1.2.1 pytz>=2018.3 pyRFC3339>=1.0 +alembic_utils>=0.5.6 youtube_dl diff --git a/server/szurubooru/api/post_api.py b/server/szurubooru/api/post_api.py index daba7f7e..fe5d1a3a 100644 --- a/server/szurubooru/api/post_api.py +++ b/server/szurubooru/api/post_api.py @@ -284,6 +284,19 @@ def get_posts_around( ) +@rest.routes.get("/post/(?P[^/]+)/pool-posts-around/?") +def get_pool_posts_around( + ctx: rest.Context, params: Dict[str, str] +) -> rest.Response: + auth.verify_privilege(ctx.user, "posts:list") + auth.verify_privilege(ctx.user, "pools:list") + auth.verify_privilege(ctx.user, "pools:view") + _search_executor_config.user = ctx.user + post = _get_post(params) + results = posts.get_pool_posts_around(post) + return posts.serialize_pool_posts_around(results) + + @rest.routes.post("/posts/reverse-search/?") def get_posts_by_image( ctx: rest.Context, _params: Dict[str, str] = {} diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 0493681e..cc553167 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -2,6 +2,7 @@ import hmac import logging from datetime import datetime from typing import Any, Callable, Dict, List, Optional, Tuple +from collections import namedtuple import sqlalchemy as sa @@ -134,12 +135,16 @@ def get_post_content_path(post: model.Post) -> str: ) +def get_post_thumbnail_path_from_id(post_id: int) -> str: + return "generated-thumbnails/%d_%s.jpg" % ( + post_id, + get_post_security_hash(post_id), + ) + + def get_post_thumbnail_path(post: model.Post) -> str: assert post - return "generated-thumbnails/%d_%s.jpg" % ( - post.post_id, - get_post_security_hash(post.post_id), - ) + return get_post_thumbnail_path_from_id(post.post_id) def get_post_thumbnail_backup_path(post: model.Post) -> str: @@ -967,3 +972,69 @@ def search_by_image(image_content: bytes) -> List[Tuple[float, model.Post]]: ] else: return [] + + +PoolPostsAround = namedtuple('PoolPostsAround', 'pool prev_post next_post') + +def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: + around = dict() + pool_ids = set() + post_ids = set() + + dbquery = """ + SELECT around.ord, around.pool_id, around.post_id, around.delta + FROM pool_post pp, + LATERAL get_pool_posts_around(pp.pool_id, pp.post_id) around + WHERE pp.post_id = :post_id; + """ + + for order, pool_id, post_id, delta in db.session.execute(dbquery, {"post_id": post.post_id}): + if pool_id not in around: + around[pool_id] = [None, None] + if delta < 0: + around[pool_id][0] = post_id + elif delta > 0: + around[pool_id][1] = post_id + pool_ids.add(pool_id) + post_ids.add(post_id) + + pools = dict() + posts = dict() + + for pool in db.session.query(model.Pool).filter(model.Pool.pool_id.in_(pool_ids)).all(): + pools[pool.pool_id] = pool + + for result in db.session.query(model.Post.post_id).filter(model.Post.post_id.in_(post_ids)).all(): + post_id = result[0] + posts[post_id] = { "id": post_id, "thumbnailUrl": get_post_thumbnail_path_from_id(post_id) } + + results = [] + + for pool_id, entry in around.items(): + prev_post = None + next_post = None + if entry[0] is not None: + prev_post = posts[entry[0]] + if entry[1] is not None: + next_post = posts[entry[1]] + results.append(PoolPostsAround(pools[pool_id], prev_post, next_post)) + + return results + + +def sort_pool_posts_around(around: List[PoolPostsAround]) -> List[PoolPostsAround]: + return sorted( + around, + key=lambda entry: entry.pool.pool_id, + ) + + +def serialize_pool_posts_around(around: List[PoolPostsAround]) -> Optional[rest.Response]: + return [ + { + "pool": pools.serialize_micro_pool(entry.pool), + "prev_post": entry.prev_post, + "next_post": entry.next_post + } + for entry in sort_pool_posts_around(around) + ] diff --git a/server/szurubooru/migrations/env.py b/server/szurubooru/migrations/env.py index cd4f6ad1..6b155888 100644 --- a/server/szurubooru/migrations/env.py +++ b/server/szurubooru/migrations/env.py @@ -11,6 +11,7 @@ import sys from time import sleep import alembic +from alembic_utils.replaceable_entity import register_entities import sqlalchemy as sa @@ -21,6 +22,9 @@ sys.path.append(os.path.join(dir_to_self, *[os.pardir] * 2)) import szurubooru.config # noqa: E402 import szurubooru.model.base # noqa: E402 + +from szurubooru.migrations.functions import get_pool_posts_around # noqa: E402 +register_entities([get_pool_posts_around]) # fmt: on diff --git a/server/szurubooru/migrations/functions.py b/server/szurubooru/migrations/functions.py new file mode 100644 index 00000000..b6ebb069 --- /dev/null +++ b/server/szurubooru/migrations/functions.py @@ -0,0 +1,46 @@ +from alembic_utils.pg_function import PGFunction + +get_pool_posts_around = PGFunction.from_sql(""" +CREATE OR REPLACE FUNCTION public.get_pool_posts_around( + P_POOL_ID int, + P_POST_ID int +) + RETURNS TABLE ( + ORD int, + POOL_ID int, + POST_ID int, + DELTA int + ) + LANGUAGE PLPGSQL +AS $$ +BEGIN + RETURN QUERY WITH main AS ( + SELECT * FROM pool_post WHERE pool_post.pool_id = P_POOL_ID AND pool_post.post_id = P_POST_ID + ), + around AS ( + (SELECT pool_post.ord, + pool_post.pool_id, + pool_post.post_id, + 1 as delta, + main.ord AS target_ord, + main.pool_id AS target_pool_id + FROM pool_post, main + WHERE pool_post.ord > main.ord + AND pool_post.pool_id = main.pool_id + ORDER BY pool_post.ord ASC LIMIT 1) + UNION + (SELECT pool_post.ord, + pool_post.pool_id, + pool_post.post_id, + -1 as delta, + main.ord AS target_ord, + main.pool_id AS target_pool_id + FROM pool_post, main + WHERE pool_post.ord < main.ord + AND pool_post.pool_id = main.pool_id + ORDER BY pool_post.ord DESC LIMIT 1) + ) + SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around; +END +$$ +""") diff --git a/server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py b/server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py new file mode 100644 index 00000000..3882a3af --- /dev/null +++ b/server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py @@ -0,0 +1,33 @@ +''' +add get pool posts around function + +Revision ID: f0b8a4298dc7 +Created at: 2021-05-08 21:23:48.782025 +''' + +import sqlalchemy as sa +from alembic import op + +from alembic_utils.pg_function import PGFunction +from sqlalchemy import text as sql_text + +revision = 'f0b8a4298dc7' +down_revision = 'adcd63ff76a2' +branch_labels = None +depends_on = None + +def upgrade(): + public_get_pool_posts_around = PGFunction( + schema="public", + signature="get_pool_posts_around( P_POOL_ID int, P_POST_ID int )", + definition='returns TABLE (\n ORD int,\n POOL_ID int,\n POST_ID int,\n DELTA int\n )\n LANGUAGE PLPGSQL\nAS $$\nBEGIN\n RETURN QUERY WITH main AS (\n SELECT * FROM pool_post WHERE pool_post.pool_id = P_POOL_ID AND pool_post.post_id = P_POST_ID\n ),\n around AS (\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n 1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord > main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord ASC LIMIT 1)\n UNION\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n -1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord < main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord DESC LIMIT 1)\n )\n SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around;\nEND\n$$' + ) + op.create_entity(public_get_pool_posts_around) + +def downgrade(): + public_get_pool_posts_around = PGFunction( + schema="public", + signature="get_pool_posts_around( P_POOL_ID int, P_POST_ID int )", + definition='returns TABLE (\n ORD int,\n POOL_ID int,\n POST_ID int,\n DELTA int\n )\n LANGUAGE PLPGSQL\nAS $$\nBEGIN\n RETURN QUERY WITH main AS (\n SELECT * FROM pool_post WHERE pool_post.pool_id = P_POOL_ID AND pool_post.post_id = P_POST_ID\n ),\n around AS (\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n 1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord > main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord ASC LIMIT 1)\n UNION\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n -1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord < main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord DESC LIMIT 1)\n )\n SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around;\nEND\n$$' + ) + op.drop_entity(public_get_pool_posts_around) From 7750e43714e777efa9c3e7f8fd2f94dfba2542d8 Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 23:15:19 -0700 Subject: [PATCH 09/35] Initial implementation of pool navigation inside posts --- client/css/colors.styl | 2 + client/css/pool-navigator-control.styl | 31 +++++++++++-- client/html/pool_navigator.tpl | 29 ++++++++++++ client/js/controllers/post_main_controller.js | 18 +++++--- client/js/controls/pool_navigator_control.js | 27 ++++++----- .../controls/pool_navigator_list_control.js | 45 +++++++++++-------- client/js/views/post_main_view.js | 9 ++-- server/szurubooru/func/posts.py | 4 +- 8 files changed, 123 insertions(+), 42 deletions(-) diff --git a/client/css/colors.styl b/client/css/colors.styl index cf7e7caf..e9458139 100644 --- a/client/css/colors.styl +++ b/client/css/colors.styl @@ -18,6 +18,8 @@ $message-error-border-color = #FCC $message-error-background-color = #FFF5F5 $message-success-border-color = #D3E3D3 $message-success-background-color = #F5FFF5 +$pool-navigator-border-color = #888 +$pool-navigator-background-color = #EEE $input-bad-border-color = #FCC $input-bad-background-color = #FFF5F5 $input-good-border-color = #D3E3D3 diff --git a/client/css/pool-navigator-control.styl b/client/css/pool-navigator-control.styl index aaf6fac2..c9f96a53 100644 --- a/client/css/pool-navigator-control.styl +++ b/client/css/pool-navigator-control.styl @@ -1,9 +1,34 @@ @import colors -$pool-navigator-header-background-color = $top-navigation-color -$pool-navigator-header-background-color-darktheme = $top-navigation-color-darktheme .pool-navigator-container - padding: 0 0 0 60px + padding: 0 0.50em 0 0.50em + margin: 0 auto + + .pool-info-wrapper + box-sizing: border-box + width: 100% + max-width: 40em + margin: 0 0 1em 0 + display: flex + padding: 0.5em 1em + border: 1px solid $pool-navigator-border-color + background: $pool-navigator-background-color + &.active + font-weight: bold + + .pool-name + flex: 1 1; + text-align: center; + overflow: hidden; + white-space: nowrap; + -o-text-overflow: ellipsis; + text-overflow: ellipsis; + + .prev, .next + flex: 0 1; + margin: 0 .25em; + white-space: nowrap; + .darktheme .pool-navigator-container background: $pool-navigator-header-background-color-darktheme diff --git a/client/html/pool_navigator.tpl b/client/html/pool_navigator.tpl index e71c4194..88e7b53e 100644 --- a/client/html/pool_navigator.tpl +++ b/client/html/pool_navigator.tpl @@ -1,2 +1,31 @@
    +
    + + <% if (ctx.canViewPosts && ctx.prevPost) { %> + + <% } %> + ‹ prev + <% if (ctx.canViewPosts && ctx.prevPost) { %> + + <% } %> + + + <% if (ctx.canViewPools) { %> + + <% } %> + <%- ctx.pool.names[0] %> + <% if (ctx.canViewPools) { %> + + <% } %> + + + <% if (ctx.canViewPosts && ctx.nextPost) { %> + + <% } %> + next › + <% if (ctx.canViewPosts && ctx.nextPost) { %> + + <% } %> + +
    diff --git a/client/js/controllers/post_main_controller.js b/client/js/controllers/post_main_controller.js index a55acb9d..a2717e85 100644 --- a/client/js/controllers/post_main_controller.js +++ b/client/js/controllers/post_main_controller.js @@ -16,10 +16,10 @@ class PostMainController extends BasePostController { constructor(ctx, editMode) { super(ctx); - let poolPostsAround = Promise.resolve({results: [], activePool: null}) - if (api.hasPrivilege("pools.list") && api.hasPrivilege("pools.view")) { + let poolPostsAround = Promise.resolve({results: [], activePool: null}); + if (api.hasPrivilege("pools:list") && api.hasPrivilege("pools:view")) { poolPostsAround = PostList.getPoolPostsAround( - ctxt.parameters.id, + ctx.parameters.id, parameters ? parameters.query : null ); } @@ -35,6 +35,7 @@ class PostMainController extends BasePostController { ]).then( (responses) => { const [post, aroundResponse, poolPostsAroundResponse] = responses; + let activePool = null; // remove junk from query, but save it into history so that it can // be still accessed after history navigation / page refresh @@ -48,13 +49,20 @@ class PostMainController extends BasePostController { ) : uri.formatClientLink("post", ctx.parameters.id); router.replace(url, ctx.state, false); + console.log(parameters.query); + parameters.query.split(" ").forEach((item) => { + const found = item.match(/^pool:([0-9]+)/i); + if (found) { + activePool = parseInt(found[1]); + } + }); } this._post = post; this._view = new PostMainView({ post: post, - poolPostsAround: poolPostsAroundResponse.results, - activePool: poolPostsAroundResponse.activePool, + poolPostsAround: poolPostsAroundResponse, + activePool: activePool, editMode: editMode, prevPostId: aroundResponse.prev ? aroundResponse.prev.id diff --git a/client/js/controls/pool_navigator_control.js b/client/js/controls/pool_navigator_control.js index afaa45ee..54de6c7a 100644 --- a/client/js/controls/pool_navigator_control.js +++ b/client/js/controls/pool_navigator_control.js @@ -8,19 +8,26 @@ const views = require("../util/views.js"); const template = views.getTemplate("pool-navigator"); class PoolNavigatorControl extends events.EventTarget { - constructor(hostNode, pool) { + constructor(hostNode, poolPostAround, isActivePool) { super(); this._hostNode = hostNode; - this._pool = pool; + this._poolPostAround = poolPostAround; + this._isActivePool = isActivePool; + + views.replaceContent( + this._hostNode, + template({ + pool: poolPostAround.pool, + parameters: { query: `pool:${poolPostAround.pool.id}` }, + linkClass: misc.makeCssName(poolPostAround.pool.category, "pool"), + canViewPosts: api.hasPrivilege("posts:view"), + canViewPools: api.hasPrivilege("pools:view"), + prevPost: poolPostAround.prevPost, + nextPost: poolPostAround.nextPost, + isActivePool: isActivePool + }) + ); } - - // get _formNode() { - // return this._hostNode.querySelector("form"); - // } - - // get _scoreContainerNode() { - // return this._hostNode.querySelector(".score-container"); - // } } module.exports = PoolNavigatorControl; diff --git a/client/js/controls/pool_navigator_list_control.js b/client/js/controls/pool_navigator_list_control.js index dd6a09b7..bc2ff221 100644 --- a/client/js/controls/pool_navigator_list_control.js +++ b/client/js/controls/pool_navigator_list_control.js @@ -7,44 +7,53 @@ const PoolNavigatorControl = require("../controls/pool_navigator_control.js"); const template = views.getTemplate("pool-navigator-list"); class PoolNavigatorListControl extends events.EventTarget { - constructor(hostNode, pools) { + constructor(hostNode, poolPostsAround, activePool) { super(); this._hostNode = hostNode; + this._poolPostsAround = poolPostsAround; + this._activePool = activePool; + this._indexToNode = {}; - const poolList = []; - for (let pool of poolList) { - this._installPoolNavigatorNode(pool); + for (let [i, entry] of this._poolPostsAround.entries()) { + this._installPoolNavigatorNode(entry, i); } } get _poolNavigatorListNode() { - return this._hostNode.querySelector("ul"); + return this._hostNode; } - _installPoolNavigatorNode(pool) { - const poolListItemNode = document.createElement("li"); + _installPoolNavigatorNode(poolPostAround, i) { + const isActivePool = poolPostAround.pool.id == this._activePool + const poolListItemNode = document.createElement("div"); const poolControl = new PoolNavigatorControl( - pool + poolListItemNode, + poolPostAround, + isActivePool ); // events.proxyEvent(commentControl, this, "submit"); // events.proxyEvent(commentControl, this, "score"); // events.proxyEvent(commentControl, this, "delete"); - // this._commentIdToNode[comment.id] = commentListItemNode; - this._poolNavigatorListNode.appendChild(poolListItemNode); + this._indexToNode[poolPostAround.id] = poolListItemNode; + if (isActivePool) { + this._poolNavigatorListNode.insertBefore(poolListItemNode, this._poolNavigatorListNode.firstChild); + } else { + this._poolNavigatorListNode.appendChild(poolListItemNode); + } } - _uninstallCommentNode(pool) { - const poolListItemNode = this._commentIdToNode[pool.id]; + _uninstallPoolNavigatorNode(index) { + const poolListItemNode = this._indexToNode[index]; poolListItemNode.parentNode.removeChild(poolListItemNode); } - // _evtAdd(e) { - // this._installPoolNode(e.detail.comment); - // } + _evtAdd(e) { + this._installPoolNavigatorNode(e.detail.index); + } - // _evtRemove(e) { - // this._uninstallPoolNode(e.detail.comment); - // } + _evtRemove(e) { + this._uninstallPoolNavigatorNode(e.detail.index); + } } module.exports = PoolNavigatorListControl; diff --git a/client/js/views/post_main_view.js b/client/js/views/post_main_view.js index 5c031243..d8168ad4 100644 --- a/client/js/views/post_main_view.js +++ b/client/js/views/post_main_view.js @@ -59,7 +59,7 @@ class PostMainView { this._installSidebar(ctx); this._installCommentForm(); this._installComments(ctx.post.comments); - this._installPoolNavigators(ctx); + this._installPoolNavigators(ctx.poolPostsAround, ctx.activePool); const showPreviousImage = () => { if (ctx.prevPostId) { @@ -140,9 +140,9 @@ class PostMainView { } } - _installPoolNavigators(ctx) { + _installPoolNavigators(poolPostsAround, activePool) { const poolNavigatorsContainerNode = document.querySelector( - "#content-holder .poolnavigators-container" + "#content-holder .pool-navigators-container" ); if (!poolNavigatorsContainerNode) { return; @@ -150,7 +150,8 @@ class PostMainView { this.poolNavigatorsControl = new PoolNavigatorListControl( poolNavigatorsContainerNode, - null + poolPostsAround, + activePool ); } diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index cc553167..7b0e81e2 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -1033,8 +1033,8 @@ def serialize_pool_posts_around(around: List[PoolPostsAround]) -> Optional[rest. return [ { "pool": pools.serialize_micro_pool(entry.pool), - "prev_post": entry.prev_post, - "next_post": entry.next_post + "prevPost": entry.prev_post, + "nextPost": entry.next_post } for entry in sort_pool_posts_around(around) ] From e1c97049da986d8ff0a463d0ddcdf784fc8e29ad Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sat, 8 May 2021 23:27:09 -0700 Subject: [PATCH 10/35] Add pool posts around test --- client/css/pool-navigator-control.styl | 1 - client/html/pool_navigator.tpl | 2 +- server/szurubooru/tests/func/test_posts.py | 23 ++++++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/client/css/pool-navigator-control.styl b/client/css/pool-navigator-control.styl index c9f96a53..e34bdb4e 100644 --- a/client/css/pool-navigator-control.styl +++ b/client/css/pool-navigator-control.styl @@ -7,7 +7,6 @@ .pool-info-wrapper box-sizing: border-box width: 100% - max-width: 40em margin: 0 0 1em 0 display: flex padding: 0.5em 1em diff --git a/client/html/pool_navigator.tpl b/client/html/pool_navigator.tpl index 88e7b53e..8841da39 100644 --- a/client/html/pool_navigator.tpl +++ b/client/html/pool_navigator.tpl @@ -13,7 +13,7 @@ <% if (ctx.canViewPools) { %> <% } %> - <%- ctx.pool.names[0] %> + Pool: <%- ctx.pool.names[0] %> <% if (ctx.canViewPools) { %> <% } %> diff --git a/server/szurubooru/tests/func/test_posts.py b/server/szurubooru/tests/func/test_posts.py index 6139555b..360f4022 100644 --- a/server/szurubooru/tests/func/test_posts.py +++ b/server/szurubooru/tests/func/test_posts.py @@ -1147,3 +1147,26 @@ def test_search_by_image(post_factory, config_injector, read_asset): result2 = posts.search_by_image(read_asset("png.png")) assert not result2 + + +def test_get_pool_posts_around(post_factory, pool_factory, config_injector): + from szurubooru.migrations.functions import get_pool_posts_around + db.session.execute(get_pool_posts_around.to_sql_statement_create()) + db.session.flush() + + config_injector({"allow_broken_uploads": False, "secret": "test"}) + post1 = post_factory(id=1) + post2 = post_factory(id=2) + post3 = post_factory(id=3) + post4 = post_factory(id=4) + pool1 = pool_factory(id=1) + pool2 = pool_factory(id=2) + pool1.posts = [post1, post2, post3, post4] + pool2.posts = [post3, post4, post2] + db.session.add_all([post1, post2, post3, post4, pool1, pool2]) + db.session.flush() + around = posts.get_pool_posts_around(post2) + assert around[0].prev_post["id"] == post1.post_id + assert around[0].next_post["id"] == post3.post_id + assert around[1].prev_post["id"] == post4.post_id + assert around[1].next_post == None From 28eaf53dfda522bf71b3fa4276649f53385d033c Mon Sep 17 00:00:00 2001 From: Ruin0x11 Date: Sun, 9 May 2021 00:49:23 -0700 Subject: [PATCH 11/35] Add first/last pool post to pool navigator --- client/css/colors.styl | 2 +- client/css/pool-navigator-control.styl | 9 ++++-- client/css/post-main-view.styl | 5 +++- client/html/pool_navigator.tpl | 18 ++++++++++++ client/js/controls/pool_navigator_control.js | 2 ++ server/szurubooru/func/posts.py | 30 ++++++++++++++------ server/szurubooru/migrations/functions.py | 24 +++++++++++++++- server/szurubooru/tests/func/test_posts.py | 4 +++ 8 files changed, 80 insertions(+), 14 deletions(-) diff --git a/client/css/colors.styl b/client/css/colors.styl index e9458139..51ae30ed 100644 --- a/client/css/colors.styl +++ b/client/css/colors.styl @@ -18,7 +18,7 @@ $message-error-border-color = #FCC $message-error-background-color = #FFF5F5 $message-success-border-color = #D3E3D3 $message-success-background-color = #F5FFF5 -$pool-navigator-border-color = #888 +$pool-navigator-border-color = #AAA $pool-navigator-background-color = #EEE $input-bad-border-color = #FCC $input-bad-background-color = #FFF5F5 diff --git a/client/css/pool-navigator-control.styl b/client/css/pool-navigator-control.styl index e34bdb4e..ed6cd16a 100644 --- a/client/css/pool-navigator-control.styl +++ b/client/css/pool-navigator-control.styl @@ -1,7 +1,7 @@ @import colors .pool-navigator-container - padding: 0 0.50em 0 0.50em + padding: 0 margin: 0 auto .pool-info-wrapper @@ -14,6 +14,8 @@ background: $pool-navigator-background-color &.active font-weight: bold + font-size: 1.10em; + padding: 0.58em 1em .pool-name flex: 1 1; @@ -23,7 +25,10 @@ -o-text-overflow: ellipsis; text-overflow: ellipsis; - .prev, .next + .first, .last + flex-basis: 1em; + + .first, .prev, .next, .last flex: 0 1; margin: 0 .25em; white-space: nowrap; diff --git a/client/css/post-main-view.styl b/client/css/post-main-view.styl index 48f3c158..c372e4a2 100644 --- a/client/css/post-main-view.styl +++ b/client/css/post-main-view.styl @@ -40,11 +40,14 @@ width: 100% .post-container - margin-bottom: 2em + margin-bottom: 1em .post-content margin: 0 + .pool-navigators-container + margin-bottom: 2em + @media (max-width: 800px) .post-view flex-wrap: wrap diff --git a/client/html/pool_navigator.tpl b/client/html/pool_navigator.tpl index 8841da39..b70c238d 100644 --- a/client/html/pool_navigator.tpl +++ b/client/html/pool_navigator.tpl @@ -1,5 +1,14 @@
    + + <% if (ctx.canViewPosts && ctx.firstPost) { %> + + <% } %> + « + <% if (ctx.canViewPosts && ctx.firstPost) { %> + + <% } %> + <% if (ctx.canViewPosts && ctx.prevPost) { %> @@ -27,5 +36,14 @@ <% } %> + + <% if (ctx.canViewPosts && ctx.lastPost) { %> + + <% } %> + » + <% if (ctx.canViewPosts && ctx.lastPost) { %> + + <% } %> +
    diff --git a/client/js/controls/pool_navigator_control.js b/client/js/controls/pool_navigator_control.js index 54de6c7a..5961ac70 100644 --- a/client/js/controls/pool_navigator_control.js +++ b/client/js/controls/pool_navigator_control.js @@ -22,8 +22,10 @@ class PoolNavigatorControl extends events.EventTarget { linkClass: misc.makeCssName(poolPostAround.pool.category, "pool"), canViewPosts: api.hasPrivilege("posts:view"), canViewPools: api.hasPrivilege("pools:view"), + firstPost: poolPostAround.firstPost, prevPost: poolPostAround.prevPost, nextPost: poolPostAround.nextPost, + lastPost: poolPostAround.lastPost, isActivePool: isActivePool }) ); diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 7b0e81e2..9dd3b1c6 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -974,7 +974,7 @@ def search_by_image(image_content: bytes) -> List[Tuple[float, model.Post]]: return [] -PoolPostsAround = namedtuple('PoolPostsAround', 'pool prev_post next_post') +PoolPostsAround = namedtuple('PoolPostsAround', 'pool first_post prev_post next_post last_post') def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: around = dict() @@ -990,11 +990,15 @@ def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: for order, pool_id, post_id, delta in db.session.execute(dbquery, {"post_id": post.post_id}): if pool_id not in around: - around[pool_id] = [None, None] - if delta < 0: + around[pool_id] = [None, None, None, None] + if delta == -2: around[pool_id][0] = post_id - elif delta > 0: + elif delta == -1: around[pool_id][1] = post_id + elif delta == 1: + around[pool_id][2] = post_id + elif delta == 2: + around[pool_id][3] = post_id pool_ids.add(pool_id) post_ids.add(post_id) @@ -1011,13 +1015,19 @@ def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: results = [] for pool_id, entry in around.items(): + first_post = None prev_post = None next_post = None - if entry[0] is not None: - prev_post = posts[entry[0]] + last_post = None if entry[1] is not None: - next_post = posts[entry[1]] - results.append(PoolPostsAround(pools[pool_id], prev_post, next_post)) + prev_post = posts[entry[1]] + if entry[0] is not None: + first_post = posts[entry[0]] + if entry[2] is not None: + next_post = posts[entry[2]] + if entry[3] is not None: + last_post = posts[entry[3]] + results.append(PoolPostsAround(pools[pool_id], first_post, prev_post, next_post, last_post)) return results @@ -1033,8 +1043,10 @@ def serialize_pool_posts_around(around: List[PoolPostsAround]) -> Optional[rest. return [ { "pool": pools.serialize_micro_pool(entry.pool), + "firstPost": entry.first_post, "prevPost": entry.prev_post, - "nextPost": entry.next_post + "nextPost": entry.next_post, + "lastPost": entry.last_post } for entry in sort_pool_posts_around(around) ] diff --git a/server/szurubooru/migrations/functions.py b/server/szurubooru/migrations/functions.py index b6ebb069..ae39e62b 100644 --- a/server/szurubooru/migrations/functions.py +++ b/server/szurubooru/migrations/functions.py @@ -39,8 +39,30 @@ BEGIN WHERE pool_post.ord < main.ord AND pool_post.pool_id = main.pool_id ORDER BY pool_post.ord DESC LIMIT 1) + UNION + (SELECT pool_post.ord, + pool_post.pool_id, + pool_post.post_id, + 2 as delta, + main.ord AS target_ord, + main.pool_id AS target_pool_id + FROM pool_post, main + WHERE pool_post.ord = (SELECT MAX(pool_post.ord) FROM pool_post) + AND pool_post.pool_id = main.pool_id + ORDER BY pool_post.ord DESC LIMIT 1) + UNION + (SELECT pool_post.ord, + pool_post.pool_id, + pool_post.post_id, + -2 as delta, + main.ord AS target_ord, + main.pool_id AS target_pool_id + FROM pool_post, main + WHERE pool_post.ord = (SELECT MIN(pool_post.ord) FROM pool_post) + AND pool_post.pool_id = main.pool_id + ORDER BY pool_post.ord DESC LIMIT 1) ) - SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around; + SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around; END $$ """) diff --git a/server/szurubooru/tests/func/test_posts.py b/server/szurubooru/tests/func/test_posts.py index 360f4022..5e08a4a8 100644 --- a/server/szurubooru/tests/func/test_posts.py +++ b/server/szurubooru/tests/func/test_posts.py @@ -1166,7 +1166,11 @@ def test_get_pool_posts_around(post_factory, pool_factory, config_injector): db.session.add_all([post1, post2, post3, post4, pool1, pool2]) db.session.flush() around = posts.get_pool_posts_around(post2) + assert around[0].first_post["id"] == post1.post_id assert around[0].prev_post["id"] == post1.post_id assert around[0].next_post["id"] == post3.post_id + assert around[0].last_post["id"] == post4.post_id + assert around[1].first_post["id"] == post3.post_id assert around[1].prev_post["id"] == post4.post_id assert around[1].next_post == None + assert around[1].last_post == None From 6a0d5741c3d5415c0f7c7f1fd1e40a847eb4ec36 Mon Sep 17 00:00:00 2001 From: Rebecca Nelson Date: Sat, 13 Apr 2024 23:07:57 -0500 Subject: [PATCH 12/35] slap alembic-utils into install process quick and let ci do the work for now --- server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/server/Dockerfile b/server/Dockerfile index 3e4dadfb..47b30b25 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -26,6 +26,7 @@ RUN apk --no-cache add \ py3-pyrfc3339 RUN pip3 install --no-cache-dir --disable-pip-version-check \ "alembic>=0.8.5" \ + "alembic_utils>=0.5.6" \ "coloredlogs==5.0" \ "pyheif==0.6.1" \ "heif-image-plugin>=0.3.2" \ From 719f6dc741c7cba507ec0a04c6aa809c2548ab8c Mon Sep 17 00:00:00 2001 From: Rebecca Nelson Date: Sat, 13 Apr 2024 23:50:56 -0500 Subject: [PATCH 13/35] Impl post_search_config.py's create_around_query() --- .../search/configs/post_search_config.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index 645f4b2d..1884dbeb 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -220,8 +220,20 @@ class PostSearchConfig(BaseSearchConfig): if token.name == "pool" and isinstance(token.criterion, criteria.PlainCriterion): self.pool_id = token.criterion.value - def create_around_query(self) -> SaQuery: - return db.session.query(model.Post).options(sa.orm.lazyload("*")) + def create_around_query(self, filter_query: SaQuery, entity_id: int) -> SaQuery: + prev_filter_query = ( + filter_query.filter(self.config.id_column > entity_id) + .order_by(None) + .order_by(sa.func.abs(self.config.id_column - entity_id).asc()) + .limit(1) + ) + next_filter_query = ( + filter_query.filter(self.config.id_column < entity_id) + .order_by(None) + .order_by(sa.func.abs(self.config.id_column - entity_id).asc()) + .limit(1) + ) + return prev_filter_query, next_filter_query def create_around_filter_queries(self, filter_query: SaQuery, entity_id: int) -> Tuple[SaQuery, SaQuery]: if self.pool_id is not None: From e39781748be978d3cddd5707e5da2a0c8caa2f8b Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Sun, 14 Apr 2024 10:14:37 -0500 Subject: [PATCH 14/35] temp rearrange server Dockerfile so cache is used for fast dev cycle --- .gitignore | 3 +++ client/Dockerfile | 1 + server/Dockerfile | 14 ++++++++------ server/requirements.txt | 1 - server/szurubooru/migrations/env.py | 6 +++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index b21e3adf..4a59f062 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ server/**/lib/ server/**/bin/ server/**/pyvenv.cfg __pycache__/ + +.vscode +docker-compose.dev.yml diff --git a/client/Dockerfile b/client/Dockerfile index ea5151fa..7dbad250 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -1,3 +1,4 @@ +ARG BUILDPLATFORM=linux/amd64 FROM --platform=$BUILDPLATFORM node:lts as builder WORKDIR /opt/app diff --git a/server/Dockerfile b/server/Dockerfile index 47b30b25..39c4bb95 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,5 +1,5 @@ ARG ALPINE_VERSION=3.13 - +ARG BUILDPLATFORM=linux/amd64 FROM alpine:$ALPINE_VERSION as prereqs WORKDIR /opt/app @@ -26,7 +26,6 @@ RUN apk --no-cache add \ py3-pyrfc3339 RUN pip3 install --no-cache-dir --disable-pip-version-check \ "alembic>=0.8.5" \ - "alembic_utils>=0.5.6" \ "coloredlogs==5.0" \ "pyheif==0.6.1" \ "heif-image-plugin>=0.3.2" \ @@ -34,8 +33,7 @@ RUN pip3 install --no-cache-dir --disable-pip-version-check \ "pillow-avif-plugin~=1.1.0" RUN apk --no-cache del py3-pip -COPY ./ /opt/app/ -RUN rm -rf /opt/app/szurubooru/tests + FROM --platform=$BUILDPLATFORM prereqs as testing @@ -74,8 +72,12 @@ RUN apk --no-cache add \ py3-waitress \ && mkdir -p /opt/app /data \ && addgroup -g ${PGID} app \ - && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app \ - && chown -R app:app /opt/app /data + && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app + + +COPY ./ /opt/app/ + +RUN chown -R app:app /opt/app /data USER app CMD ["/opt/app/docker-start.sh"] diff --git a/server/requirements.txt b/server/requirements.txt index 1eabef8a..ffe18f0c 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -13,4 +13,3 @@ pytz>=2018.3 pyyaml>=3.11 SQLAlchemy>=1.0.12, <1.4 yt-dlp -alembic_utils>=0.5.6 diff --git a/server/szurubooru/migrations/env.py b/server/szurubooru/migrations/env.py index 6b155888..32a02deb 100644 --- a/server/szurubooru/migrations/env.py +++ b/server/szurubooru/migrations/env.py @@ -11,7 +11,7 @@ import sys from time import sleep import alembic -from alembic_utils.replaceable_entity import register_entities +#from alembic_utils.replaceable_entity import register_entities import sqlalchemy as sa @@ -23,8 +23,8 @@ sys.path.append(os.path.join(dir_to_self, *[os.pardir] * 2)) import szurubooru.config # noqa: E402 import szurubooru.model.base # noqa: E402 -from szurubooru.migrations.functions import get_pool_posts_around # noqa: E402 -register_entities([get_pool_posts_around]) +#from szurubooru.migrations.functions import get_pool_posts_around # noqa: E402 +#register_entities([get_pool_posts_around]) # fmt: on From fa95a988eb95d1adcfc252aebd19df91d4feae7b Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Sun, 14 Apr 2024 10:16:57 -0500 Subject: [PATCH 15/35] remove initial pgfunction migration bc endgoal is to not have it --- ...8dc7_add_get_pool_posts_around_function.py | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py diff --git a/server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py b/server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py deleted file mode 100644 index 3882a3af..00000000 --- a/server/szurubooru/migrations/versions/f0b8a4298dc7_add_get_pool_posts_around_function.py +++ /dev/null @@ -1,33 +0,0 @@ -''' -add get pool posts around function - -Revision ID: f0b8a4298dc7 -Created at: 2021-05-08 21:23:48.782025 -''' - -import sqlalchemy as sa -from alembic import op - -from alembic_utils.pg_function import PGFunction -from sqlalchemy import text as sql_text - -revision = 'f0b8a4298dc7' -down_revision = 'adcd63ff76a2' -branch_labels = None -depends_on = None - -def upgrade(): - public_get_pool_posts_around = PGFunction( - schema="public", - signature="get_pool_posts_around( P_POOL_ID int, P_POST_ID int )", - definition='returns TABLE (\n ORD int,\n POOL_ID int,\n POST_ID int,\n DELTA int\n )\n LANGUAGE PLPGSQL\nAS $$\nBEGIN\n RETURN QUERY WITH main AS (\n SELECT * FROM pool_post WHERE pool_post.pool_id = P_POOL_ID AND pool_post.post_id = P_POST_ID\n ),\n around AS (\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n 1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord > main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord ASC LIMIT 1)\n UNION\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n -1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord < main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord DESC LIMIT 1)\n )\n SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around;\nEND\n$$' - ) - op.create_entity(public_get_pool_posts_around) - -def downgrade(): - public_get_pool_posts_around = PGFunction( - schema="public", - signature="get_pool_posts_around( P_POOL_ID int, P_POST_ID int )", - definition='returns TABLE (\n ORD int,\n POOL_ID int,\n POST_ID int,\n DELTA int\n )\n LANGUAGE PLPGSQL\nAS $$\nBEGIN\n RETURN QUERY WITH main AS (\n SELECT * FROM pool_post WHERE pool_post.pool_id = P_POOL_ID AND pool_post.post_id = P_POST_ID\n ),\n around AS (\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n 1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord > main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord ASC LIMIT 1)\n UNION\n (SELECT pool_post.ord,\n pool_post.pool_id,\n pool_post.post_id,\n -1 as delta,\n main.ord AS target_ord,\n main.pool_id AS target_pool_id\n FROM pool_post, main\n WHERE pool_post.ord < main.ord\n AND pool_post.pool_id = main.pool_id\n ORDER BY pool_post.ord DESC LIMIT 1)\n )\n SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around;\nEND\n$$' - ) - op.drop_entity(public_get_pool_posts_around) From eabf637736d128dc9bd4a0fd9918c2fed079ebcf Mon Sep 17 00:00:00 2001 From: Rebecca Nelson Date: Sun, 14 Apr 2024 19:12:49 -0500 Subject: [PATCH 16/35] Add todo msg --- .gitignore | 2 ++ server/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4a59f062..625b3ed8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # User-specific configuration config.yaml .env +sql/ +data/ # Client Development Artifacts */*_modules/ diff --git a/server/Dockerfile b/server/Dockerfile index 39c4bb95..ee6c9505 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -74,7 +74,7 @@ RUN apk --no-cache add \ && addgroup -g ${PGID} app \ && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app - +# TODO: put this back when done modifying things COPY ./ /opt/app/ RUN chown -R app:app /opt/app /data From fa14bea4ac73c11c3a185845d75395ef8f9783de Mon Sep 17 00:00:00 2001 From: Rebecca Nelson Date: Sun, 14 Apr 2024 19:36:55 -0500 Subject: [PATCH 17/35] rewire post search config to actual function --- server/szurubooru/func/posts.py | 2 ++ .../search/configs/post_search_config.py | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 629cff9d..ad298c49 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -978,6 +978,8 @@ def search_by_image(image_content: bytes) -> List[Tuple[float, model.Post]]: PoolPostsAround = namedtuple('PoolPostsAround', 'pool first_post prev_post next_post last_post') def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: + return [] + around = dict() pool_ids = set() post_ids = set() diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index 1884dbeb..3bda9b46 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -220,25 +220,26 @@ class PostSearchConfig(BaseSearchConfig): if token.name == "pool" and isinstance(token.criterion, criteria.PlainCriterion): self.pool_id = token.criterion.value - def create_around_query(self, filter_query: SaQuery, entity_id: int) -> SaQuery: - prev_filter_query = ( - filter_query.filter(self.config.id_column > entity_id) - .order_by(None) - .order_by(sa.func.abs(self.config.id_column - entity_id).asc()) - .limit(1) - ) - next_filter_query = ( - filter_query.filter(self.config.id_column < entity_id) - .order_by(None) - .order_by(sa.func.abs(self.config.id_column - entity_id).asc()) - .limit(1) - ) - return prev_filter_query, next_filter_query - + def create_around_query(self) -> SaQuery: + return db.session.query(model.Post).options(sa.orm.lazyload("*")) + def create_around_filter_queries(self, filter_query: SaQuery, entity_id: int) -> Tuple[SaQuery, SaQuery]: if self.pool_id is not None: return _posts_around_pool(filter_query, entity_id, self.pool_id) - return super(PostSearchConfig, self).create_around_filter_queries(filter_query, entity_id) + + prev_filter_query = ( + filter_query.filter(self.id_column > entity_id) + .order_by(None) + .order_by(sa.func.abs(self.id_column - entity_id).asc()) + .limit(1) + ) + next_filter_query = ( + filter_query.filter(self.id_column < entity_id) + .order_by(None) + .order_by(sa.func.abs(self.id_column - entity_id).asc()) + .limit(1) + ) + return prev_filter_query, next_filter_query def create_filter_query(self, disable_eager_loads: bool) -> SaQuery: strategy = ( From 1c189fc9c2231dcdbd54fc06909553c20a996e64 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Mon, 15 Apr 2024 10:16:45 -0500 Subject: [PATCH 18/35] fix defaulted abstraction --- .../search/configs/post_search_config.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index 3bda9b46..ad98421a 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -226,20 +226,7 @@ class PostSearchConfig(BaseSearchConfig): def create_around_filter_queries(self, filter_query: SaQuery, entity_id: int) -> Tuple[SaQuery, SaQuery]: if self.pool_id is not None: return _posts_around_pool(filter_query, entity_id, self.pool_id) - - prev_filter_query = ( - filter_query.filter(self.id_column > entity_id) - .order_by(None) - .order_by(sa.func.abs(self.id_column - entity_id).asc()) - .limit(1) - ) - next_filter_query = ( - filter_query.filter(self.id_column < entity_id) - .order_by(None) - .order_by(sa.func.abs(self.id_column - entity_id).asc()) - .limit(1) - ) - return prev_filter_query, next_filter_query + return super(PostSearchConfig, self).create_around_filter_queries(filter_query, entity_id) def create_filter_query(self, disable_eager_loads: bool) -> SaQuery: strategy = ( From 85f012b02f0f87e28a4ba1ebe18e81a8257a923c Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:24:40 -0500 Subject: [PATCH 19/35] Rewrite get_pool_posts_around to not use raw sql --- server/szurubooru/api/post_api.py | 4 +- server/szurubooru/func/posts.py | 90 +++++++++---------------------- 2 files changed, 28 insertions(+), 66 deletions(-) diff --git a/server/szurubooru/api/post_api.py b/server/szurubooru/api/post_api.py index fe5d1a3a..96588ce5 100644 --- a/server/szurubooru/api/post_api.py +++ b/server/szurubooru/api/post_api.py @@ -291,10 +291,10 @@ def get_pool_posts_around( auth.verify_privilege(ctx.user, "posts:list") auth.verify_privilege(ctx.user, "pools:list") auth.verify_privilege(ctx.user, "pools:view") - _search_executor_config.user = ctx.user + _search_executor_config.user = ctx.user # never calling _search_executor so why are we setting user? post = _get_post(params) results = posts.get_pool_posts_around(post) - return posts.serialize_pool_posts_around(results) + return posts.serialize_pool_posts_around(ctx, results) @rest.routes.post("/posts/reverse-search/?") diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index ad298c49..21136773 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -135,16 +135,12 @@ def get_post_content_path(post: model.Post) -> str: ) -def get_post_thumbnail_path_from_id(post_id: int) -> str: - return "generated-thumbnails/%d_%s.jpg" % ( - post_id, - get_post_security_hash(post_id), - ) - - def get_post_thumbnail_path(post: model.Post) -> str: assert post - return get_post_thumbnail_path_from_id(post.post_id) + return "generated-thumbnails/%d_%s.jpg" % ( + post.post_id, + get_post_security_hash(post.post_id), + ) def get_post_thumbnail_backup_path(post: model.Post) -> str: @@ -977,61 +973,27 @@ def search_by_image(image_content: bytes) -> List[Tuple[float, model.Post]]: PoolPostsAround = namedtuple('PoolPostsAround', 'pool first_post prev_post next_post last_post') + def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: - return [] - - around = dict() - pool_ids = set() - post_ids = set() - - dbquery = """ - SELECT around.ord, around.pool_id, around.post_id, around.delta - FROM pool_post pp, - LATERAL get_pool_posts_around(pp.pool_id, pp.post_id) around - WHERE pp.post_id = :post_id; - """ - - for order, pool_id, post_id, delta in db.session.execute(dbquery, {"post_id": post.post_id}): - if pool_id not in around: - around[pool_id] = [None, None, None, None] - if delta == -2: - around[pool_id][0] = post_id - elif delta == -1: - around[pool_id][1] = post_id - elif delta == 1: - around[pool_id][2] = post_id - elif delta == 2: - around[pool_id][3] = post_id - pool_ids.add(pool_id) - post_ids.add(post_id) - - pools = dict() - posts = dict() - - for pool in db.session.query(model.Pool).filter(model.Pool.pool_id.in_(pool_ids)).all(): - pools[pool.pool_id] = pool - - for result in db.session.query(model.Post.post_id).filter(model.Post.post_id.in_(post_ids)).all(): - post_id = result[0] - posts[post_id] = { "id": post_id, "thumbnailUrl": get_post_thumbnail_path_from_id(post_id) } - results = [] + for pool in post.pools: + first_post, prev_post, next_post, last_post = None, None, None, None - for pool_id, entry in around.items(): - first_post = None - prev_post = None - next_post = None - last_post = None - if entry[1] is not None: - prev_post = posts[entry[1]] - if entry[0] is not None: - first_post = posts[entry[0]] - if entry[2] is not None: - next_post = posts[entry[2]] - if entry[3] is not None: - last_post = posts[entry[3]] - results.append(PoolPostsAround(pools[pool_id], first_post, prev_post, next_post, last_post)) + # find index of current post: + index_in_pool = list(map(lambda p: p.post_id, pool.posts)).index(post.post_id) + # collect first, prev, next, last post: + if index_in_pool > 0: + first_post = pool.posts[0] + prev_post = pool.posts[index_in_pool - 1] + if index_in_pool < len(pool.posts) - 1: + next_post = pool.posts[index_in_pool + 1] + last_post = pool.posts[-1] + + around = PoolPostsAround(pool, first_post, prev_post, next_post, last_post) + logger.info("===============> WE NOW HAVE: %s", around) + results.append(around) + return results @@ -1042,14 +1004,14 @@ def sort_pool_posts_around(around: List[PoolPostsAround]) -> List[PoolPostsAroun ) -def serialize_pool_posts_around(around: List[PoolPostsAround]) -> Optional[rest.Response]: +def serialize_pool_posts_around(ctx: rest.Context, around: List[PoolPostsAround]) -> Optional[rest.Response]: return [ { "pool": pools.serialize_micro_pool(entry.pool), - "firstPost": entry.first_post, - "prevPost": entry.prev_post, - "nextPost": entry.next_post, - "lastPost": entry.last_post + "firstPost": serialize_micro_post(ctx.user, entry.first_post), + "prevPost": serialize_micro_post(ctx.user, entry.prev_post), + "nextPost": serialize_micro_post(ctx.user, entry.next_post), + "lastPost": serialize_micro_post(ctx.user, entry.last_post) } for entry in sort_pool_posts_around(around) ] From dde52f1009a0c6544f47fcf034e0bafc4037901e Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:26:59 -0500 Subject: [PATCH 20/35] Fix pool post serializer --- server/szurubooru/func/posts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 21136773..069d4009 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -1008,10 +1008,10 @@ def serialize_pool_posts_around(ctx: rest.Context, around: List[PoolPostsAround] return [ { "pool": pools.serialize_micro_pool(entry.pool), - "firstPost": serialize_micro_post(ctx.user, entry.first_post), - "prevPost": serialize_micro_post(ctx.user, entry.prev_post), - "nextPost": serialize_micro_post(ctx.user, entry.next_post), - "lastPost": serialize_micro_post(ctx.user, entry.last_post) + "firstPost": serialize_micro_post(entry.first_post, ctx.user), + "prevPost": serialize_micro_post(entry.prev_post, ctx.user), + "nextPost": serialize_micro_post(entry.next_post, ctx.user), + "lastPost": serialize_micro_post(entry.last_post, ctx.user) } for entry in sort_pool_posts_around(around) ] From 75840f2ba51442a671bf67d97657e12845dad428 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:31:17 -0500 Subject: [PATCH 21/35] Remove unneeded files --- server/szurubooru/api/post_api.py | 1 - server/szurubooru/func/posts.py | 1 - server/szurubooru/migrations/functions.py | 68 ------------------- .../search/configs/post_search_config.py | 1 - 4 files changed, 71 deletions(-) delete mode 100644 server/szurubooru/migrations/functions.py diff --git a/server/szurubooru/api/post_api.py b/server/szurubooru/api/post_api.py index 96588ce5..ea034a8f 100644 --- a/server/szurubooru/api/post_api.py +++ b/server/szurubooru/api/post_api.py @@ -291,7 +291,6 @@ def get_pool_posts_around( auth.verify_privilege(ctx.user, "posts:list") auth.verify_privilege(ctx.user, "pools:list") auth.verify_privilege(ctx.user, "pools:view") - _search_executor_config.user = ctx.user # never calling _search_executor so why are we setting user? post = _get_post(params) results = posts.get_pool_posts_around(post) return posts.serialize_pool_posts_around(ctx, results) diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 069d4009..7098e894 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -991,7 +991,6 @@ def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: last_post = pool.posts[-1] around = PoolPostsAround(pool, first_post, prev_post, next_post, last_post) - logger.info("===============> WE NOW HAVE: %s", around) results.append(around) return results diff --git a/server/szurubooru/migrations/functions.py b/server/szurubooru/migrations/functions.py deleted file mode 100644 index ae39e62b..00000000 --- a/server/szurubooru/migrations/functions.py +++ /dev/null @@ -1,68 +0,0 @@ -from alembic_utils.pg_function import PGFunction - -get_pool_posts_around = PGFunction.from_sql(""" -CREATE OR REPLACE FUNCTION public.get_pool_posts_around( - P_POOL_ID int, - P_POST_ID int -) - RETURNS TABLE ( - ORD int, - POOL_ID int, - POST_ID int, - DELTA int - ) - LANGUAGE PLPGSQL -AS $$ -BEGIN - RETURN QUERY WITH main AS ( - SELECT * FROM pool_post WHERE pool_post.pool_id = P_POOL_ID AND pool_post.post_id = P_POST_ID - ), - around AS ( - (SELECT pool_post.ord, - pool_post.pool_id, - pool_post.post_id, - 1 as delta, - main.ord AS target_ord, - main.pool_id AS target_pool_id - FROM pool_post, main - WHERE pool_post.ord > main.ord - AND pool_post.pool_id = main.pool_id - ORDER BY pool_post.ord ASC LIMIT 1) - UNION - (SELECT pool_post.ord, - pool_post.pool_id, - pool_post.post_id, - -1 as delta, - main.ord AS target_ord, - main.pool_id AS target_pool_id - FROM pool_post, main - WHERE pool_post.ord < main.ord - AND pool_post.pool_id = main.pool_id - ORDER BY pool_post.ord DESC LIMIT 1) - UNION - (SELECT pool_post.ord, - pool_post.pool_id, - pool_post.post_id, - 2 as delta, - main.ord AS target_ord, - main.pool_id AS target_pool_id - FROM pool_post, main - WHERE pool_post.ord = (SELECT MAX(pool_post.ord) FROM pool_post) - AND pool_post.pool_id = main.pool_id - ORDER BY pool_post.ord DESC LIMIT 1) - UNION - (SELECT pool_post.ord, - pool_post.pool_id, - pool_post.post_id, - -2 as delta, - main.ord AS target_ord, - main.pool_id AS target_pool_id - FROM pool_post, main - WHERE pool_post.ord = (SELECT MIN(pool_post.ord) FROM pool_post) - AND pool_post.pool_id = main.pool_id - ORDER BY pool_post.ord DESC LIMIT 1) - ) - SELECT around.ord, around.pool_id, around.post_id, around.delta FROM around; -END -$$ -""") diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index ad98421a..b1de2476 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -114,7 +114,6 @@ def _pool_filter( query: SaQuery, criterion: Optional[criteria.BaseCriterion], negated: bool ) -> SaQuery: assert criterion - from szurubooru.search.configs import util as search_util subquery = db.session.query(model.PoolPost.post_id.label("foreign_id")) subquery = subquery.options(sa.orm.lazyload("*")) subquery = search_util.create_num_filter(model.PoolPost.pool_id)(subquery, criterion, False) From fdb36ed097739a8971029f994d6696bfba678f46 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:32:01 -0500 Subject: [PATCH 22/35] Remove unneeded new SQL function lines --- server/szurubooru/migrations/env.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/szurubooru/migrations/env.py b/server/szurubooru/migrations/env.py index 32a02deb..9c3bcd61 100644 --- a/server/szurubooru/migrations/env.py +++ b/server/szurubooru/migrations/env.py @@ -23,8 +23,6 @@ sys.path.append(os.path.join(dir_to_self, *[os.pardir] * 2)) import szurubooru.config # noqa: E402 import szurubooru.model.base # noqa: E402 -#from szurubooru.migrations.functions import get_pool_posts_around # noqa: E402 -#register_entities([get_pool_posts_around]) # fmt: on From 75d9fb2dc7a51b62f0a8a947547b7107d3b64a59 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:46:58 -0500 Subject: [PATCH 23/35] Restore dev env convenience changes --- server/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index ee6c9505..5001a48f 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -33,7 +33,8 @@ RUN pip3 install --no-cache-dir --disable-pip-version-check \ "pillow-avif-plugin~=1.1.0" RUN apk --no-cache del py3-pip - +COPY ./ /opt/app/ +RUN rm -rf /opt/app/szurubooru/tests FROM --platform=$BUILDPLATFORM prereqs as testing @@ -72,14 +73,16 @@ RUN apk --no-cache add \ py3-waitress \ && mkdir -p /opt/app /data \ && addgroup -g ${PGID} app \ - && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app + && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app \ + && chown -R app:app /opt/app /data # TODO: put this back when done modifying things COPY ./ /opt/app/ -RUN chown -R app:app /opt/app /data - USER app + +# TODO: put this back when done modifying things +COPY ./ /opt/app/ CMD ["/opt/app/docker-start.sh"] ARG PORT=6666 From 2ea36ceb7f8ae6211083ceffebb1fd5c9e5541ca Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:47:51 -0500 Subject: [PATCH 24/35] Restore more dev env convenience changes --- server/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 5001a48f..d0c86aa0 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -76,13 +76,8 @@ RUN apk --no-cache add \ && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app \ && chown -R app:app /opt/app /data -# TODO: put this back when done modifying things -COPY ./ /opt/app/ - USER app -# TODO: put this back when done modifying things -COPY ./ /opt/app/ CMD ["/opt/app/docker-start.sh"] ARG PORT=6666 From cb0377a8eb392dc8bc0663b6957c70d55f31f059 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 10:48:09 -0500 Subject: [PATCH 25/35] Spacing --- server/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/server/Dockerfile b/server/Dockerfile index d0c86aa0..5140e8cd 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -77,7 +77,6 @@ RUN apk --no-cache add \ && chown -R app:app /opt/app /data USER app - CMD ["/opt/app/docker-start.sh"] ARG PORT=6666 From 5607664397de6749701a6b140c29af411948bbad Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 11:29:41 -0500 Subject: [PATCH 26/35] Update dockerfile to cache package installs before copying in code --- server/Dockerfile | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 5140e8cd..480936b5 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -48,12 +48,14 @@ RUN apk --no-cache add \ && pip3 install --no-cache-dir --disable-pip-version-check \ pytest-pgsql \ freezegun \ - && apk --no-cache del py3-pip \ - && addgroup app \ + && apk --no-cache del py3-pip + +COPY ./ /opt/app/ + +RUN addgroup app \ && adduser -SDH -h /opt/app -g '' -G app app \ && chown app:app /opt/app -COPY --chown=app:app ./szurubooru/tests /opt/app/szurubooru/tests/ ENV TEST_ENVIRONMENT="true" USER app @@ -70,8 +72,12 @@ ARG PGID=1000 RUN apk --no-cache add \ dumb-init \ py3-setuptools \ - py3-waitress \ - && mkdir -p /opt/app /data \ + py3-waitress + + +COPY ./ /opt/app/ + +RUN mkdir -p /opt/app /data \ && addgroup -g ${PGID} app \ && adduser -SDH -h /opt/app -g '' -G app -u ${PUID} app \ && chown -R app:app /opt/app /data From 076a3a0b4420c6e0ce2c79b233713338573d3d7d Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 11:35:04 -0500 Subject: [PATCH 27/35] Don't copy codebase twice in dockerfile --- server/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 480936b5..a0792c4e 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -33,9 +33,6 @@ RUN pip3 install --no-cache-dir --disable-pip-version-check \ "pillow-avif-plugin~=1.1.0" RUN apk --no-cache del py3-pip -COPY ./ /opt/app/ -RUN rm -rf /opt/app/szurubooru/tests - FROM --platform=$BUILDPLATFORM prereqs as testing WORKDIR /opt/app @@ -74,7 +71,6 @@ RUN apk --no-cache add \ py3-setuptools \ py3-waitress - COPY ./ /opt/app/ RUN mkdir -p /opt/app /data \ From e7031b3b5d3d905b0f43e1e28706d17497f6bea5 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Thu, 18 Apr 2024 11:36:47 -0500 Subject: [PATCH 28/35] Fix tests/func/test_get_pools_around to use new format --- server/szurubooru/tests/func/test_posts.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/server/szurubooru/tests/func/test_posts.py b/server/szurubooru/tests/func/test_posts.py index 019f40ac..1c187fe1 100644 --- a/server/szurubooru/tests/func/test_posts.py +++ b/server/szurubooru/tests/func/test_posts.py @@ -1224,10 +1224,6 @@ def test_search_by_image(post_factory, config_injector, read_asset): def test_get_pool_posts_around(post_factory, pool_factory, config_injector): - from szurubooru.migrations.functions import get_pool_posts_around - db.session.execute(get_pool_posts_around.to_sql_statement_create()) - db.session.flush() - config_injector({"allow_broken_uploads": False, "secret": "test"}) post1 = post_factory(id=1) post2 = post_factory(id=2) @@ -1240,11 +1236,11 @@ def test_get_pool_posts_around(post_factory, pool_factory, config_injector): db.session.add_all([post1, post2, post3, post4, pool1, pool2]) db.session.flush() around = posts.get_pool_posts_around(post2) - assert around[0].first_post["id"] == post1.post_id - assert around[0].prev_post["id"] == post1.post_id - assert around[0].next_post["id"] == post3.post_id - assert around[0].last_post["id"] == post4.post_id - assert around[1].first_post["id"] == post3.post_id - assert around[1].prev_post["id"] == post4.post_id + assert around[0].first_post.post_id == post1.post_id + assert around[0].prev_post.post_id == post1.post_id + assert around[0].next_post.post_id == post3.post_id + assert around[0].last_post.post_id == post4.post_id + assert around[1].first_post.post_id == post3.post_id + assert around[1].prev_post.post_id == post4.post_id assert around[1].next_post == None assert around[1].last_post == None From d0f6a36f5e2e2e86bb8a9c74d4282f8e1f53499d Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 07:43:50 -0500 Subject: [PATCH 29/35] Make verify_unpage for test_pool_search ignore order unless specified --- .../search/configs/test_pool_search_config.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/server/szurubooru/tests/search/configs/test_pool_search_config.py b/server/szurubooru/tests/search/configs/test_pool_search_config.py index 1103ec40..112a74aa 100644 --- a/server/szurubooru/tests/search/configs/test_pool_search_config.py +++ b/server/szurubooru/tests/search/configs/test_pool_search_config.py @@ -12,13 +12,16 @@ def executor(): @pytest.fixture def verify_unpaged(executor): - def verify(input, expected_pool_names): + def verify(input, expected_pool_names, test_order=False): actual_count, actual_pools = executor.execute( input, offset=0, limit=100 ) actual_pool_names = [u.names[0].name for u in actual_pools] - assert actual_count == len(expected_pool_names) + if not test_order: + actual_pool_names = sorted(actual_pool_names) + expected_pool_names = sorted(expected_pool_names) assert actual_pool_names == expected_pool_names + assert actual_count == len(expected_pool_names) return verify @@ -338,7 +341,7 @@ def test_sort_by_name( db.session.add(pool_factory(id=2, names=["t2"])) db.session.add(pool_factory(id=1, names=["t1"])) db.session.flush() - verify_unpaged(input, expected_pool_names) + verify_unpaged(input, expected_pool_names, test_order=True) @pytest.mark.parametrize( @@ -360,7 +363,7 @@ def test_sort_by_creation_time( pool3.creation_time = datetime(1991, 1, 3) db.session.add_all([pool3, pool1, pool2]) db.session.flush() - verify_unpaged(input, expected_pool_names) + verify_unpaged(input, expected_pool_names, test_order=True) @pytest.mark.parametrize( @@ -384,7 +387,7 @@ def test_sort_by_last_edit_time( pool3.last_edit_time = datetime(1991, 1, 3) db.session.add_all([pool3, pool1, pool2]) db.session.flush() - verify_unpaged(input, expected_pool_names) + verify_unpaged(input, expected_pool_names, test_order=True) @pytest.mark.parametrize( @@ -405,7 +408,7 @@ def test_sort_by_post_count( pool2.posts.append(post1) pool2.posts.append(post2) db.session.flush() - verify_unpaged(input, expected_pool_names) + verify_unpaged(input, expected_pool_names, test_order=True) @pytest.mark.parametrize( @@ -428,4 +431,4 @@ def test_sort_by_category( pool3 = pool_factory(id=3, names=["t3"], category=cat1) db.session.add_all([pool1, pool2, pool3]) db.session.flush() - verify_unpaged(input, expected_pool_names) + verify_unpaged(input, expected_pool_names, test_order=True) From 0955ed0f36cf0b305206661e743924dbe75e7a5a Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 07:49:33 -0500 Subject: [PATCH 30/35] Remove redundant no-query tests from pool search sort tests --- .../szurubooru/tests/search/configs/test_pool_search_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/szurubooru/tests/search/configs/test_pool_search_config.py b/server/szurubooru/tests/search/configs/test_pool_search_config.py index 112a74aa..e497ddaa 100644 --- a/server/szurubooru/tests/search/configs/test_pool_search_config.py +++ b/server/szurubooru/tests/search/configs/test_pool_search_config.py @@ -326,7 +326,6 @@ def test_filter_by_invalid_input(executor, input): @pytest.mark.parametrize( "input,expected_pool_names", [ - ("", ["t1", "t2"]), ("sort:name", ["t1", "t2"]), ("-sort:name", ["t2", "t1"]), ("sort:name,asc", ["t1", "t2"]), @@ -347,7 +346,6 @@ def test_sort_by_name( @pytest.mark.parametrize( "input,expected_pool_names", [ - ("", ["t1", "t2", "t3"]), ("sort:creation-date", ["t3", "t2", "t1"]), ("sort:creation-time", ["t3", "t2", "t1"]), ], @@ -369,7 +367,6 @@ def test_sort_by_creation_time( @pytest.mark.parametrize( "input,expected_pool_names", [ - ("", ["t1", "t2", "t3"]), ("sort:last-edit-date", ["t3", "t2", "t1"]), ("sort:last-edit-time", ["t3", "t2", "t1"]), ("sort:edit-date", ["t3", "t2", "t1"]), From 674979cfca16c974f9f03a56ae5fde2f82eb6346 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 07:53:56 -0500 Subject: [PATCH 31/35] Fix test_pool_search_config.test_sort_by_category to assume default last-created secondary ordering --- .../tests/search/configs/test_pool_search_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/szurubooru/tests/search/configs/test_pool_search_config.py b/server/szurubooru/tests/search/configs/test_pool_search_config.py index e497ddaa..5df9b2e5 100644 --- a/server/szurubooru/tests/search/configs/test_pool_search_config.py +++ b/server/szurubooru/tests/search/configs/test_pool_search_config.py @@ -423,8 +423,8 @@ def test_sort_by_category( ): cat1 = pool_category_factory(name="cat1") cat2 = pool_category_factory(name="cat2") - pool1 = pool_factory(id=1, names=["t1"], category=cat2) - pool2 = pool_factory(id=2, names=["t2"], category=cat2) + pool2 = pool_factory(id=1, names=["t2"], category=cat2) + pool1 = pool_factory(id=2, names=["t1"], category=cat2) pool3 = pool_factory(id=3, names=["t3"], category=cat1) db.session.add_all([pool1, pool2, pool3]) db.session.flush() From 2ff6d7485853fcd44b38a7b66925f73bd4fc789b Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 08:20:48 -0500 Subject: [PATCH 32/35] Remove unnecessary comma from pool list controller --- client/js/controllers/pool_list_controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/js/controllers/pool_list_controller.js b/client/js/controllers/pool_list_controller.js index 37a45cfc..67608684 100644 --- a/client/js/controllers/pool_list_controller.js +++ b/client/js/controllers/pool_list_controller.js @@ -43,7 +43,7 @@ class PoolListController { }); this._headerView.addEventListener( "submit", - (e) => this._evtSubmit(e), + (e) => this._evtSubmit(e) ); this._headerView.addEventListener( "navigate", From 4ddbbcb40f2bcf441944c8a85487b9bead204c1d Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 08:27:34 -0500 Subject: [PATCH 33/35] Spacing in server Dockerfile --- server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/server/Dockerfile b/server/Dockerfile index a0792c4e..ae57f686 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,6 +1,7 @@ ARG ALPINE_VERSION=3.13 ARG BUILDPLATFORM=linux/amd64 + FROM alpine:$ALPINE_VERSION as prereqs WORKDIR /opt/app From dff6b655172f6983eeb25ea6d016812d945d03d4 Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 08:43:32 -0500 Subject: [PATCH 34/35] Tidy up unused symbols, spacing --- server/szurubooru/migrations/env.py | 2 -- server/szurubooru/search/configs/util.py | 1 - server/szurubooru/search/typing.py | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/server/szurubooru/migrations/env.py b/server/szurubooru/migrations/env.py index 9c3bcd61..cd4f6ad1 100644 --- a/server/szurubooru/migrations/env.py +++ b/server/szurubooru/migrations/env.py @@ -11,7 +11,6 @@ import sys from time import sleep import alembic -#from alembic_utils.replaceable_entity import register_entities import sqlalchemy as sa @@ -22,7 +21,6 @@ sys.path.append(os.path.join(dir_to_self, *[os.pardir] * 2)) import szurubooru.config # noqa: E402 import szurubooru.model.base # noqa: E402 - # fmt: on diff --git a/server/szurubooru/search/configs/util.py b/server/szurubooru/search/configs/util.py index fd40b43d..58e6ebe5 100644 --- a/server/szurubooru/search/configs/util.py +++ b/server/szurubooru/search/configs/util.py @@ -205,7 +205,6 @@ def create_subquery_filter( filter_column: SaColumn, filter_factory: SaColumn, subquery_decorator: Callable[[SaQuery], None] = None, - order: SaQuery = None, ) -> Filter: filter_func = filter_factory(filter_column) diff --git a/server/szurubooru/search/typing.py b/server/szurubooru/search/typing.py index 011f7eae..686c2cb6 100644 --- a/server/szurubooru/search/typing.py +++ b/server/szurubooru/search/typing.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Union +from typing import Any, Callable SaColumn = Any SaQuery = Any From ebb46e53a4a8989229523ff960957806334d7dfa Mon Sep 17 00:00:00 2001 From: Deka Jello Date: Fri, 19 Apr 2024 10:32:11 -0500 Subject: [PATCH 35/35] apply pre-commit --- client/css/post-upload.styl | 2 +- server/Dockerfile | 2 +- server/szurubooru/func/posts.py | 4 ++-- server/szurubooru/search/configs/post_search_config.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/css/post-upload.styl b/client/css/post-upload.styl index cb6b0067..2b38d928 100644 --- a/client/css/post-upload.styl +++ b/client/css/post-upload.styl @@ -65,7 +65,7 @@ $cancel-button-color = tomato img width: 100% height: 100% - + video width: 100% height: 100% diff --git a/server/Dockerfile b/server/Dockerfile index ae57f686..ad01406c 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -49,7 +49,7 @@ RUN apk --no-cache add \ && apk --no-cache del py3-pip COPY ./ /opt/app/ - + RUN addgroup app \ && adduser -SDH -h /opt/app -g '' -G app app \ && chown app:app /opt/app diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 7098e894..b8c608ab 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -989,10 +989,10 @@ def get_pool_posts_around(post: model.Post) -> List[PoolPostsAround]: if index_in_pool < len(pool.posts) - 1: next_post = pool.posts[index_in_pool + 1] last_post = pool.posts[-1] - + around = PoolPostsAround(pool, first_post, prev_post, next_post, last_post) results.append(around) - + return results diff --git a/server/szurubooru/search/configs/post_search_config.py b/server/szurubooru/search/configs/post_search_config.py index b1de2476..6dea1262 100644 --- a/server/szurubooru/search/configs/post_search_config.py +++ b/server/szurubooru/search/configs/post_search_config.py @@ -221,7 +221,7 @@ class PostSearchConfig(BaseSearchConfig): def create_around_query(self) -> SaQuery: return db.session.query(model.Post).options(sa.orm.lazyload("*")) - + def create_around_filter_queries(self, filter_query: SaQuery, entity_id: int) -> Tuple[SaQuery, SaQuery]: if self.pool_id is not None: return _posts_around_pool(filter_query, entity_id, self.pool_id)