From 0e5fbde09740c33a26c155787b2d8237ec52f82e Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Sat, 24 Feb 2018 23:45:00 -0600 Subject: [PATCH 01/13] Changed password setup to use libsodium and argon2id * regular SHA256 is not secure * added code to auto migrate old passwords to the new password_hash if the existing password_hash matches either of the old password generation schemes. --- server/requirements.txt | 1 + server/szurubooru/func/auth.py | 39 ++++++++++++++++++++++------- server/szurubooru/migrations/env.py | 10 ++++++-- server/szurubooru/model/user.py | 2 +- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/server/requirements.txt b/server/requirements.txt index 2cd15ec1..7cc47868 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -11,3 +11,4 @@ scipy>=0.18.1 elasticsearch>=5.0.0 elasticsearch-dsl>=5.0.0 scikit-image>=0.12 +pynacl>=1.2.1 \ No newline at end of file diff --git a/server/szurubooru/func/auth.py b/server/szurubooru/func/auth.py index 25c991c4..d2bb69ac 100644 --- a/server/szurubooru/func/auth.py +++ b/server/szurubooru/func/auth.py @@ -1,8 +1,11 @@ import hashlib import random from collections import OrderedDict -from szurubooru import config, model, errors +from nacl.exceptions import InvalidkeyError + +from szurubooru import config, model, errors, db from szurubooru.func import util +from nacl.pwhash import argon2id, verify RANK_MAP = OrderedDict([ @@ -17,7 +20,14 @@ RANK_MAP = OrderedDict([ def get_password_hash(salt: str, password: str) -> str: - ''' Retrieve new-style password hash. ''' + """ Retrieve argon2id password hash.""" + return argon2id.str( + (config.config['secret'] + salt + password).encode('utf8') + ).decode('utf8') + + +def get_sha256_legacy_password_hash(salt: str, password: str) -> str: + """ Retrieve old-style sha256 password hash.""" digest = hashlib.sha256() digest.update(config.config['secret'].encode('utf8')) digest.update(salt.encode('utf8')) @@ -25,8 +35,8 @@ def get_password_hash(salt: str, password: str) -> str: return digest.hexdigest() -def get_legacy_password_hash(salt: str, password: str) -> str: - ''' Retrieve old-style password hash. ''' +def get_sha1_legacy_password_hash(salt: str, password: str) -> str: + """ Retrieve old-style sha1 password hash.""" digest = hashlib.sha1() digest.update(b'1A2/$_4xVa') digest.update(salt.encode('utf8')) @@ -47,11 +57,22 @@ def create_password() -> str: def is_valid_password(user: model.User, password: str) -> bool: assert user salt, valid_hash = user.password_salt, user.password_hash - possible_hashes = [ - get_password_hash(salt, password), - get_legacy_password_hash(salt, password) - ] - return valid_hash in possible_hashes + + try: + return verify(user.password_hash.encode('utf8'), + (config.config['secret'] + salt + password).encode('utf8')) + except InvalidkeyError: + possible_hashes = [ + get_sha256_legacy_password_hash(salt, password), + get_sha1_legacy_password_hash(salt, password) + ] + if valid_hash in possible_hashes: + # Convert the user password hash to the new hash + user.password_hash = get_password_hash(salt, password) + db.session.commit() + return True + + return False def has_privilege(user: model.User, privilege_name: str) -> bool: diff --git a/server/szurubooru/migrations/env.py b/server/szurubooru/migrations/env.py index 7065a69e..61b20f4e 100644 --- a/server/szurubooru/migrations/env.py +++ b/server/szurubooru/migrations/env.py @@ -35,7 +35,11 @@ def run_migrations_offline(): ''' url = alembic_config.get_main_option('sqlalchemy.url') alembic.context.configure( - url=url, target_metadata=target_metadata, literal_binds=True) + url=url, + target_metadata=target_metadata, + literal_binds=True, + compare_type=True + ) with alembic.context.begin_transaction(): alembic.context.run_migrations() @@ -56,7 +60,9 @@ def run_migrations_online(): with connectable.connect() as connection: alembic.context.configure( connection=connection, - target_metadata=target_metadata) + target_metadata=target_metadata, + compare_type=True + ) with alembic.context.begin_transaction(): alembic.context.run_migrations() diff --git a/server/szurubooru/model/user.py b/server/szurubooru/model/user.py index dd7c0629..d355a143 100644 --- a/server/szurubooru/model/user.py +++ b/server/szurubooru/model/user.py @@ -23,7 +23,7 @@ class User(Base): last_login_time = sa.Column('last_login_time', sa.DateTime) version = sa.Column('version', sa.Integer, default=1, nullable=False) name = sa.Column('name', sa.Unicode(50), nullable=False, unique=True) - password_hash = sa.Column('password_hash', sa.Unicode(64), nullable=False) + password_hash = sa.Column('password_hash', sa.Unicode(128), nullable=False) password_salt = sa.Column('password_salt', sa.Unicode(32)) email = sa.Column('email', sa.Unicode(64), nullable=True) rank = sa.Column('rank', sa.Unicode(32), nullable=False) From d6ee744777c35a4934d0027a68f1a6417b312e7f Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Sun, 25 Feb 2018 00:05:15 -0600 Subject: [PATCH 02/13] Added migration to support new password_hash format --- .gitignore | 3 ++ ...increase_password_hash_max_field_length.py | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 server/szurubooru/migrations/versions/9ef1a1643c2a_increase_password_hash_max_field_length.py diff --git a/.gitignore b/.gitignore index a683cefa..30cd78cf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ config.yaml */*_modules/ .coverage .cache +__pycache__ +.idea/ +*.iml \ No newline at end of file diff --git a/server/szurubooru/migrations/versions/9ef1a1643c2a_increase_password_hash_max_field_length.py b/server/szurubooru/migrations/versions/9ef1a1643c2a_increase_password_hash_max_field_length.py new file mode 100644 index 00000000..137b5911 --- /dev/null +++ b/server/szurubooru/migrations/versions/9ef1a1643c2a_increase_password_hash_max_field_length.py @@ -0,0 +1,30 @@ +''' +Alter the password_hash field to work with larger output. Particularly libsodium output for greater password security. + +Revision ID: 9ef1a1643c2a +Created at: 2018-02-24 23:00:32.848575 +''' + +import sqlalchemy as sa +from alembic import op + + +revision = '9ef1a1643c2a' +down_revision = '02ef5f73f4ab' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.alter_column('user', 'password_hash', + existing_type=sa.VARCHAR(length=64), + type_=sa.Unicode(length=128), + existing_nullable=False) + + +def downgrade(): + op.alter_column('user', 'password_hash', + existing_type=sa.Unicode(length=128), + type_=sa.VARCHAR(length=64), + existing_nullable=False) From a526a567672aca037a5994c3884e798a41a30839 Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Sun, 25 Feb 2018 04:44:02 -0600 Subject: [PATCH 03/13] Users are only authenticated against their password on login, and to retrieve a token. * Passwords are wiped from the app and cookies after login and token retrieval * Tokens are revoked at the end of the session/logout * If the user chooses the "remember me" option, the token is stored in the cookie * A user interface to revoke tokens will be added --- client/js/api.js | 77 +++++++++++++++++-- config.yaml.dist | 4 + server/szurubooru/api/__init__.py | 1 + server/szurubooru/api/user_api.py | 2 +- server/szurubooru/api/user_token_api.py | 38 +++++++++ server/szurubooru/func/auth.py | 9 +++ server/szurubooru/func/user_tokens.py | 74 ++++++++++++++++++ server/szurubooru/func/users.py | 18 +++-- server/szurubooru/middleware/authenticator.py | 24 ++++-- .../a39c7f98a7fa_add_user_token_table.py | 34 ++++++++ server/szurubooru/model/__init__.py | 4 +- server/szurubooru/model/user.py | 18 +++++ server/szurubooru/rest/__init__.py | 1 + server/szurubooru/rest/routes.py | 16 ++++ 14 files changed, 298 insertions(+), 22 deletions(-) create mode 100644 server/szurubooru/api/user_token_api.py create mode 100644 server/szurubooru/func/user_tokens.py create mode 100644 server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py diff --git a/client/js/api.js b/client/js/api.js index abfeb6f0..608fc6f7 100644 --- a/client/js/api.js +++ b/client/js/api.js @@ -15,6 +15,7 @@ class Api extends events.EventTarget { this.user = null; this.userName = null; this.userPassword = null; + this.userToken = null; this.cache = {}; this.allRanks = [ 'anonymous', @@ -87,11 +88,70 @@ class Api extends events.EventTarget { loginFromCookies() { const auth = cookies.getJSON('auth'); - return auth && auth.user && auth.password ? - this.login(auth.user, auth.password, true) : + return auth && auth.user && auth.token ? + this.login_with_token(auth.user, auth.token, true) : Promise.resolve(); } + login_with_token(userName, token, doRemember) { + this.cache = {}; + return new Promise((resolve, reject) => { + this.userName = userName; + this.userToken = token; + this.get('/user/' + userName + '?bump-login=true') + .then(response => { + const options = {}; + if (doRemember) { + options.expires = 365; + } + cookies.set( + 'auth', + {'user': userName, 'token': token}, + options); + this.user = response; + resolve(); + this.dispatchEvent(new CustomEvent('login')); + }, error => { + reject(error); + this.logout(); + }); + }); + } + + get_token(userName, options) { + return new Promise((resolve, reject) => { + this.post('/user-tokens', {}) + .then(response => { + cookies.set( + 'auth', + {'user': userName, 'token': response.token}, + options); + this.userName = userName; + this.userToken = response.token; + this.userPassword = null; + }, error => { + reject(error); + }); + }); + } + + delete_token(userName, userToken) { + return new Promise((resolve, reject) => { + this.delete('/user-tokens/' + userToken, {}) + .then(response => { + const options = {}; + cookies.set( + 'auth', + {'user': userName, 'token': null}, + options); + this.userName = userName; + this.userToken = null; + }, error => { + reject(error); + }); + }); + } + login(userName, userPassword, doRemember) { this.cache = {}; return new Promise((resolve, reject) => { @@ -103,10 +163,7 @@ class Api extends events.EventTarget { if (doRemember) { options.expires = 365; } - cookies.set( - 'auth', - {'user': userName, 'password': userPassword}, - options); + this.get_token(this.userName, options); this.user = response; resolve(); this.dispatchEvent(new CustomEvent('login')); @@ -118,9 +175,11 @@ class Api extends events.EventTarget { } logout() { + this.delete_token(this.userName, this.userToken); this.user = null; this.userName = null; this.userPassword = null; + this.userToken = null; this.dispatchEvent(new CustomEvent('logout')); } @@ -258,7 +317,11 @@ class Api extends events.EventTarget { } try { - if (this.userName && this.userPassword) { + if (this.userName && this.userToken) { + req.auth = null; + req.set('Authorization', 'Token ' + new Buffer(this.userName + ":" + this.userToken).toString('base64')) + } + else if (this.userName && this.userPassword) { req.auth( this.userName, encodeURIComponent(this.userPassword) diff --git a/config.yaml.dist b/config.yaml.dist index 18eb8611..0b10dbcf 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -86,6 +86,10 @@ privileges: 'users:delete:any': administrator 'users:delete:self': regular + 'user_token:list': regular + 'user_token:create': regular + 'user_token:delete': regular + 'posts:create:anonymous': regular 'posts:create:identified': regular 'posts:list': anonymous diff --git a/server/szurubooru/api/__init__.py b/server/szurubooru/api/__init__.py index 2a2d5af7..0d7f75f8 100644 --- a/server/szurubooru/api/__init__.py +++ b/server/szurubooru/api/__init__.py @@ -1,5 +1,6 @@ import szurubooru.api.info_api import szurubooru.api.user_api +import szurubooru.api.user_token_api import szurubooru.api.post_api import szurubooru.api.tag_api import szurubooru.api.tag_category_api diff --git a/server/szurubooru/api/user_api.py b/server/szurubooru/api/user_api.py index 95d8e4fe..aa0dbb9b 100644 --- a/server/szurubooru/api/user_api.py +++ b/server/szurubooru/api/user_api.py @@ -1,5 +1,5 @@ from typing import Any, Dict -from szurubooru import model, search, rest, config, errors +from szurubooru import model, search, rest from szurubooru.func import auth, users, serialization, versions diff --git a/server/szurubooru/api/user_token_api.py b/server/szurubooru/api/user_token_api.py new file mode 100644 index 00000000..8936d721 --- /dev/null +++ b/server/szurubooru/api/user_token_api.py @@ -0,0 +1,38 @@ +from typing import Dict + +from szurubooru import model, rest +from szurubooru.func import auth, user_tokens, serialization + + +def _serialize( + ctx: rest.Context, user_token: model.UserToken) -> rest.Response: + return user_tokens.serialize_user_token( + user_token, + ctx.user, + options=serialization.get_serialization_options(ctx)) + + +@rest.routes.get('/user-tokens/?') +def get_user_tokens(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Response: + auth.verify_privilege(ctx.user, 'user_token:list') + user_token_list = user_tokens.get_user_tokens(ctx.user) + return { + "tokens": [_serialize(ctx, token) for token in user_token_list] + } + + +@rest.routes.post('/user-tokens/?') +def create_user_token(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Response: + auth.verify_privilege(ctx.user, 'user_token:create') + user_token = user_tokens.create_user_token(ctx.user) + return _serialize(ctx, user_token) + + +@rest.routes.delete('/user-tokens/(?P[^/]+)/?') +def create_user_token(ctx: rest.Context, params: Dict[str, str]) -> rest.Response: + user_token = user_tokens.get_user_token_by_user_and_token(ctx.user, params['user_token']) + if user_token is not None: + auth.verify_privilege(ctx.user, 'user_token:delete') + ctx.session.delete(user_token) + ctx.session.commit() + return {} diff --git a/server/szurubooru/func/auth.py b/server/szurubooru/func/auth.py index d2bb69ac..a9c40b0f 100644 --- a/server/szurubooru/func/auth.py +++ b/server/szurubooru/func/auth.py @@ -6,6 +6,7 @@ from nacl.exceptions import InvalidkeyError from szurubooru import config, model, errors, db from szurubooru.func import util from nacl.pwhash import argon2id, verify +import uuid RANK_MAP = OrderedDict([ @@ -75,6 +76,10 @@ def is_valid_password(user: model.User, password: str) -> bool: return False +def is_valid_token(user_token: model.UserToken) -> bool: + return user_token is not None and user_token.enabled + + def has_privilege(user: model.User, privilege_name: str) -> bool: assert user all_ranks = list(RANK_MAP.keys()) @@ -99,3 +104,7 @@ def generate_authentication_token(user: model.User) -> str: digest.update(config.config['secret'].encode('utf8')) digest.update(user.password_salt.encode('utf8')) return digest.hexdigest() + + +def generate_authorization_token() -> str: + return uuid.uuid4().__str__() diff --git a/server/szurubooru/func/user_tokens.py b/server/szurubooru/func/user_tokens.py new file mode 100644 index 00000000..8c9a35de --- /dev/null +++ b/server/szurubooru/func/user_tokens.py @@ -0,0 +1,74 @@ +from datetime import datetime +from typing import Any, Optional, List, Dict, Callable + +import sqlalchemy as sa + +from szurubooru import db, model, rest +from szurubooru.func import auth, serialization, users + + +class UserTokenSerializer(serialization.BaseSerializer): + def __init__( + self, + user_token: model.UserToken, + auth_user: model.User) -> None: + self.user_token = user_token + self.auth_user = auth_user + + def _serializers(self) -> Dict[str, Callable[[], Any]]: + return { + 'user': self.serialize_user, + 'token': self.serialize_token, + 'enabled': self.serialize_enabled, + 'creationTime': self.serialize_creation_time, + 'lastLoginTime': self.serialize_last_edit_time, + } + + def serialize_user(self) -> Any: + return users.serialize_micro_user(self.user_token.user, self.auth_user) + + def serialize_creation_time(self) -> Any: + return self.user_token.creation_time + + def serialize_last_edit_time(self) -> Any: + return self.user_token.last_edit_time + + def serialize_token(self) -> Any: + return self.user_token.token + + def serialize_enabled(self) -> Any: + return self.user_token.enabled + + +def serialize_user_token( + user_token: Optional[model.UserToken], + auth_user: model.User, + options: List[str] = []) -> Optional[rest.Response]: + if not user_token: + return None + return UserTokenSerializer(user_token, auth_user).serialize(options) + + +def get_user_token_by_user_and_token(user: model.User, token: str) -> model.UserToken: + return (db.session.query(model.UserToken) + .filter(model.UserToken.user_id == user.user_id, model.UserToken.token == token) + .one_or_none()) + + +def get_user_tokens(user: model.User) -> List[model.UserToken]: + assert user + return (db.session.query(model.UserToken) + .filter(sa.func.lower(model.UserToken.user_id) == sa.func.lower(user.user_id)) + .all()) + + +def create_user_token(user: model.User) -> model.UserToken: + assert user + user_token = model.UserToken() + user_token.user = user + user_token.token = auth.generate_authorization_token() + user_token.enabled = True + user_token.creation_time = datetime.utcnow() + db.session.add(user_token) + db.session.commit() + return user_token diff --git a/server/szurubooru/func/users.py b/server/szurubooru/func/users.py index ba6f67f2..19c62bf7 100644 --- a/server/szurubooru/func/users.py +++ b/server/szurubooru/func/users.py @@ -1,7 +1,9 @@ -import re -from typing import Any, Optional, Union, List, Dict, Callable from datetime import datetime +from typing import Any, Optional, Union, List, Dict, Callable + +import re import sqlalchemy as sa + from szurubooru import config, db, model, errors, rest from szurubooru.func import auth, util, serialization, files, images @@ -172,9 +174,9 @@ def get_user_count() -> int: def try_get_user_by_name(name: str) -> Optional[model.User]: return ( db.session - .query(model.User) - .filter(sa.func.lower(model.User.name) == sa.func.lower(name)) - .one_or_none()) + .query(model.User) + .filter(sa.func.lower(model.User.name) == sa.func.lower(name)) + .one_or_none()) def get_user_by_name(name: str) -> model.User: @@ -187,11 +189,11 @@ def get_user_by_name(name: str) -> model.User: def try_get_user_by_name_or_email(name_or_email: str) -> Optional[model.User]: return ( db.session - .query(model.User) - .filter( + .query(model.User) + .filter( (sa.func.lower(model.User.name) == sa.func.lower(name_or_email)) | (sa.func.lower(model.User.email) == sa.func.lower(name_or_email))) - .one_or_none()) + .one_or_none()) def get_user_by_name_or_email(name_or_email: str) -> model.User: diff --git a/server/szurubooru/middleware/authenticator.py b/server/szurubooru/middleware/authenticator.py index 644fe3b3..a2753583 100644 --- a/server/szurubooru/middleware/authenticator.py +++ b/server/szurubooru/middleware/authenticator.py @@ -1,7 +1,7 @@ import base64 from typing import Optional from szurubooru import db, model, errors, rest -from szurubooru.func import auth, users +from szurubooru.func import auth, users, user_tokens from szurubooru.rest.errors import HttpBadRequest @@ -13,19 +13,33 @@ def _authenticate(username: str, password: str) -> model.User: return user +def _authenticate_token(username: str, token: str) -> model.User: + """Try to authenticate user. Throw AuthError for invalid users.""" + user = users.get_user_by_name(username) + user_token = user_tokens.get_user_token_by_user_and_token(user, token) + if not auth.is_valid_token(user_token): + raise errors.AuthError('Invalid token.') + return user + + def _get_user(ctx: rest.Context) -> Optional[model.User]: if not ctx.has_header('Authorization'): return None try: auth_type, credentials = ctx.get_header('Authorization').split(' ', 1) - if auth_type.lower() != 'basic': + if auth_type.lower() == 'basic': + username, password = base64.decodebytes( + credentials.encode('ascii')).decode('utf8').split(':', 1) + return _authenticate(username, password) + elif auth_type.lower() == 'token': + username, token = base64.decodebytes( + credentials.encode('ascii')).decode('utf8').split(':', 1) + return _authenticate_token(username, token) + else: raise HttpBadRequest( 'ValidationError', 'Only basic HTTP authentication is supported.') - username, password = base64.decodebytes( - credentials.encode('ascii')).decode('utf8').split(':', 1) - return _authenticate(username, password) except ValueError as err: msg = ( 'Basic authentication header value are not properly formed. ' diff --git a/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py b/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py new file mode 100644 index 00000000..6e6ac5d8 --- /dev/null +++ b/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py @@ -0,0 +1,34 @@ +''' +Added a user_token table for API authorization + +Revision ID: a39c7f98a7fa +Created at: 2018-02-25 01:31:27.345595 +''' + +import sqlalchemy as sa +from alembic import op + + +revision = 'a39c7f98a7fa' +down_revision = '9ef1a1643c2a' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('user_token', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('token', sa.Unicode(length=36), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('creation_time', sa.DateTime(), nullable=False), + sa.Column('last_edit_time', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_user_token_user_id'), 'user_token', ['user_id'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_user_token_user_id'), table_name='user_token') + op.drop_table('user_token') diff --git a/server/szurubooru/model/__init__.py b/server/szurubooru/model/__init__.py index ad2231c2..9f87b489 100644 --- a/server/szurubooru/model/__init__.py +++ b/server/szurubooru/model/__init__.py @@ -1,5 +1,7 @@ from szurubooru.model.base import Base -from szurubooru.model.user import User +from szurubooru.model.user import ( + User, + UserToken) from szurubooru.model.tag_category import TagCategory from szurubooru.model.tag import Tag, TagName, TagSuggestion, TagImplication from szurubooru.model.post import ( diff --git a/server/szurubooru/model/user.py b/server/szurubooru/model/user.py index d355a143..7ca1a10e 100644 --- a/server/szurubooru/model/user.py +++ b/server/szurubooru/model/user.py @@ -84,3 +84,21 @@ class User(Base): 'version_id_col': version, 'version_id_generator': False, } + + +class UserToken(Base): + __tablename__ = 'user_token' + + user_token_id = sa.Column('id', sa.Integer, primary_key=True) + user_id = sa.Column( + 'user_id', + sa.Integer, + sa.ForeignKey('user.id', ondelete='CASCADE'), + nullable=False, + index=True) + token = sa.Column('token', sa.Unicode(36), nullable=False) + enabled = sa.Column('enabled', sa.Boolean, nullable=False, default=True) + creation_time = sa.Column('creation_time', sa.DateTime, nullable=False) + last_edit_time = sa.Column('last_edit_time', sa.DateTime) + + user = sa.orm.relationship('User') diff --git a/server/szurubooru/rest/__init__.py b/server/szurubooru/rest/__init__.py index 14a3e305..d6b3ef28 100644 --- a/server/szurubooru/rest/__init__.py +++ b/server/szurubooru/rest/__init__.py @@ -1,2 +1,3 @@ from szurubooru.rest.app import application from szurubooru.rest.context import Context, Response +import szurubooru.rest.routes diff --git a/server/szurubooru/rest/routes.py b/server/szurubooru/rest/routes.py index 569cbe1f..41477710 100644 --- a/server/szurubooru/rest/routes.py +++ b/server/szurubooru/rest/routes.py @@ -1,8 +1,12 @@ +import logging + from typing import Callable, Dict from collections import defaultdict from szurubooru.rest.context import Context, Response +logger = logging.getLogger(__name__) + # pylint: disable=invalid-name RouteHandler = Callable[[Context, Dict[str, str]], Response] routes = defaultdict(dict) # type: Dict[str, Dict[str, RouteHandler]] @@ -11,6 +15,9 @@ routes = defaultdict(dict) # type: Dict[str, Dict[str, RouteHandler]] def get(url: str) -> Callable[[RouteHandler], RouteHandler]: def wrapper(handler: RouteHandler) -> RouteHandler: routes[url]['GET'] = handler + logger.info( + 'Registered [GET] %s (user=%s, queries=%d)', + url) return handler return wrapper @@ -18,6 +25,9 @@ def get(url: str) -> Callable[[RouteHandler], RouteHandler]: def put(url: str) -> Callable[[RouteHandler], RouteHandler]: def wrapper(handler: RouteHandler) -> RouteHandler: routes[url]['PUT'] = handler + logger.info( + 'Registered [PUT] %s (user=%s, queries=%d)', + url) return handler return wrapper @@ -25,6 +35,9 @@ def put(url: str) -> Callable[[RouteHandler], RouteHandler]: def post(url: str) -> Callable[[RouteHandler], RouteHandler]: def wrapper(handler: RouteHandler) -> RouteHandler: routes[url]['POST'] = handler + logger.info( + 'Registered [POST] %s (user=%s, queries=%d)', + url) return handler return wrapper @@ -32,5 +45,8 @@ def post(url: str) -> Callable[[RouteHandler], RouteHandler]: def delete(url: str) -> Callable[[RouteHandler], RouteHandler]: def wrapper(handler: RouteHandler) -> RouteHandler: routes[url]['DELETE'] = handler + logger.info( + 'Registered [DELETE] %s (user=%s, queries=%d)', + url) return handler return wrapper From 796563f772fcfa6ae3e6b27e425f5f14bba2d78e Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Sun, 25 Feb 2018 17:30:48 -0600 Subject: [PATCH 04/13] Cleanup func imports, and small formatting changes. --- server/szurubooru/func/auth.py | 6 +- server/szurubooru/func/cache.py | 2 +- server/szurubooru/func/comments.py | 7 +- server/szurubooru/func/favorites.py | 3 +- server/szurubooru/func/file_uploads.py | 4 +- server/szurubooru/func/files.py | 2 + server/szurubooru/func/image_hash.py | 16 ++-- server/szurubooru/func/images.py | 9 +-- server/szurubooru/func/mailer.py | 3 +- server/szurubooru/func/mime.py | 5 +- server/szurubooru/func/net.py | 4 +- server/szurubooru/func/posts.py | 96 ++++++++++++------------ server/szurubooru/func/scores.py | 10 +-- server/szurubooru/func/serialization.py | 1 + server/szurubooru/func/snapshots.py | 5 +- server/szurubooru/func/tag_categories.py | 24 +++--- server/szurubooru/func/tags.py | 81 ++++++++++---------- server/szurubooru/func/util.py | 14 ++-- 18 files changed, 147 insertions(+), 145 deletions(-) diff --git a/server/szurubooru/func/auth.py b/server/szurubooru/func/auth.py index a9c40b0f..a71285ab 100644 --- a/server/szurubooru/func/auth.py +++ b/server/szurubooru/func/auth.py @@ -1,13 +1,13 @@ +import uuid + import hashlib import random from collections import OrderedDict from nacl.exceptions import InvalidkeyError +from nacl.pwhash import argon2id, verify from szurubooru import config, model, errors, db from szurubooru.func import util -from nacl.pwhash import argon2id, verify -import uuid - RANK_MAP = OrderedDict([ (model.User.RANK_ANONYMOUS, 'anonymous'), diff --git a/server/szurubooru/func/cache.py b/server/szurubooru/func/cache.py index 01e46592..260dfbab 100644 --- a/server/szurubooru/func/cache.py +++ b/server/szurubooru/func/cache.py @@ -1,5 +1,5 @@ -from typing import Any, List, Dict from datetime import datetime +from typing import Any, List, Dict class LruCacheItem: diff --git a/server/szurubooru/func/comments.py b/server/szurubooru/func/comments.py index 9f882831..97fdf0b2 100644 --- a/server/szurubooru/func/comments.py +++ b/server/szurubooru/func/comments.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Any, Optional, List, Dict, Callable + from szurubooru import db, model, errors, rest from szurubooru.func import users, scores, serialization @@ -75,9 +76,9 @@ def try_get_comment_by_id(comment_id: int) -> Optional[model.Comment]: comment_id = int(comment_id) return ( db.session - .query(model.Comment) - .filter(model.Comment.comment_id == comment_id) - .one_or_none()) + .query(model.Comment) + .filter(model.Comment.comment_id == comment_id) + .one_or_none()) def get_comment_by_id(comment_id: int) -> model.Comment: diff --git a/server/szurubooru/func/favorites.py b/server/szurubooru/func/favorites.py index f567bfad..6e19a59b 100644 --- a/server/szurubooru/func/favorites.py +++ b/server/szurubooru/func/favorites.py @@ -1,5 +1,6 @@ -from typing import Any, Optional, Callable, Tuple from datetime import datetime +from typing import Any, Optional, Callable, Tuple + from szurubooru import db, model, errors diff --git a/server/szurubooru/func/file_uploads.py b/server/szurubooru/func/file_uploads.py index e7f93d83..ece9a252 100644 --- a/server/szurubooru/func/file_uploads.py +++ b/server/szurubooru/func/file_uploads.py @@ -1,7 +1,7 @@ -from typing import Optional from datetime import datetime, timedelta -from szurubooru.func import files, util +from typing import Optional +from szurubooru.func import files, util MAX_MINUTES = 60 diff --git a/server/szurubooru/func/files.py b/server/szurubooru/func/files.py index fa9f36fd..097ac309 100644 --- a/server/szurubooru/func/files.py +++ b/server/szurubooru/func/files.py @@ -1,5 +1,7 @@ from typing import Any, Optional, List + import os + from szurubooru import config diff --git a/server/szurubooru/func/image_hash.py b/server/szurubooru/func/image_hash.py index dae84435..456752bb 100644 --- a/server/szurubooru/func/image_hash.py +++ b/server/szurubooru/func/image_hash.py @@ -1,12 +1,14 @@ import logging -from io import BytesIO from datetime import datetime from typing import Any, Optional, Tuple, Set, List, Callable + import elasticsearch import elasticsearch_dsl import numpy as np -from skimage.color import rgb2gray from PIL import Image +from io import BytesIO +from skimage.color import rgb2gray + from szurubooru import config, errors # pylint: disable=invalid-name @@ -133,7 +135,7 @@ def _compute_differentials(grey_level_matrix: NpMatrix) -> NpMatrix: np.diff(grey_level_matrix), ( np.zeros(grey_level_matrix.shape[0]) - .reshape((grey_level_matrix.shape[0], 1)) + .reshape((grey_level_matrix.shape[0], 1)) ) ), axis=1) down_neighbors = -np.concatenate( @@ -141,7 +143,7 @@ def _compute_differentials(grey_level_matrix: NpMatrix) -> NpMatrix: np.diff(grey_level_matrix, axis=0), ( np.zeros(grey_level_matrix.shape[1]) - .reshape((1, grey_level_matrix.shape[1])) + .reshape((1, grey_level_matrix.shape[1])) ) )) left_neighbors = -np.concatenate( @@ -207,7 +209,7 @@ def _get_words(array: NpMatrix, k: int, n: int) -> NpMatrix: def _words_to_int(word_array: NpMatrix) -> NpMatrix: width = word_array.shape[1] - coding_vector = 3**np.arange(width) + coding_vector = 3 ** np.arange(width) return np.dot(word_array + 1, coding_vector) @@ -247,7 +249,9 @@ def _safety_blanket(default_param_factory: Callable[[], Any]) -> Callable: raise errors.ProcessingError('Not an image.') except Exception as ex: raise errors.ThirdPartyError('Unknown error (%s).' % ex) + return wrapper_inner + return wrapper_outer @@ -349,5 +353,5 @@ def get_all_paths() -> Set[str]: using=_get_session(), index=config.config['elasticsearch']['index'], doc_type=ES_DOC_TYPE) - .source(['path'])) + .source(['path'])) return set(h.path for h in search.scan()) diff --git a/server/szurubooru/func/images.py b/server/szurubooru/func/images.py index 0bf84ed5..749d0898 100644 --- a/server/szurubooru/func/images.py +++ b/server/szurubooru/func/images.py @@ -1,16 +1,15 @@ -from typing import List -import logging import json +import logging +import math import shlex import subprocess -import math +from typing import List + from szurubooru import errors from szurubooru.func import mime, util - logger = logging.getLogger(__name__) - _SCALE_FIT_FMT = ( r'scale=iw*max({width}/iw\,{height}/ih):ih*max({width}/iw\,{height}/ih)') diff --git a/server/szurubooru/func/mailer.py b/server/szurubooru/func/mailer.py index 76682f11..fbf96927 100644 --- a/server/szurubooru/func/mailer.py +++ b/server/szurubooru/func/mailer.py @@ -1,5 +1,6 @@ -import smtplib import email.mime.text +import smtplib + from szurubooru import config diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 12e358c0..327ad8b5 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -1,6 +1,7 @@ -import re from typing import Optional +import re + APPLICATION_SWF = 'application/x-shockwave-flash' IMAGE_JPEG = 'image/jpeg' IMAGE_PNG = 'image/png' @@ -63,4 +64,4 @@ def is_image(mime_type: str) -> bool: def is_animated_gif(content: bytes) -> bool: pattern = b'\x21\xF9\x04[\x00-\xFF]{4}\x00[\x2C\x21]' return get_mime_type(content) == IMAGE_GIF \ - and len(re.findall(pattern, content)) > 1 + and len(re.findall(pattern, content)) > 1 diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py index e6326c06..c8651bf5 100644 --- a/server/szurubooru/func/net.py +++ b/server/szurubooru/func/net.py @@ -1,6 +1,6 @@ import urllib.request -from szurubooru import config -from szurubooru import errors + +from szurubooru import config, errors def download(url: str) -> bytes: diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 4e524387..70a87d21 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -1,13 +1,14 @@ -import hmac -from typing import Any, Optional, Tuple, List, Dict, Callable from datetime import datetime +from typing import Any, Optional, Tuple, List, Dict, Callable + +import hmac import sqlalchemy as sa + from szurubooru import config, db, model, errors, rest from szurubooru.func import ( users, scores, comments, tags, util, mime, images, files, image_hash, serialization, snapshots) - EMPTY_PIXEL = ( b'\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00' b'\xff\xff\xff\x21\xf9\x04\x01\x00\x00\x01\x00\x2c\x00\x00\x00\x00' @@ -237,8 +238,8 @@ class PostSerializer(serialization.BaseSerializer): { post['id']: post for post in [ - serialize_micro_post(rel, self.auth_user) - for rel in self.post.relations] + serialize_micro_post(rel, self.auth_user) + for rel in self.post.relations] }.values(), key=lambda post: post['id']) @@ -322,9 +323,9 @@ def get_post_count() -> int: def try_get_post_by_id(post_id: int) -> Optional[model.Post]: return ( db.session - .query(model.Post) - .filter(model.Post.post_id == post_id) - .one_or_none()) + .query(model.Post) + .filter(model.Post.post_id == post_id) + .one_or_none()) def get_post_by_id(post_id: int) -> model.Post: @@ -337,9 +338,9 @@ def get_post_by_id(post_id: int) -> model.Post: def try_get_current_post_feature() -> Optional[model.PostFeature]: return ( db.session - .query(model.PostFeature) - .order_by(model.PostFeature.time.desc()) - .first()) + .query(model.PostFeature) + .order_by(model.PostFeature.time.desc()) + .first()) def try_get_featured_post() -> Optional[model.Post]: @@ -486,10 +487,10 @@ def update_post_content(post: model.Post, content: Optional[bytes]) -> None: post.checksum = util.get_sha1(content) other_post = ( db.session - .query(model.Post) - .filter(model.Post.checksum == post.checksum) - .filter(model.Post.post_id != post.post_id) - .one_or_none()) + .query(model.Post) + .filter(model.Post.checksum == post.checksum) + .filter(model.Post.post_id != post.post_id) + .one_or_none()) if other_post \ and other_post.post_id \ and other_post.post_id != post.post_id: @@ -553,9 +554,9 @@ def update_post_relations(post: model.Post, new_post_ids: List[int]) -> None: if new_post_ids: new_posts = ( db.session - .query(model.Post) - .filter(model.Post.post_id.in_(new_post_ids)) - .all()) + .query(model.Post) + .filter(model.Post.post_id.in_(new_post_ids)) + .all()) else: new_posts = [] if len(new_posts) != len(new_post_ids): @@ -654,15 +655,13 @@ def merge_posts( alias2 = sa.orm.util.aliased(table) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.post_id == source_post_id)) + .where(alias1.post_id == source_post_id)) if anti_dup_func is not None: update_stmt = ( - update_stmt - .where( - ~sa.exists() - .where(anti_dup_func(alias1, alias2)) - .where(alias2.post_id == target_post_id))) + update_stmt.where(~sa.exists() + .where(anti_dup_func(alias1, alias2)) + .where(alias2.post_id == target_post_id))) update_stmt = update_stmt.values(post_id=target_post_id) db.session.execute(update_stmt) @@ -696,24 +695,24 @@ def merge_posts( alias2 = sa.orm.util.aliased(model.PostRelation) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.parent_id == source_post_id) - .where(alias1.child_id != target_post_id) - .where( + .where(alias1.parent_id == source_post_id) + .where(alias1.child_id != target_post_id) + .where( ~sa.exists() - .where(alias2.child_id == alias1.child_id) - .where(alias2.parent_id == target_post_id)) - .values(parent_id=target_post_id)) + .where(alias2.child_id == alias1.child_id) + .where(alias2.parent_id == target_post_id)) + .values(parent_id=target_post_id)) db.session.execute(update_stmt) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.child_id == source_post_id) - .where(alias1.parent_id != target_post_id) - .where( + .where(alias1.child_id == source_post_id) + .where(alias1.parent_id != target_post_id) + .where( ~sa.exists() - .where(alias2.parent_id == alias1.parent_id) - .where(alias2.child_id == target_post_id)) - .values(child_id=target_post_id)) + .where(alias2.parent_id == alias1.parent_id) + .where(alias2.child_id == target_post_id)) + .values(child_id=target_post_id)) db.session.execute(update_stmt) merge_tags(source_post.post_id, target_post.post_id) @@ -734,10 +733,9 @@ def merge_posts( def search_by_image_exact(image_content: bytes) -> Optional[model.Post]: checksum = util.get_sha1(image_content) return ( - db.session - .query(model.Post) - .filter(model.Post.checksum == checksum) - .one_or_none()) + db.session.query(model.Post) + .filter(model.Post.checksum == checksum) + .one_or_none()) def search_by_image(image_content: bytes) -> List[PostLookalike]: @@ -756,21 +754,19 @@ def populate_reverse_search() -> None: excluded_post_ids = image_hash.get_all_paths() post_ids_to_hash = ( - db.session - .query(model.Post.post_id) - .filter( + db.session.query(model.Post.post_id) + .filter( (model.Post.type == model.Post.TYPE_IMAGE) | (model.Post.type == model.Post.TYPE_ANIMATION)) - .filter(~model.Post.post_id.in_(excluded_post_ids)) - .order_by(model.Post.post_id.asc()) - .all()) + .filter(~model.Post.post_id.in_(excluded_post_ids)) + .order_by(model.Post.post_id.asc()) + .all()) for post_ids_chunk in util.chunks(post_ids_to_hash, 100): posts_chunk = ( - db.session - .query(model.Post) - .filter(model.Post.post_id.in_(post_ids_chunk)) - .all()) + db.session.query(model.Post) + .filter(model.Post.post_id.in_(post_ids_chunk)) + .all()) for post in posts_chunk: content_path = get_post_content_path(post) if files.has(content_path): diff --git a/server/szurubooru/func/scores.py b/server/szurubooru/func/scores.py index 615fd981..a35206bc 100644 --- a/server/szurubooru/func/scores.py +++ b/server/szurubooru/func/scores.py @@ -1,5 +1,6 @@ import datetime from typing import Any, Tuple, Callable + from szurubooru import db, model, errors @@ -40,11 +41,10 @@ def get_score(entity: model.Base, user: model.User) -> int: assert user table, get_column = _get_table_info(entity) row = ( - db.session - .query(table.score) - .filter(get_column(table) == get_column(entity)) - .filter(table.user_id == user.user_id) - .one_or_none()) + db.session.query(table.score) + .filter(get_column(table) == get_column(entity)) + .filter(table.user_id == user.user_id) + .one_or_none()) return row[0] if row else 0 diff --git a/server/szurubooru/func/serialization.py b/server/szurubooru/func/serialization.py index 699fb473..42a5413b 100644 --- a/server/szurubooru/func/serialization.py +++ b/server/szurubooru/func/serialization.py @@ -1,4 +1,5 @@ from typing import Any, List, Dict, Callable + from szurubooru import model, rest, errors diff --git a/server/szurubooru/func/snapshots.py b/server/szurubooru/func/snapshots.py index 240c3bce..dded0eda 100644 --- a/server/szurubooru/func/snapshots.py +++ b/server/szurubooru/func/snapshots.py @@ -1,5 +1,6 @@ -from typing import Any, Optional, Dict, Callable from datetime import datetime +from typing import Any, Optional, Dict, Callable + from szurubooru import db, model from szurubooru.func import diff, users @@ -96,7 +97,7 @@ def modify(entity: model.Base, auth_user: Optional[model.User]) -> None: cls for cls in model.Base._decl_class_registry.values() if hasattr(cls, '__table__') - and cls.__table__.fullname == entity.__table__.fullname + and cls.__table__.fullname == entity.__table__.fullname ), None) assert table diff --git a/server/szurubooru/func/tag_categories.py b/server/szurubooru/func/tag_categories.py index f1951a8c..3a6a3853 100644 --- a/server/szurubooru/func/tag_categories.py +++ b/server/szurubooru/func/tag_categories.py @@ -1,10 +1,11 @@ -import re from typing import Any, Optional, Dict, List, Callable + +import re import sqlalchemy as sa + from szurubooru import config, db, model, errors, rest from szurubooru.func import util, serialization, cache - DEFAULT_CATEGORY_NAME_CACHE_KEY = 'default-tag-category' @@ -88,9 +89,9 @@ def update_category_name(category: model.TagCategory, name: str) -> None: expr = sa.func.lower(model.TagCategory.name) == name.lower() if category.tag_category_id: expr = expr & ( - model.TagCategory.tag_category_id != category.tag_category_id) + model.TagCategory.tag_category_id != category.tag_category_id) already_exists = ( - db.session.query(model.TagCategory).filter(expr).count() > 0) + db.session.query(model.TagCategory).filter(expr).count() > 0) if already_exists: raise TagCategoryAlreadyExistsError( 'A category with this name already exists.') @@ -115,9 +116,8 @@ def update_category_color(category: model.TagCategory, color: str) -> None: def try_get_category_by_name( name: str, lock: bool = False) -> Optional[model.TagCategory]: query = ( - db.session - .query(model.TagCategory) - .filter(sa.func.lower(model.TagCategory.name) == name.lower())) + db.session.query(model.TagCategory) + .filter(sa.func.lower(model.TagCategory.name) == name.lower())) if lock: query = query.with_lockmode('update') return query.one_or_none() @@ -141,9 +141,8 @@ def get_all_categories() -> List[model.TagCategory]: def try_get_default_category( lock: bool = False) -> Optional[model.TagCategory]: query = ( - db.session - .query(model.TagCategory) - .filter(model.TagCategory.default)) + db.session.query(model.TagCategory) + .filter(model.TagCategory.default)) if lock: query = query.with_lockmode('update') category = query.first() @@ -151,9 +150,8 @@ def try_get_default_category( # category, get the first record available. if not category: query = ( - db.session - .query(model.TagCategory) - .order_by(model.TagCategory.tag_category_id.asc())) + db.session.query(model.TagCategory) + .order_by(model.TagCategory.tag_category_id.asc())) if lock: query = query.with_lockmode('update') category = query.first() diff --git a/server/szurubooru/func/tags.py b/server/szurubooru/func/tags.py index 7d92f1e7..0064da4f 100644 --- a/server/szurubooru/func/tags.py +++ b/server/szurubooru/func/tags.py @@ -1,9 +1,9 @@ -import json -import os -import re -from typing import Any, Optional, Tuple, List, Dict, Callable from datetime import datetime +from typing import Any, Optional, Tuple, List, Dict, Callable + +import re import sqlalchemy as sa + from szurubooru import config, db, model, errors, rest from szurubooru.func import util, tag_categories, serialization @@ -138,11 +138,10 @@ def serialize_tag( def try_get_tag_by_name(name: str) -> Optional[model.Tag]: return ( - db.session - .query(model.Tag) - .join(model.TagName) - .filter(sa.func.lower(model.TagName.name) == name.lower()) - .one_or_none()) + db.session.query(model.Tag) + .join(model.TagName) + .filter(sa.func.lower(model.TagName.name) == name.lower()) + .one_or_none()) def get_tag_by_name(name: str) -> model.Tag: @@ -158,12 +157,12 @@ def get_tags_by_names(names: List[str]) -> List[model.Tag]: return [] return ( db.session.query(model.Tag) - .join(model.TagName) - .filter( + .join(model.TagName) + .filter( sa.sql.or_( sa.func.lower(model.TagName.name) == name.lower() for name in names)) - .all()) + .all()) def get_or_create_tags_by_names( @@ -196,16 +195,15 @@ def get_tag_siblings(tag: model.Tag) -> List[model.Tag]: pt_alias1 = sa.orm.aliased(model.PostTag) pt_alias2 = sa.orm.aliased(model.PostTag) result = ( - db.session - .query(tag_alias, sa.func.count(pt_alias2.post_id)) - .join(pt_alias1, pt_alias1.tag_id == tag_alias.tag_id) - .join(pt_alias2, pt_alias2.post_id == pt_alias1.post_id) - .filter(pt_alias2.tag_id == tag.tag_id) - .filter(pt_alias1.tag_id != tag.tag_id) - .group_by(tag_alias.tag_id) - .order_by(sa.func.count(pt_alias2.post_id).desc()) - .order_by(tag_alias.first_name) - .limit(50)) + db.session.query(tag_alias, sa.func.count(pt_alias2.post_id)) + .join(pt_alias1, pt_alias1.tag_id == tag_alias.tag_id) + .join(pt_alias2, pt_alias2.post_id == pt_alias1.post_id) + .filter(pt_alias2.tag_id == tag.tag_id) + .filter(pt_alias1.tag_id != tag.tag_id) + .group_by(tag_alias.tag_id) + .order_by(sa.func.count(pt_alias2.post_id).desc()) + .order_by(tag_alias.first_name) + .limit(50)) return result @@ -213,10 +211,10 @@ def delete(source_tag: model.Tag) -> None: assert source_tag db.session.execute( sa.sql.expression.delete(model.TagSuggestion) - .where(model.TagSuggestion.child_id == source_tag.tag_id)) + .where(model.TagSuggestion.child_id == source_tag.tag_id)) db.session.execute( sa.sql.expression.delete(model.TagImplication) - .where(model.TagImplication.child_id == source_tag.tag_id)) + .where(model.TagImplication.child_id == source_tag.tag_id)) db.session.delete(source_tag) @@ -231,13 +229,12 @@ def merge_tags(source_tag: model.Tag, target_tag: model.Tag) -> None: alias2 = sa.orm.util.aliased(model.PostTag) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.tag_id == source_tag_id)) + .where(alias1.tag_id == source_tag_id)) update_stmt = ( update_stmt - .where( - ~sa.exists() - .where(alias1.post_id == alias2.post_id) - .where(alias2.tag_id == target_tag_id))) + .where(~sa.exists() + .where(alias1.post_id == alias2.post_id) + .where(alias2.tag_id == target_tag_id))) update_stmt = update_stmt.values(tag_id=target_tag_id) db.session.execute(update_stmt) @@ -247,24 +244,22 @@ def merge_tags(source_tag: model.Tag, target_tag: model.Tag) -> None: alias2 = sa.orm.util.aliased(table) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.parent_id == source_tag_id) - .where(alias1.child_id != target_tag_id) - .where( - ~sa.exists() - .where(alias2.child_id == alias1.child_id) - .where(alias2.parent_id == target_tag_id)) - .values(parent_id=target_tag_id)) + .where(alias1.parent_id == source_tag_id) + .where(alias1.child_id != target_tag_id) + .where(~sa.exists() + .where(alias2.child_id == alias1.child_id) + .where(alias2.parent_id == target_tag_id)) + .values(parent_id=target_tag_id)) db.session.execute(update_stmt) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.child_id == source_tag_id) - .where(alias1.parent_id != target_tag_id) - .where( - ~sa.exists() - .where(alias2.parent_id == alias1.parent_id) - .where(alias2.child_id == target_tag_id)) - .values(child_id=target_tag_id)) + .where(alias1.child_id == source_tag_id) + .where(alias1.parent_id != target_tag_id) + .where(~sa.exists() + .where(alias2.parent_id == alias1.parent_id) + .where(alias2.child_id == target_tag_id)) + .values(child_id=target_tag_id)) db.session.execute(update_stmt) def merge_suggestions(source_tag_id: int, target_tag_id: int) -> None: diff --git a/server/szurubooru/func/util.py b/server/szurubooru/func/util.py index ba2d4dc9..164b2545 100644 --- a/server/szurubooru/func/util.py +++ b/server/szurubooru/func/util.py @@ -1,12 +1,13 @@ -import os +from contextlib import contextmanager +from datetime import datetime, timedelta +from typing import Any, Optional, Union, Tuple, List, Dict, Generator, TypeVar + import hashlib +import os import re import tempfile -from typing import Any, Optional, Union, Tuple, List, Dict, Generator, TypeVar -from datetime import datetime, timedelta -from contextlib import contextmanager -from szurubooru import errors +from szurubooru import errors T = TypeVar('T') @@ -14,7 +15,7 @@ T = TypeVar('T') def snake_case_to_lower_camel_case(text: str) -> str: components = text.split('_') return components[0].lower() + \ - ''.join(word[0].upper() + word[1:].lower() for word in components[1:]) + ''.join(word[0].upper() + word[1:].lower() for word in components[1:]) def snake_case_to_upper_train_case(text: str) -> str: @@ -86,6 +87,7 @@ def is_valid_email(email: Optional[str]) -> bool: class dotdict(dict): # pylint: disable=invalid-name ''' dot.notation access to dictionary attributes. ''' + def __getattr__(self, attr: str) -> Any: return self.get(attr) From f11f4e9313ccef85badd07cc3defcdb3d2aa06a1 Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Mon, 26 Feb 2018 19:52:02 -0600 Subject: [PATCH 05/13] Revert "Cleanup func imports, and small formatting changes." This reverts commit 796563f --- server/szurubooru/func/auth.py | 6 +- server/szurubooru/func/cache.py | 2 +- server/szurubooru/func/comments.py | 7 +- server/szurubooru/func/favorites.py | 3 +- server/szurubooru/func/file_uploads.py | 4 +- server/szurubooru/func/files.py | 2 - server/szurubooru/func/image_hash.py | 16 ++-- server/szurubooru/func/images.py | 9 ++- server/szurubooru/func/mailer.py | 3 +- server/szurubooru/func/mime.py | 5 +- server/szurubooru/func/net.py | 4 +- server/szurubooru/func/posts.py | 96 ++++++++++++------------ server/szurubooru/func/scores.py | 10 +-- server/szurubooru/func/serialization.py | 1 - server/szurubooru/func/snapshots.py | 5 +- server/szurubooru/func/tag_categories.py | 24 +++--- server/szurubooru/func/tags.py | 81 ++++++++++---------- server/szurubooru/func/util.py | 14 ++-- 18 files changed, 145 insertions(+), 147 deletions(-) diff --git a/server/szurubooru/func/auth.py b/server/szurubooru/func/auth.py index a71285ab..a9c40b0f 100644 --- a/server/szurubooru/func/auth.py +++ b/server/szurubooru/func/auth.py @@ -1,13 +1,13 @@ -import uuid - import hashlib import random from collections import OrderedDict from nacl.exceptions import InvalidkeyError -from nacl.pwhash import argon2id, verify from szurubooru import config, model, errors, db from szurubooru.func import util +from nacl.pwhash import argon2id, verify +import uuid + RANK_MAP = OrderedDict([ (model.User.RANK_ANONYMOUS, 'anonymous'), diff --git a/server/szurubooru/func/cache.py b/server/szurubooru/func/cache.py index 260dfbab..01e46592 100644 --- a/server/szurubooru/func/cache.py +++ b/server/szurubooru/func/cache.py @@ -1,5 +1,5 @@ -from datetime import datetime from typing import Any, List, Dict +from datetime import datetime class LruCacheItem: diff --git a/server/szurubooru/func/comments.py b/server/szurubooru/func/comments.py index 97fdf0b2..9f882831 100644 --- a/server/szurubooru/func/comments.py +++ b/server/szurubooru/func/comments.py @@ -1,6 +1,5 @@ from datetime import datetime from typing import Any, Optional, List, Dict, Callable - from szurubooru import db, model, errors, rest from szurubooru.func import users, scores, serialization @@ -76,9 +75,9 @@ def try_get_comment_by_id(comment_id: int) -> Optional[model.Comment]: comment_id = int(comment_id) return ( db.session - .query(model.Comment) - .filter(model.Comment.comment_id == comment_id) - .one_or_none()) + .query(model.Comment) + .filter(model.Comment.comment_id == comment_id) + .one_or_none()) def get_comment_by_id(comment_id: int) -> model.Comment: diff --git a/server/szurubooru/func/favorites.py b/server/szurubooru/func/favorites.py index 6e19a59b..f567bfad 100644 --- a/server/szurubooru/func/favorites.py +++ b/server/szurubooru/func/favorites.py @@ -1,6 +1,5 @@ -from datetime import datetime from typing import Any, Optional, Callable, Tuple - +from datetime import datetime from szurubooru import db, model, errors diff --git a/server/szurubooru/func/file_uploads.py b/server/szurubooru/func/file_uploads.py index ece9a252..e7f93d83 100644 --- a/server/szurubooru/func/file_uploads.py +++ b/server/szurubooru/func/file_uploads.py @@ -1,8 +1,8 @@ -from datetime import datetime, timedelta from typing import Optional - +from datetime import datetime, timedelta from szurubooru.func import files, util + MAX_MINUTES = 60 diff --git a/server/szurubooru/func/files.py b/server/szurubooru/func/files.py index 097ac309..fa9f36fd 100644 --- a/server/szurubooru/func/files.py +++ b/server/szurubooru/func/files.py @@ -1,7 +1,5 @@ from typing import Any, Optional, List - import os - from szurubooru import config diff --git a/server/szurubooru/func/image_hash.py b/server/szurubooru/func/image_hash.py index 456752bb..dae84435 100644 --- a/server/szurubooru/func/image_hash.py +++ b/server/szurubooru/func/image_hash.py @@ -1,14 +1,12 @@ import logging +from io import BytesIO from datetime import datetime from typing import Any, Optional, Tuple, Set, List, Callable - import elasticsearch import elasticsearch_dsl import numpy as np -from PIL import Image -from io import BytesIO from skimage.color import rgb2gray - +from PIL import Image from szurubooru import config, errors # pylint: disable=invalid-name @@ -135,7 +133,7 @@ def _compute_differentials(grey_level_matrix: NpMatrix) -> NpMatrix: np.diff(grey_level_matrix), ( np.zeros(grey_level_matrix.shape[0]) - .reshape((grey_level_matrix.shape[0], 1)) + .reshape((grey_level_matrix.shape[0], 1)) ) ), axis=1) down_neighbors = -np.concatenate( @@ -143,7 +141,7 @@ def _compute_differentials(grey_level_matrix: NpMatrix) -> NpMatrix: np.diff(grey_level_matrix, axis=0), ( np.zeros(grey_level_matrix.shape[1]) - .reshape((1, grey_level_matrix.shape[1])) + .reshape((1, grey_level_matrix.shape[1])) ) )) left_neighbors = -np.concatenate( @@ -209,7 +207,7 @@ def _get_words(array: NpMatrix, k: int, n: int) -> NpMatrix: def _words_to_int(word_array: NpMatrix) -> NpMatrix: width = word_array.shape[1] - coding_vector = 3 ** np.arange(width) + coding_vector = 3**np.arange(width) return np.dot(word_array + 1, coding_vector) @@ -249,9 +247,7 @@ def _safety_blanket(default_param_factory: Callable[[], Any]) -> Callable: raise errors.ProcessingError('Not an image.') except Exception as ex: raise errors.ThirdPartyError('Unknown error (%s).' % ex) - return wrapper_inner - return wrapper_outer @@ -353,5 +349,5 @@ def get_all_paths() -> Set[str]: using=_get_session(), index=config.config['elasticsearch']['index'], doc_type=ES_DOC_TYPE) - .source(['path'])) + .source(['path'])) return set(h.path for h in search.scan()) diff --git a/server/szurubooru/func/images.py b/server/szurubooru/func/images.py index 749d0898..0bf84ed5 100644 --- a/server/szurubooru/func/images.py +++ b/server/szurubooru/func/images.py @@ -1,15 +1,16 @@ -import json +from typing import List import logging -import math +import json import shlex import subprocess -from typing import List - +import math from szurubooru import errors from szurubooru.func import mime, util + logger = logging.getLogger(__name__) + _SCALE_FIT_FMT = ( r'scale=iw*max({width}/iw\,{height}/ih):ih*max({width}/iw\,{height}/ih)') diff --git a/server/szurubooru/func/mailer.py b/server/szurubooru/func/mailer.py index fbf96927..76682f11 100644 --- a/server/szurubooru/func/mailer.py +++ b/server/szurubooru/func/mailer.py @@ -1,6 +1,5 @@ -import email.mime.text import smtplib - +import email.mime.text from szurubooru import config diff --git a/server/szurubooru/func/mime.py b/server/szurubooru/func/mime.py index 327ad8b5..12e358c0 100644 --- a/server/szurubooru/func/mime.py +++ b/server/szurubooru/func/mime.py @@ -1,6 +1,5 @@ -from typing import Optional - import re +from typing import Optional APPLICATION_SWF = 'application/x-shockwave-flash' IMAGE_JPEG = 'image/jpeg' @@ -64,4 +63,4 @@ def is_image(mime_type: str) -> bool: def is_animated_gif(content: bytes) -> bool: pattern = b'\x21\xF9\x04[\x00-\xFF]{4}\x00[\x2C\x21]' return get_mime_type(content) == IMAGE_GIF \ - and len(re.findall(pattern, content)) > 1 + and len(re.findall(pattern, content)) > 1 diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py index c8651bf5..e6326c06 100644 --- a/server/szurubooru/func/net.py +++ b/server/szurubooru/func/net.py @@ -1,6 +1,6 @@ import urllib.request - -from szurubooru import config, errors +from szurubooru import config +from szurubooru import errors def download(url: str) -> bytes: diff --git a/server/szurubooru/func/posts.py b/server/szurubooru/func/posts.py index 70a87d21..4e524387 100644 --- a/server/szurubooru/func/posts.py +++ b/server/szurubooru/func/posts.py @@ -1,14 +1,13 @@ -from datetime import datetime -from typing import Any, Optional, Tuple, List, Dict, Callable - import hmac +from typing import Any, Optional, Tuple, List, Dict, Callable +from datetime import datetime import sqlalchemy as sa - from szurubooru import config, db, model, errors, rest from szurubooru.func import ( users, scores, comments, tags, util, mime, images, files, image_hash, serialization, snapshots) + EMPTY_PIXEL = ( b'\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00' b'\xff\xff\xff\x21\xf9\x04\x01\x00\x00\x01\x00\x2c\x00\x00\x00\x00' @@ -238,8 +237,8 @@ class PostSerializer(serialization.BaseSerializer): { post['id']: post for post in [ - serialize_micro_post(rel, self.auth_user) - for rel in self.post.relations] + serialize_micro_post(rel, self.auth_user) + for rel in self.post.relations] }.values(), key=lambda post: post['id']) @@ -323,9 +322,9 @@ def get_post_count() -> int: def try_get_post_by_id(post_id: int) -> Optional[model.Post]: return ( db.session - .query(model.Post) - .filter(model.Post.post_id == post_id) - .one_or_none()) + .query(model.Post) + .filter(model.Post.post_id == post_id) + .one_or_none()) def get_post_by_id(post_id: int) -> model.Post: @@ -338,9 +337,9 @@ def get_post_by_id(post_id: int) -> model.Post: def try_get_current_post_feature() -> Optional[model.PostFeature]: return ( db.session - .query(model.PostFeature) - .order_by(model.PostFeature.time.desc()) - .first()) + .query(model.PostFeature) + .order_by(model.PostFeature.time.desc()) + .first()) def try_get_featured_post() -> Optional[model.Post]: @@ -487,10 +486,10 @@ def update_post_content(post: model.Post, content: Optional[bytes]) -> None: post.checksum = util.get_sha1(content) other_post = ( db.session - .query(model.Post) - .filter(model.Post.checksum == post.checksum) - .filter(model.Post.post_id != post.post_id) - .one_or_none()) + .query(model.Post) + .filter(model.Post.checksum == post.checksum) + .filter(model.Post.post_id != post.post_id) + .one_or_none()) if other_post \ and other_post.post_id \ and other_post.post_id != post.post_id: @@ -554,9 +553,9 @@ def update_post_relations(post: model.Post, new_post_ids: List[int]) -> None: if new_post_ids: new_posts = ( db.session - .query(model.Post) - .filter(model.Post.post_id.in_(new_post_ids)) - .all()) + .query(model.Post) + .filter(model.Post.post_id.in_(new_post_ids)) + .all()) else: new_posts = [] if len(new_posts) != len(new_post_ids): @@ -655,13 +654,15 @@ def merge_posts( alias2 = sa.orm.util.aliased(table) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.post_id == source_post_id)) + .where(alias1.post_id == source_post_id)) if anti_dup_func is not None: update_stmt = ( - update_stmt.where(~sa.exists() - .where(anti_dup_func(alias1, alias2)) - .where(alias2.post_id == target_post_id))) + update_stmt + .where( + ~sa.exists() + .where(anti_dup_func(alias1, alias2)) + .where(alias2.post_id == target_post_id))) update_stmt = update_stmt.values(post_id=target_post_id) db.session.execute(update_stmt) @@ -695,24 +696,24 @@ def merge_posts( alias2 = sa.orm.util.aliased(model.PostRelation) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.parent_id == source_post_id) - .where(alias1.child_id != target_post_id) - .where( + .where(alias1.parent_id == source_post_id) + .where(alias1.child_id != target_post_id) + .where( ~sa.exists() - .where(alias2.child_id == alias1.child_id) - .where(alias2.parent_id == target_post_id)) - .values(parent_id=target_post_id)) + .where(alias2.child_id == alias1.child_id) + .where(alias2.parent_id == target_post_id)) + .values(parent_id=target_post_id)) db.session.execute(update_stmt) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.child_id == source_post_id) - .where(alias1.parent_id != target_post_id) - .where( + .where(alias1.child_id == source_post_id) + .where(alias1.parent_id != target_post_id) + .where( ~sa.exists() - .where(alias2.parent_id == alias1.parent_id) - .where(alias2.child_id == target_post_id)) - .values(child_id=target_post_id)) + .where(alias2.parent_id == alias1.parent_id) + .where(alias2.child_id == target_post_id)) + .values(child_id=target_post_id)) db.session.execute(update_stmt) merge_tags(source_post.post_id, target_post.post_id) @@ -733,9 +734,10 @@ def merge_posts( def search_by_image_exact(image_content: bytes) -> Optional[model.Post]: checksum = util.get_sha1(image_content) return ( - db.session.query(model.Post) - .filter(model.Post.checksum == checksum) - .one_or_none()) + db.session + .query(model.Post) + .filter(model.Post.checksum == checksum) + .one_or_none()) def search_by_image(image_content: bytes) -> List[PostLookalike]: @@ -754,19 +756,21 @@ def populate_reverse_search() -> None: excluded_post_ids = image_hash.get_all_paths() post_ids_to_hash = ( - db.session.query(model.Post.post_id) - .filter( + db.session + .query(model.Post.post_id) + .filter( (model.Post.type == model.Post.TYPE_IMAGE) | (model.Post.type == model.Post.TYPE_ANIMATION)) - .filter(~model.Post.post_id.in_(excluded_post_ids)) - .order_by(model.Post.post_id.asc()) - .all()) + .filter(~model.Post.post_id.in_(excluded_post_ids)) + .order_by(model.Post.post_id.asc()) + .all()) for post_ids_chunk in util.chunks(post_ids_to_hash, 100): posts_chunk = ( - db.session.query(model.Post) - .filter(model.Post.post_id.in_(post_ids_chunk)) - .all()) + db.session + .query(model.Post) + .filter(model.Post.post_id.in_(post_ids_chunk)) + .all()) for post in posts_chunk: content_path = get_post_content_path(post) if files.has(content_path): diff --git a/server/szurubooru/func/scores.py b/server/szurubooru/func/scores.py index a35206bc..615fd981 100644 --- a/server/szurubooru/func/scores.py +++ b/server/szurubooru/func/scores.py @@ -1,6 +1,5 @@ import datetime from typing import Any, Tuple, Callable - from szurubooru import db, model, errors @@ -41,10 +40,11 @@ def get_score(entity: model.Base, user: model.User) -> int: assert user table, get_column = _get_table_info(entity) row = ( - db.session.query(table.score) - .filter(get_column(table) == get_column(entity)) - .filter(table.user_id == user.user_id) - .one_or_none()) + db.session + .query(table.score) + .filter(get_column(table) == get_column(entity)) + .filter(table.user_id == user.user_id) + .one_or_none()) return row[0] if row else 0 diff --git a/server/szurubooru/func/serialization.py b/server/szurubooru/func/serialization.py index 42a5413b..699fb473 100644 --- a/server/szurubooru/func/serialization.py +++ b/server/szurubooru/func/serialization.py @@ -1,5 +1,4 @@ from typing import Any, List, Dict, Callable - from szurubooru import model, rest, errors diff --git a/server/szurubooru/func/snapshots.py b/server/szurubooru/func/snapshots.py index dded0eda..240c3bce 100644 --- a/server/szurubooru/func/snapshots.py +++ b/server/szurubooru/func/snapshots.py @@ -1,6 +1,5 @@ -from datetime import datetime from typing import Any, Optional, Dict, Callable - +from datetime import datetime from szurubooru import db, model from szurubooru.func import diff, users @@ -97,7 +96,7 @@ def modify(entity: model.Base, auth_user: Optional[model.User]) -> None: cls for cls in model.Base._decl_class_registry.values() if hasattr(cls, '__table__') - and cls.__table__.fullname == entity.__table__.fullname + and cls.__table__.fullname == entity.__table__.fullname ), None) assert table diff --git a/server/szurubooru/func/tag_categories.py b/server/szurubooru/func/tag_categories.py index 3a6a3853..f1951a8c 100644 --- a/server/szurubooru/func/tag_categories.py +++ b/server/szurubooru/func/tag_categories.py @@ -1,11 +1,10 @@ -from typing import Any, Optional, Dict, List, Callable - import re +from typing import Any, Optional, Dict, List, Callable import sqlalchemy as sa - from szurubooru import config, db, model, errors, rest from szurubooru.func import util, serialization, cache + DEFAULT_CATEGORY_NAME_CACHE_KEY = 'default-tag-category' @@ -89,9 +88,9 @@ def update_category_name(category: model.TagCategory, name: str) -> None: expr = sa.func.lower(model.TagCategory.name) == name.lower() if category.tag_category_id: expr = expr & ( - model.TagCategory.tag_category_id != category.tag_category_id) + model.TagCategory.tag_category_id != category.tag_category_id) already_exists = ( - db.session.query(model.TagCategory).filter(expr).count() > 0) + db.session.query(model.TagCategory).filter(expr).count() > 0) if already_exists: raise TagCategoryAlreadyExistsError( 'A category with this name already exists.') @@ -116,8 +115,9 @@ def update_category_color(category: model.TagCategory, color: str) -> None: def try_get_category_by_name( name: str, lock: bool = False) -> Optional[model.TagCategory]: query = ( - db.session.query(model.TagCategory) - .filter(sa.func.lower(model.TagCategory.name) == name.lower())) + db.session + .query(model.TagCategory) + .filter(sa.func.lower(model.TagCategory.name) == name.lower())) if lock: query = query.with_lockmode('update') return query.one_or_none() @@ -141,8 +141,9 @@ def get_all_categories() -> List[model.TagCategory]: def try_get_default_category( lock: bool = False) -> Optional[model.TagCategory]: query = ( - db.session.query(model.TagCategory) - .filter(model.TagCategory.default)) + db.session + .query(model.TagCategory) + .filter(model.TagCategory.default)) if lock: query = query.with_lockmode('update') category = query.first() @@ -150,8 +151,9 @@ def try_get_default_category( # category, get the first record available. if not category: query = ( - db.session.query(model.TagCategory) - .order_by(model.TagCategory.tag_category_id.asc())) + db.session + .query(model.TagCategory) + .order_by(model.TagCategory.tag_category_id.asc())) if lock: query = query.with_lockmode('update') category = query.first() diff --git a/server/szurubooru/func/tags.py b/server/szurubooru/func/tags.py index 0064da4f..7d92f1e7 100644 --- a/server/szurubooru/func/tags.py +++ b/server/szurubooru/func/tags.py @@ -1,9 +1,9 @@ -from datetime import datetime -from typing import Any, Optional, Tuple, List, Dict, Callable - +import json +import os import re +from typing import Any, Optional, Tuple, List, Dict, Callable +from datetime import datetime import sqlalchemy as sa - from szurubooru import config, db, model, errors, rest from szurubooru.func import util, tag_categories, serialization @@ -138,10 +138,11 @@ def serialize_tag( def try_get_tag_by_name(name: str) -> Optional[model.Tag]: return ( - db.session.query(model.Tag) - .join(model.TagName) - .filter(sa.func.lower(model.TagName.name) == name.lower()) - .one_or_none()) + db.session + .query(model.Tag) + .join(model.TagName) + .filter(sa.func.lower(model.TagName.name) == name.lower()) + .one_or_none()) def get_tag_by_name(name: str) -> model.Tag: @@ -157,12 +158,12 @@ def get_tags_by_names(names: List[str]) -> List[model.Tag]: return [] return ( db.session.query(model.Tag) - .join(model.TagName) - .filter( + .join(model.TagName) + .filter( sa.sql.or_( sa.func.lower(model.TagName.name) == name.lower() for name in names)) - .all()) + .all()) def get_or_create_tags_by_names( @@ -195,15 +196,16 @@ def get_tag_siblings(tag: model.Tag) -> List[model.Tag]: pt_alias1 = sa.orm.aliased(model.PostTag) pt_alias2 = sa.orm.aliased(model.PostTag) result = ( - db.session.query(tag_alias, sa.func.count(pt_alias2.post_id)) - .join(pt_alias1, pt_alias1.tag_id == tag_alias.tag_id) - .join(pt_alias2, pt_alias2.post_id == pt_alias1.post_id) - .filter(pt_alias2.tag_id == tag.tag_id) - .filter(pt_alias1.tag_id != tag.tag_id) - .group_by(tag_alias.tag_id) - .order_by(sa.func.count(pt_alias2.post_id).desc()) - .order_by(tag_alias.first_name) - .limit(50)) + db.session + .query(tag_alias, sa.func.count(pt_alias2.post_id)) + .join(pt_alias1, pt_alias1.tag_id == tag_alias.tag_id) + .join(pt_alias2, pt_alias2.post_id == pt_alias1.post_id) + .filter(pt_alias2.tag_id == tag.tag_id) + .filter(pt_alias1.tag_id != tag.tag_id) + .group_by(tag_alias.tag_id) + .order_by(sa.func.count(pt_alias2.post_id).desc()) + .order_by(tag_alias.first_name) + .limit(50)) return result @@ -211,10 +213,10 @@ def delete(source_tag: model.Tag) -> None: assert source_tag db.session.execute( sa.sql.expression.delete(model.TagSuggestion) - .where(model.TagSuggestion.child_id == source_tag.tag_id)) + .where(model.TagSuggestion.child_id == source_tag.tag_id)) db.session.execute( sa.sql.expression.delete(model.TagImplication) - .where(model.TagImplication.child_id == source_tag.tag_id)) + .where(model.TagImplication.child_id == source_tag.tag_id)) db.session.delete(source_tag) @@ -229,12 +231,13 @@ def merge_tags(source_tag: model.Tag, target_tag: model.Tag) -> None: alias2 = sa.orm.util.aliased(model.PostTag) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.tag_id == source_tag_id)) + .where(alias1.tag_id == source_tag_id)) update_stmt = ( update_stmt - .where(~sa.exists() - .where(alias1.post_id == alias2.post_id) - .where(alias2.tag_id == target_tag_id))) + .where( + ~sa.exists() + .where(alias1.post_id == alias2.post_id) + .where(alias2.tag_id == target_tag_id))) update_stmt = update_stmt.values(tag_id=target_tag_id) db.session.execute(update_stmt) @@ -244,22 +247,24 @@ def merge_tags(source_tag: model.Tag, target_tag: model.Tag) -> None: alias2 = sa.orm.util.aliased(table) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.parent_id == source_tag_id) - .where(alias1.child_id != target_tag_id) - .where(~sa.exists() - .where(alias2.child_id == alias1.child_id) - .where(alias2.parent_id == target_tag_id)) - .values(parent_id=target_tag_id)) + .where(alias1.parent_id == source_tag_id) + .where(alias1.child_id != target_tag_id) + .where( + ~sa.exists() + .where(alias2.child_id == alias1.child_id) + .where(alias2.parent_id == target_tag_id)) + .values(parent_id=target_tag_id)) db.session.execute(update_stmt) update_stmt = ( sa.sql.expression.update(alias1) - .where(alias1.child_id == source_tag_id) - .where(alias1.parent_id != target_tag_id) - .where(~sa.exists() - .where(alias2.parent_id == alias1.parent_id) - .where(alias2.child_id == target_tag_id)) - .values(child_id=target_tag_id)) + .where(alias1.child_id == source_tag_id) + .where(alias1.parent_id != target_tag_id) + .where( + ~sa.exists() + .where(alias2.parent_id == alias1.parent_id) + .where(alias2.child_id == target_tag_id)) + .values(child_id=target_tag_id)) db.session.execute(update_stmt) def merge_suggestions(source_tag_id: int, target_tag_id: int) -> None: diff --git a/server/szurubooru/func/util.py b/server/szurubooru/func/util.py index 164b2545..ba2d4dc9 100644 --- a/server/szurubooru/func/util.py +++ b/server/szurubooru/func/util.py @@ -1,21 +1,20 @@ -from contextlib import contextmanager -from datetime import datetime, timedelta -from typing import Any, Optional, Union, Tuple, List, Dict, Generator, TypeVar - -import hashlib import os +import hashlib import re import tempfile - +from typing import Any, Optional, Union, Tuple, List, Dict, Generator, TypeVar +from datetime import datetime, timedelta +from contextlib import contextmanager from szurubooru import errors + T = TypeVar('T') def snake_case_to_lower_camel_case(text: str) -> str: components = text.split('_') return components[0].lower() + \ - ''.join(word[0].upper() + word[1:].lower() for word in components[1:]) + ''.join(word[0].upper() + word[1:].lower() for word in components[1:]) def snake_case_to_upper_train_case(text: str) -> str: @@ -87,7 +86,6 @@ def is_valid_email(email: Optional[str]) -> bool: class dotdict(dict): # pylint: disable=invalid-name ''' dot.notation access to dictionary attributes. ''' - def __getattr__(self, attr: str) -> Any: return self.get(attr) From deb70e5f28251979555c1ea9dad262514dca61eb Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Mon, 26 Feb 2018 19:53:56 -0600 Subject: [PATCH 06/13] Tokens now correctly delete themselves --- client/js/api.js | 11 +++++++++-- server/szurubooru/api/user_token_api.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/client/js/api.js b/client/js/api.js index 608fc6f7..86687cad 100644 --- a/client/js/api.js +++ b/client/js/api.js @@ -175,11 +175,18 @@ class Api extends events.EventTarget { } logout() { - this.delete_token(this.userName, this.userToken); + this.delete_token(this.userName, this.userToken).then(response => { + this._logout() + }, error => { + this._logout() + }); + + } + + _logout() { this.user = null; this.userName = null; this.userPassword = null; - this.userToken = null; this.dispatchEvent(new CustomEvent('logout')); } diff --git a/server/szurubooru/api/user_token_api.py b/server/szurubooru/api/user_token_api.py index 8936d721..1b4b6115 100644 --- a/server/szurubooru/api/user_token_api.py +++ b/server/szurubooru/api/user_token_api.py @@ -30,9 +30,9 @@ def create_user_token(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.R @rest.routes.delete('/user-tokens/(?P[^/]+)/?') def create_user_token(ctx: rest.Context, params: Dict[str, str]) -> rest.Response: + auth.verify_privilege(ctx.user, 'user_token:delete') user_token = user_tokens.get_user_token_by_user_and_token(ctx.user, params['user_token']) if user_token is not None: - auth.verify_privilege(ctx.user, 'user_token:delete') ctx.session.delete(user_token) ctx.session.commit() return {} From d0b423e91c40483d2c23f648b701e17439cf91a3 Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Mon, 26 Feb 2018 20:45:51 -0600 Subject: [PATCH 07/13] Updated API documentation for the new user-token endpoints --- API.md | 127 +++++++++++++++++++++++- server/szurubooru/api/user_token_api.py | 2 +- 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/API.md b/API.md index 62e0eb22..6d15d551 100644 --- a/API.md +++ b/API.md @@ -7,6 +7,7 @@ 1. [General rules](#general-rules) - [Authentication](#authentication) + - [User token authentication](#user-token-authentication) - [Basic requests](#basic-requests) - [File uploads](#file-uploads) - [Error handling](#error-handling) @@ -56,6 +57,10 @@ - [Updating user](#updating-user) - [Getting user](#getting-user) - [Deleting user](#deleting-user) + - User Tokens + - [Listing tokens](#listing-tokens) + - [Creating token](#creating-token) + - [Deleting token](#deleting-token) - Password reset - [Password reset - step 1: mail request](#password-reset---step-2-confirmation) - [Password reset - step 2: confirmation](#password-reset---step-2-confirmation) @@ -70,6 +75,7 @@ - [User](#user) - [Micro user](#micro-user) + - [User token](#user-token) - [Tag category](#tag-category) - [Tag](#tag) - [Micro tag](#micro-tag) @@ -91,16 +97,35 @@ ## Authentication Authentication is achieved by means of [basic HTTP -auth](https://en.wikipedia.org/wiki/Basic_access_authentication). For this -reason, it is recommended to connect through HTTPS. There are no sessions, so -every privileged request must be authenticated. Available privileges depend on -the user's rank. The way how rank translates to privileges is defined in the -server's configuration. +auth](https://en.wikipedia.org/wiki/Basic_access_authentication) or through the +use of [user token authentication](#user-token-authentication). For this reason, +it is recommended to connect through HTTPS. There are no sessions, so every +privileged request must be authenticated. Available privileges depend on the +user's rank. The way how rank translates to privileges is defined in the server's +configuration. It is recommended to add `?bump-login` GET parameter to the first request in a client "session" (where the definition of a session is up to the client), so that the user's last login time is kept up to date. +## User token authentication + +User token authentication works similarly to [basic HTTP +auth](https://en.wikipedia.org/wiki/Basic_access_authentication). Because it +operates similarly to ***basic HTTP auth*** it is still recommended to connect +through HTTPS. The authorization header uses the type of Token and the username +and token are encoded as Base64 and sent as the second parameter. + +Example header for user1:token-is-more-secure +``` +Authorization: Token dXNlcjE6dG9rZW4taXMtbW9yZS1zZWN1cmU= +``` + +The benefit of token authentication is that beyond the initial login to acquire +the first token, there is no need to transmit the user password in plaintext via +basic auth. Additionally tokens can be revoked at anytime allowing a cleaner +interface for isolating clients from user credentials. + ## Basic requests Every request must use `Content-Type: application/json` and `Accept: @@ -1469,6 +1494,74 @@ data. Deletes existing user. +## Listing tokens +- **Request** + + `GET /user-tokens/` + +- **Output** + + An [unpaged search result resource](#unpaged-search-result), for which + `` is a [user token resource](#user-token). + +- **Errors** + + - privileges are too low + +- **Description** + + Searches for users tokens for the currently logged in user. + +## Creating token +- **Request** + + `POST /user-token` + +- **Input** + + ```json5 + {} + ``` + +- **Output** + + A [user token resource](#user-token). + +- **Errors** + + - privileges are too low + +- **Description** + + Creates a new user token that can be used for authentication of api + endpoints instead of a password. + +## Deleting token +- **Request** + + `DELETE /user-token/` + +- **Input** + + ```json5 + {} + ``` + +- **Output** + + ```json5 + {} + ``` + +- **Errors** + + - the token does not exist + - privileges are too low + +- **Description** + + Deletes existing user token. + ## Password reset - step 1: mail request - **Request** @@ -1701,6 +1794,30 @@ A single user. A [user resource](#user) stripped down to `name` and `avatarUrl` fields. +## User token +**Description** + +A single user token. + +**Structure** + +```json5 +{ + "user": , + "token": , + "enabled": , + "creationTime": , + "lastEditTime": , +} +``` + +**Field meaning** +- ``: micro user. See [micro user](#micro-user). +- ``: the token that can be used to authenticate the user. +- ``: whether the token is still valid for authentication. +- ``: time the user token was created , formatted as per RFC 3339. +- ``: time the user token was edited, formatted as per RFC 3339. + ## Tag category **Description** diff --git a/server/szurubooru/api/user_token_api.py b/server/szurubooru/api/user_token_api.py index 1b4b6115..7559a9fc 100644 --- a/server/szurubooru/api/user_token_api.py +++ b/server/szurubooru/api/user_token_api.py @@ -17,7 +17,7 @@ def get_user_tokens(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Res auth.verify_privilege(ctx.user, 'user_token:list') user_token_list = user_tokens.get_user_tokens(ctx.user) return { - "tokens": [_serialize(ctx, token) for token in user_token_list] + "results": [_serialize(ctx, token) for token in user_token_list] } From 22cf806220fc0ed89e04f93321612c2292e3533e Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Mon, 26 Feb 2018 21:47:01 -0600 Subject: [PATCH 08/13] Added versioning, cleaned up API documentation, fixed endpoints, resolved logout diplay update issue --- API.md | 33 +++++++++++++++++++ client/js/api.js | 25 +++++++------- config.yaml.dist | 1 + server/szurubooru/api/user_token_api.py | 17 +++++++--- server/szurubooru/func/user_tokens.py | 4 +++ .../a39c7f98a7fa_add_user_token_table.py | 1 + server/szurubooru/model/user.py | 1 + 7 files changed, 66 insertions(+), 16 deletions(-) diff --git a/API.md b/API.md index 6d15d551..ae2bb6b3 100644 --- a/API.md +++ b/API.md @@ -60,6 +60,7 @@ - User Tokens - [Listing tokens](#listing-tokens) - [Creating token](#creating-token) + - [Updating token](#updating-token) - [Deleting token](#deleting-token) - Password reset - [Password reset - step 1: mail request](#password-reset---step-2-confirmation) @@ -1535,6 +1536,36 @@ data. Creates a new user token that can be used for authentication of api endpoints instead of a password. + +## Updating user +- **Request** + + `PUT /user-token/` + +- **Input** + + ```json5 + { + "version": , + "enabled": , // optional + } + ``` + +- **Output** + + A [user token resource](#user-token). + +- **Errors** + + - the version is outdated + - the user token does not exist + - privileges are too low + +- **Description** + + Updates an existing user token using specified parameters. All fields + except the [`version`](#versioning) are optional - update concerns only + provided fields. ## Deleting token - **Request** @@ -1806,6 +1837,7 @@ A single user token. "user": , "token": , "enabled": , + "version": , "creationTime": , "lastEditTime": , } @@ -1815,6 +1847,7 @@ A single user token. - ``: micro user. See [micro user](#micro-user). - ``: the token that can be used to authenticate the user. - ``: whether the token is still valid for authentication. +- ``: resource version. See [versioning](#versioning). - ``: time the user token was created , formatted as per RFC 3339. - ``: time the user token was edited, formatted as per RFC 3339. diff --git a/client/js/api.js b/client/js/api.js index 86687cad..67dbdc22 100644 --- a/client/js/api.js +++ b/client/js/api.js @@ -118,9 +118,9 @@ class Api extends events.EventTarget { }); } - get_token(userName, options) { + create_token(userName, options) { return new Promise((resolve, reject) => { - this.post('/user-tokens', {}) + this.post('/user-token', {}) .then(response => { cookies.set( 'auth', @@ -137,15 +137,14 @@ class Api extends events.EventTarget { delete_token(userName, userToken) { return new Promise((resolve, reject) => { - this.delete('/user-tokens/' + userToken, {}) + this.delete('/user-token/' + userToken, {}) .then(response => { const options = {}; cookies.set( 'auth', {'user': userName, 'token': null}, options); - this.userName = userName; - this.userToken = null; + resolve(); }, error => { reject(error); }); @@ -163,7 +162,7 @@ class Api extends events.EventTarget { if (doRemember) { options.expires = 365; } - this.get_token(this.userName, options); + this.create_token(this.userName, options); this.user = response; resolve(); this.dispatchEvent(new CustomEvent('login')); @@ -175,18 +174,20 @@ class Api extends events.EventTarget { } logout() { - this.delete_token(this.userName, this.userToken).then(response => { - this._logout() - }, error => { - this._logout() - }); - + let self = this; + this.delete_token(this.userName, this.userToken) + .then(response => { + self._logout(); + }, error => { + self._logout(); + }); } _logout() { this.user = null; this.userName = null; this.userPassword = null; + this.userToken = null; this.dispatchEvent(new CustomEvent('logout')); } diff --git a/config.yaml.dist b/config.yaml.dist index 0b10dbcf..e3409b6f 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -88,6 +88,7 @@ privileges: 'user_token:list': regular 'user_token:create': regular + 'user_token:edit': regular 'user_token:delete': regular 'posts:create:anonymous': regular diff --git a/server/szurubooru/api/user_token_api.py b/server/szurubooru/api/user_token_api.py index 7559a9fc..bd3037be 100644 --- a/server/szurubooru/api/user_token_api.py +++ b/server/szurubooru/api/user_token_api.py @@ -1,7 +1,7 @@ from typing import Dict from szurubooru import model, rest -from szurubooru.func import auth, user_tokens, serialization +from szurubooru.func import auth, user_tokens, serialization, versions def _serialize( @@ -21,15 +21,24 @@ def get_user_tokens(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Res } -@rest.routes.post('/user-tokens/?') +@rest.routes.post('/user-token/?') def create_user_token(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Response: auth.verify_privilege(ctx.user, 'user_token:create') user_token = user_tokens.create_user_token(ctx.user) return _serialize(ctx, user_token) -@rest.routes.delete('/user-tokens/(?P[^/]+)/?') -def create_user_token(ctx: rest.Context, params: Dict[str, str]) -> rest.Response: +@rest.routes.put('/user-token/(?P[^/]+)/?') +def edit_user_token(ctx: rest.Context, _params: Dict[str, str] = {}) -> rest.Response: + auth.verify_privilege(ctx.user, 'user_token:edit') + user_token = user_tokens.get_user_token_by_user_and_token(ctx.user, params['user_token']) + versions.verify_version(user_token, ctx) + versions.bump_version(user_token) + return _serialize(ctx, user_token) + + +@rest.routes.delete('/user-token/(?P[^/]+)/?') +def delete_user_token(ctx: rest.Context, params: Dict[str, str]) -> rest.Response: auth.verify_privilege(ctx.user, 'user_token:delete') user_token = user_tokens.get_user_token_by_user_and_token(ctx.user, params['user_token']) if user_token is not None: diff --git a/server/szurubooru/func/user_tokens.py b/server/szurubooru/func/user_tokens.py index 8c9a35de..3492c860 100644 --- a/server/szurubooru/func/user_tokens.py +++ b/server/szurubooru/func/user_tokens.py @@ -20,6 +20,7 @@ class UserTokenSerializer(serialization.BaseSerializer): 'user': self.serialize_user, 'token': self.serialize_token, 'enabled': self.serialize_enabled, + 'version': self.serialize_version, 'creationTime': self.serialize_creation_time, 'lastLoginTime': self.serialize_last_edit_time, } @@ -39,6 +40,9 @@ class UserTokenSerializer(serialization.BaseSerializer): def serialize_enabled(self) -> Any: return self.user_token.enabled + def serialize_version(self) -> Any: + return self.user_token.version + def serialize_user_token( user_token: Optional[model.UserToken], diff --git a/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py b/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py index 6e6ac5d8..c61ee3ea 100644 --- a/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py +++ b/server/szurubooru/migrations/versions/a39c7f98a7fa_add_user_token_table.py @@ -23,6 +23,7 @@ def upgrade(): sa.Column('enabled', sa.Boolean(), nullable=False), sa.Column('creation_time', sa.DateTime(), nullable=False), sa.Column('last_edit_time', sa.DateTime(), nullable=True), + sa.Column('version', sa.Integer(), nullable=False), sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='CASCADE'), sa.PrimaryKeyConstraint('id') ) diff --git a/server/szurubooru/model/user.py b/server/szurubooru/model/user.py index 7ca1a10e..3e31c81e 100644 --- a/server/szurubooru/model/user.py +++ b/server/szurubooru/model/user.py @@ -100,5 +100,6 @@ class UserToken(Base): enabled = sa.Column('enabled', sa.Boolean, nullable=False, default=True) creation_time = sa.Column('creation_time', sa.DateTime, nullable=False) last_edit_time = sa.Column('last_edit_time', sa.DateTime) + version = sa.Column('version', sa.Integer, default=1, nullable=False) user = sa.orm.relationship('User') From 23268ded75d519b3bfbde13e72340cefc178bba8 Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Tue, 27 Feb 2018 11:26:38 -0600 Subject: [PATCH 09/13] ignoring local data folder --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 30cd78cf..89183272 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ config.yaml .cache __pycache__ .idea/ -*.iml \ No newline at end of file +*.iml +data/ \ No newline at end of file From 05d2785ec6425c7d43eb5c6b41cd9c3729c37727 Mon Sep 17 00:00:00 2001 From: ReAnzu Date: Tue, 27 Feb 2018 18:14:07 -0600 Subject: [PATCH 10/13] Added a Manage tokens tab to the user panel --- client/css/user-view.styl | 20 +++++- client/html/user.tpl | 3 + client/html/user_tokens.tpl | 29 ++++++++ client/js/api.js | 4 +- client/js/controllers/user_controller.js | 78 ++++++++++++++++++++- client/js/models/user_token.js | 76 +++++++++++++++++++++ client/js/views/user_tokens_view.js | 86 ++++++++++++++++++++++++ client/js/views/user_view.js | 12 +++- config.yaml.dist | 12 ++-- server/szurubooru/api/user_token_api.py | 32 +++++---- server/szurubooru/func/user_tokens.py | 2 +- 11 files changed, 332 insertions(+), 22 deletions(-) create mode 100644 client/html/user_tokens.tpl create mode 100644 client/js/models/user_token.js create mode 100644 client/js/views/user_tokens_view.js diff --git a/client/css/user-view.styl b/client/css/user-view.styl index 12cba75e..46f3044a 100644 --- a/client/css/user-view.styl +++ b/client/css/user-view.styl @@ -1,6 +1,6 @@ #user width: 100% - max-width: 35em + max-width: 45em nav.text-nav margin-bottom: 1.5em @@ -37,6 +37,24 @@ height: 1px clear: both + #user-tokens + .token-flex-container + width: 100% + display: flex; + flex-direction column; + padding-bottom: 0.5em; + + .token-flex-row + display: flex; + flex-direction: row; + justify-content: space-between; + padding-top: 0.25em; + padding-bottom: 0.25em; + border-bottom: black solid 1px; + + form + width: auto; + #user-delete form width: 100% diff --git a/client/html/user.tpl b/client/html/user.tpl index 28e34e67..cbf4f18f 100644 --- a/client/html/user.tpl +++ b/client/html/user.tpl @@ -6,6 +6,9 @@ --><% if (ctx.canEditAnything) { %>
  • '>Account settings
    • <% } %><% if (ctx.canListTokens) { %>
  • '>Manage tokens
    • <% } %><% if (ctx.canDelete) { %>
  • '>Account deletion
    • <% } %>