Merge remote-tracking branch 'upstream/master'

INSTALL.md

@@ -84,29 +84,32 @@ user@host:szuru/server$ source python_modules/bin/activate # enters the sandbox
 
 ### Preparing `szurubooru` for first run
 
-1. Configure things:
+1. Compile the frontend:
 
     ```console
+    user@host:szuru$ cd client
+    user@host:szuru/client$ node build.js
+    ```
+
+    You can include the flags `--no-transpile` to disable the JavaScript
+    transpiler, which provides compatibility with older browsers, and
+    `--debug` to generate JS source mappings.
+
+2. Configure things:
+
+    ```console
+    user@host:szuru/client$ cd ..
     user@host:szuru$ cp config.yaml.dist config.yaml
     user@host:szuru$ vim config.yaml
     ```
 
     Pay extra attention to these fields:
 
-    - base URL,
-    - API URL,
     - data directory,
     - data URL,
     - database,
     - the `smtp` section.
 
-2. Compile the frontend:
-
-    ```console
-    user@host:szuru$ cd client
-    user@host:szuru/client$ npm run build
-    ```
-
 3. Upgrade the database:
 
     ```console
@@ -121,7 +124,7 @@ user@host:szuru/server$ source python_modules/bin/activate # enters the sandbox
 4. Run the tests:
 
     ```console
-    (python_modules) user@host:szuru/server$ ./test
+    (python_modules) user@host:szuru/server$ pytest
     ```
 
 It is recommended to rebuild the frontend after each change to configuration.
@@ -140,6 +143,11 @@ meant to be exposed directly to the end users.
 The API should be exposed using WSGI server such as `waitress`, `gunicorn` or
 similar. Other configurations might be possible but I didn't pursue them.
 
+API calls are made to the relative URL `/api/`. Your HTTP server should be
+configured to proxy this URL format to the WSGI server. Some users may prefer
+to use a dedicated reverse proxy for this, to incorporate additional features
+such as load balancing and SSL.
+
 Note that the API URL in the virtual host configuration needs to be the same as
 the one in the `config.yaml`, so that client knows how to access the backend!
 
@@ -177,8 +185,6 @@ server {
 **`config.yaml`**:
 
 ```yaml
-api_url: 'http://example.com/api/'
-base_url: 'http://example.com/'
 data_url: 'http://example.com/data/'
 data_dir: '/srv/www/booru/client/public/data'
 ```

client/.babelrc

@@ -1 +1 @@
-{ "presets": ["es2015"] }
+{ "presets": ["env"] }

client/build.js

@@ -5,20 +5,6 @@ const glob = require('glob');
 const path = require('path');
 const util = require('util');
 const execSync = require('child_process').execSync;
-const camelcase = require('camelcase');
-
-function convertKeysToCamelCase(input) {
-    let result = {};
-    Object.keys(input).map((key, _) => {
-        const value = input[key];
-        if (value !== null && value.constructor == Object) {
-            result[camelcase(key)] = convertKeysToCamelCase(value);
-        } else {
-            result[camelcase(key)] = value;
-        }
-    });
-    return result;
-}
 
 function readTextFile(path) {
     return fs.readFileSync(path, 'utf-8');
@@ -29,37 +15,27 @@ function writeFile(path, content) {
 }
 
 function getVersion() {
-    return execSync('git describe --always --dirty --long --tags')
+    let build_info = process.env.BUILD_INFO;
-        .toString()
+    if (build_info) {
-        .trim();
+        return build_info.trim();
+    } else {
+        try {
+            build_info = execSync('git describe --always --dirty --long --tags')
+                .toString();
+        } catch (e) {
+            console.warn('Cannot find build version');
+            return 'unknown';
+        }
+        return build_info.trim();
+    }
 }
 
 function getConfig() {
-    const yaml = require('js-yaml');
+    let config = {
-    const merge = require('merge');
+        meta: {
-    const camelcaseKeys = require('camelcase-keys');
-
-    function parseConfigFile(path) {
-        let result = yaml.load(readTextFile(path, 'utf-8'));
-        return convertKeysToCamelCase(result);
-    }
-
-    let config = parseConfigFile('../config.yaml.dist');
-
-    try {
-        const localConfig = parseConfigFile('../config.yaml');
-        config = merge.recursive(config, localConfig);
-    } catch (e) {
-        console.warn('Local config does not exist, ignoring');
-    }
-
-    config.canSendMails = !!config.smtp.host;
-    delete config.secret;
-    delete config.smtp;
-    delete config.database;
-    config.meta = {
           version: getVersion(),
-        buildDate: new Date().toUTCString(),
+          buildDate: new Date().toUTCString()
+        }
     };
 
     return config;
@@ -70,11 +46,11 @@ function copyFile(source, target) {
 }
 
 function minifyJs(path) {
-    return require('uglify-es').minify(fs.readFileSync(path, 'utf-8'), {compress: {unused: false}}).code;
+    return require('terser').minify(fs.readFileSync(path, 'utf-8'), {compress: {unused: false}}).code;
 }
 
 function minifyCss(css) {
-    return require('csso').minify(css);
+    return require('csso').minify(css).css;
 }
 
 function minifyHtml(html) {
@@ -85,15 +61,11 @@ function minifyHtml(html) {
     }).trim();
 }
 
-function bundleHtml(config) {
+function bundleHtml() {
     const underscore = require('underscore');
     const babelify = require('babelify');
     const baseHtml = readTextFile('./html/index.htm', 'utf-8');
-    const finalHtml = baseHtml
+    writeFile('./public/index.htm', minifyHtml(baseHtml));
-        .replace(
-            /(<title>)(.*)(<\/title>)/,
-            util.format('$1%s$3', config.name));
-    writeFile('./public/index.htm', minifyHtml(finalHtml));
 
     glob('./html/**/*.tpl', {}, (er, files) => {
         let compiledTemplateJs = '\'use strict\'\n';
@@ -143,7 +115,7 @@ function bundleCss() {
     });
 }
 
-function bundleJs(config) {
+function bundleJs() {
     const browserify = require('browserify');
     const external = [
         'underscore',
@@ -170,7 +142,7 @@ function bundleJs(config) {
             for (let lib of external) {
                 b.require(lib);
             }
-            if (config.transpile) {
+            if (!process.argv.includes('--no-transpile')) {
                 b.add(require.resolve('babel-polyfill'));
             }
             writeJsBundle(
@@ -179,15 +151,15 @@ function bundleJs(config) {
 
         if (!process.argv.includes('--no-app-js')) {
             let outputFile = fs.createWriteStream('./public/js/app.min.js');
-            let b = browserify({debug: config.debug});
+            let b = browserify({debug: process.argv.includes('--debug')});
-            if (config.transpile) {
+            if (!process.argv.includes('--no-transpile')) {
                 b = b.transform('babelify');
             }
             writeJsBundle(
                 b.external(external).add(files),
                 './public/js/app.min.js',
                 'Bundled app JS',
-                !config.debug);
+                !process.argv.includes('--debug'));
         }
     });
 }
@@ -217,11 +189,11 @@ const config = getConfig();
 bundleConfig(config);
 bundleBinaryAssets();
 if (!process.argv.includes('--no-html')) {
-    bundleHtml(config);
+    bundleHtml();
 }
 if (!process.argv.includes('--no-css')) {
     bundleCss();
 }
 if (!process.argv.includes('--no-js')) {
-    bundleJs(config);
+    bundleJs();
 }

client/html/index.htm

@@ -3,7 +3,7 @@
 <head>
     <meta charset='utf-8'/>
     <meta name='viewport' content='width=device-width, initial-scale=1, maximum-scale=1'>
-    <title><!-- configured in the config file --></title>
+    <title>Loading...</title>
     <link href='/css/app.min.css' rel='stylesheet' type='text/css'/>
     <link href='/css/vendor.min.css' rel='stylesheet' type='text/css'/>
     <link rel='shortcut icon' type='image/png' href='/img/favicon.png'/>

client/html/post_readonly_sidebar.tpl

@@ -36,8 +36,8 @@
 
         <section class='search'>
             Search on
-            <a href='http://iqdb.org/?url=<%- encodeURIComponent(ctx.post.contentUrl) %>'>IQDB</a> &middot;
+            <a href='http://iqdb.org/?url=<%- encodeURIComponent(ctx.post.fullContentUrl) %>'>IQDB</a> &middot;
-            <a href='https://www.google.com/searchbyimage?&image_url=<%- encodeURIComponent(ctx.post.contentUrl) %>'>Google Images</a>
+            <a href='https://www.google.com/searchbyimage?&image_url=<%- encodeURIComponent(ctx.post.fullContentUrl) %>'>Google Images</a>
         </section>
 
         <section class='social'>

client/js/api.js

@@ -2,7 +2,6 @@
 
 const cookies = require('js-cookie');
 const request = require('superagent');
-const config = require('./config.js');
 const events = require('./events.js');
 const progress = require('./util/progress.js');
 const uri = require('./util/uri.js');
@@ -257,7 +256,7 @@ class Api extends events.EventTarget {
 
     _getFullUrl(url) {
         const fullUrl =
-            (config.apiUrl + '/' + url).replace(/([^:])\/+/g, '$1/');
+            ('/api/' + url).replace(/([^:])\/+/g, '$1/');
         const matches = fullUrl.match(/^([^?]*)\??(.*)$/);
         const baseUrl = matches[1];
         const request = matches[2];

client/js/controls/post_readonly_sidebar_control.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const api = require('../api.js');
-const config = require('../config.js');
 const events = require('../events.js');
 const views = require('../util/views.js');
 

client/js/models/post.js

@@ -30,6 +30,7 @@ class Post extends events.EventTarget {
     get user()               { return this._user; }
     get safety()             { return this._safety; }
     get contentUrl()         { return this._contentUrl; }
+    get fullContentUrl()     { return this._fullContentUrl; }
     get thumbnailUrl()       { return this._thumbnailUrl; }
     get canvasWidth()        { return this._canvasWidth || 800; }
     get canvasHeight()       { return this._canvasHeight || 450; }
@@ -275,6 +276,7 @@ class Post extends events.EventTarget {
             _user:          response.user,
             _safety:        response.safety,
             _contentUrl:    response.contentUrl,
+            _fullContentUrl: new URL(response.contentUrl, window.location.href).href,
             _thumbnailUrl:  response.thumbnailUrl,
             _canvasWidth:   response.canvasWidth,
             _canvasHeight:  response.canvasHeight,

client/js/models/post_list.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const settings = require('../models/settings.js');
-const config = require('../config.js');
 const api = require('../api.js');
 const uri = require('../util/uri.js');
 const AbstractList = require('./abstract_list.js');

client/js/util/markdown.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const marked = require('marked');
-const config = require('../config.js');
 
 class BaseMarkdownWrapper {
     preprocess(text) {
@@ -64,15 +63,12 @@ class TagPermalinkFixWrapper extends BaseMarkdownWrapper {
 class EntityPermalinkWrapper extends BaseMarkdownWrapper {
     preprocess(text) {
         // URL-based permalinks
-        let baseUrl = config.baseUrl.replace(/\/+$/, '');
         text = text.replace(
-            new RegExp('\\b' + baseUrl + '/post/(\\d+)/?\\b', 'g'), '@$1');
+            new RegExp('\\b/post/(\\d+)/?\\b', 'g'), '@$1');
         text = text.replace(
-            new RegExp('\\b' + baseUrl + '/tag/([a-zA-Z0-9_-]+?)/?', 'g'),
+            new RegExp('\\b/tag/([a-zA-Z0-9_-]+?)/?', 'g'), '#$1');
-            '#$1');
         text = text.replace(
-            new RegExp('\\b' + baseUrl + '/user/([a-zA-Z0-9_-]+?)/?', 'g'),
+            new RegExp('\\b/user/([a-zA-Z0-9_-]+?)/?', 'g'), '+$1');
-            '+$1');
 
         text = text.replace(
             /(^|^\(|(?:[^\]])\(|[\s<>\[\]\)])([+#@][a-zA-Z0-9_-]+)/g,

client/package.json

@@ -6,26 +6,25 @@
     "watch": "c1=\"\";while :;do c2=$(find html js css img -type f -and -not -iname '*autogen*'|sort|xargs cat|md5sum);[[ $c1 != $c2 ]]&&npm run build -- --no-vendor-js;c1=$c2;sleep 1;done"
   },
   "dependencies": {
-    "babel-polyfill": "^6.26.0",
+    "font-awesome": "^4.7.0",
-    "babel-preset-es2015": "^6.24.1",
-    "babelify": "^7.2.0",
-    "browserify": "^13.0.0",
-    "camelcase": "^2.1.1",
-    "camelcase-keys": "^4.2.0",
-    "csso": "^1.8.0",
-    "font-awesome": "^4.6.1",
-    "glob": "^7.1.2",
-    "html-minifier": "^1.3.1",
     "ios-inner-height": "^1.0.3",
     "js-cookie": "^2.2.0",
-    "js-yaml": "^3.10.0",
+    "marked": "^0.4.0",
-    "marked": "^0.3.9",
+    "mousetrap": "^1.6.2",
-    "merge": "^1.2.0",
-    "mousetrap": "^1.6.1",
     "nprogress": "^0.2.0",
-    "stylus": "^0.54.2",
+    "superagent": "^3.8.3"
-    "superagent": "^1.8.3",
+  },
-    "uglify-es": "^3.3.4",
+  "devDependencies": {
-    "underscore": "^1.8.3"
+    "babel-core": "^6.26.3",
+    "babel-polyfill": "^6.26.0",
+    "babel-preset-env": "^1.7.0",
+    "babelify": "^8.0.0",
+    "browserify": "^16.2.2",
+    "csso": "^3.5.1",
+    "glob": "^7.1.2",
+    "html-minifier": "^3.5.18",
+    "stylus": "^0.54.5",
+    "terser": "^3.7.7",
+    "underscore": "^1.9.1"
   }
 }

config.yaml.dist

@@ -2,12 +2,9 @@
 # and override only what you need.
 
 name: szurubooru # shown in the website title and on the front page
-debug: 0 # generate source maps for JS debugging?
+debug: 0 # generate server logs?
 show_sql: 0 # show sql in server logs?
-transpile: 1 # generate bigger JS to support older browsers?
 secret: change # used to salt the users' password hashes
-api_url: # where frontend connects to, example: http://api.example.com/
-base_url: # used to form links to frontend, example: http://example.com/
 data_url: # used to form links to posts and avatars, example: http://example.com/data/
 data_dir: # absolute path for posts and avatars storage, example: /srv/www/booru/client/public/data/
 user_agent: # user agent name used to download files from the web on behalf of the api users
@@ -45,10 +42,10 @@ smtp:
     port: # example: 25
     user: # example: bot
     pass: # example: groovy123
-    # host can be left empty, in which case it is recommended to fill contactEmail.
+    # host can be left empty, in which case it is recommended to fill contact_email.
 
 
-contactEmail: # example: bob@example.com. Meant for manual password reset procedures
+contact_email: # example: bob@example.com. Meant for manual password reset procedures
 
 
 # used for reverse image search

server/szurubooru/api/info_api.py

@@ -42,7 +42,7 @@ def get_info(
             'tagCategoryNameRegex': config.config['tag_category_name_regex'],
             'defaultUserRank': config.config['default_rank'],
             'enableSafety': config.config['enable_safety'],
-            'contactEmail': config.config['contactEmail'],
+            'contactEmail': config.config['contact_email'],
             'canSendMails': bool(config.config['smtp']['host']),
             'privileges':
                 util.snake_case_to_lower_camel_case_keys(

server/szurubooru/api/password_reset_api.py

@@ -13,7 +13,7 @@ MAIL_BODY = (
 
 @rest.routes.get('/password-reset/(?P<user_name>[^/]+)/?')
 def start_password_reset(
-        _ctx: rest.Context, params: Dict[str, str]) -> rest.Response:
+        ctx: rest.Context, params: Dict[str, str]) -> rest.Response:
     user_name = params['user_name']
     user = users.get_user_by_name_or_email(user_name)
     if not user.email:
@@ -21,13 +21,19 @@ def start_password_reset(
             'User %r hasn\'t supplied email. Cannot reset password.' % (
                 user_name))
     token = auth.generate_authentication_token(user)
-    url = '%s/password-reset/%s:%s' % (
+
-        config.config['base_url'].rstrip('/'), user.name, token)
+    if 'HTTP_ORIGIN' in ctx.env:
+        url = ctx.env['HTTP_ORIGIN'].rstrip('/')
+    else:
+        url = ''
+    url += '/password-reset/%s:%s' % (user.name, token)
+
     mailer.send_mail(
         'noreply@%s' % config.config['name'],
         user.email,
         MAIL_SUBJECT.format(name=config.config['name']),
         MAIL_BODY.format(name=config.config['name'], url=url))
+
     return {}
 
 

server/szurubooru/facade.py

@@ -79,7 +79,7 @@ def validate_config() -> None:
             'Default rank %r is not on the list of known ranks' % (
                 config.config['default_rank']))
 
-    for key in ['base_url', 'api_url', 'data_url', 'data_dir']:
+    for key in ['data_url', 'data_dir']:
         if not config.config[key]:
             raise errors.ConfigError(
                 'Service is not configured: %r is missing' % key)

server/szurubooru/func/tag_categories.py

@@ -105,7 +105,7 @@ def update_category_color(category: model.TagCategory, color: str) -> None:
     assert category
     if not color:
         raise InvalidTagCategoryColorError('Color cannot be empty.')
-    if not re.match(r'^#?[0-9A-Za-z]+$', color):
+    if not re.match(r'^#?[0-9a-z]+$', color):
         raise InvalidTagCategoryColorError('Invalid color.')
     if util.value_exceeds_column_size(color, model.TagCategory.color):
         raise InvalidTagCategoryColorError('Color is too long.')

server/szurubooru/rest/app.py

@@ -63,7 +63,7 @@ def _create_context(env: Dict[str, Any]) -> context.Context:
                 'Could not decode the request body. The JSON '
                 'was incorrect or was not encoded as UTF-8.')
 
-    return context.Context(method, path, headers, params, files)
+    return context.Context(env, method, path, headers, params, files)
 
 
 def application(

server/szurubooru/rest/context.py

@@ -11,11 +11,13 @@ Response = Optional[Dict[str, Any]]
 class Context:
     def __init__(
             self,
+            env: Dict[str, Any],
             method: str,
             url: str,
             headers: Dict[str, str] = None,
             params: Request = None,
             files: Dict[str, bytes] = None) -> None:
+        self.env = env
         self.method = method
         self.url = url
         self._headers = headers or {}

server/szurubooru/tests/api/test_info.py

@@ -10,6 +10,9 @@ def test_info_api(
     auth_user = user_factory(rank=model.User.RANK_REGULAR)
     anon_user = user_factory(rank=model.User.RANK_ANONYMOUS)
     config_injector({
+        'name': 'test installation',
+        'contact_email': 'test@example.com',
+        'enable_safety': True,
         'data_dir': str(directory),
         'user_name_regex': '1',
         'password_regex': '2',
@@ -21,11 +24,17 @@ def test_info_api(
             'test_key2': 'test_value2',
             'posts:view:featured': 'regular',
         },
+        'smtp': {
+            'host': 'example.com',
+        }
     })
     db.session.add_all([post_factory(), post_factory()])
     db.session.flush()
 
     expected_config_key = {
+        'name': 'test installation',
+        'contactEmail': 'test@example.com',
+        'enableSafety': True,
         'userNameRegex': '1',
         'passwordRegex': '2',
         'tagNameRegex': '3',
@@ -36,6 +45,7 @@ def test_info_api(
             'testKey2': 'test_value2',
             'posts:view:featured': 'regular',
         },
+        'canSendMails': True
     }
 
     with fake_datetime('2016-01-01 13:00'):

server/szurubooru/tests/conftest.py

@@ -95,6 +95,7 @@ def session(query_logger):  # pylint: disable=unused-argument
 def context_factory(session):
     def factory(params=None, files=None, user=None, headers=None):
         ctx = rest.Context(
+            env={'HTTP_ORIGIN': 'http://example.com'},
             method=None,
             url=None,
             headers=headers or {},

server/szurubooru/tests/rest/test_context.py

@@ -6,13 +6,14 @@ from szurubooru.func import net
 
 
 def test_has_param():
-    ctx = rest.Context(method=None, url=None, params={'key': 'value'})
+    ctx = rest.Context(env={}, method=None, url=None, params={'key': 'value'})
     assert ctx.has_param('key')
     assert not ctx.has_param('non-existing')
 
 
 def test_get_file():
-    ctx = rest.Context(method=None, url=None, files={'key': b'content'})
+    ctx = rest.Context(
+        env={}, method=None, url=None, files={'key': b'content'})
     assert ctx.get_file('key') == b'content'
     with pytest.raises(errors.ValidationError):
         ctx.get_file('non-existing')
@@ -22,7 +23,7 @@ def test_get_file_from_url():
     with unittest.mock.patch('szurubooru.func.net.download'):
         net.download.return_value = b'content'
         ctx = rest.Context(
-            method=None, url=None, params={'keyUrl': 'example.com'})
+            env={}, method=None, url=None, params={'keyUrl': 'example.com'})
         assert ctx.get_file('key') == b'content'
         net.download.assert_called_once_with('example.com')
         with pytest.raises(errors.ValidationError):
@@ -31,6 +32,7 @@ def test_get_file_from_url():
 
 def test_getting_list_parameter():
     ctx = rest.Context(
+        env={},
         method=None,
         url=None,
         params={'key': 'value', 'list': ['1', '2', '3']})
@@ -43,6 +45,7 @@ def test_getting_list_parameter():
 
 def test_getting_string_parameter():
     ctx = rest.Context(
+        env={},
         method=None,
         url=None,
         params={'key': 'value', 'list': ['1', '2', '3']})
@@ -55,6 +58,7 @@ def test_getting_string_parameter():
 
 def test_getting_int_parameter():
     ctx = rest.Context(
+        env={},
         method=None,
         url=None,
         params={'key': '50', 'err': 'invalid', 'list': [1, 2, 3]})
@@ -76,7 +80,8 @@ def test_getting_int_parameter():
 
 def test_getting_bool_parameter():
     def test(value):
-        ctx = rest.Context(method=None, url=None, params={'key': value})
+        ctx = rest.Context(
+            env={}, method=None, url=None, params={'key': value})
         return ctx.get_param_as_bool('key')
 
     assert test('1') is True
@@ -104,7 +109,7 @@ def test_getting_bool_parameter():
     with pytest.raises(errors.ValidationError):
         test(['1', '2'])
 
-    ctx = rest.Context(method=None, url=None)
+    ctx = rest.Context(env={}, method=None, url=None)
     with pytest.raises(errors.ValidationError):
         ctx.get_param_as_bool('non-existing')
     assert ctx.get_param_as_bool('non-existing', default=True) is True