Improve compilation speed for development builds (#402)

* Improve incremental build times
* Live-reloading in development mode

client/build.js

@@ -57,6 +57,11 @@ const glob = require('glob');
 const path = require('path');
 const util = require('util');
 const execSync = require('child_process').execSync;
+const browserify = require('browserify');
+const chokidar = require('chokidar');
+const WebSocket = require('ws');
+var PrettyError = require('pretty-error');
+var pe = new PrettyError();
 
 function readTextFile(path) {
     return fs.readFileSync(path, 'utf-8');
@@ -148,9 +153,6 @@ function bundleCss() {
     console.info('Bundled CSS');
 }
 
-function bundleJs() {
-    const browserify = require('browserify');
-
 function minifyJs(path) {
     return require('terser').minify(
         fs.readFileSync(path, 'utf-8'), { compress: { unused: false } }).code;
@@ -158,7 +160,7 @@ function bundleJs() {
 
 function writeJsBundle(b, path, compress, callback) {
     let outputFile = fs.createWriteStream(path);
-        b.bundle().pipe(outputFile);
+    b.bundle().on('error', (e) => console.error(pe.render(e))).pipe(outputFile);
     outputFile.on('finish', () => {
         if (compress) {
             fs.writeFileSync(path, minifyJs(path));
@@ -167,7 +169,7 @@ function bundleJs() {
     });
 }
 
-    if (!process.argv.includes('--no-vendor-js')) {
+function bundleVendorJs(compress) {
     let b = browserify();
     for (let lib of external_js) {
         b.require(lib);
@@ -176,7 +178,7 @@ function bundleJs() {
         b.add(require.resolve('babel-polyfill'));
     }
     const file = './public/js/vendor.min.js';
-        writeJsBundle(b, file, true, () => {
+    writeJsBundle(b, file, compress, () => {
         if (process.argv.includes('--gzip')) {
             gzipFile(file);
         }
@@ -184,23 +186,36 @@ function bundleJs() {
     });
 }
 
-    if (!process.argv.includes('--no-app-js')) {
-        let b = browserify({debug: process.argv.includes('--debug')});
-        if (!process.argv.includes('--no-transpile')) {
-            b = b.transform('babelify');
-        }
-        b = b.external(external_js).add(glob.sync('./js/**/*.js'));
-        const compress = !process.argv.includes('--debug');
+function bundleAppJs(b, compress, callback) {
     const file = './public/js/app.min.js';
     writeJsBundle(b, file, compress, () => {
         if (process.argv.includes('--gzip')) {
             gzipFile(file);
         }
         console.info('Bundled app JS');
+        callback();
     });
 }
+
+function bundleJs() {
+    if (!process.argv.includes('--no-vendor-js')) {
+        bundleVendorJs(true);
     }
 
+    if (!process.argv.includes('--no-app-js')) {
+        let watchify = require('watchify');
+        let b = browserify({ debug: process.argv.includes('--debug') });
+        if (!process.argv.includes('--no-transpile')) {
+            b = b.transform('babelify');
+        }
+        b = b.external(external_js).add(glob.sync('./js/**/*.js'));
+        const compress = !process.argv.includes('--debug');
+        bundleAppJs(b, compress, () => { });
+    }
+}
+
+const environment = process.argv.includes('--watch') ? "development" : "production";
+
 function bundleConfig() {
     function getVersion() {
         let build_info = process.env.BUILD_INFO;
@@ -218,7 +233,8 @@ function bundleConfig() {
         meta: {
             version: getVersion(),
             buildDate: new Date().toUTCString()
-        }
+        },
+        environment: environment
     };
 
     fs.writeFileSync('./js/.config.autogen.json', JSON.stringify(config));
@@ -298,10 +314,98 @@ function makeOutputDirs() {
     }
 }
 
+function watch() {
+    let wss = new WebSocket.Server({ port: 8080 });
+    const liveReload = !process.argv.includes('--no-live-reload');
+
+    function emitReload() {
+        if (liveReload) {
+            console.log("Requesting live reload.")
+            wss.clients.forEach((client) => {
+                if (client.readyState === WebSocket.OPEN) {
+                    client.send("reload");
+                }
+            });
+        }
+    }
+
+    chokidar.watch('./fonts/**/*').on('change', () => {
+        try {
+            bundleBinaryAssets();
+            emitReload();
+        } catch (e) {
+            console.error(pe.render(e));
+        }
+    });
+    chokidar.watch('./img/**/*').on('change', () => {
+        try {
+            bundleWebAppFiles();
+            emitReload();
+        } catch (e) {
+            console.error(pe.render(e));
+        }
+    });
+    chokidar.watch('./html/**/*.tpl').on('change', () => {
+        try {
+            bundleHtml();
+        } catch (e) {
+            console.error(pe.render(e));
+        }
+    });
+    chokidar.watch('./css/**/*.styl').on('change', () => {
+        try {
+            bundleCss()
+            emitReload();
+        } catch (e) {
+            console.error(pe.render(e));
+        }
+    });
+
+    bundleBinaryAssets();
+    bundleWebAppFiles();
+    bundleCss();
+    bundleHtml();
+
+    bundleVendorJs(true);
+
+    let watchify = require('watchify');
+    let b = browserify({
+        debug: process.argv.includes('--debug'),
+        entries: ['js/main.js'],
+        cache: {},
+        packageCache: {},
+    });
+
+    b.plugin(watchify);
+
+    if (!process.argv.includes('--no-transpile')) {
+        b = b.transform('babelify');
+    }
+    b = b.external(external_js).add(glob.sync('./js/**/*.js'));
+    const compress = false;
+
+    function bundle(id) {
+        console.info("Rebundling app JS...");
+        let start = new Date();
+        bundleAppJs(b, compress, () => {
+            let end = new Date() - start;
+            console.info('Rebundled in %ds.', end / 1000)
+            emitReload();
+        });
+    }
+
+    b.on('update', bundle);
+    bundle();
+}
+
 // -------------------------------------------------
 
+console.log("Building for '" + environment + "' environment.");
 makeOutputDirs();
 bundleConfig();
+if (process.argv.includes('--watch')) {
+    watch();
+} else {
     if (!process.argv.includes('--no-binary-assets')) {
         bundleBinaryAssets();
     }
@@ -317,3 +421,4 @@ if (!process.argv.includes('--no-css')) {
     if (!process.argv.includes('--no-js')) {
         bundleJs();
     }
+}

client/html/home_footer.tpl

@@ -1,7 +1,7 @@
 <ul>
     <li><%- ctx.postCount %> posts</li><span class='sep'>
     </span><li><%= ctx.makeFileSize(ctx.diskUsage) %></li><span class='sep'>
-    </span><li>Build <a class='version' href='https://github.com/rr-/szurubooru/commits/master'><%- ctx.version %></a> from <%= ctx.makeRelativeTime(ctx.buildDate) %></li><span class='sep'>
+    </span><li>Build <a class='version' href='https://github.com/rr-/szurubooru/commits/master'><%- ctx.version %></a><%- ctx.isDevelopmentMode ? " (DEV MODE)" : "" %> from <%= ctx.makeRelativeTime(ctx.buildDate) %></li><span class='sep'>
     </span><% if (ctx.canListSnapshots) { %><li><a href='<%- ctx.formatClientLink('history') %>'>History</a></li><span class='sep'>
     </span><% } %>
 </ul>

client/js/controllers/home_controller.js

@@ -17,6 +17,7 @@ class HomeController {
             buildDate: config.meta.buildDate,
             canListSnapshots: api.hasPrivilege("snapshots:list"),
             canListPosts: api.hasPrivilege("posts:list"),
+            isDevelopmentMode: config.environment == "development"
         });
 
         Info.get().then(

client/js/main.js

@@ -1,5 +1,20 @@
 "use strict";
 
+const config = require("./config.js");
+
+if (config.environment == "development") {
+    var ws = new WebSocket("ws://" + location.hostname + ":8080");
+    ws.addEventListener('open', function (event) {
+        console.log("Live-reloading websocket connected.");
+    });
+    ws.addEventListener('message', (event) => {
+        console.log(event);
+        if (event.data == 'reload'){
+            location.reload();
+        }
+    });
+}
+
 require("./util/polyfill.js");
 const misc = require("./util/misc.js");
 const views = require("./util/views.js");

client/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "scripts": {
     "build": "node build.js",
-    "watch": "c1=\"\";while :;do c2=$(find html js css img -type f -and -not -iname '*autogen*'|sort|xargs cat|md5sum);[[ $c1 != $c2 ]]&&npm run build -- --debug --no-vendor-js;c1=$c2;sleep 1;done"
+    "watch": "node build.js --watch"
   },
   "dependencies": {
     "dompurify": "^2.0.17",
@@ -21,12 +21,16 @@
     "babel-preset-env": "^1.7.0",
     "babelify": "^8.0.0",
     "browserify": "^16.2.2",
+    "chokidar": "^3.5.1",
     "csso": "^3.5.1",
     "glob": "^7.1.2",
     "html-minifier": "^3.5.18",
     "jimp": "^0.13.0",
+    "pretty-error": "^3.0.3",
     "stylus": "^0.54.5",
     "terser": "^3.7.7",
-    "underscore": "^1.9.1"
+    "underscore": "^1.9.1",
+    "watchify": "^4.0.0",
+    "ws": "^7.4.5"
   }
 }