From 0716d918acf406ec60dc690769a72728d0b19514 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 8 Feb 2023 20:55:33 +0100 Subject: [PATCH] cmd/hiveview: JS/CSS bundling with esbuild (#708) This should fix issues with browser caching in the frontend. Ever since the app was split up into multiple JS files, the page would occasionally fail to load because the browser did not refresh some of the JS files when they were modified. In `hiveview -serve` mode, the new bundling system works like this: - In deploy.go, we maintain a static list of 'bundle targets'. The targets correspond with the app's entry point JS/CSS files. - Whenever one of the HTML files is requested, the server traverses the document and replaces references to bundle targets with a path to the built bundle. For example, +
@@ -19,14 +19,12 @@
-

Recent results - +

+ Recent results +

- - - diff --git a/cmd/hiveview/assets/lib/app-index.js b/cmd/hiveview/assets/lib/app-index.js index 51d248a4b4..e89297bd43 100644 --- a/cmd/hiveview/assets/lib/app-index.js +++ b/cmd/hiveview/assets/lib/app-index.js @@ -1,12 +1,8 @@ -import '../extlib/bootstrap.module.js' -import '../extlib/dataTables.module.js' -import { $ } from '../extlib/jquery.module.js' +import { $ } from 'jquery' import { html, format, nav } from './utils.js' -import * as app from './app.js' - -$(document).ready(function() { - app.init(); +import * as routes from './routes.js' +export default function navigate() { $('#loading').show(); console.log("Loading file list..."); $.ajax("listing.jsonl", { @@ -21,7 +17,7 @@ $(document).ready(function() { $('#loading').hide(); }, }); -}); +} function resultStats(fails, success, total) { f = parseInt(fails), s = parseInt(success); @@ -35,7 +31,7 @@ function resultStats(fails, success, total) { } function linkToSuite(suiteID, suiteName, linkText) { - let url = app.route.suite(suiteID, suiteName); + let url = routes.suite(suiteID, suiteName); return html.get_link(url, linkText); } diff --git a/cmd/hiveview/assets/lib/app-suite.js b/cmd/hiveview/assets/lib/app-suite.js index 9320a1ed57..5ad9eee613 100644 --- a/cmd/hiveview/assets/lib/app-suite.js +++ b/cmd/hiveview/assets/lib/app-suite.js @@ -1,12 +1,9 @@ -import '../extlib/bootstrap.module.js' -import '../extlib/dataTables.module.js' -import { $ } from '../extlib/jquery.module.js' +import 'datatables.net' +import { $ } from 'jquery' import { html, nav, format, loader } from './utils.js' -import * as app from './app.js' - -$(document).ready(function () { - app.init(); +import * as routes from './routes.js' +export default function navigate() { let name = nav.load("suitename"); if (name) { showSuiteName(name); @@ -25,7 +22,7 @@ $(document).ready(function () { $.ajax({ xhr: loader.newXhrWithProgressBar, type: 'GET', - url: app.resultsRoot + filename, + url: routes.resultsRoot + filename, dataType: 'json', success: function(suiteData) { showSuiteData(suiteData, filename); @@ -37,7 +34,7 @@ $(document).ready(function () { showError("error fetching " + filename + " : " + error); }, }); -}) +} // showSuiteName displays the suite title. function showSuiteName(name) { @@ -118,8 +115,8 @@ function showSuiteData(data, suiteID) { let suiteTimes = testSuiteTimes(cases); $("#testsuite_start").html("🕒 " + suiteTimes.start.toLocaleString()); $("#testsuite_duration").html("⌛️ " + format.duration(suiteTimes.duration)); - let logfile = app.resultsRoot + data.simLog; - let url = app.route.simulatorLog(suiteID, suiteName, logfile); + let logfile = routes.resultsRoot + data.simLog; + let url = routes.simulatorLog(suiteID, suiteName, logfile); $("#sim-log-link").attr("href", url); $("#sim-log-link").text("simulator log"); $("#testsuite_info").show(); @@ -306,8 +303,8 @@ function formatClientLogsList(suiteData, testIndex, clientInfo) { let links = []; for (let instanceID in clientInfo) { let instanceInfo = clientInfo[instanceID] - let logfile = app.resultsRoot + instanceInfo.logFile; - let url = app.route.clientLog(suiteData.suiteID, suiteData.name, testIndex, logfile); + let logfile = routes.resultsRoot + instanceInfo.logFile; + let url = routes.clientLog(suiteData.suiteID, suiteData.name, testIndex, logfile); let link = html.get_link(url, instanceInfo.name); link.classList.add('log-link'); links.push(link.outerHTML); @@ -443,7 +440,7 @@ function formatTestLog(suiteData, test) { if (hiddenLines > 0) { // Create the truncation marker. let linkText = "..." + hiddenLines + " lines hidden: click for full output..."; - let linkURL = app.route.testLog(suiteData.suiteID, suiteData.name, test.testIndex); + let linkURL = routes.testLog(suiteData.suiteID, suiteData.name, test.testIndex); let trunc = html.get_link(linkURL, linkText); trunc.classList.add("output-trunc"); output.appendChild(trunc); diff --git a/cmd/hiveview/assets/lib/app-viewer.js b/cmd/hiveview/assets/lib/app-viewer.js index 2bb2f65362..9b762b22bb 100644 --- a/cmd/hiveview/assets/lib/app-viewer.js +++ b/cmd/hiveview/assets/lib/app-viewer.js @@ -1,10 +1,8 @@ -import { $ } from '../extlib/jquery.module.js' +import { $ } from 'jquery' import { html, nav, format, loader } from './utils.js' -import * as app from './app.js' - -function navigate() { - app.init(); +import * as routes from './routes.js' +export default function navigate() { // Check for line number in hash. var line = null; if (window.location.hash.substr(1, 1) == "L") { @@ -26,7 +24,7 @@ function navigate() { showError("Invalid parameters! Missing 'suitefile' or 'testid' in URL."); return; } - fetchTestLog(app.resultsRoot + suiteFile, testIndex, line); + fetchTestLog(routes.resultsRoot + suiteFile, testIndex, line); return; } @@ -43,8 +41,6 @@ function navigate() { showText(document.getElementById("exampletext").innerHTML); } -$(document).ready(navigate); - // setHL sets the highlight on a line number. function setHL(num, scroll) { // out with the old @@ -74,10 +70,10 @@ function showLinkBack(suiteID, suiteName, testID) { var text, url; if (testID) { text = "Back to test " + testID + " in suite ‘" + suiteName + "’"; - url = app.route.testInSuite(suiteID, suiteName, testID); + url = routes.testInSuite(suiteID, suiteName, testID); } else { text = "Back to test suite ‘" + suiteName + "’"; - url = app.route.suite(suiteID, suiteName); + url = routes.suite(suiteID, suiteName); } $('#link-back').html(html.get_link(url, text)); } @@ -148,7 +144,7 @@ function lineNumberClicked() { // fetchFile loads up a new file to view function fetchFile(url, line /* optional jump to line */ ) { - let resultsRE = new RegExp("^" + app.resultsRoot); + let resultsRE = new RegExp("^" + routes.resultsRoot); $.ajax({ xhr: loader.newXhrWithProgressBar, url: url, diff --git a/cmd/hiveview/assets/lib/app.css b/cmd/hiveview/assets/lib/app.css index 4bd140e3c8..f3593c0855 100644 --- a/cmd/hiveview/assets/lib/app.css +++ b/cmd/hiveview/assets/lib/app.css @@ -1,6 +1,6 @@ -@import "../extlib/bootstrap-5.2.3.min.css"; -@import "../extlib/dataTables-1.13.1.bootstrap5.min.css"; -@import "../extlib/responsive-2.4.0.bootstrap5.min.css"; +@import "../extlib/bootstrap-5.2.3.css"; +@import "../extlib/dataTables-1.13.1.bootstrap5.css"; +@import "../extlib/responsive-2.4.0.bootstrap5.css"; main { margin: 8px 8px; diff --git a/cmd/hiveview/assets/lib/app.js b/cmd/hiveview/assets/lib/app.js index 60aad01fe5..5e3b941558 100644 --- a/cmd/hiveview/assets/lib/app.js +++ b/cmd/hiveview/assets/lib/app.js @@ -1,74 +1,48 @@ -import { $ } from '../extlib/jquery.module.js' - -export const resultsRoot = "/results/" - -// This object has constructor function for various app-internal URLs. -export let route = { - simulatorLog: function(suiteID, suiteName, file) { - let params = new URLSearchParams({ - "suiteid": suiteID, - "suitename": suiteName, - "file": file, - }); - return "/viewer.html?" + params.toString(); - }, - - testLog: function(suiteID, suiteName, testIndex) { - let params = new URLSearchParams({ - "suiteid": suiteID, - "suitename": suiteName, - "testid": testIndex, - "showtestlog": "1", - }); - return "/viewer.html?" + params.toString(); - }, - - clientLog: function(suiteID, suiteName, testIndex, file) { - let params = new URLSearchParams({ - "suiteid": suiteID, - "suitename": suiteName, - "testid": testIndex, - "file": file, - }); - return "/viewer.html?" + params.toString(); - }, - - suite: function(suiteID, suiteName) { - let params = new URLSearchParams({"suiteid": suiteID, "suitename": suiteName}); - return "/suite.html?" + params.toString(); - }, - - testInSuite: function(suiteID, suiteName, testIndex) { - return route.suite(suiteID, suiteName) + "#test-" + escape(testIndex); - }, -} - -export function init() { - // Update the header. - $.ajax({ - type: 'GET', - url: resultsRoot + "hive.json", - dataType: 'json', - success: function(data) { - console.log("hive.json:", data); - $("#hive-instance-info").html(hiveInfoHTML(data)); - }, - error: function(xhr, status, error) { - console.log("error fetching hive.json:", error); - }, - }); -} +// Pull in dependencies. +import 'datatables.net' +import 'datatables.net-bs5' +import 'datatables.net-responsive' +import 'datatables.net-responsive-bs5' +import 'bootstrap' +import { $ } from 'jquery' + +// Pull in app files. +import * as routes from './routes.js' +import { default as index } from './app-index.js' +import { default as suite } from './app-suite.js' +import { default as viewer } from './app-viewer.js' + +$(document).ready(function() { + // Kick off the page main function. + let pages = { index, suite, viewer }; + let name = $('script[type=module]').attr('data-main'); + pages[name](); + + // Update the header with version info from hive.json. + $.ajax({ + type: 'GET', + url: routes.resultsRoot + "hive.json", + dataType: 'json', + success: function(data) { + console.log("hive.json:", data); + $("#hive-instance-info").html(hiveInfoHTML(data)); + }, + error: function(xhr, status, error) { + console.log("error fetching hive.json:", error); + }, + }); +}) function hiveInfoHTML(data) { - var txt = ""; - if (data.buildDate) { - let date = new Date(data.buildDate).toLocaleString(); - txt += 'built: ' + date + ''; - } - if (data.sourceCommit) { - let url = "https://github.com/ethereum/hive/commit/" + escape(data.sourceCommit); - let link = '' + data.sourceCommit.substring(0, 8) + ''; - txt += 'commit: ' + link + ''; - } - return txt; + var txt = ""; + if (data.buildDate) { + let date = new Date(data.buildDate).toLocaleString(); + txt += 'built: ' + date + ''; + } + if (data.sourceCommit) { + let url = "https://github.com/ethereum/hive/commit/" + escape(data.sourceCommit); + let link = '' + data.sourceCommit.substring(0, 8) + ''; + txt += 'commit: ' + link + ''; + } + return txt; } diff --git a/cmd/hiveview/assets/lib/routes.js b/cmd/hiveview/assets/lib/routes.js new file mode 100644 index 0000000000..1193d96e90 --- /dev/null +++ b/cmd/hiveview/assets/lib/routes.js @@ -0,0 +1,40 @@ +export const resultsRoot = "/results/" + +// This object has constructor function for various app-internal URLs. +export function simulatorLog(suiteID, suiteName, file) { + let params = new URLSearchParams({ + "suiteid": suiteID, + "suitename": suiteName, + "file": file, + }); + return "/viewer.html?" + params.toString(); +} + +export function testLog(suiteID, suiteName, testIndex) { + let params = new URLSearchParams({ + "suiteid": suiteID, + "suitename": suiteName, + "testid": testIndex, + "showtestlog": "1", + }); + return "/viewer.html?" + params.toString(); +} + +export function clientLog(suiteID, suiteName, testIndex, file) { + let params = new URLSearchParams({ + "suiteid": suiteID, + "suitename": suiteName, + "testid": testIndex, + "file": file, + }); + return "/viewer.html?" + params.toString(); +} + +export function suite(suiteID, suiteName) { + let params = new URLSearchParams({"suiteid": suiteID, "suitename": suiteName}); + return "/suite.html?" + params.toString(); +} + +export function testInSuite(suiteID, suiteName, testIndex) { + return suite(suiteID, suiteName) + "#test-" + escape(testIndex); +} diff --git a/cmd/hiveview/assets/lib/utils.js b/cmd/hiveview/assets/lib/utils.js index 8c5a3c87ee..7526c31760 100644 --- a/cmd/hiveview/assets/lib/utils.js +++ b/cmd/hiveview/assets/lib/utils.js @@ -1,4 +1,4 @@ -import { $ } from '../extlib/jquery.module.js' +import { $ } from 'jquery' export let html = { // encode does HTML-encoding/escaping. diff --git a/cmd/hiveview/assets/suite.html b/cmd/hiveview/assets/suite.html index f7aeeaa7c3..91f91ee29b 100644 --- a/cmd/hiveview/assets/suite.html +++ b/cmd/hiveview/assets/suite.html @@ -8,7 +8,7 @@ - +
@@ -40,7 +40,4 @@

Results:

- - - diff --git a/cmd/hiveview/assets/viewer.html b/cmd/hiveview/assets/viewer.html index 63639225ce..3c91f2b4d9 100644 --- a/cmd/hiveview/assets/viewer.html +++ b/cmd/hiveview/assets/viewer.html @@ -6,7 +6,7 @@ - + +
@@ -46,7 +46,4 @@

Loading file...

- - - diff --git a/cmd/hiveview/deploy.go b/cmd/hiveview/deploy.go new file mode 100644 index 0000000000..a947b307f1 --- /dev/null +++ b/cmd/hiveview/deploy.go @@ -0,0 +1,264 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "io/fs" + "sort" + "strings" + "time" + + "golang.org/x/net/html" +) + +// hiveviewBundler creates the esbuild bundler and registers JS/CSS targets. +func hiveviewBundler(fsys fs.FS) *bundler { + b := newBundler(fsys) + b.add("lib/app.js") + b.add("lib/app.css") + b.add("lib/viewer.css") + return b +} + +// moduleAliases maps ES module names to files. +var moduleAliases = map[string]string{ + "jquery": "./extlib/jquery-3.6.3.esm.js", + "@popper/core": "./extlib/popper-2.9.2.js", + "bootstrap": "./extlib/bootstrap-5.2.3.mjs", + "datatables.net": "./extlib/dataTables-1.13.1.mjs", + "datatables.net-bs5": "./extlib/dataTables-1.13.1.bootstrap5.mjs", + "datatables.net-responsive": "./extlib/responsive-2.4.0.mjs", + "datatables.net-responsive-bs5": "./extlib/responsive-2.4.0.bootstrap5.mjs", +} + +func importMapScript() string { + im := map[string]any{"imports": moduleAliases} + imdata, _ := json.Marshal(im) + return `` +} + +// deployFS is a virtual overlay file system for the assets directory. +// It mostly acts as a pass-through for the 'assets' file system, except for +// two special cases: +// +// - The overlay adds a bundle/ directory containing built JS/CSS bundles. +// - For .html files in the root directory, all JS/CSS references are checked +// against the bundler, and URLs in the HTML will be replaced by references to +// bundle files. +type deployFS struct { + assets fs.FS + bundler *bundler + useBundle bool +} + +func newDeployFS(assets fs.FS, useBundle bool) *deployFS { + return &deployFS{ + assets: assets, + bundler: hiveviewBundler(assets), + useBundle: useBundle, + } +} + +func isBundlePath(name string) bool { + return name == "bundle" || strings.HasPrefix(name, "bundle/") +} + +var _ fs.FS = (*deployFS)(nil) + +// Open opens a file. +func (dfs *deployFS) Open(name string) (f fs.File, err error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "Open", Path: name, Err: fs.ErrInvalid} + } + switch { + case !strings.Contains(name, "/") && strings.HasSuffix(name, ".html"): + return dfs.openHTML(name) + case isBundlePath(name): + return dfs.bundler.fs().Open(name) + default: + return dfs.assets.Open(name) + } +} + +var _ fs.ReadDirFS = (*deployFS)(nil) + +// ReadDir reads a directory. +func (dfs *deployFS) ReadDir(name string) ([]fs.DirEntry, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "ReadDir", Path: name, Err: fs.ErrInvalid} + } + switch { + case name == ".": + return dfs.readDirRoot() + case isBundlePath(name): + return fs.ReadDir(dfs.bundler.fs(), name) + default: + return fs.ReadDir(dfs.assets, name) + } +} + +func (dfs *deployFS) readDirRoot() ([]fs.DirEntry, error) { + entries, err := fs.ReadDir(dfs.assets, ".") + if err != nil { + return nil, err + } + bundleEntries, _ := fs.ReadDir(dfs.bundler.fs(), ".") + entries = append(entries, bundleEntries...) + sort.Slice(entries, func(i, j int) bool { + return entries[i].Name() < entries[j].Name() + }) + return entries, nil +} + +// openHTML opens a HTML file in the root directory and modifies it to use +// bundled JS/CSS files. +func (dfs *deployFS) openHTML(name string) (fs.File, error) { + inputFile, err := dfs.assets.Open(name) + if err != nil { + return nil, err + } + defer inputFile.Close() + + inputInfo, err := inputFile.Stat() + if err != nil { + return nil, err + } + output := new(bytes.Buffer) + modTime := inputInfo.ModTime() + + if !dfs.useBundle { + // JS bundle is disabled. To make ES module loading work without the bundle, + // the document needs an importmap. + insertAfterTag(inputFile, output, "head", importMapScript()) + modTime = time.Now() + } else { + // Replace script/style references with bundle paths, if possible. + var errorShown bool + modifyHTML(inputFile, output, func(token *html.Token, errlog io.Writer) { + ref := scriptOrStyleReference(token) + if ref == nil { + return // not script + } + bundle, buildmsg, err := dfs.bundler.build(ref.Val) + if errors.Is(err, fs.ErrNotExist) { + return + } else if err != nil { + if !errorShown { + io.WriteString(errlog, "** ESBUILD ERRORS **\n\n") + errorShown = true + } + renderBuildMsg(buildmsg, errlog) + modTime = time.Now() + return + } + if bundle.buildTime.After(modTime) { + modTime = bundle.buildTime + } + ref.Val = "/bundle/" + bundle.name() + }) + } + + content := output.Bytes() + file := newMemFile(inputInfo.Name(), modTime, content) + return file, nil +} + +// insertAfterTag adds content to the document after the first occurrence of a HTML tag. +// The resulting document is written to w. +func insertAfterTag(r io.Reader, w io.Writer, tagName, content string) error { + var done bool + z := html.NewTokenizer(r) + for { + tt := z.Next() + if tt == html.ErrorToken { + if z.Err() == io.EOF { + return nil + } + return z.Err() + } else { + w.Write(z.Raw()) + if !done && tt == html.StartTagToken { + name, _ := z.TagName() + if string(name) == tagName { + io.WriteString(w, content) + done = true + } + } + } + } +} + +// modifyScripts changes the 'src' URL of all script tags using the given function. +func modifyHTML(r io.Reader, w io.Writer, modify func(tag *html.Token, errlog io.Writer)) error { + var errlog bytes.Buffer + z := html.NewTokenizer(r) + for { + tt := z.Next() + switch tt { + case html.ErrorToken: + if z.Err() == io.EOF { + return nil + } + return z.Err() + case html.StartTagToken, html.SelfClosingTagToken: + token := z.Token() + modify(&token, &errlog) + io.WriteString(w, token.String()) + case html.EndTagToken: + // Insert the build error log at end of body. + tag, _ := z.TagName() + if string(tag) == "body" && errlog.Len() > 0 { + logHTML := `
`
+				logHTML += html.EscapeString(errlog.String())
+				logHTML += `
` + io.WriteString(w, logHTML) + } + w.Write(z.Raw()) + default: + w.Write(z.Raw()) + } + } +} + +func scriptOrStyleReference(token *html.Token) *html.Attribute { + switch token.Data { + case "script": + return findAttr(token, "src") + case "link": + rel := findAttr(token, "rel") + if rel != nil && rel.Val == "stylesheet" { + return findAttr(token, "href") + } + } + return nil +} + +func findAttr(token *html.Token, key string) *html.Attribute { + for i, a := range token.Attr { + if a.Namespace == "" && a.Key == key { + return &token.Attr[i] + } + } + return nil +} + +// memFile is a virtual fs.File backed by []byte. +type memFile struct { + *bytes.Reader + modTime time.Time + name string +} + +func newMemFile(name string, modTime time.Time, content []byte) *memFile { + return &memFile{bytes.NewReader(content), modTime, name} +} + +func (f *memFile) Close() error { return nil } +func (f *memFile) Stat() (fs.FileInfo, error) { return f, nil } +func (f *memFile) Name() string { return f.name } +func (f *memFile) Mode() fs.FileMode { return 0644 } +func (f *memFile) ModTime() time.Time { return f.modTime } +func (f *memFile) IsDir() bool { return false } +func (f *memFile) Sys() any { return nil } diff --git a/cmd/hiveview/deploy_test.go b/cmd/hiveview/deploy_test.go new file mode 100644 index 0000000000..1fb7e5ed6c --- /dev/null +++ b/cmd/hiveview/deploy_test.go @@ -0,0 +1,19 @@ +package main + +import ( + "io/fs" + "strings" + "testing" +) + +func TestBuildAllBundles(t *testing.T) { + assets, _ := fs.Sub(embeddedAssets, "assets") + b := hiveviewBundler(assets) + + var output strings.Builder + msg, err := b.buildAll() + if err != nil { + renderBuildMsg(msg, &output) + t.Fatal("esbuild errors:\n\n", output.String()) + } +} diff --git a/cmd/hiveview/esbuild.go b/cmd/hiveview/esbuild.go new file mode 100644 index 0000000000..c11f9c720d --- /dev/null +++ b/cmd/hiveview/esbuild.go @@ -0,0 +1,274 @@ +package main + +import ( + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "log" + "path" + "strings" + "sync" + "time" + + esbuild "github.com/evanw/esbuild/pkg/api" + "github.com/liamg/memoryfs" +) + +// bundler creates JS/CSS bundles and caches them in memory. +type bundler struct { + fsys fs.FS + options *esbuild.BuildOptions + mu sync.Mutex + files map[string]*bundleFile + mem *memoryfs.FS +} + +type bundleFile struct { + inputPath string + hash [32]byte + inputFiles []string + buildTime time.Time + buildMsg []esbuild.Message +} + +func newBundler(fsys fs.FS) *bundler { + mem := memoryfs.New() + mem.MkdirAll("bundle", 0755) + return &bundler{ + mem: mem, + fsys: fsys, + files: make(map[string]*bundleFile), + } +} + +// add adds a file target. +func (b *bundler) add(name string) { + b.mu.Lock() + defer b.mu.Unlock() + + name = path.Clean(strings.TrimPrefix(name, "/")) + if _, ok := b.files[name]; !ok { + b.files[name] = &bundleFile{inputPath: name} + } +} + +// fs returns a virtual filesystem containing built bundle files. +// Calling this also ensures all bundles are up-to-date! +func (b *bundler) fs() fs.FS { + b.mu.Lock() + defer b.mu.Unlock() + + b.buildAll() + return b.mem +} + +func (b *bundler) buildAll() ([]esbuild.Message, error) { + var allmsg []esbuild.Message + var firsterr error + for _, bf := range b.files { + msg, err := bf.rebuild(b) + if err != nil && firsterr == nil { + firsterr = err + } + allmsg = append(allmsg, msg...) + } + return allmsg, firsterr +} + +// build ensures the given bundle file is built, and returns the bundle. +func (b *bundler) build(name string) (bf *bundleFile, buildmsg []esbuild.Message, err error) { + b.mu.Lock() + defer b.mu.Unlock() + + name = path.Clean(strings.TrimPrefix(name, "/")) + bf, ok := b.files[name] + if !ok { + return nil, nil, fs.ErrNotExist + } + buildmsg, err = bf.rebuild(b) + if err != nil { + return nil, buildmsg, err + } + cpy := *bf + return &cpy, nil, nil +} + +// rebuild builds the bundle if necessary. +func (bf *bundleFile) rebuild(b *bundler) ([]esbuild.Message, error) { + if !bf.needsBuild(b) { + return nil, nil + } + return bf.build(b) +} + +// name returns the output file name (including the hash). +func (bf *bundleFile) name() string { + base := path.Base(bf.inputPath) + baseNoExt := strings.TrimSuffix(base, path.Ext(base)) + return fmt.Sprintf("%s.%x%s", baseNoExt, bf.hash[:], path.Ext(base)) +} + +// needsBuild reports whether the bundle needs to be rebuilt. +func (bf *bundleFile) needsBuild(b *bundler) bool { + if bf.buildTime.IsZero() { + return true + } + for _, f := range bf.inputFiles { + stat, err := fs.Stat(b.fsys, f) + if err != nil || stat.ModTime().After(bf.buildTime) { + return true + } + } + return false +} + +// build creates/updates a bundle. +func (bf *bundleFile) build(b *bundler) ([]esbuild.Message, error) { + log.Printf("esbuild: %s", bf.inputPath) + + prevName := bf.name() + startTime := time.Now() + + var loadedFiles []string + loader := fsLoaderPlugin(b.fsys, &loadedFiles) + options := esbuild.BuildOptions{ + Bundle: true, + LogLevel: esbuild.LogLevelInfo, + EntryPoints: []string{bf.inputPath}, + Plugins: []esbuild.Plugin{loader}, + Platform: esbuild.PlatformBrowser, + Target: esbuild.ES2020, + MinifyIdentifiers: true, + MinifyWhitespace: true, + MinifySyntax: true, + Alias: moduleAliases, + } + res := esbuild.Build(options) + msg := append(res.Errors, res.Warnings...) + if len(res.Errors) > 0 { + return msg, fmt.Errorf("error in %s", bf.inputPath) + } + content := res.OutputFiles[0].Contents + if len(content) == 0 { + panic("empty build output") + } + + // Update the result. + bf.hash = sha256.Sum256(content) + bf.buildTime = startTime + bf.inputFiles = loadedFiles + + // Write output to memfs. + // fmt.Println("store:", path.Join("bundle", bf.name())) + err := b.mem.WriteFile(path.Join("bundle", bf.name()), content, 0644) + if err != nil { + panic("can't write to memfs: " + err.Error()) + } + // Ensure the previous output file is gone. + if prevName != bf.name() { + b.mem.Remove(path.Join("bundle", prevName)) + } + + return msg, nil +} + +// fsLoaderPlugin constructs an esbuild loader plugin that wraps a filesystem. +// The plugin does two things: +// +// - All file loads are done through the given filesystem. +// - Loaded paths are appended to the 'loadedFiles' list. Note that it is +// not safe to access loadedFiles until esbuild.Build has returned. +func fsLoaderPlugin(fsys fs.FS, loadedFiles *[]string) esbuild.Plugin { + var addedFile = make(map[string]bool) + var fileListMutex sync.Mutex + addToLoadedFiles := func(path string) { + fileListMutex.Lock() + defer fileListMutex.Unlock() + if !addedFile[path] { + *loadedFiles = append(*loadedFiles, path) + addedFile[path] = true + } + } + + return esbuild.Plugin{ + Name: "fsLoader", + Setup: func(build esbuild.PluginBuild) { + resOpt := esbuild.OnResolveOptions{Filter: ".*"} + build.OnResolve(resOpt, func(args esbuild.OnResolveArgs) (esbuild.OnResolveResult, error) { + var p string + switch args.Kind { + case esbuild.ResolveCSSURLToken: + // url(...) in CSS is always considered an external resource. + return esbuild.OnResolveResult{Path: args.Path, External: true}, nil + + case esbuild.ResolveEntryPoint: + // For the initial entry point in the bundle, args.Importer is set + // to the absolute working directory, which can't be used, so just treat + // it as a raw path into the FS. + p = strings.TrimPrefix(args.Path, "/") + + default: + // All other import paths are resolved relative to the importing + // file's location, unless the name is defined as an alias. + alias, ok := build.InitialOptions.Alias[args.Path] + if ok { + p = path.Clean(alias) + // fmt.Println("resolved alias:", args.Path, "=>", p) + } else { + p = path.Join(path.Dir(args.Importer), args.Path) + } + } + + res := esbuild.OnResolveResult{Path: p, Namespace: "fsLoader"} + _, err := fs.Stat(fsys, p) + if errors.Is(err, fs.ErrNotExist) && !strings.HasPrefix(args.Path, ".") { + err = fmt.Errorf("File %s does not exist. Missing definition in moduleAliases?", p) + } + return res, err + }) + + loadOpt := esbuild.OnLoadOptions{Filter: ".*", Namespace: "fsLoader"} + build.OnLoad(loadOpt, func(args esbuild.OnLoadArgs) (esbuild.OnLoadResult, error) { + addToLoadedFiles(args.Path) + text, err := fs.ReadFile(fsys, args.Path) + if err != nil { + return esbuild.OnLoadResult{}, err + } + str := string(text) + return esbuild.OnLoadResult{ + Contents: &str, + ResolveDir: path.Dir(args.Path), + Loader: loaderFromExt(args.Path), + }, nil + }) + }, + } +} + +func loaderFromExt(name string) esbuild.Loader { + switch path.Ext(name) { + case ".css": + return esbuild.LoaderCSS + case ".js", ".mjs": + return esbuild.LoaderJS + case ".ts": + return esbuild.LoaderTS + case ".json": + return esbuild.LoaderJSON + default: + return esbuild.LoaderNone + } +} + +func renderBuildMsg(msgs []esbuild.Message, w io.Writer) { + for _, msg := range msgs { + file := strings.Replace(msg.Location.File, "fsLoader:", "assets/", 1) + fmt.Fprintf(w, "%s:%d %s\n", file, msg.Location.Line, msg.Text) + fmt.Fprintln(w, " |") + fmt.Fprintln(w, " |", msg.Location.LineText) + fmt.Fprintln(w, " |") + fmt.Fprintln(w, "") + } +} diff --git a/cmd/hiveview/main.go b/cmd/hiveview/main.go index 66a33e6237..9024ec66b1 100644 --- a/cmd/hiveview/main.go +++ b/cmd/hiveview/main.go @@ -4,8 +4,11 @@ package main import ( "flag" + "io" + "io/fs" "log" "os" + "path/filepath" "time" ) @@ -18,6 +21,7 @@ func main() { var ( serve = flag.Bool("serve", false, "Enables the HTTP server") listing = flag.Bool("listing", false, "Generates listing JSON to stdout") + deploy = flag.Bool("deploy", false, "Compiles the frontend to a static directory") gc = flag.Bool("gc", false, "Deletes old log files") gcKeepInterval = flag.Duration("keep", 5*durationMonth, "Time interval of past log files to keep (for -gc)") gcKeepMin = flag.Int("keep-min", 10, "Minmum number of suite outputs to keep (for -gc)") @@ -26,6 +30,7 @@ func main() { flag.StringVar(&config.listenAddr, "addr", "0.0.0.0:8080", "HTTP server listen address") flag.StringVar(&config.logDir, "logdir", "workspace/logs", "Path to hive simulator log directory") flag.StringVar(&config.assetsDir, "assets", "", "Path to static files directory. Serves baked-in assets when not set.") + flag.BoolVar(&config.disableBundle, "assets.nobundle", false, "Disables JS/CSS bundling (for development).") flag.Parse() log.SetFlags(log.LstdFlags) @@ -38,7 +43,55 @@ func main() { case *gc: cutoff := time.Now().Add(-*gcKeepInterval) logdirGC(config.logDir, cutoff, *gcKeepMin) + case *deploy: + doDeploy(&config) default: log.Fatalf("Use -serve or -listing to select mode") } } + +// doDeploy writes the UI to a directory. +func doDeploy(config *serverConfig) { + if flag.NArg() != 1 { + log.Fatalf("-deploy requires output directory as argument") + } + outputDir := flag.Arg(0) + assetFS, err := config.assetFS() + if err != nil { + log.Fatalf("-assets: %v", err) + } + if err := os.MkdirAll(outputDir, 0755); err != nil { + log.Fatal(err) + } + + deploy := newDeployFS(assetFS, !config.disableBundle) + if err := copyFS(outputDir, deploy); err != nil { + log.Fatal(err) + } +} + +// copyFS walks the specified root directory on src and copies directories and +// files to dest filesystem. +func copyFS(dest string, src fs.FS) error { + return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil || d == nil { + return err + } + destPath := filepath.Join(dest, filepath.FromSlash(path)) + if d.IsDir() { + return os.MkdirAll(destPath, 0755) + } + srcFile, err := src.Open(path) + if err != nil { + return err + } + destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer destFile.Close() + log.Println("copy", path) + _, err = io.Copy(destFile, srcFile) + return err + }) +} diff --git a/cmd/hiveview/serve.go b/cmd/hiveview/serve.go index c4e579af28..686a1c482b 100644 --- a/cmd/hiveview/serve.go +++ b/cmd/hiveview/serve.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "os" + "strings" "github.com/gorilla/mux" ) @@ -16,34 +17,42 @@ import ( var embeddedAssets embed.FS type serverConfig struct { - listenAddr string - logDir string - assetsDir string + listenAddr string + logDir string + assetsDir string + disableBundle bool } -func runServer(config serverConfig) { - var assetFS fs.FS - if config.assetsDir != "" { - if stat, _ := os.Stat(config.assetsDir); stat == nil || !stat.IsDir() { - log.Fatalf("-assets: %q is not a directory", config.assetsDir) - } - assetFS = os.DirFS(config.assetsDir) - } else { - sub, err := fs.Sub(embeddedAssets, "assets") - if err != nil { - panic(err) +func (cfg *serverConfig) assetFS() (fs.FS, error) { + if cfg.assetsDir != "" { + if stat, _ := os.Stat(cfg.assetsDir); stat == nil || !stat.IsDir() { + return nil, fmt.Errorf("%q is not a directory", cfg.assetsDir) } - assetFS = sub + return os.DirFS(cfg.assetsDir), nil + } + sub, err := fs.Sub(embeddedAssets, "assets") + if err != nil { + panic(err) + } + return sub, nil +} + +func runServer(config serverConfig) { + assetFS, err := config.assetFS() + if err != nil { + log.Fatalf("-assets: %v", err) } // Create handlers. + deployFS := newDeployFS(assetFS, !config.disableBundle) logDirFS := os.DirFS(config.logDir) logHandler := http.FileServer(http.FS(logDirFS)) listingHandler := serveListing{fsys: logDirFS} + mux := mux.NewRouter() mux.Handle("/listing.jsonl", listingHandler).Methods("GET") mux.PathPrefix("/results").Handler(http.StripPrefix("/results/", logHandler)) - mux.PathPrefix("/").Handler(http.FileServer(http.FS(assetFS))) + mux.PathPrefix("/").Handler(serveFiles{deployFS}) // Start the server. l, err := net.Listen("tcp", config.listenAddr) @@ -64,3 +73,17 @@ func (h serveListing) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) } } + +type serveFiles struct{ fsys fs.FS } + +func (h serveFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Add caching-related headers. + path := r.URL.Path + if path == "/" || strings.HasSuffix(path, ".html") { + w.Header().Set("cache-control", "no-cache") + } + + srv := http.FileServer(http.FS(h.fsys)) + srv.ServeHTTP(w, r) + +} diff --git a/go.mod b/go.mod index 852b11ed35..6e8406e1c9 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,11 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/ethereum/go-ethereum v1.10.26 github.com/ethereum/hive/hiveproxy v0.0.0-20220708193637-ec524d7345a1 + github.com/evanw/esbuild v0.17.6 github.com/fsouza/go-dockerclient v1.8.1 github.com/gorilla/mux v1.8.0 + github.com/liamg/memoryfs v1.6.0 + golang.org/x/net v0.5.0 gopkg.in/inconshreveable/log15.v2 v2.0.0-20200109203555-b30bc20e4fd1 gopkg.in/yaml.v3 v3.0.1 ) @@ -64,8 +67,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/crypto v0.4.0 // indirect - golang.org/x/net v0.4.0 // indirect - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.4.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect ) diff --git a/go.sum b/go.sum index a90ec2fd56..89e88d25f1 100644 --- a/go.sum +++ b/go.sum @@ -312,6 +312,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/ethereum/go-ethereum v1.10.26 h1:i/7d9RBBwiXCEuyduBQzJw/mKmnvzsN14jqBmytw72s= github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanw/esbuild v0.17.6 h1:4TFYbndxF6+KQwLTyZ4PGIal5Pc6g6AK6jp0IU4Elf0= +github.com/evanw/esbuild v0.17.6/go.mod h1:iINY06rn799hi48UqEnaQvVfZWe6W9bET78LbvN8VWk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -542,6 +544,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/liamg/memoryfs v1.6.0 h1:jAFec2HI1PgMTem5gR7UT8zi9u4BfG5jorCRlLH06W8= +github.com/liamg/memoryfs v1.6.0/go.mod h1:z7mfqXFQS8eSeBBsFjYLlxYRMRyiPktytvYCYTb3BSk= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= @@ -966,8 +970,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1067,11 +1071,11 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1080,7 +1084,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=