From d4db464460e78c5a8fafa50a272157678cf4f683 Mon Sep 17 00:00:00 2001 From: Christoph Tavan Date: Fri, 25 Oct 2019 14:58:06 +0200 Subject: [PATCH 01/10] Pass config.experimental.cpus to export during build Currently, there is no way of specifying the number of worker threads of `next export` when run as part of `next build`. I suggest a sane default should be to just use the same amount of workers that were used during the build process which currently seems to be configured through `config.experimental.cpus`. This setting is already respected in the two other places where jest-workers are in use: The TerserPlugin and the staticCheckWorkers in `next build`. --- packages/next/build/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 693fc4803d2da..3feb83a48b348 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -471,6 +471,7 @@ export default async function build(dir: string, conf = null): Promise { sprPages, silent: true, buildExport: true, + threads: config.experimental.cpus, pages: combinedPages, outdir: path.join(distDir, 'export'), } From 1516361e1f021a8bfcfbf2a3fa158b9d0038b682 Mon Sep 17 00:00:00 2001 From: Christoph Tavan Date: Fri, 25 Oct 2019 15:06:41 +0200 Subject: [PATCH 02/10] Only enable worker threads if there is more than 1 worker Multiple worker threads can cause problems when certain dependencies are being used, see e.g. https://github.com/zeit/next.js/issues/7894 This patch allows disabling of worker threads by setting `config.experimental.cpus = 1`. The benefit of spawning 1 worker thread, if there is any at all, should very limited anyways, so the workload can just as well be processed in the main thread. --- packages/next/build/index.ts | 2 +- .../webpack/plugins/terser-webpack-plugin/src/TaskRunner.js | 2 +- packages/next/export/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 3feb83a48b348..2a2cd8d1b0845 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -337,7 +337,7 @@ export default async function build(dir: string, conf = null): Promise { const staticCheckWorkers = new Worker(staticCheckWorker, { numWorkers: config.experimental.cpus, - enableWorkerThreads: true, + enableWorkerThreads: config.experimental.cpus > 1, }) const analysisBegin = process.hrtime() diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js index b306966292ede..cb905786e3ea2 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js @@ -28,8 +28,8 @@ export default class TaskRunner { if (this.maxConcurrentWorkers > 1) { this.workers = new Worker(worker, { - enableWorkerThreads: true, numWorkers: this.maxConcurrentWorkers, + enableWorkerThreads: this.maxConcurrentWorkers > 1, }) this.boundWorkers = options => this.workers.default(options) } else { diff --git a/packages/next/export/index.ts b/packages/next/export/index.ts index dba1116a3f3d1..0792904475bf4 100644 --- a/packages/next/export/index.ts +++ b/packages/next/export/index.ts @@ -267,7 +267,7 @@ export default async function( { maxRetries: 0, numWorkers: threads, - enableWorkerThreads: true, + enableWorkerThreads: threads > 1, exposedMethods: ['default'], } ) as any From b45c011fb2eb341a6ea13a4ddea40fbfe8077469 Mon Sep 17 00:00:00 2001 From: Christoph Tavan Date: Fri, 25 Oct 2019 16:52:15 +0200 Subject: [PATCH 03/10] Disable parallel build for firebase authentication example --- examples/with-firebase-authentication/next.config.js | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 examples/with-firebase-authentication/next.config.js diff --git a/examples/with-firebase-authentication/next.config.js b/examples/with-firebase-authentication/next.config.js new file mode 100644 index 0000000000000..ff566d92fe871 --- /dev/null +++ b/examples/with-firebase-authentication/next.config.js @@ -0,0 +1,9 @@ +// Parallel builds are currently broken due to a limitation in grpc (a transitive firebase +// dependency). Until this is fixed, a workaround is to set the number of available CPUs to 1. +// https://github.com/zeit/next.js/issues/7894 +// https://github.com/grpc/grpc-node/issues/778 +module.exports = { + experimental: { + cpus: 1 + } +} From 684c8ff4d0ef2e5d7046195750ecc65d14a108a1 Mon Sep 17 00:00:00 2001 From: Christoph Tavan Date: Mon, 28 Oct 2019 20:24:05 +0100 Subject: [PATCH 04/10] Add integration test to cover #7894 --- .../pages/page-1.js | 2 + .../pages/page-2.js | 2 + .../test/index.test.js | 37 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 test/integration/build-firebase-only-sequentially/pages/page-1.js create mode 100644 test/integration/build-firebase-only-sequentially/pages/page-2.js create mode 100644 test/integration/build-firebase-only-sequentially/test/index.test.js diff --git a/test/integration/build-firebase-only-sequentially/pages/page-1.js b/test/integration/build-firebase-only-sequentially/pages/page-1.js new file mode 100644 index 0000000000000..dd9c28611eea2 --- /dev/null +++ b/test/integration/build-firebase-only-sequentially/pages/page-1.js @@ -0,0 +1,2 @@ +import 'firebase/firestore' +export default () =>
Firebase
diff --git a/test/integration/build-firebase-only-sequentially/pages/page-2.js b/test/integration/build-firebase-only-sequentially/pages/page-2.js new file mode 100644 index 0000000000000..dd9c28611eea2 --- /dev/null +++ b/test/integration/build-firebase-only-sequentially/pages/page-2.js @@ -0,0 +1,2 @@ +import 'firebase/firestore' +export default () =>
Firebase
diff --git a/test/integration/build-firebase-only-sequentially/test/index.test.js b/test/integration/build-firebase-only-sequentially/test/index.test.js new file mode 100644 index 0000000000000..c6ab580e20385 --- /dev/null +++ b/test/integration/build-firebase-only-sequentially/test/index.test.js @@ -0,0 +1,37 @@ +/* eslint-env jest */ +/* global jasmine */ +import path from 'path' +import fs from 'fs-extra' +import { nextBuild } from 'next-test-utils' + +jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 1 +const appDir = path.join(__dirname, '..') +const nextConfig = path.join(appDir, 'next.config.js') + +describe('Builds with firebase dependency only sequentially', () => { + it('Throws an error when building with firebase dependency in parallel', async () => { + await fs.writeFile( + nextConfig, + `module.exports = { experimental: { cpus: 2 } }` + ) + const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) + expect(results.stdout + results.stderr).toMatch(/Build error occurred/) + expect(results.stdout + results.stderr).toMatch( + /grpc_node\.node\. Module did not self-register\./ + ) + await fs.remove(nextConfig) + }) + + it('Throws no error when building with firebase dependency in sequence', async () => { + await fs.writeFile( + nextConfig, + `module.exports = { experimental: { cpus: 1 } }` + ) + const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) + expect(results.stdout + results.stderr).not.toMatch(/Build error occurred/) + expect(results.stdout + results.stderr).not.toMatch( + /grpc_node\.node\. Module did not self-register\./ + ) + await fs.remove(nextConfig) + }) +}) From 6d305749045f3a4f6a8dc6dc75cff3b304abcebe Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 28 Oct 2019 14:53:36 -0500 Subject: [PATCH 05/10] Rename test suite and add worker_threads config --- packages/next/build/index.ts | 2 +- .../terser-webpack-plugin/src/TaskRunner.js | 2 +- packages/next/export/index.ts | 2 +- packages/next/next-server/server/config.ts | 1 + .../pages/page-1.js | 0 .../pages/page-2.js | 0 .../test/index.test.js | 14 +++++--------- 7 files changed, 9 insertions(+), 12 deletions(-) rename test/integration/{build-firebase-only-sequentially => firebase-grpc}/pages/page-1.js (100%) rename test/integration/{build-firebase-only-sequentially => firebase-grpc}/pages/page-2.js (100%) rename test/integration/{build-firebase-only-sequentially => firebase-grpc}/test/index.test.js (75%) diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 2a2cd8d1b0845..1d6eefdc45f3f 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -337,7 +337,7 @@ export default async function build(dir: string, conf = null): Promise { const staticCheckWorkers = new Worker(staticCheckWorker, { numWorkers: config.experimental.cpus, - enableWorkerThreads: config.experimental.cpus > 1, + enableWorkerThreads: config.experimental.workerThreads, }) const analysisBegin = process.hrtime() diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js index cb905786e3ea2..469de3051b486 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js @@ -29,7 +29,7 @@ export default class TaskRunner { if (this.maxConcurrentWorkers > 1) { this.workers = new Worker(worker, { numWorkers: this.maxConcurrentWorkers, - enableWorkerThreads: this.maxConcurrentWorkers > 1, + enableWorkerThreads: true, }) this.boundWorkers = options => this.workers.default(options) } else { diff --git a/packages/next/export/index.ts b/packages/next/export/index.ts index 0792904475bf4..2a0ae8b13958c 100644 --- a/packages/next/export/index.ts +++ b/packages/next/export/index.ts @@ -267,7 +267,7 @@ export default async function( { maxRetries: 0, numWorkers: threads, - enableWorkerThreads: threads > 1, + enableWorkerThreads: nextConfig.experimental.workerThreads, exposedMethods: ['default'], } ) as any diff --git a/packages/next/next-server/server/config.ts b/packages/next/next-server/server/config.ts index 3fe9b13f1a7b3..7c742c14889c3 100644 --- a/packages/next/next-server/server/config.ts +++ b/packages/next/next-server/server/config.ts @@ -48,6 +48,7 @@ const defaultConfig: { [key: string]: any } = { publicDirectory: false, sprFlushToDisk: true, deferScripts: false, + workerThreads: true, }, future: { excludeDefaultMomentLocales: false, diff --git a/test/integration/build-firebase-only-sequentially/pages/page-1.js b/test/integration/firebase-grpc/pages/page-1.js similarity index 100% rename from test/integration/build-firebase-only-sequentially/pages/page-1.js rename to test/integration/firebase-grpc/pages/page-1.js diff --git a/test/integration/build-firebase-only-sequentially/pages/page-2.js b/test/integration/firebase-grpc/pages/page-2.js similarity index 100% rename from test/integration/build-firebase-only-sequentially/pages/page-2.js rename to test/integration/firebase-grpc/pages/page-2.js diff --git a/test/integration/build-firebase-only-sequentially/test/index.test.js b/test/integration/firebase-grpc/test/index.test.js similarity index 75% rename from test/integration/build-firebase-only-sequentially/test/index.test.js rename to test/integration/firebase-grpc/test/index.test.js index c6ab580e20385..270d3d2b93699 100644 --- a/test/integration/build-firebase-only-sequentially/test/index.test.js +++ b/test/integration/firebase-grpc/test/index.test.js @@ -9,29 +9,25 @@ const appDir = path.join(__dirname, '..') const nextConfig = path.join(appDir, 'next.config.js') describe('Builds with firebase dependency only sequentially', () => { - it('Throws an error when building with firebase dependency in parallel', async () => { - await fs.writeFile( - nextConfig, - `module.exports = { experimental: { cpus: 2 } }` - ) + it('Throws an error when building with firebase dependency with worker_threads', async () => { + await fs.remove(nextConfig) const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) expect(results.stdout + results.stderr).toMatch(/Build error occurred/) expect(results.stdout + results.stderr).toMatch( /grpc_node\.node\. Module did not self-register\./ ) - await fs.remove(nextConfig) }) - it('Throws no error when building with firebase dependency in sequence', async () => { + it('Throws no error when building with firebase dependency without worker_threads', async () => { await fs.writeFile( nextConfig, - `module.exports = { experimental: { cpus: 1 } }` + `module.exports = { experimental: { workerThreads: false } }` ) const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) + await fs.remove(nextConfig) expect(results.stdout + results.stderr).not.toMatch(/Build error occurred/) expect(results.stdout + results.stderr).not.toMatch( /grpc_node\.node\. Module did not self-register\./ ) - await fs.remove(nextConfig) }) }) From d0979f368eed1f11d88e763f98c846d9b4636752 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 28 Oct 2019 15:05:40 -0500 Subject: [PATCH 06/10] Disable worker_threads by default --- examples/with-firebase-authentication/next.config.js | 9 --------- .../plugins/terser-webpack-plugin/src/TaskRunner.js | 2 +- packages/next/next-server/server/config.ts | 2 +- test/integration/firebase-grpc/test/index.test.js | 11 +++++------ 4 files changed, 7 insertions(+), 17 deletions(-) delete mode 100644 examples/with-firebase-authentication/next.config.js diff --git a/examples/with-firebase-authentication/next.config.js b/examples/with-firebase-authentication/next.config.js deleted file mode 100644 index ff566d92fe871..0000000000000 --- a/examples/with-firebase-authentication/next.config.js +++ /dev/null @@ -1,9 +0,0 @@ -// Parallel builds are currently broken due to a limitation in grpc (a transitive firebase -// dependency). Until this is fixed, a workaround is to set the number of available CPUs to 1. -// https://github.com/zeit/next.js/issues/7894 -// https://github.com/grpc/grpc-node/issues/778 -module.exports = { - experimental: { - cpus: 1 - } -} diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js index 469de3051b486..b306966292ede 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js @@ -28,8 +28,8 @@ export default class TaskRunner { if (this.maxConcurrentWorkers > 1) { this.workers = new Worker(worker, { - numWorkers: this.maxConcurrentWorkers, enableWorkerThreads: true, + numWorkers: this.maxConcurrentWorkers, }) this.boundWorkers = options => this.workers.default(options) } else { diff --git a/packages/next/next-server/server/config.ts b/packages/next/next-server/server/config.ts index 7c742c14889c3..97a8745c9ab9f 100644 --- a/packages/next/next-server/server/config.ts +++ b/packages/next/next-server/server/config.ts @@ -48,7 +48,7 @@ const defaultConfig: { [key: string]: any } = { publicDirectory: false, sprFlushToDisk: true, deferScripts: false, - workerThreads: true, + workerThreads: false, }, future: { excludeDefaultMomentLocales: false, diff --git a/test/integration/firebase-grpc/test/index.test.js b/test/integration/firebase-grpc/test/index.test.js index 270d3d2b93699..b3aa63e39a882 100644 --- a/test/integration/firebase-grpc/test/index.test.js +++ b/test/integration/firebase-grpc/test/index.test.js @@ -10,7 +10,10 @@ const nextConfig = path.join(appDir, 'next.config.js') describe('Builds with firebase dependency only sequentially', () => { it('Throws an error when building with firebase dependency with worker_threads', async () => { - await fs.remove(nextConfig) + await fs.writeFile( + nextConfig, + `module.exports = { experimental: { workerThreads: true } }` + ) const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) expect(results.stdout + results.stderr).toMatch(/Build error occurred/) expect(results.stdout + results.stderr).toMatch( @@ -19,12 +22,8 @@ describe('Builds with firebase dependency only sequentially', () => { }) it('Throws no error when building with firebase dependency without worker_threads', async () => { - await fs.writeFile( - nextConfig, - `module.exports = { experimental: { workerThreads: false } }` - ) - const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) await fs.remove(nextConfig) + const results = await nextBuild(appDir, [], { stdout: true, stderr: true }) expect(results.stdout + results.stderr).not.toMatch(/Build error occurred/) expect(results.stdout + results.stderr).not.toMatch( /grpc_node\.node\. Module did not self-register\./ From d8547029e2552b9fa2312ee098310c3030f0ef0a Mon Sep 17 00:00:00 2001 From: Joe Haddad Date: Mon, 28 Oct 2019 16:18:28 -0400 Subject: [PATCH 07/10] Update index.test.js --- test/integration/firebase-grpc/test/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/firebase-grpc/test/index.test.js b/test/integration/firebase-grpc/test/index.test.js index b3aa63e39a882..68481a8ba063b 100644 --- a/test/integration/firebase-grpc/test/index.test.js +++ b/test/integration/firebase-grpc/test/index.test.js @@ -8,7 +8,7 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 1 const appDir = path.join(__dirname, '..') const nextConfig = path.join(appDir, 'next.config.js') -describe('Builds with firebase dependency only sequentially', () => { +describe('Building Firebase', () => { it('Throws an error when building with firebase dependency with worker_threads', async () => { await fs.writeFile( nextConfig, From 1f953f263a5effc713df048800af255847bafe49 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 28 Oct 2019 15:19:16 -0500 Subject: [PATCH 08/10] Use workerThreads config for TerserPlugin --- packages/next/build/webpack-config.ts | 5 +++-- .../webpack/plugins/terser-webpack-plugin/src/TaskRunner.js | 3 ++- .../build/webpack/plugins/terser-webpack-plugin/src/index.js | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts index ec50bea7f612c..74b044cdaed42 100644 --- a/packages/next/build/webpack-config.ts +++ b/packages/next/build/webpack-config.ts @@ -171,11 +171,12 @@ export default async function getBaseWebpackConfig( const webpackMode = dev ? 'development' : 'production' const terserPluginConfig = { - parallel: true, - sourceMap: false, cache: true, cpus: config.experimental.cpus, distDir: distDir, + parallel: true, + sourceMap: false, + workerThreads: config.experimental.workerThreads, } const terserOptions = { parse: { diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js index b306966292ede..5bad31ce4f8f8 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js @@ -10,13 +10,14 @@ const writeFileP = promisify(writeFile) const readFileP = promisify(readFile) export default class TaskRunner { - constructor({ distDir, cpus, cache }) { + constructor({ distDir, cpus, cache, workerThreads }) { if (cache) { mkdirp.sync((this.cacheDir = join(distDir, 'cache', 'next-minifier'))) } // In some cases cpus() returns undefined // https://github.com/nodejs/node/issues/19022 this.maxConcurrentWorkers = cpus + this.useWorkerThreads = workerThreads } run(tasks, callback) { diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/index.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/index.js index 1acafe76f9891..4302cb344e4d1 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/index.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/index.js @@ -21,10 +21,12 @@ export class TerserPlugin { cache = false, cpus, distDir, + workerThreads, } = options this.cpus = cpus this.distDir = distDir + this.workerThreads = workerThreads this.options = { warningsFilter, sourceMap, @@ -134,6 +136,7 @@ export class TerserPlugin { distDir: this.distDir, cpus: this.cpus, cache: this.options.cache, + workerThreads: this.workerThreads, }) const processedAssets = new WeakSet() From 95747e198bb88efce0546249ef9786df7f6d3599 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 28 Oct 2019 15:27:39 -0500 Subject: [PATCH 09/10] Update to use workerThreads config in TerserPlugin for consistency --- .../webpack/plugins/terser-webpack-plugin/src/TaskRunner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js index 5bad31ce4f8f8..d8c0b48df39fd 100644 --- a/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js +++ b/packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js @@ -29,7 +29,7 @@ export default class TaskRunner { if (this.maxConcurrentWorkers > 1) { this.workers = new Worker(worker, { - enableWorkerThreads: true, + enableWorkerThreads: this.useWorkerThreads, numWorkers: this.maxConcurrentWorkers, }) this.boundWorkers = options => this.workers.default(options) From 8cf099681469dbce573d1ac1e8eb203ac5fac1b1 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 28 Oct 2019 18:49:17 -0500 Subject: [PATCH 10/10] Disable node 12 specific test --- test/integration/firebase-grpc/test/index.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/firebase-grpc/test/index.test.js b/test/integration/firebase-grpc/test/index.test.js index 68481a8ba063b..dc24e5fba36de 100644 --- a/test/integration/firebase-grpc/test/index.test.js +++ b/test/integration/firebase-grpc/test/index.test.js @@ -9,7 +9,8 @@ const appDir = path.join(__dirname, '..') const nextConfig = path.join(appDir, 'next.config.js') describe('Building Firebase', () => { - it('Throws an error when building with firebase dependency with worker_threads', async () => { + // TODO: investigate re-enabling this test in node 12 environment + xit('Throws an error when building with firebase dependency with worker_threads', async () => { await fs.writeFile( nextConfig, `module.exports = { experimental: { workerThreads: true } }`