feat(babel): add parallel processing via worker threads

packages/babel/README.md

@@ -94,7 +94,7 @@ A [picomatch pattern](https://github.com/micromatch/picomatch), or array of patt
 
 ### `filter`
 
-Type: (id: string) => boolean<br>
+Type: (id: string, code: string) => boolean<br>
 
 Custom [filter function](https://github.com/rollup/plugins/tree/master/packages/pluginutils#createfilter) can be used to determine whether or not certain modules should be operated upon.
 
@@ -135,6 +135,15 @@ Default: `false`
 
 Before transpiling your input files this plugin also transpile a short piece of code **for each** input file. This is used to validate some misconfiguration errors, but for sufficiently big projects it can slow your build times so if you are confident about your configuration then you might disable those checks with this option.
 
+### `parallel`
+
+Type: `Boolean | number`
+Default: `false`
+
+Enable parallel processing of files in worker threads. This has a setup cost, so is best suited for larger projects. Pass an integer to set the number of workers. Set `true` for the default number of workers (4).
+
+This option cannot be used alongside custom overrides or non-serializable Babel options.
+
 ### External dependencies
 
 Ideally, you should only be transforming your source code, rather than running all of your external dependencies through Babel (to ignore external dependencies from being handled by this plugin you might use `exclude: 'node_modules/**'` option). If you have a dependency that exposes untranspiled ES6 source code that doesn't run in your target environment, then you may need to break this rule, but it often causes problems with unusual `.babelrc` files or mismatched versions of Babel.

packages/babel/package.json

@@ -55,7 +55,7 @@
   "peerDependencies": {
     "@babel/core": "^7.0.0",
     "@types/babel__core": "^7.1.9",
-    "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0"
+    "rollup": "^2.0.0||^3.0.0||^4.0.0"
   },
   "peerDependenciesMeta": {
     "rollup": {

packages/babel/rollup.config.mjs

@@ -1,14 +1,37 @@
 import { readFileSync } from 'fs';
 
-import { createConfig } from '../../shared/rollup.config.mjs';
+import { createConfig, emitModulePackageFile } from '../../shared/rollup.config.mjs';
 
 import { babel } from './src/index.js';
 
 const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf8'));
 
 export default {
   ...createConfig({ pkg }),
-  input: './src/index.js',
+  input: {
+    index: './src/index.js',
+    worker: './src/worker.js'
+  },
+  output: [
+    {
+      format: 'cjs',
+      dir: 'dist/cjs',
+      exports: 'named',
+      footer(chunkInfo) {
+        if (chunkInfo.name === 'index') {
+          return 'module.exports = Object.assign(exports.default, exports);';
+        }
+        return null;
+      },
+      sourcemap: true
+    },
+    {
+      format: 'es',
+      dir: 'dist/es',
+      plugins: [emitModulePackageFile()],
+      sourcemap: true
+    }
+  ],
   plugins: [
     babel({
       presets: [['@babel/preset-env', { targets: { node: 14 } }]],

packages/babel/src/index.js

@@ -2,10 +2,9 @@ import * as babel from '@babel/core';
 import { createFilter } from '@rollup/pluginutils';
 
 import { BUNDLED, HELPERS } from './constants.js';
-import bundledHelpersPlugin from './bundledHelpersPlugin.js';
-import preflightCheck from './preflightCheck.js';
 import transformCode from './transformCode.js';
-import { addBabelPlugin, escapeRegExpCharacters, warnOnce, stripQuery } from './utils.js';
+import { escapeRegExpCharacters, warnOnce } from './utils.js';
+import WorkerPool from './workerPool.js';
 
 const unpackOptions = ({
   extensions = babel.DEFAULT_EXTENSIONS,
@@ -37,7 +36,7 @@ const warnAboutDeprecatedHelpersOption = ({ deprecatedOption, suggestion }) => {
   );
 };
 
-const unpackInputPluginOptions = ({ skipPreflightCheck = false, ...rest }, rollupVersion) => {
+const unpackInputPluginOptions = ({ skipPreflightCheck = false, ...rest }) => {
   if ('runtimeHelpers' in rest) {
     warnAboutDeprecatedHelpersOption({
       deprecatedOption: 'runtimeHelpers',
@@ -63,8 +62,7 @@ const unpackInputPluginOptions = ({ skipPreflightCheck = false, ...rest }, rollu
       supportsStaticESM: true,
       supportsDynamicImport: true,
       supportsTopLevelAwait: true,
-      // todo: remove version checks for 1.20 - 1.25 when we bump peer deps
-      supportsExportNamespaceFrom: !rollupVersion.match(/^1\.2[0-5]\./),
+      supportsExportNamespaceFrom: true,
       ...rest.caller
     }
   });
@@ -101,6 +99,24 @@ const returnObject = () => {
   return {};
 };
 
+function isSerializable(value) {
+  if (value === null) {
+    return true;
+  } else if (Array.isArray(value)) {
+    return value.every(isSerializable);
+  }
+  switch (typeof value) {
+    case 'string':
+    case 'number':
+    case 'boolean':
+      return true;
+    case 'object':
+      return Object.keys(value).every((key) => isSerializable(value[key]));
+    default:
+      return false;
+  }
+}
+
 function createBabelInputPluginFactory(customCallback = returnObject) {
   const overrides = customCallback(babel);
 
@@ -110,77 +126,113 @@ function createBabelInputPluginFactory(customCallback = returnObject) {
       overrides
     );
 
-    let babelHelpers;
-    let babelOptions;
-    let filter;
-    let skipPreflightCheck;
-    return {
-      name: 'babel',
-
-      options() {
-        // todo: remove options hook and hoist declarations when version checks are removed
-        let exclude;
-        let include;
-        let extensions;
-        let customFilter;
+    let workerPool;
+
+    const {
+      exclude,
+      extensions,
+      babelHelpers,
+      include,
+      filter: customFilter,
+      skipPreflightCheck,
+      parallel,
+      ...babelOptions
+    } = unpackInputPluginOptions(pluginOptionsWithOverrides);
+
+    const extensionRegExp = new RegExp(
+      `(${extensions.map(escapeRegExpCharacters).join('|')})(\\?.*)?(#.*)?$`
+    );
+    if (customFilter && (include || exclude)) {
+      throw new Error('Could not handle include or exclude with custom filter together');
+    }
+    const userDefinedFilter =
+      typeof customFilter === 'function' ? customFilter : createFilter(include, exclude);
+    const filter = (id, code) => extensionRegExp.test(id) && userDefinedFilter(id, code);
 
-        ({
-          exclude,
-          extensions,
-          babelHelpers,
-          include,
-          filter: customFilter,
-          skipPreflightCheck,
-          ...babelOptions
-        } = unpackInputPluginOptions(pluginOptionsWithOverrides, this.meta.rollupVersion));
+    if (parallel) {
+      const parallelAllowed =
+        isSerializable(babelOptions) && !overrides?.config && !overrides?.result;
 
-        const extensionRegExp = new RegExp(
-          `(${extensions.map(escapeRegExpCharacters).join('|')})$`
+      if (!parallelAllowed) {
+        throw new Error(
+          'Cannot use "parallel" mode alongside custom overrides or non-serializable Babel options.'
         );
-        if (customFilter && (include || exclude)) {
-          throw new Error('Could not handle include or exclude with custom filter together');
-        }
-        const userDefinedFilter =
-          typeof customFilter === 'function' ? customFilter : createFilter(include, exclude);
-        filter = (id) => extensionRegExp.test(stripQuery(id).bareId) && userDefinedFilter(id);
+      }
+
+      const parallelWorkerCount = typeof parallel === 'number' ? parallel : 4;
+      workerPool = new WorkerPool(
+        new URL('./worker.js', import.meta.url).pathname,
+        parallelWorkerCount
+      );
+    }
 
-        return null;
+    const helpersFilter = { id: new RegExp(`^${escapeRegExpCharacters(HELPERS)}$`) };
+
+    return {
+      name: 'babel',
+
+      resolveId: {
+        filter: helpersFilter,
+        handler(id) {
+          if (id !== HELPERS) {
+            return null;
+          }
+          return id;
+        }
       },
 
-      resolveId(id) {
-        if (id !== HELPERS) {
-          return null;
+      load: {
+        filter: helpersFilter,
+        handler(id) {
+          if (id !== HELPERS) {
+            return null;
+          }
+          return babel.buildExternalHelpers(null, 'module');
         }
-        return id;
       },
 
-      load(id) {
-        if (id !== HELPERS) {
-          return null;
+      transform: {
+        filter: {
+          id: extensionRegExp
+        },
+        async handler(code, filename) {
+          if (!(await filter(filename, code))) return null;
+          if (filename === HELPERS) return null;
+
+          if (parallel) {
+            return workerPool.runTask({
+              inputCode: code,
+              babelOptions: { ...babelOptions, filename },
+              runPreflightCheck: !skipPreflightCheck,
+              babelHelpers
+            });
+          }
+
+          return transformCode({
+            inputCode: code,
+            babelOptions: { ...babelOptions, filename },
+            overrides: {
+              config: overrides.config?.bind(this),
+              result: overrides.result?.bind(this)
+            },
+            customOptions,
+            error: this.error.bind(this),
+            runPreflightCheck: !skipPreflightCheck,
+            babelHelpers
+          });
         }
-        return babel.buildExternalHelpers(null, 'module');
       },
 
-      transform(code, filename) {
-        if (!filter(filename)) return null;
-        if (filename === HELPERS) return null;
+      async closeBundle() {
+        if (parallel && !this.meta.watchMode) {
+          await workerPool.terminate();
+        }
+      },
 
-        return transformCode(
-          code,
-          { ...babelOptions, filename },
-          overrides,
-          customOptions,
-          this,
-          async (transformOptions) => {
-            if (!skipPreflightCheck) {
-              await preflightCheck(this, babelHelpers, transformOptions);
-            }
-
-            return babelHelpers === BUNDLED
-              ? addBabelPlugin(transformOptions, bundledHelpersPlugin)
-              : transformOptions;
-          }
-        );
+      async closeWatcher() {
+        if (parallel) {
+          await workerPool.terminate();
+        }
       }
     };
   };
@@ -259,7 +311,16 @@ function createBabelOutputPluginFactory(customCallback = returnObject) {
           }
         }
 
-        return transformCode(code, babelOptions, overrides, customOptions, this);
+        return transformCode({
+          inputCode: code,
+          babelOptions,
+          overrides: {
+            config: overrides.config?.bind(this),
+            result: overrides.result?.bind(this)
+          },
+          customOptions,
+          error: this.error.bind(this)
+        });
       }
     };
   };

packages/babel/src/preflightCheck.js

@@ -37,27 +37,27 @@ const mismatchError = (actual, expected, filename) =>
 // Revert to /\/helpers\/(esm\/)?inherits/ when Babel 8 gets released, this was fixed in https://github.com/babel/babel/issues/14185
 const inheritsHelperRe = /[\\/]+helpers[\\/]+(esm[\\/]+)?inherits/;
 
-export default async function preflightCheck(ctx, babelHelpers, transformOptions) {
+export default async function preflightCheck(error, babelHelpers, transformOptions) {
   const finalOptions = addBabelPlugin(transformOptions, helpersTestTransform);
   const check = (await babel.transformAsync(PREFLIGHT_INPUT, finalOptions)).code;
 
   // Babel sometimes splits ExportDefaultDeclaration into 2 statements, so we also check for ExportNamedDeclaration
   if (!/export (d|{)/.test(check)) {
-    ctx.error(MODULE_ERROR);
+    error(MODULE_ERROR);
   }
 
   if (inheritsHelperRe.test(check)) {
     if (babelHelpers === RUNTIME) {
       return;
     }
-    ctx.error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename));
+    error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename));
   }
 
   if (check.includes('babelHelpers.inherits')) {
     if (babelHelpers === EXTERNAL) {
       return;
     }
-    ctx.error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename));
+    error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename));
   }
 
   // test unminifiable string content
@@ -66,12 +66,12 @@ export default async function preflightCheck(ctx, babelHelpers, transformOptions
       return;
     }
     if (babelHelpers === RUNTIME && !transformOptions.plugins.length) {
-      ctx.error(
+      error(
         `You must use the \`@babel/plugin-transform-runtime\` plugin when \`babelHelpers\` is "${RUNTIME}".\n`
       );
     }
-    ctx.error(mismatchError(INLINE, babelHelpers, transformOptions.filename));
+    error(mismatchError(INLINE, babelHelpers, transformOptions.filename));
   }
 
-  ctx.error(UNEXPECTED_ERROR);
+  error(UNEXPECTED_ERROR);
 }