Implement foundations for additional bundler option

This implements core code from https://github.com/vercel/next.js/tree/wbinnssmith/try-ci-test for rspack. It does not include code for the built-in flight or app loader.

These code paths are currently taken when `NEXT_RSPACK` is set when running Next.js. Ultimately, this will be set with a new Next.js plugin, which will also install the necessary dependencies.

Co-authored-by: hardfist <[email protected]>
Co-authored-by: JJ Kasper <[email protected]>
Tim Neutkens <[email protected]>

package.json

@@ -16,16 +16,20 @@
     "test-types": "tsc",
     "test-unit": "jest test/unit/ packages/next/ packages/font",
     "test-dev": "cross-env NEXT_TEST_MODE=dev pnpm testheadless",
+    "test-dev-rspack": "cross-env NEXT_TEST_MODE=dev NEXT_RSPACK=1 BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 BUILTIN_APP_LOADER=1 BUILTIN_SWC_LOADER=1 pnpm testheadless",
     "test-dev-turbo": "cross-env NEXT_TEST_MODE=dev TURBOPACK=1 TURBOPACK_DEV=1 pnpm testheadless",
     "test-start": "cross-env NEXT_TEST_MODE=start pnpm testheadless",
+    "test-start-rspack": "cross-env NEXT_TEST_MODE=start NEXT_RSPACK=1 BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 BUILTIN_APP_LOADER=1 BUILTIN_SWC_LOADER=1 pnpm testheadless",
     "test-start-turbo": "cross-env NEXT_TEST_MODE=start TURBOPACK=1 TURBOPACK_BUILD=1 pnpm testheadless",
     "test-deploy": "cross-env NEXT_TEST_MODE=deploy pnpm testheadless",
-    "testonly-dev": "cross-env NEXT_TEST_MODE=dev pnpm testonly",
+    "testonly-dev": "cross-env NEXT_TEST_MODE=dev  NEXT_RSPACK=1 pnpm testonly",
     "testonly-dev-turbo": "cross-env NEXT_TEST_MODE=dev TURBOPACK=1 TURBOPACK_DEV=1 pnpm testonly",
     "testonly-start": "cross-env NEXT_TEST_MODE=start pnpm testonly",
+    "testonly-start-rspack": "cross-env NEXT_TEST_MODE=start NEXT_RSPACK=1 pnpm testonly",
     "testonly-start-turbo": "cross-env NEXT_TEST_MODE=start TURBOPACK=1 TURBOPACK_BUILD=1 pnpm testonly",
     "testonly-deploy": "cross-env NEXT_TEST_MODE=deploy pnpm testonly",
     "test": "pnpm testheadless",
+    "test-rspack": "cross-env NEXT_RSPACK=1 pnpm testheadless",
     "test-turbo": "cross-env TURBOPACK=1 TURBOPACK_DEV=1 TURBOPACK_BUILD=1 pnpm testheadless",
     "testonly": "jest --runInBand",
     "testheadless": "cross-env HEADLESS=true pnpm testonly",

packages/next/src/bin/next.ts

@@ -1,5 +1,10 @@
 #!/usr/bin/env node
 
+if (process.env.NEXT_RSPACK) {
+  // silent rspack's schema check
+  process.env.RSPACK_CONFIG_VALIDATE = 'loose-silent'
+}
+
 import '../server/require-hook'
 
 import { Argument, Command, Option } from 'next/dist/compiled/commander'

packages/next/src/build/collect-build-traces.ts

@@ -395,6 +395,14 @@ export async function collectBuildTraces({
         const { entryNameFilesMap } = buildTraceContext?.chunksTrace || {}
 
         const cachedLookupIgnoreRoutes = new Map<string, boolean>()
+        let chunks: string[] = []
+
+        // we aren't getting all chunks in the trace-entrypoint plugin
+        // with rspack currently so for now just add them manually for
+        // all trace files
+        if (process.env.NEXT_RSPACK) {
+          chunks = await fs.readdir(path.join(distDir, 'server', 'chunks'))
+        }
 
         await Promise.all(
           [
@@ -463,6 +471,19 @@ export async function collectBuildTraces({
               curTracedFiles.add(file)
             }
 
+            if (process.env.NEXT_RSPACK) {
+              for (const file of chunks) {
+                curTracedFiles.add(
+                  path
+                    .relative(
+                      traceOutputDir,
+                      path.join(distDir, 'server/chunks', file)
+                    )
+                    .replace(/\\/g, '/')
+                )
+              }
+            }
+
             await fs.writeFile(
               traceOutputPath,
               JSON.stringify({

packages/next/src/build/compiler.ts

@@ -1,5 +1,6 @@
 import { webpack } from 'next/dist/compiled/webpack/webpack'
 import type { Span } from '../trace'
+import { getRspackCore } from '../shared/lib/get-rspack'
 
 export type CompilerResult = {
   errors: webpack.StatsError[]
@@ -35,7 +36,7 @@ function closeCompiler(compiler: webpack.Compiler | webpack.MultiCompiler) {
   })
 }
 
-export function runCompiler(
+export async function runCompiler(
   config: webpack.Configuration,
   {
     runWebpackSpan,
@@ -50,6 +51,10 @@ export function runCompiler(
     inputFileSystem?: webpack.Compiler['inputFileSystem'],
   ]
 > {
+  if (process.env.NEXT_RSPACK_OTEL) {
+    await getRspackCore().experiments.globalTrace.register('trace', 'otel', '')
+  }
+
   return new Promise((resolve, reject) => {
     const compiler = webpack(config)
     // Ensure we use the previous inputFileSystem
@@ -58,34 +63,43 @@ export function runCompiler(
     }
     compiler.fsStartTime = Date.now()
     compiler.run((err, stats) => {
+      const compilerResult = runWebpackSpan
+        .traceChild('webpack-generate-error-stats')
+        .traceFn(() =>
+          generateStats({ errors: [], warnings: [], stats }, stats!)
+        )
+
       const webpackCloseSpan = runWebpackSpan.traceChild('webpack-close', {
         name: config.name || 'unknown',
       })
-      webpackCloseSpan
-        .traceAsyncFn(() => closeCompiler(compiler))
-        .then(() => {
-          if (err) {
-            const reason = err.stack ?? err.toString()
-            if (reason) {
-              return resolve([
-                {
-                  errors: [{ message: reason, details: (err as any).details }],
-                  warnings: [],
-                  stats,
-                },
-                compiler.inputFileSystem,
-              ])
-            }
-            return reject(err)
-          } else if (!stats) throw new Error('No Stats from webpack')
 
-          const result = webpackCloseSpan
-            .traceChild('webpack-generate-error-stats')
-            .traceFn(() =>
-              generateStats({ errors: [], warnings: [], stats }, stats)
-            )
-          return resolve([result, compiler.inputFileSystem])
+      let closePromise = webpackCloseSpan.traceAsyncFn(() =>
+        closeCompiler(compiler)
+      )
+
+      if (process.env.NEXT_RSPACK) {
+        closePromise = closePromise.then(() => {
+          return getRspackCore().experiments.globalTrace.cleanup()
         })
+      }
+
+      closePromise.then(() => {
+        if (err) {
+          const reason = err.stack ?? err.toString()
+          if (reason) {
+            return resolve([
+              {
+                errors: [{ message: reason, details: (err as any).details }],
+                warnings: [],
+                stats,
+              },
+              compiler.inputFileSystem,
+            ])
+          }
+          return reject(err)
+        } else if (!stats) throw new Error('No Stats from webpack')
+        return resolve([compilerResult, compiler.inputFileSystem])
+      })
     })
   })
 }

packages/next/src/build/entries.ts

@@ -13,7 +13,7 @@ import * as Log from './output/log'
 import type { LoadedEnvFiles } from '@next/env'
 import type { AppLoaderOptions } from './webpack/loaders/next-app-loader'
 
-import { posix, join, dirname, extname } from 'path'
+import { posix, join, dirname, extname, normalize } from 'path'
 import { stringify } from 'querystring'
 import fs from 'fs'
 import {

packages/next/src/build/webpack-build/index.ts

@@ -103,17 +103,17 @@ async function webpackBuildWithWorker(
         combinedResult.buildTraceContext.entriesTrace!.entryNameMap =
           entryNameMap
       }
+    }
 
-      if (curResult.buildTraceContext?.chunksTrace) {
-        const { entryNameFilesMap } = curResult.buildTraceContext.chunksTrace!
+    if (curResult.buildTraceContext?.chunksTrace) {
+      const { entryNameFilesMap } = curResult.buildTraceContext.chunksTrace!
 
-        if (entryNameFilesMap) {
-          combinedResult.buildTraceContext.chunksTrace =
-            curResult.buildTraceContext.chunksTrace!
+      if (entryNameFilesMap) {
+        combinedResult.buildTraceContext.chunksTrace =
+          curResult.buildTraceContext.chunksTrace!
 
-          combinedResult.buildTraceContext.chunksTrace!.entryNameFilesMap =
-            entryNameFilesMap
-        }
+        combinedResult.buildTraceContext.chunksTrace!.entryNameFilesMap =
+          entryNameFilesMap
       }
     }
   }

packages/next/src/build/webpack/config/blocks/base.ts

@@ -34,6 +34,8 @@ export const base = curry(function base(
   if (ctx.isDevelopment) {
     if (process.env.__NEXT_TEST_MODE && !process.env.__NEXT_TEST_WITH_DEVTOOL) {
       config.devtool = false
+    } else if (process.env.NEXT_RSPACK) {
+      config.devtool = 'source-map'
     } else {
       // `eval-source-map` provides full-fidelity source maps for the
       // original source, including columns and original variable names.
@@ -73,7 +75,7 @@ export const base = curry(function base(
         shouldIgnorePath,
       })
     )
-  } else if (config.devtool === 'eval-source-map') {
+  } else if (config.devtool === 'eval-source-map' && !process.env.NEXT_RSPACK) {
     // We're using a fork of `eval-source-map`
     config.devtool = false
     config.plugins.push(

packages/next/src/build/webpack/config/blocks/css/index.ts

@@ -14,6 +14,7 @@ import {
 import { getPostCssPlugins } from './plugins'
 import { nonNullable } from '../../../../../lib/non-nullable'
 import { WEBPACK_LAYERS } from '../../../../../lib/constants'
+import { getRspackCore } from '../../../../../shared/lib/get-rspack'
 
 // RegExps for all Style Sheet variants
 export const regexLikeCss = /\.(css|scss|sass)$/
@@ -147,6 +148,7 @@ export const css = curry(async function css(
   ctx: ConfigurationContext,
   config: webpack.Configuration
 ) {
+  const isRspack = Boolean(process.env.NEXT_RSPACK)
   const {
     prependData: sassPrependData,
     additionalData: sassAdditionalData,
@@ -576,6 +578,8 @@ export const css = curry(async function css(
             // Exclude extensions that webpack handles by default
             exclude: [
               /\.(js|mjs|jsx|ts|tsx)$/,
+              // TODO investigate why this is needed
+              ...(isRspack ? [/^$/] : []),
               /\.html$/,
               /\.json$/,
               /\.webpack\[[^\]]+\]$/,
@@ -592,8 +596,10 @@ export const css = curry(async function css(
   // Enable full mini-css-extract-plugin hmr for prod mode pages or app dir
   if (ctx.isClient && (ctx.isProduction || ctx.hasAppDir)) {
     // Extract CSS as CSS file(s) in the client-side production bundle.
-    const MiniCssExtractPlugin =
-      require('../../../plugins/mini-css-extract-plugin').default
+    const MiniCssExtractPlugin = isRspack
+      ? getRspackCore().CssExtractRspackPlugin
+      : require('../../../plugins/mini-css-extract-plugin').default
+
     fns.push(
       plugin(
         // @ts-ignore webpack 5 compat

packages/next/src/build/webpack/config/blocks/css/loaders/client.ts

@@ -1,4 +1,5 @@
 import type { webpack } from 'next/dist/compiled/webpack/webpack'
+import { getRspackCore } from '../../../../../../shared/lib/get-rspack'
 
 export function getClientStyleLoader({
   hasAppDir,
@@ -11,6 +12,7 @@ export function getClientStyleLoader({
   isDevelopment: boolean
   assetPrefix: string
 }): webpack.RuleSetUseItem {
+  const isRspack = Boolean(process.env.NEXT_RSPACK)
   const shouldEnableApp = typeof isAppDir === 'boolean' ? isAppDir : hasAppDir
 
   // Keep next-style-loader for development mode in `pages/`
@@ -41,8 +43,10 @@ export function getClientStyleLoader({
     }
   }
 
-  const MiniCssExtractPlugin =
-    require('../../../../plugins/mini-css-extract-plugin').default
+  const MiniCssExtractPlugin = isRspack
+    ? getRspackCore().rspack.CssExtractRspackPlugin
+    : require('../../../../plugins/mini-css-extract-plugin').default
+
   return {
     loader: MiniCssExtractPlugin.loader,
     options: {