add support for esm externals (vercel#27069)

add `experimental.esmExternals: boolean | 'loose'` config option remove `output.environment` configuration in favor of `target` | | `esmExternals: false` (default) | `esmExternals: 'loose'` | `esmExternals: true` | | ------------------------ | ------------------------------- | ----------------------- | -------------------- | | import cjs package | `require()` | `require()` | `require()` | | require cjs package | `require()` | `require()` | `require()` | | import mixed package | `require()` *** | `import()` | `import()` | | require mixed package | `require()` | `require()` | `require()` | | import pure esm package | `import()` | `import()` | `import()` | | require pure esm package | Error ** | `import()` * | Error ** | | import pure cjs package | `require()` | `require()` | Resolving error | | require pure cjs package | `require()` | `require()` | `require()` | cjs package: Offers only CJS implementation (may not even have an `exports` field) mixed package: Offers CJS and ESM implementation via `exports` field pure esm package: Only offers an ESM implementation (may not even have an `exports` field) pure cjs package: CommonJs package that prevents importing via `exports` field when `import` is used. `*` This case will behave a bit unexpected for now, since `require` will return a Promise. So that need to be awaited. This will be fixed once the whole next.js bundle is ESM. It didn't work at all before this PR. `**` This is a new Error when trying to require an esm package. `***` For mixed packages we prefer the CommonJS variant to avoid a breaking change. ## Feature - [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. - [ ] Related issues linked using `fixes #number` - [x] Integration tests added - [ ] Documentation added - [ ] Telemetry added. In case of a feature if it's used or not. - [x] Errors have helpful link attached, see `contributing.md` ## Documentation / Examples - [ ] Make sure the linting passes
filoblu · Jul 10, 2021 · 7a8da97 · 7a8da97
1 parent 6eaa685
commit 7a8da97
Show file tree

Hide file tree

Showing 33 changed files with 591 additions and 100 deletions.
diff --git a/errors/import-esm-externals.md b/errors/import-esm-externals.md
@@ -0,0 +1,19 @@
+# ESM packages need to be imported
+
+#### Why This Error Occurred
+
+Packages in node_modules that are published as EcmaScript Module, need to be `import`ed via `import ... from 'package'` or `import('package')`.
+
+You get this error when using a different way to reference the package, e. g. `require()`.
+
+#### Possible Ways to Fix It
+
+1. Use `import` or `import()` to reference the package instead. (Recommended)
+
+2. If you are already using `import`, make sure that this is not changed by a transpiler, e. g. TypeScript or Babel.
+
+3. Switch to loose mode (`experimental.esmExternals: 'loose'`), which tries to automatically correct this error.
+
+### Useful Links
+
+- [Node.js ESM require docs](https://nodejs.org/dist/latest-v16.x/docs/api/esm.html#esm_require)
diff --git a/errors/manifest.json b/errors/manifest.json
@@ -406,6 +406,10 @@
         {
           "title": "placeholder-blur-data-url",
           "path": "/errors/placeholder-blur-data-url.md"
+        },
+        {
+          "title": "import-esm-externals",
+          "path": "/errors/import-esm-externals.md"
         }
       ]
     }

diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts
@@ -180,6 +180,11 @@ const WEBPACK_RESOLVE_OPTIONS = {
   symlinks: true,
 }
 
+const WEBPACK_ESM_RESOLVE_OPTIONS = {
+  dependencyType: 'esm',
+  symlinks: true,
+}
+
 const NODE_RESOLVE_OPTIONS = {
   dependencyType: 'commonjs',
   modules: ['node_modules'],
@@ -201,6 +206,13 @@ const NODE_RESOLVE_OPTIONS = {
   restrictions: [],
 }
 
+const NODE_ESM_RESOLVE_OPTIONS = {
+  ...NODE_RESOLVE_OPTIONS,
+  dependencyType: 'esm',
+  conditionNames: ['node', 'import', 'module'],
+  fullySpecified: true,
+}
+
 export default async function getBaseWebpackConfig(
   dir: string,
   {
@@ -653,12 +665,19 @@ export default async function getBaseWebpackConfig(
     config.conformance
   )
 
+  const esmExternals = !!config.experimental?.esmExternals
+  const looseEsmExternals = config.experimental?.esmExternals === 'loose'
+
   async function handleExternals(
     context: string,
     request: string,
+    dependencyType: string,
     getResolve: (
       options: any
-    ) => (resolveContext: string, resolveRequest: string) => Promise<string>
+    ) => (
+      resolveContext: string,
+      resolveRequest: string
+    ) => Promise<[string | null, boolean]>
   ) {
     // We need to externalize internal requests for files intended to
     // not be bundled.
@@ -687,27 +706,53 @@ export default async function getBaseWebpackConfig(
       }
     }
 
-    const resolve = getResolve(WEBPACK_RESOLVE_OPTIONS)
+    // When in esm externals mode, and using import, we resolve with
+    // ESM resolving options.
+    const isEsmRequested = dependencyType === 'esm'
+    const preferEsm = esmExternals && isEsmRequested
+
+    const resolve = getResolve(
+      preferEsm ? WEBPACK_ESM_RESOLVE_OPTIONS : WEBPACK_RESOLVE_OPTIONS
+    )
 
     // Resolve the import with the webpack provided context, this
     // ensures we're resolving the correct version when multiple
     // exist.
-    let res: string
+    let res: string | null
+    let isEsm: boolean = false
     try {
-      res = await resolve(context, request)
+      ;[res, isEsm] = await resolve(context, request)
     } catch (err) {
-      // If the request cannot be resolved, we need to tell webpack to
-      // "bundle" it so that webpack shows an error (that it cannot be
-      // resolved).
-      return
+      res = null
     }
 
-    // Same as above, if the request cannot be resolved we need to have
+    // If resolving fails, and we can use an alternative way
+    // try the alternative resolving options.
+    if (!res && (isEsmRequested || looseEsmExternals)) {
+      const resolveAlternative = getResolve(
+        preferEsm ? WEBPACK_RESOLVE_OPTIONS : WEBPACK_ESM_RESOLVE_OPTIONS
+      )
+      try {
+        ;[res, isEsm] = await resolveAlternative(context, request)
+      } catch (err) {
+        res = null
+      }
+    }
+
+    // If the request cannot be resolved we need to have
     // webpack "bundle" it so it surfaces the not found error.
     if (!res) {
       return
     }
 
+    // ESM externals can only be imported (and not required).
+    // Make an exception in loose mode.
+    if (!isEsmRequested && isEsm && !looseEsmExternals) {
+      throw new Error(
+        `ESM packages (${request}) need to be imported. Use 'import' to reference the package instead. https://nextjs.org/docs/messages/import-esm-externals`
+      )
+    }
+
     if (isLocal) {
       // Makes sure dist/shared and dist/server are not bundled
       // we need to process shared/lib/router/router so that
@@ -741,26 +786,32 @@ export default async function getBaseWebpackConfig(
     // package that'll be available at runtime. If it's not identical,
     // we need to bundle the code (even if it _should_ be external).
     let baseRes: string | null
+    let baseIsEsm: boolean
     try {
-      const baseResolve = getResolve(NODE_RESOLVE_OPTIONS)
-      baseRes = await baseResolve(dir, request)
+      const baseResolve = getResolve(
+        isEsm ? NODE_ESM_RESOLVE_OPTIONS : NODE_RESOLVE_OPTIONS
+      )
+      ;[baseRes, baseIsEsm] = await baseResolve(dir, request)
     } catch (err) {
       baseRes = null
+      baseIsEsm = false
     }
 
     // Same as above: if the package, when required from the root,
     // would be different from what the real resolution would use, we
     // cannot externalize it.
-    // if res or baseRes are symlinks they could point to the the same file,
-    // but the resolver will resolve symlinks so this is already handled
-    if (baseRes !== res) {
+    // if request is pointing to a symlink it could point to the the same file,
+    // the resolver will resolve symlinks so this is handled
+    if (baseRes !== res || isEsm !== baseIsEsm) {
       return
     }
 
+    const externalType = isEsm ? 'module' : 'commonjs'
+
     if (
       res.match(/next[/\\]dist[/\\]shared[/\\](?!lib[/\\]router[/\\]router)/)
     ) {
-      return `commonjs ${request}`
+      return `${externalType} ${request}`
     }
 
     // Default pages have to be transpiled
@@ -783,7 +834,7 @@ export default async function getBaseWebpackConfig(
     // Anything else that is standard JavaScript within `node_modules`
     // can be externalized.
     if (/node_modules[/\\].*\.c?js$/.test(res)) {
-      return `commonjs ${request}`
+      return `${externalType} ${request}`
     }
 
     // Default behavior: bundle the code!
@@ -803,17 +854,43 @@ export default async function getBaseWebpackConfig(
             ? ({
                 context,
                 request,
+                dependencyType,
                 getResolve,
               }: {
                 context: string
                 request: string
+                dependencyType: string
                 getResolve: (
                   options: any
                 ) => (
                   resolveContext: string,
-                  resolveRequest: string
-                ) => Promise<string>
-              }) => handleExternals(context, request, getResolve)
+                  resolveRequest: string,
+                  callback: (
+                    err?: Error,
+                    result?: string,
+                    resolveData?: { descriptionFileData?: { type?: any } }
+                  ) => void
+                ) => void
+              }) =>
+                handleExternals(context, request, dependencyType, (options) => {
+                  const resolveFunction = getResolve(options)
+                  return (resolveContext: string, requestToResolve: string) =>
+                    new Promise((resolve, reject) => {
+                      resolveFunction(
+                        resolveContext,
+                        requestToResolve,
+                        (err, result, resolveData) => {
+                          if (err) return reject(err)
+                          if (!result) return resolve([null, false])
+                          const isEsm = /\.js$/i.test(result)
+                            ? resolveData?.descriptionFileData?.type ===
+                              'module'
+                            : /\.mjs$/i.test(result)
+                          resolve([result, isEsm])
+                        }
+                      )
+                    })
+                })
             : (
                 context: string,
                 request: string,
@@ -822,13 +899,15 @@ export default async function getBaseWebpackConfig(
                 handleExternals(
                   context,
                   request,
+                  'commonjs',
                   () => (resolveContext: string, requestToResolve: string) =>
                     new Promise((resolve) =>
-                      resolve(
+                      resolve([
                         require.resolve(requestToResolve, {
                           paths: [resolveContext],
-                        })
-                      )
+                        }),
+                        false,
+                      ])
                     )
                 ).then((result) => callback(undefined, result), callback),
         ]
@@ -920,19 +999,6 @@ export default async function getBaseWebpackConfig(
       ],
     },
     output: {
-      ...(isWebpack5
-        ? {
-            environment: {
-              arrowFunction: false,
-              bigIntLiteral: false,
-              const: false,
-              destructuring: false,
-              dynamicImport: false,
-              forOf: false,
-              module: false,
-            },
-          }
-        : {}),
       // we must set publicPath to an empty value to override the default of
       // auto which doesn't work in IE11
       publicPath: `${config.assetPrefix || ''}/_next/`,

diff --git a/packages/next/build/webpack/config/blocks/base.ts b/packages/next/build/webpack/config/blocks/base.ts
@@ -11,7 +11,12 @@ export const base = curry(function base(
 ) {
   config.mode = ctx.isDevelopment ? 'development' : 'production'
   config.name = ctx.isServer ? 'server' : 'client'
-  config.target = ctx.isServer ? 'node' : 'web'
+  if (isWebpack5) {
+    // @ts-ignore TODO webpack 5 typings
+    config.target = ctx.isServer ? 'node12.17' : ['web', 'es5']
+  } else {
+    config.target = ctx.isServer ? 'node' : 'web'
+  }
 
   // Stop compilation early in a production build when an error is encountered.
   // This behavior isn't desirable in development due to how the HMR system

diff --git a/packages/next/bundles/package.json b/packages/next/bundles/package.json
@@ -2,7 +2,7 @@
   "dependencies": {
     "schema-utils3": "npm:[email protected]",
     "webpack-sources2": "npm:[email protected]",
-    "webpack5": "npm:webpack@5.43.0"
+    "webpack5": "npm:webpack@5.44.0"
   },
   "resolutions": {
     "browserslist": "4.16.6",

diff --git a/packages/next/bundles/yarn.lock b/packages/next/bundles/yarn.lock
@@ -18,16 +18,11 @@
     "@types/estree" "*"
     "@types/json-schema" "*"
 
-"@types/estree@*":
+"@types/estree@*", "@types/estree@^0.0.50":
   version "0.0.50"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.50.tgz#1e0caa9364d3fccd2931c3ed96fdbeaa5d4cca83"
   integrity sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==
 
-"@types/estree@^0.0.49":
-  version "0.0.49"
-  resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.49.tgz#3facb98ebcd4114a4ecef74e0de2175b56fd4464"
-  integrity sha512-K1AFuMe8a+pXmfHTtnwBvqoEylNKVeaiKYkjmcEAdytMQVJ/i9Fu7sc13GxgXdO49gkE7Hy8SyJonUZUn+eVaw==
-
 "@types/json-schema@*", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.7":
   version "7.0.8"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.8.tgz#edf1bf1dbf4e04413ca8e5b17b3b7d7d54b59818"
@@ -483,13 +478,13 @@ watchpack@^2.2.0:
     source-list-map "^2.0.1"
     source-map "^0.6.1"
 
-"webpack5@npm:webpack@5.43.0":
-  version "5.43.0"
-  resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.43.0.tgz#36a122d6e9bac3836273857f56ed7801d40c9145"
-  integrity sha512-ex3nB9uxNI0azzb0r3xGwi+LS5Gw1RCRSKk0kg3kq9MYdIPmLS6UI3oEtG7esBaB51t9I+5H+vHmL3htaxqMSw==
+"webpack5@npm:webpack@5.44.0":
+  version "5.44.0"
+  resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.44.0.tgz#97b13a02bd79fb71ac6301ce697920660fa214a1"
+  integrity sha512-I1S1w4QLoKmH19pX6YhYN0NiSXaWY8Ou00oA+aMcr9IUGeF5azns+IKBkfoAAG9Bu5zOIzZt/mN35OffBya8AQ==
   dependencies:
     "@types/eslint-scope" "^3.7.0"
-    "@types/estree" "^0.0.49"
+    "@types/estree" "^0.0.50"
     "@webassemblyjs/ast" "1.11.1"
     "@webassemblyjs/wasm-edit" "1.11.1"
     "@webassemblyjs/wasm-parser" "1.11.1"