Remove node-fetch (#11266)

* Revert "Revert "Remove `node-fetch` (#11122)" (#11261)" This reverts commit 7d122f4. * Update [[...variant]].page.tsx * Remove node-fetch references * Update package.json * Add cert checks to `fetchDataFromBFF` * Update bundleSizeConfig.js * Update bundleSizeConfig.js * Update bundleSizeConfig.js * Update package.json * update * yarn * yarn fix * Update getPageData.test.ts * undici update * cache * cache * cache * cache * Update bundleSizeConfig.js * cache * yarn cache updates * Merge branch 'latest' into revert-11261-revert-11122-remove-nodefetch * Create undici-npm-6.11.1-7a6c072033-1294806846.zip * cache * Create undici-npm-6.18.2-c1115b72ab-27c66d84d0.zip * Update undici * Consolidate Nextjs pages fetch * Update bundleSizeConfig.js * Remove `JEST_WORKER_ID` checks * Keep BFF path check * Create fsevents-patch-6b67494872-10.zip * Merge conflicts * Fix av-embeds agent import * fix getAgent mock import * Remove `isomorphic-fetch` dep * Remove types/node-fetch that crept back in * undici update * Update `optHeader` setting logic * Remove `accept` header setting when running locally * Re-add `isLocal` check
bbc · Oct 24, 2024 · df4043e · df4043e
1 parent cb27e04
commit df4043e
Show file tree

Hide file tree

Showing 63 changed files with 315 additions and 449 deletions.
diff --git a/.yarn/cache/@esbuild-darwin-x64-npm-0.21.5-491c2ae06c-10.zip b/.yarn/cache/@esbuild-darwin-x64-npm-0.21.5-491c2ae06c-10.zip
diff --git a/...nux-x64-gnu-npm-14.2.10-c8787ddfc6-10.zip → ...-darwin-x64-npm-14.2.10-a1396b1353-10.zip b/...nux-x64-gnu-npm-14.2.10-c8787ddfc6-10.zip → ...-darwin-x64-npm-14.2.10-a1396b1353-10.zip
diff --git a/.yarn/cache/@types-jsdom-npm-21.1.6-29114a659c-423fcaf3ec.zip b/.yarn/cache/@types-jsdom-npm-21.1.6-29114a659c-423fcaf3ec.zip
diff --git a/.yarn/cache/@types-jsdom-npm-21.1.7-b9d35cbe67-a5ee54aec8.zip b/.yarn/cache/@types-jsdom-npm-21.1.7-b9d35cbe67-a5ee54aec8.zip
diff --git a/.yarn/cache/@types-node-fetch-npm-2.6.11-55289bb1a8-c416df8f18.zip b/.yarn/cache/@types-node-fetch-npm-2.6.11-55289bb1a8-c416df8f18.zip
diff --git a/.yarn/cache/fsevents-patch-6b67494872-10.zip b/.yarn/cache/fsevents-patch-6b67494872-10.zip
diff --git a/...eware-npm-2.0.7-886a673e85-4a51bf612b.zip → ...eware-npm-2.0.6-3bb17658ee-768e7ae5a4.zip b/...eware-npm-2.0.7-886a673e85-4a51bf612b.zip → ...eware-npm-2.0.6-3bb17658ee-768e7ae5a4.zip
diff --git a/.yarn/cache/isomorphic-fetch-npm-3.0.0-bce711adff-568fe03075.zip b/.yarn/cache/isomorphic-fetch-npm-3.0.0-bce711adff-568fe03075.zip
diff --git a/...jake-npm-10.9.2-3bf2173aed-3be324708f.zip → ...jake-npm-10.8.7-1caf9b4534-ad1cfe3988.zip b/...jake-npm-10.9.2-3bf2173aed-3be324708f.zip → ...jake-npm-10.8.7-1caf9b4534-ad1cfe3988.zip
diff --git a/.yarn/cache/micromatch-npm-4.0.5-cfab5d7669-a749888789.zip b/.yarn/cache/micromatch-npm-4.0.5-cfab5d7669-a749888789.zip
diff --git a/.yarn/cache/micromatch-npm-4.0.8-c9570e4aca-6bf2a01672.zip b/.yarn/cache/micromatch-npm-4.0.8-c9570e4aca-6bf2a01672.zip
diff --git a/.yarn/cache/node-fetch-npm-2.6.12-48619ce9d6-370ed4d906.zip b/.yarn/cache/node-fetch-npm-2.6.12-48619ce9d6-370ed4d906.zip
diff --git a/.yarn/cache/whatwg-fetch-npm-3.6.20-a6f79b98c4-2b4ed92acd.zip b/.yarn/cache/whatwg-fetch-npm-3.6.20-a6f79b98c4-2b4ed92acd.zip
diff --git a/package.json b/package.json
@@ -45,14 +45,14 @@
     "cypress:adhoc": "yarn cypress:interactive -- --project ./AdHocCypress/.",
     "setupDevEnv": "cp envConfig/local.env .env && rm -rf envConfig/secret.env && ./scripts/checkSecretEnvVariables.sh",
     "dependencyFreshness": "node ./scripts/dependencyFreshness",
+    "dev": "yarn setupDevEnv && rm -rf build && run-p webpack:dev:client webpack:dev:server",
     "esmDependencyCheck": "node ./scripts/esmDependencyCheck.js",
-    "dev": "yarn setupDevEnv && rm -rf build && NODE_OPTIONS=--no-experimental-fetch run-p webpack:dev:client webpack:dev:server",
     "lighthouse": "./scripts/lighthouseRun.sh",
     "mostReadAndWatchedCollectionData": "node ./scripts/mostReadAndWatchedCollectionData",
     "postshrinkwrap": "test -z $CI && ./scripts/packagelockHttps.sh; git update-index --assume-unchanged .env",
     "prepare": "husky install",
     "preinstall": "node scripts/check-package-manager.js",
-    "start": "NODE_ENV=production node --no-experimental-fetch --max-old-space-size=3500 build/server.js",
+    "start": "NODE_ENV=production node --max-old-space-size=3500 build/server.js",
     "stop": "./scripts/killApp.sh",
     "storybook": "node .storybook/buildFontPreloads && storybook dev -p 9001 -c .storybook",
     "test": "yarn build && yarn test:local",
@@ -109,7 +109,6 @@
     "helmet": "7.1.0",
     "helmet-csp": "4.0.0",
     "intersection-observer": "0.12.2",
-    "isomorphic-fetch": "3.0.0",
     "jalaali-js": "1.2.7",
     "js-cookie": "3.0.5",
     "lru-cache": "11.0.1",
@@ -125,6 +124,7 @@
     "react-lazyload": "3.2.1",
     "react-router-config": "5.1.1",
     "react-router-dom": "5.3.4",
+    "undici": "6.20.1",
     "url-parse": "1.5.10",
     "uuid": "8.3.2",
     "winston": "patch:[email protected]#./patches/winston-file-descriptor.patch"
@@ -182,7 +182,6 @@
     "@types/js-cookie": "^3.0.3",
     "@types/jsdom": "^21.0.0",
     "@types/loadable__component": "5.13.9",
-    "@types/node-fetch": "^2.6.11",
     "@types/ramda": "0.30.0",
     "@types/react": "18.3.3",
     "@types/react-dom": "18.3.1",
@@ -235,7 +234,6 @@
     "minimist": "1.2.8",
     "mkdirp": "3.0.1",
     "mocha-junit-reporter": "2.2.1",
-    "node-fetch": "2.6.12",
     "npm-run-all2": "6.2.2",
     "prettier": "3.2.5",
     "retry": "0.13.1",

diff --git a/scripts/ampHtmlValidator/index.js b/scripts/ampHtmlValidator/index.js
@@ -1,5 +1,4 @@
 /* eslint-disable no-console */
-const fetch = require('isomorphic-fetch');
 const amphtmlValidator = require('amphtml-validator');
 const { getPageUrls } = require('../../cypress/support/helpers/getPageUrls');
 

diff --git a/scripts/ampHtmlValidator/index.test.js b/scripts/ampHtmlValidator/index.test.js
@@ -1,6 +1,3 @@
-const fetch = require('isomorphic-fetch');
-
-jest.mock('isomorphic-fetch');
 fetch.mockImplementation(() => ({ text: () => '<html amp></html>' }));
 const log = jest.spyOn(global.console, 'log');
 log.mockImplementation(jest.fn);

diff --git a/scripts/bundleSize/bundleSizeConfig.js b/scripts/bundleSize/bundleSizeConfig.js
@@ -7,5 +7,5 @@
  * We are allowing a variance of -5 on `MIN_SIZE` and +5 on `MAX_SIZE` to avoid the need for frequent changes, as bundle sizes can fluctuate
  */
 
-export const MIN_SIZE = 646 - 5;
-export const MAX_SIZE = 1186 + 5;
+export const MIN_SIZE = 635 - 5;
+export const MAX_SIZE = 1175 + 5;
diff --git a/scripts/dependencyFreshness.js b/scripts/dependencyFreshness.js
@@ -1,7 +1,6 @@
 /* eslint-disable no-console */
 const { exec } = require('child_process');
 const fs = require('fs');
-const fetch = require('node-fetch');
 const { dependencies, devDependencies } = require('../package.json');
 
 const allDependencies = { ...dependencies, ...devDependencies };

diff --git a/scripts/linkeyTest.js b/scripts/linkeyTest.js
@@ -1,5 +1,3 @@
-import fetch from 'node-fetch';
-
 /* eslint-disable no-undef */
 const getServiceName = serviceConfig => {
   if (serviceConfig.default) {

diff --git a/scripts/mostReadAndWatchedCollectionData.js b/scripts/mostReadAndWatchedCollectionData.js
@@ -1,6 +1,7 @@
 const fs = require('fs');
-const { Agent } = require('https');
-const fetch = require('node-fetch');
+const fsPromises = require('fs/promises');
+const { Agent } = require('undici');
+const { createSecureContext } = require('tls');
 const allServices = require('../cypress/support/config/settings')();
 
 const badServices = [
@@ -19,18 +20,37 @@ const services = Object.keys(allServices);
 
 console.log('services', services);
 
+const loadCerts = ({ caPath, certChainPath, keyPath }) =>
+  Promise.all([
+    fsPromises.readFile(caPath, 'UTF-8'),
+    fsPromises.readFile(certChainPath, 'UTF-8'),
+    fsPromises.readFile(keyPath, 'UTF-8'),
+  ]);
+
 const fetchWithCert = async (url, options) => {
-  const cert = fs.readFileSync(
-    process.env.CERT_CHAIN_PATH || '/etc/pki/tls/certs/client.crt',
-  );
-  const ca = fs.readFileSync(
-    process.env.CA_PATH || '/etc/pki/tls/certs/ca-bundle.crt',
-  );
-  const key = fs.readFileSync(
-    process.env.KEY_PATH || '/etc/pki/tls/private/client.key',
-  );
-  const agent = new Agent({ cert, ca, key });
-  return fetch(url, { agent, ...options });
+  const caPath = process.env.CA_PATH || '/etc/pki/tls/certs/ca-bundle.crt';
+  const certChainPath =
+    process.env.CERT_CHAIN_PATH || '/etc/pki/tls/certs/client.crt';
+  const keyPath = process.env.KEY_PATH || '/etc/pki/tls/private/client.key';
+
+  const [ca, certChain, key] = await loadCerts({
+    caPath,
+    certChainPath,
+    keyPath,
+  });
+
+  return fetch(url, {
+    dispatcher: new Agent({
+      connect: {
+        secureContext: createSecureContext({
+          cert: certChain,
+          key,
+          ca,
+        }),
+      },
+    }),
+    ...options,
+  });
 };
 
 const timeTable = [];
@@ -41,9 +61,9 @@ const collectResults = async (link, service, type) => {
   await fetchWithCert(link).then(response => {
     if (response.ok) {
       response.json().then(json => {
-        const jsondata = json.hasOwnProperty('data') ? json.data : json;
-        if (!(json.hasOwnProperty('status') && json.status === '404')) {
-          if (!jsondata || !jsondata.hasOwnProperty('generated')) {
+        const jsondata = 'data' in json ? json.data : json;
+        if (!('status' in json && json.status === '404')) {
+          if (!jsondata || !('generated' in jsondata)) {
             console.log('json where no data', jsondata);
           }
           const generatedDateTime = new Date(jsondata.generated);
@@ -52,9 +72,8 @@ const collectResults = async (link, service, type) => {
           const minutesSinceGenerated = Math.floor(
             timeDifference / (1000 * 60),
           );
-          const records = jsondata.hasOwnProperty('items')
-            ? jsondata.items
-            : jsondata.records;
+          const records =
+            'items' in jsondata ? jsondata.items : jsondata.records;
           const counts = records
             .map(record => record.count)
             .slice(0, 10)
@@ -97,7 +116,7 @@ const collectResults = async (link, service, type) => {
 
   services
     .filter(service => !badServices.includes(service))
-    .forEach((service, i) => {
+    .forEach(service => {
       let serviceToCall = service;
       if (servicesWithVariants.some(variant => service.includes(variant))) {
         serviceToCall = service.replace(/([A-Z])/, '_$1').toLowerCase();

diff --git a/scripts/mostReadCollectionTimes.js b/scripts/mostReadCollectionTimes.js
@@ -1,6 +1,7 @@
 const fs = require('fs');
-const { Agent } = require('https');
-const fetch = require('node-fetch');
+const fsPromises = require('fs/promises');
+const { Agent } = require('undici');
+const { createSecureContext } = require('tls');
 const allServices = require('../cypress/support/config/settings')();
 
 const badServices = [
@@ -19,20 +20,38 @@ const services = Object.keys(allServices);
 
 console.log('services', services);
 
+const loadCerts = ({ caPath, certChainPath, keyPath }) =>
+  Promise.all([
+    fsPromises.readFile(caPath, 'UTF-8'),
+    fsPromises.readFile(certChainPath, 'UTF-8'),
+    fsPromises.readFile(keyPath, 'UTF-8'),
+  ]);
+
 const fetchWithCert = async (url, options) => {
-  const cert = fs.readFileSync(
-    process.env.CERT_CHAIN_PATH || '/etc/pki/tls/certs/client.crt',
-  );
-  const ca = fs.readFileSync(
-    process.env.CA_PATH || '/etc/pki/tls/certs/ca-bundle.crt',
-  );
-  const key = fs.readFileSync(
-    process.env.KEY_PATH || '/etc/pki/tls/private/client.key',
-  );
-  const agent = new Agent({ cert, ca, key });
-  return fetch(url, { agent, ...options });
-};
+  const caPath = process.env.CA_PATH || '/etc/pki/tls/certs/ca-bundle.crt';
+  const certChainPath =
+    process.env.CERT_CHAIN_PATH || '/etc/pki/tls/certs/client.crt';
+  const keyPath = process.env.KEY_PATH || '/etc/pki/tls/private/client.key';
+
+  const [ca, certChain, key] = await loadCerts({
+    caPath,
+    certChainPath,
+    keyPath,
+  });
 
+  return fetch(url, {
+    dispatcher: new Agent({
+      connect: {
+        secureContext: createSecureContext({
+          cert: certChain,
+          key,
+          ca,
+        }),
+      },
+    }),
+    ...options,
+  });
+};
 const timeTable = [];
 
 let csvContents;
@@ -41,9 +60,9 @@ const collectResults = async (link, service, type) => {
   await fetchWithCert(link).then(response => {
     if (response.ok) {
       response.json().then(json => {
-        const jsondata = json.hasOwnProperty('data') ? json.data : json;
-        if (!(json.hasOwnProperty('status') && json.status === '404')) {
-          if (!jsondata || !jsondata.hasOwnProperty('generated')) {
+        const jsondata = 'data' in json ? json.data : json;
+        if (!('status' in json && json.status === '404')) {
+          if (!jsondata || !('generated' in jsondata)) {
             console.log('json where no data', jsondata);
           }
           const generatedDateTime = new Date(jsondata.generated);
@@ -52,9 +71,8 @@ const collectResults = async (link, service, type) => {
           const minutesSinceGenerated = Math.floor(
             timeDifference / (1000 * 60),
           );
-          const records = jsondata.hasOwnProperty('items')
-            ? jsondata.items
-            : jsondata.records;
+          const records =
+            'items' in jsondata ? jsondata.items : jsondata.records;
           const counts = records
             .map(record => record.count)
             .slice(0, 10)
@@ -81,21 +99,23 @@ const collectResults = async (link, service, type) => {
           if (timeTable.length === (services.length - badServices.length) * 2) {
             csvContents =
               'service, type, link, generated, timeSinceGenerated, lastRecordTimeStamp, firstRecordTimeStamp, sequence, totalRecords, rank1, rank2, rank3, rank4, rank5, rank6, rank7, rank8, rank9, rank10';
-              timeTable.sort((a, b) => {
+            timeTable
+              .sort((a, b) => {
                 if (a.service > b.service) {
                   return 1;
                 }
-                else if (a.service < b.service) {
+                if (a.service < b.service) {
                   return -1;
                 }
-                else if (a.type < b.type) {
+                if (a.type < b.type) {
                   return -1;
                 }
-                else if (a.type > b.type) {
+                if (a.type > b.type) {
                   return 1;
                 }
                 return 0;
-              }).forEach(result => {
+              })
+              .forEach(result => {
                 csvContents += `\n${result.service},${result.type},${result.link},${result.generated},${result.minutesSinceGenerated},${result.lastRecordTimestamp},${result.firstRecordTimestamp},${result.sequence},${result.totalRecords},${result.counts}`;
               });
             fs.writeFileSync('./mostReadCollectionTimes.csv', csvContents);
@@ -111,7 +131,7 @@ const collectResults = async (link, service, type) => {
 
   services
     .filter(service => !badServices.includes(service))
-    .forEach((service, i) => {
+    .forEach(service => {
       let serviceToCall = service;
       if (servicesWithVariants.some(variant => service.includes(variant))) {
         serviceToCall = service.replace(/([A-Z])/, '_$1').toLowerCase();
@@ -139,4 +159,4 @@ const collectResults = async (link, service, type) => {
     });
 
   await Promise.all(promises);
-})();
+})();
diff --git a/src/app/legacy/containers/ConsentBanner/useConsentBanners/setCookieOven.js b/src/app/legacy/containers/ConsentBanner/useConsentBanners/setCookieOven.js
@@ -1,4 +1,3 @@
-import 'isomorphic-fetch';
 import path from 'ramda/src/path';
 
 import getCookieOvenEndpoints from './getCookieOvenEndpoints';

diff --git a/src/app/legacy/containers/RadioSchedule/Canonical/index.jsx b/src/app/legacy/containers/RadioSchedule/Canonical/index.jsx
@@ -1,6 +1,5 @@
 import React, { useEffect, useState, useContext } from 'react';
 import { useTheme } from '@emotion/react';
-import 'isomorphic-fetch';
 import styled from '@emotion/styled';
 import moment from 'moment';
 import {

diff --git a/src/app/lib/analyticsUtils/sendBeacon/index.js b/src/app/lib/analyticsUtils/sendBeacon/index.js
@@ -1,7 +1,6 @@
 import onClient from '../../utilities/onClient';
 import nodeLogger from '../../logger.node';
 import { ATI_LOGGING_ERROR } from '../../logger.const';
-import 'isomorphic-fetch';
 
 const logger = nodeLogger(__filename);
 

diff --git a/src/app/models/types/fetch.ts b/src/app/models/types/fetch.ts
@@ -1,3 +1,7 @@
+import { Agent } from 'undici';
+
 export interface FetchError extends Error {
   status: number;
 }
+
+export type GetAgent = () => Promise<Agent>;
diff --git a/src/app/models/types/initialData.ts b/src/app/models/types/initialData.ts
@@ -1,3 +1,4 @@
+import { GetAgent } from './fetch';
 import { PageTypes, Services, Toggles, Variants } from './global';
 
 export type InitialDataProps = {
@@ -6,4 +7,5 @@ export type InitialDataProps = {
   pageType: PageTypes;
   variant?: Variants;
   toggles?: Toggles;
+  getAgent?: GetAgent;
 };
diff --git a/src/app/models/types/pageDataParams.ts b/src/app/models/types/pageDataParams.ts
@@ -1,5 +1,5 @@
 import { ParsedUrlQuery } from 'querystring';
-import { Services, Variants } from './global';
+import { PageTypes, Services, Variants } from './global';
 
 export default interface PageDataParams extends ParsedUrlQuery {
   id: string;
@@ -9,4 +9,5 @@ export default interface PageDataParams extends ParsedUrlQuery {
   // eslint-disable-next-line camelcase
   renderer_env?: string;
   resolvedUrl: string;
+  pageType: PageTypes;
 }
diff --git a/src/app/pages/HomePage/index.stories.tsx b/src/app/pages/HomePage/index.stories.tsx
@@ -2,7 +2,6 @@
 import React, { useEffect, useState } from 'react';
 import Url from 'url-parse';
 import { HOME_PAGE } from '#app/routes/utils/pageTypes';
-import fetch from 'node-fetch';
 import { Curation } from '#app/models/types/curationData';
 import { Services } from '#app/models/types/global';
 import withServicesDecorator from '#storybook/withServicesDecorator';
@@ -44,7 +43,7 @@ const Component = ({ service, variant }: StoryProps) => {
   useEffect(() => {
     const loadPageData = async () => {
       const response = await fetch(
-        new Url(`data/${service}/homePage/index.json`),
+        new Url(`data/${service}/homePage/index.json`).toString(),
       );
       const { data } = await response.json();
 

diff --git a/src/app/pages/MostReadPage/index.stories.jsx b/src/app/pages/MostReadPage/index.stories.jsx
@@ -1,5 +1,4 @@
 import React, { useEffect, useState } from 'react';
-import fetch from 'node-fetch';
 import Url from 'url-parse';
 import { BrowserRouter } from 'react-router-dom';
 import { MOST_READ_PAGE } from '#app/routes/utils/pageTypes';