use fresher node and debian version

.github/workflows/docker_latest.yml

@@ -50,7 +50,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         image:
           # We build two images, `grist-oss` and `grist`.
           # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images

.github/workflows/main.yml

@@ -18,19 +18,19 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         tests:
           - ':lint:python:client:common:smoke:stubs:'
           - ':server-1-of-2:'
           - ':server-2-of-2:'
-          - ':nbrowser-^[A-G]:'
-          - ':nbrowser-^[H-L]:'
-          - ':nbrowser-^[M-O]:'
-          - ':nbrowser-^[P-S]:'
-          - ':nbrowser-^[^A-S]:'
+          - ':nbrowser-^[A-D]:'
+          - ':nbrowser-^[E-L]:'
+          - ':nbrowser-^[M-N]:'
+          - ':nbrowser-^[O-R]:'
+          - ':nbrowser-^[^A-R]:'
         include:
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 18.x
+            node-version: 22.x
             python-version: '3.10'
     steps:
       - uses: actions/checkout@v3
@@ -125,6 +125,7 @@ jobs:
           ARTIFACT_NAME=logs-$(echo $TESTS | sed 's/[^-a-zA-Z0-9]/_/g')
           echo "Artifact name is '$ARTIFACT_NAME'"
           echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV
+          mkdir -p $TESTDIR
           find $TESTDIR -iname "*.socket" -exec rm {} \;
         env:
           TESTS: ${{ matrix.tests }}

Dockerfile

@@ -10,7 +10,7 @@ FROM scratch AS ext
 ## Javascript build stage
 ################################################################################
 
-FROM node:18-buster AS builder
+FROM node:22-bookworm AS builder
 
 # Install all node dependencies.
 WORKDIR /grist
@@ -45,19 +45,30 @@ RUN \
 ## Python collection stage
 ################################################################################
 
-# Fetch python3.11 and python2.7
-FROM python:3.11-slim-buster AS collector
+# Fetch python3.11
+FROM python:3.11-slim-bookworm AS collector-py3
+ADD sandbox/requirements3.txt requirements3.txt
+RUN \
+  pip3 install -r requirements3.txt
 
-# Install all python dependencies.
+# Fetch <shame>python2.7</shame>
+# This is to support users with old documents.
+# If you have documents with python2.7 formulas, try switching
+# to python3 in the document settings. It'll probably work fine!
+# And we'll be forced to turn off python2 support eventually,
+# the workarounds needed to keep it are getting silly.
+# It doesn't exist in recent Debian, so we need to reach back
+# to buster.
+FROM python:2.7-slim-buster AS collector-py2
 ADD sandbox/requirements.txt requirements.txt
-ADD sandbox/requirements3.txt requirements3.txt
 RUN \
   apt update && \
   apt install -y --no-install-recommends python2 python-pip python-setuptools \
   build-essential libxml2-dev libxslt-dev python-dev zlib1g-dev && \
   pip2 install wheel && \
   pip2 install -r requirements.txt && \
-  pip3 install -r requirements3.txt
+  pip2 install six && \
+  find /usr/lib -iname "libffi.so.6*" -exec cp {} /usr/local/lib \;
 
 ################################################################################
 ## Sandbox collection stage
@@ -66,14 +77,16 @@ RUN \
 # Fetch gvisor-based sandbox. Note, to enable it to run within default
 # unprivileged docker, layers of protection that require privilege have
 # been stripped away, see https://github.com/google/gvisor/issues/4371
+# The sandbox binary is built on buster, but remains compatible with recent
+# Debian.
 FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox
 
 ################################################################################
 ## Run-time stage
 ################################################################################
 
 # Now, start preparing final image.
-FROM node:18-buster-slim
+FROM node:22-bookworm-slim
 
 # Install libexpat1, libsqlite3-0 for python3 library binary dependencies.
 # Install pgrep for managing gvisor processes.
@@ -91,13 +104,23 @@ COPY --from=builder /grist/node_modules /grist/node_modules
 COPY --from=builder /grist/_build /grist/_build
 COPY --from=builder /grist/static /grist/static-built
 
-# Copy python files.
-COPY --from=collector /usr/bin/python2.7 /usr/bin/python2.7
-COPY --from=collector /usr/lib/python2.7 /usr/lib/python2.7
-COPY --from=collector /usr/local/lib/python2.7 /usr/local/lib/python2.7
-COPY --from=collector /usr/local/bin/python3.11 /usr/bin/python3.11
-COPY --from=collector /usr/local/lib/python3.11 /usr/local/lib/python3.11
-COPY --from=collector /usr/local/lib/libpython3.11.* /usr/local/lib/
+# Copy python2 files.
+COPY --from=collector-py2 /usr/bin/python2.7 /usr/bin/python2.7
+COPY --from=collector-py2 /usr/lib/python2.7 /usr/lib/python2.7
+COPY --from=collector-py2 /usr/local/lib/python2.7 /usr/local/lib/python2.7
+# Make a small python2 tweak so that material in /usr/local/lib is found.
+RUN \
+  mkdir /etc/python2.7 && \
+  echo "import sys\nsys.path.append('/usr/local/lib/python2.7/site-packages')" > /etc/python2.7/sitecustomize.py
+# Copy across an older libffi library binary needed by python2.
+# We moved it a bit sleazily to a predictable location to avoid awkward
+# architecture-dependent logic.
+COPY --from=collector-py2 /usr/local/lib/libffi.so.6* /usr/local/lib
+
+# Copy python3 files.
+COPY --from=collector-py3 /usr/local/bin/python3.11 /usr/bin/python3.11
+COPY --from=collector-py3 /usr/local/lib/python3.11 /usr/local/lib/python3.11
+COPY --from=collector-py3 /usr/local/lib/libpython3.11.* /usr/local/lib/
 # Set default to python3
 RUN \
   ln -s /usr/bin/python3.11 /usr/bin/python && \

app/client/components/GristClientSocket.ts

@@ -71,7 +71,13 @@ export class GristClientSocket {
   }
 
   private _createWSSocket() {
-    if (typeof WebSocket !== 'undefined') {
+    // We used to check if WebSocket was defined here, and use it
+    // if so, secure in the fact that we were in the browser and
+    // the browser would pass along cookie information. But recent
+    // node defines WebSocket, so we narrow down this path to when
+    // a global document is defined (window doesn't work because
+    // some tests mock it).
+    if (typeof document !== 'undefined') {
       this._wsSocket = new WebSocket(this._url);
     } else {
       this._wsSocket = new WS(this._url, undefined, this._options);

test/common/NumberFormat.ts

@@ -6,7 +6,17 @@ describe("NumberFormat", function() {
     locale: 'en-US'
   };
 
+  // useGrouping became more nuanced in recent node.
+  // Its old 'true' value may now be 'always' or 'auto'.
+  const useGroupingAlways = buildNumberFormat(
+    {numMode: 'decimal'},
+    defaultDocSettings
+  ).resolvedOptions().useGrouping as boolean|string;
+  const useGroupingAuto = (useGroupingAlways === 'always') ? 'auto' : true;
+
   it("should convert Grist options into Intr.NumberFormat", function() {
+    assert.include([true, 'always'], String(useGroupingAlways));
+
     assert.ownInclude(buildNumberFormat({}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 10,
@@ -17,21 +27,21 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 0,
       maximumFractionDigits: 3,
       style: 'decimal',
-      useGrouping: true,
+      useGrouping: useGroupingAlways,
     });
     assert.ownInclude(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 0,
       // style: 'percent',  // In node v14.17.0 style is 'decimal' (unclear why)
                             // so we check final formatting instead in this case.
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
     });
     assert.equal(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).format(0.5), '50%');
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'USD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'scientific'}, defaultDocSettings).resolvedOptions(), {
@@ -73,42 +83,42 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'EUR',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-NZ'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'NZD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'de-CH'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'CHF',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'es-AR'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'ARS',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'zh-TW'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'TWD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-AU'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'AUD',
     });
   });

test/server/lib/DocApi.ts

@@ -76,6 +76,8 @@ function makeConfig(username: string): AxiosRequestConfig {
 }
 
 describe('DocApi', function () {
+  const webhooksTestPort = Number(process.env.WEBHOOK_TEST_PORT || 34365);
+
   this.timeout(30000);
   testUtils.setTmpLogLevel('error');
   let oldEnv: testUtils.EnvironmentSnapshot;
@@ -121,7 +123,7 @@ describe('DocApi', function () {
       homeUrl = serverUrl = home.serverUrl;
       hasHomeApi = true;
     });
-    testDocApi();
+    testDocApi({webhooksTestPort});
   });
 
   describe('With GRIST_ANON_PLAYGROUND disabled', async () => {
@@ -157,7 +159,7 @@ describe('DocApi', function () {
         homeUrl = serverUrl = home.serverUrl;
         hasHomeApi = true;
       });
-      testDocApi();
+      testDocApi({webhooksTestPort});
     });
 
     describe('behind a reverse-proxy', function () {
@@ -206,7 +208,7 @@ describe('DocApi', function () {
 
         after(() => tearDown(proxy, [home, docs]));
 
-        testDocApi();
+        testDocApi({webhooksTestPort});
       });
 
       async function testCompareDocs(proxy: TestServerReverseProxy, home: TestServer) {
@@ -261,7 +263,7 @@ describe('DocApi', function () {
         serverUrl = docs.serverUrl;
         hasHomeApi = false;
       });
-      testDocApi();
+      testDocApi({webhooksTestPort});
     });
   }
 
@@ -323,7 +325,10 @@ describe('DocApi', function () {
 });
 
 // Contains the tests. This is where you want to add more test.
-function testDocApi() {
+function testDocApi(settings: {
+  webhooksTestPort: number,
+}) {
+  const { webhooksTestPort } = settings;
   let chimpy: AxiosRequestConfig, kiwi: AxiosRequestConfig,
     charon: AxiosRequestConfig, nobody: AxiosRequestConfig, support: AxiosRequestConfig;
 
@@ -3478,13 +3483,20 @@ function testDocApi() {
   });
 
   describe('webhooks related endpoints', async function () {
-    const serving: Serving = await serveSomething(app => {
-      app.use(express.json());
-      app.post('/200', ({body}, res) => {
-        res.sendStatus(200);
-        res.end();
-      });
-    }, webhooksTestPort);
+    let serving: Serving;
+    before(async function () {
+      serving = await serveSomething(app => {
+        app.use(express.json());
+        app.post('/200', ({body}, res) => {
+          res.sendStatus(200);
+          res.end();
+        });
+      }, webhooksTestPort);
+    });
+
+    after(async function () {
+      await serving.shutdown();
+    });
 
     /*
       Regression test for old _subscribe endpoint. /docs/{did}/webhooks should be used instead to subscribe
@@ -3577,7 +3589,8 @@ function testDocApi() {
           }
         }]
       },
-        403, /Column not found notExisting/);
+        // this check was previously just wrong, was the test not running somehow??
+        404, /Column not found "notExisting"/);
 
     });
 
@@ -5385,8 +5398,6 @@ async function getWorkspaceId(api: UserAPIImpl, name: string) {
   return workspaces.find((w) => w.name === name)!.id;
 }
 
-const webhooksTestPort = Number(process.env.WEBHOOK_TEST_PORT || 34365);
-
 async function setupDataDir(dir: string) {
   // we'll be serving Hello.grist content for various document ids, so let's make copies of it in
   // tmpDir

test/server/lib/ProxyAgent.ts

@@ -84,15 +84,17 @@ describe("ProxyAgent", function () {
     it("should report error when proxy fails", async function() {
       // if the proxy isn't listening, fetches produces error messages.
       await testProxyServer.dispose();
+      // Error message depends a little on node version.
       const logMessages2 = await captureLog('warn', async () => {
-        await assert.isRejected(testFetch('/200'), /ECONNREFUSED/);
-        await assert.isRejected(testFetch('/404'), /ECONNREFUSED/);
+        await assert.isRejected(testFetch('/200'), /(request.*failed)|(ECONNREFUSED)/);
+        await assert.isRejected(testFetch('/404'), /(request.*failed)|(ECONNREFUSED)/);
       });
 
       // We rely on "ProxyAgent error" message to detect issues with the proxy server.
+      // Error message depends a little on node version.
       assertMatchArray(logMessages2, [
-        /warn: ProxyAgent error.*ECONNREFUSED/,
-        /warn: ProxyAgent error.*ECONNREFUSED/,
+        /warn: ProxyAgent error.*((request.*failed)|(ECONNREFUSED)|(AggregateError))/,
+        /warn: ProxyAgent error.*((request.*failed)|(ECONNREFUSED)|(AggregateError))/,
       ]);
     });
   });

test/server/lib/UnhandledErrors.ts

@@ -51,7 +51,8 @@ describe('UnhandledErrors', function() {
         }, 1000, 100);
 
         // We expect the server to be dead now.
-        await assert.isRejected(fetch(`${server.serverUrl}/status`), /failed.*ECONNREFUSED/);
+        // Error message depends a little on node version.
+        await assert.isRejected(fetch(`${server.serverUrl}/status`), /(request.*failed)|(ECONNREFUSED)/);
 
       } finally {
         await server.stop();