use fresher node and debian version

.github/workflows/docker_latest.yml

@@ -50,7 +50,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         image:
           # We build two images, `grist-oss` and `grist`.
           # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images

.github/workflows/main.yml

@@ -18,19 +18,19 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         tests:
           - ':lint:python:client:common:smoke:stubs:'
           - ':server-1-of-2:'
           - ':server-2-of-2:'
-          - ':nbrowser-^[A-G]:'
-          - ':nbrowser-^[H-L]:'
-          - ':nbrowser-^[M-O]:'
-          - ':nbrowser-^[P-S]:'
-          - ':nbrowser-^[^A-S]:'
+          - ':nbrowser-^[A-D]:'
+          - ':nbrowser-^[E-L]:'
+          - ':nbrowser-^[M-N]:'
+          - ':nbrowser-^[O-R]:'
+          - ':nbrowser-^[^A-R]:'
         include:
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 18.x
+            node-version: 22.x
             python-version: '3.10'
     steps:
       - uses: actions/checkout@v3
@@ -125,6 +125,7 @@ jobs:
           ARTIFACT_NAME=logs-$(echo $TESTS | sed 's/[^-a-zA-Z0-9]/_/g')
           echo "Artifact name is '$ARTIFACT_NAME'"
           echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV
+          mkdir -p $TESTDIR
           find $TESTDIR -iname "*.socket" -exec rm {} \;
         env:
           TESTS: ${{ matrix.tests }}

Dockerfile

@@ -10,7 +10,7 @@ FROM scratch AS ext
 ## Javascript build stage
 ################################################################################
 
-FROM node:18-buster AS builder
+FROM node:22-bookworm AS builder
 
 # Install all node dependencies.
 WORKDIR /grist
@@ -45,19 +45,30 @@ RUN \
 ## Python collection stage
 ################################################################################
 
-# Fetch python3.11 and python2.7
-FROM python:3.11-slim-buster AS collector
+# Fetch python3.11
+FROM python:3.11-slim-bookworm AS collector-py3
+ADD sandbox/requirements3.txt requirements3.txt
+RUN \
+  pip3 install -r requirements3.txt
 
-# Install all python dependencies.
+# Fetch <shame>python2.7</shame>
+# This is to support users with old documents.
+# If you have documents with python2.7 formulas, try switching
+# to python3 in the document settings. It'll probably work fine!
+# And we'll be forced to turn off python2 support eventually,
+# the workarounds needed to keep it are getting silly.
+# It doesn't exist in recent Debian, so we need to reach back
+# to buster.
+FROM python:2.7-slim-buster AS collector-py2
 ADD sandbox/requirements.txt requirements.txt
-ADD sandbox/requirements3.txt requirements3.txt
 RUN \
   apt update && \
   apt install -y --no-install-recommends python2 python-pip python-setuptools \
   build-essential libxml2-dev libxslt-dev python-dev zlib1g-dev && \
   pip2 install wheel && \
   pip2 install -r requirements.txt && \
-  pip3 install -r requirements3.txt
+  pip2 install six && \
+  find /usr/lib -iname "libffi.so.6*" -exec cp {} /usr/local/lib \;
 
 ################################################################################
 ## Sandbox collection stage
@@ -66,14 +77,16 @@ RUN \
 # Fetch gvisor-based sandbox. Note, to enable it to run within default
 # unprivileged docker, layers of protection that require privilege have
 # been stripped away, see https://github.com/google/gvisor/issues/4371
+# The sandbox binary is built on buster, but remains compatible with recent
+# Debian.
 FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox
 
 ################################################################################
 ## Run-time stage
 ################################################################################
 
 # Now, start preparing final image.
-FROM node:18-buster-slim
+FROM node:22-bookworm-slim
 
 # Install libexpat1, libsqlite3-0 for python3 library binary dependencies.
 # Install pgrep for managing gvisor processes.
@@ -91,13 +104,23 @@ COPY --from=builder /grist/node_modules /grist/node_modules
 COPY --from=builder /grist/_build /grist/_build
 COPY --from=builder /grist/static /grist/static-built
 
-# Copy python files.
-COPY --from=collector /usr/bin/python2.7 /usr/bin/python2.7
-COPY --from=collector /usr/lib/python2.7 /usr/lib/python2.7
-COPY --from=collector /usr/local/lib/python2.7 /usr/local/lib/python2.7
-COPY --from=collector /usr/local/bin/python3.11 /usr/bin/python3.11
-COPY --from=collector /usr/local/lib/python3.11 /usr/local/lib/python3.11
-COPY --from=collector /usr/local/lib/libpython3.11.* /usr/local/lib/
+# Copy python2 files.
+COPY --from=collector-py2 /usr/bin/python2.7 /usr/bin/python2.7
+COPY --from=collector-py2 /usr/lib/python2.7 /usr/lib/python2.7
+COPY --from=collector-py2 /usr/local/lib/python2.7 /usr/local/lib/python2.7
+# Make a small python2 tweak so that material in /usr/local/lib is found.
+RUN \
+  mkdir /etc/python2.7 && \
+  echo "import sys\nsys.path.append('/usr/local/lib/python2.7/site-packages')" > /etc/python2.7/sitecustomize.py
+# Copy across an older libffi library binary needed by python2.
+# We moved it a bit sleazily to a predictable location to avoid awkward
+# architecture-dependent logic.
+COPY --from=collector-py2 /usr/local/lib/libffi.so.6* /usr/local/lib
+
+# Copy python3 files.
+COPY --from=collector-py3 /usr/local/bin/python3.11 /usr/bin/python3.11
+COPY --from=collector-py3 /usr/local/lib/python3.11 /usr/local/lib/python3.11
+COPY --from=collector-py3 /usr/local/lib/libpython3.11.* /usr/local/lib/
 # Set default to python3
 RUN \
   ln -s /usr/bin/python3.11 /usr/bin/python && \

app/client/components/GristClientSocket.ts

@@ -71,7 +71,13 @@ export class GristClientSocket {
   }
 
   private _createWSSocket() {
-    if (typeof WebSocket !== 'undefined') {
+    // We used to check if WebSocket was defined here, and use it
+    // if so, secure in the fact that we were in the browser and
+    // the browser would pass along cookie information. But recent
+    // node defines WebSocket, so we narrow down this path to when
+    // a global document is defined (window doesn't work because
+    // some tests mock it).
+    if (typeof document !== 'undefined') {
       this._wsSocket = new WebSocket(this._url);
     } else {
       this._wsSocket = new WS(this._url, undefined, this._options);

test/common/NumberFormat.ts

@@ -6,7 +6,17 @@ describe("NumberFormat", function() {
     locale: 'en-US'
   };
 
+  // useGrouping became more nuanced in recent node.
+  // Its old 'true' value may now be 'always' or 'auto'.
+  const useGroupingAlways = buildNumberFormat(
+    {numMode: 'decimal'},
+    defaultDocSettings
+  ).resolvedOptions().useGrouping as boolean|string;
+  const useGroupingAuto = (useGroupingAlways === 'always') ? 'auto' : true;
+
   it("should convert Grist options into Intr.NumberFormat", function() {
+    assert.include([true, 'always'], String(useGroupingAlways));
+
     assert.ownInclude(buildNumberFormat({}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 10,
@@ -17,21 +27,21 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 0,
       maximumFractionDigits: 3,
       style: 'decimal',
-      useGrouping: true,
+      useGrouping: useGroupingAlways,
     });
     assert.ownInclude(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 0,
       maximumFractionDigits: 0,
       // style: 'percent',  // In node v14.17.0 style is 'decimal' (unclear why)
                             // so we check final formatting instead in this case.
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
     });
     assert.equal(buildNumberFormat({numMode: 'percent'}, defaultDocSettings).format(0.5), '50%');
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, defaultDocSettings).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'USD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'scientific'}, defaultDocSettings).resolvedOptions(), {
@@ -73,42 +83,42 @@ describe("NumberFormat", function() {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'EUR',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-NZ'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'NZD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'de-CH'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'CHF',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'es-AR'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'ARS',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'zh-TW'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'TWD',
     });
     assert.ownInclude(buildNumberFormat({numMode: 'currency'}, {locale: 'en-AU'}).resolvedOptions(), {
       minimumFractionDigits: 2,
       maximumFractionDigits: 2,
       style: 'currency',
-      useGrouping: true,
+      useGrouping: useGroupingAuto,
       currency: 'AUD',
     });
   });

test/nbrowser/Importer.ts

@@ -19,6 +19,10 @@ describe('Importer', function() {
     // have tests go faster. Each successful test case should leave the document unchanged.
     if (!docUrl || !await gu.testCurrentUrl(docUrl)) {
       const session = await gu.session().teamSite.login();
+      // TODO: tests check colors literally, so need to be in
+      // light theme - but calling gu.setGristTheme results in
+      // some problems so right now if you are a dev you just
+      // need to run these tests in light mode, sorry.
       await session.tempDoc(cleanup, 'Hello.grist');
       docUrl = await driver.getCurrentUrl();
     }
@@ -450,9 +454,9 @@ describe('Importer', function() {
       assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true);
 
       // Check that the merge field select button has a red outline.
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(208, 2, 27)'
+        /solid rgb\(208, 2, 27\)/
       );
 
       // Select a merge field, and check that the red outline is gone.
@@ -461,9 +465,9 @@ describe('Importer', function() {
         '.test-multi-select-menu .test-multi-select-menu-option',
         /Name/
       ).click();
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(217, 217, 217)'
+        /solid rgb\(217, 217, 217\)/
       );
       // Hide dropdown
       await gu.sendKeys(Key.ESCAPE);
@@ -584,9 +588,9 @@ describe('Importer', function() {
       await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click();
 
       // Check that it failed, and that the merge fields select button is outlined in red.
-      assert.equal(
+      assert.match(
         await driver.find('.test-importer-merge-fields-select').getCssValue('border'),
-        '1px solid rgb(208, 2, 27)'
+        /solid rgb\(208, 2, 27\)/
       );
       assert.equal(
         await driver.find('.test-importer-source-selected .test-importer-from').getText(),

test/nbrowser/gristUtils.ts

@@ -1464,6 +1464,7 @@ export function revertChanges(test: () => Promise<void>, invariant: () => any =
 export async function redo(optCount: number = 1, optTimeout?: number) {
   for (let i = 0; i < optCount; ++i) {
     await driver.find('.test-redo').doClick();
+    await waitForServer(optTimeout);
   }
   await waitForServer(optTimeout);
 }
@@ -2781,11 +2782,11 @@ export function addSamplesForSuite(includeTutorial = false) {
 }
 
 export async function openAccountMenu() {
-  await driver.findWait('.test-dm-account', 1000).click();
+  await driver.findWait('.test-dm-account', 2000).click();
   // Since the AccountWidget loads orgs and the user data asynchronously, the menu
   // can expand itself causing the click to land on a wrong button.
   await waitForServer();
-  await driver.findWait('.test-site-switcher-org', 1000);
+  await driver.findWait('.test-site-switcher-org', 2000);
   await driver.sleep(250);  // There's still some jitter (scroll-bar? other user accounts?)
 }
 

test/nbrowser/importerTestUtils.ts

@@ -35,7 +35,14 @@ export const getPreviewDiffCellValues = stackWrapFunc(async (cols: number[], row
 
 // Helper that waits for the diff preview to finish loading.
 export const waitForDiffPreviewToLoad = stackWrapFunc(async (): Promise<void> => {
+  await gu.waitForServer();
   await driver.wait(() => driver.find('.test-importer-preview').isPresent(), 5000);
+  await gu.waitToPass(async () => {
+    const preview = (await getPreviewDiffCellValues([0], [1]))[0];
+    if (preview[0] === undefined && preview[1] === undefined) {
+      throw new Error('sometimes data is a little slow to show up?');
+    }
+  }, 2000);
 });
 
 // Helper that gets the list of visible column matching rows to the left of the preview.