Add E2E testing framework (#98)

* Add playwright to project

* Add playright CI workflow

* Add rudimentary test for Dashboard

* Move playwright into base directory

* Update playwright.yml

* Update playwright.yml

* Update playwright.yml

* Update playwright.yml

* Update playwright.yml

* Update playwright.yml

* Update playwright.yml

* Update playwright.yml

* Fix bad yml

* Add wait-on for dev server pipeline

* Add failing test to check pipeline

* Remove failing test

* Add tests for dashboard quick navigation buttons

* Remove example test and add to readme

* Update playwright.yml

.github/workflows/playwright.yml

@@ -0,0 +1,33 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master, release/* ]
+  pull_request:
+    branches: [ main, release/* ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 16
+    - name: Install dependencies
+      run: cd src && npm ci --legacy-peer-deps
+    - name: Install wait-on
+      run: npm install -g wait-on
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run Dev Server and E2E Tests
+      run: |
+        cd src
+        npm start &
+        npx wait-on http://localhost:3000
+        npx playwright test src/e2e
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30

README.md

@@ -26,6 +26,18 @@ An embed directory will be created with the built UI, ready to be embedded in th
 
 Once you're happy with them, create a pull request **against the `release/vX.Y.Z` branch*** that you started from (***not `main`!***). Once merged, the CI will run and build the UI. It will then push it to a new tag that is compatible with the Go module rules. For example, the first customization to `v3.16.2` of Aim will end up in a tag named `v0.31602.1`.
 
+### How to run E2E tests?
+
+The E2E tests can be run locally by following these steps:
+1. Start the FastTrackML server
+2. Run the Aim UI in development mode (on localhost:3000)
+3. In another terminal, run `cd src/e2e`
+4. Run `npx playwright test` to run the test suite
+
+New tests can be added directly to the `src/e2e` directory. You may also run `npx playwright show-report` to see the test results.
+
+For a guide on how to write a test, see [Playwright's example tests](https://github.com/microsoft/playwright/blob/main/examples/todomvc/tests/integration.spec.ts).
+
 ### How is this all enforced?
 
 A GitHub app has been created with the `contents:write` permissions on this repo. Its App ID and private key are stored as secrets under the `restricted` environment. This environment is limited to the `main` and `release/v*` branches
@@ -239,3 +251,4 @@ do
   gh api /repos/G-Research/fasttrackml-ui-aim/rulesets/$rule | jq '[{name: .name, target: .target, conditions: .conditions, rules: .rules, bypass_actors: .bypass_actors}]'
 done | jq -s add
 ```
+

src/.gitignore

@@ -25,3 +25,7 @@ yarn-debug.log*
 yarn-error.log*
 
 Desktop
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

src/e2e/Dashboard.spec.ts

@@ -0,0 +1,35 @@
+import { test, expect } from '@playwright/test';
+
+const BASE_URL = 'http://localhost:3000';
+
+test.describe('Dashboard', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto(BASE_URL);
+  });
+
+  test('has title', async ({ page }) => {
+    await expect(page).toHaveTitle('FastTrackML (modern)');
+  });
+
+  test('active runs link redirects correctly', async ({ page }) => {
+    await page.getByText('Active Runs').click({ force: true });
+
+    await page.getByRole('code', { name: 'runs.active == True' });
+  });
+
+  test('archived runs link redirects correctly', async ({ page }) => {
+    await page.getByText('Archived Runs').click({ force: true });
+
+    await page.getByRole('code', { name: 'runs.active == False' });
+  });
+
+  test("last week's runs link redirects correctly", async ({ page }) => {
+    await page.getByText("Last Week's Runs").click({ force: true });
+
+    // The text varies depending on the current date:
+    // Example: datetime(2024, 6, 3) <= run.created_at < datetime(2024, 6, 10)
+    await page.getByRole('code', {
+      name: /datetime\(\d+, \d+, \d+\) <= run\.created_at < datetime\(\d+, \d+, \d+\)/,
+    });
+  });
+});

src/package.json

@@ -43,6 +43,7 @@
     "memoize-one": "^5.2.1",
     "moment": "^2.29.4",
     "monaco-editor": "^0.33.0",
+    "playwright": "^1.44.1",
     "plotly.js": "^2.7.0",
     "prop-types": "^15.7.2",
     "prosemirror-tables": "^1.1.1",
@@ -107,6 +108,7 @@
     ]
   },
   "devDependencies": {
+    "@playwright/test": "^1.44.1",
     "@storybook/addon-actions": "^6.5.12",
     "@storybook/addon-essentials": "^6.5.12",
     "@storybook/addon-interactions": "^6.5.12",

src/playwright.config.ts

@@ -0,0 +1,77 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './e2e',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});