Merge pull request #5343 from Gustry/flaky-tests

Tests - Make flaky tests failing

.github/workflows/e2e_tests.yml

@@ -44,6 +44,7 @@ jobs:
       PLAYWRIGHT_FORCE_TTY: true
       PLAYWRIGHT_LIST_PRINT_STEPS: true
       PLAYWRIGHT_JSON_OUTPUT_DIR: ${{ github.workspace }}/tests/end2end/playwright-report
+      PLAYWRIGHT_OPTIONS: --project=end2end --fail-on-flaky-tests
       FORCE_COLOR: true
     steps:
 
@@ -199,7 +200,7 @@ jobs:
         env:
           PLAYWRIGHT_JSON_OUTPUT_NAME: tests-results-readonly.json
         run: |
-          npx playwright test --grep @readonly --project=end2end
+          npx playwright test --grep @readonly ${{ env.PLAYWRIGHT_OPTIONS }}
 
       - name: Prepare the database diff from Playwright read only
         run: |
@@ -242,7 +243,7 @@ jobs:
           PLAYWRIGHT_JSON_OUTPUT_NAME: tests-results-not-readonly.json
         if: success() || steps.test-playwright-read-only.conclusion == 'failure'
         run: |
-          npx playwright test --workers 1 --grep-invert @readonly --project=end2end
+          npx playwright test --workers 1 --grep-invert @readonly ${{ env.PLAYWRIGHT_OPTIONS }}
 
       - name: Notify in case of playwright failure, from mainstream branches only
         # Check for playwright tests with and without tags "readonly"

tests/end2end/playwright/globals.js

@@ -48,6 +48,7 @@ async function CatchErrors(page, layersInTreeView = 0) {
  * @param {boolean} mapMustLoad If the loading of the map must be successful or not. Some error might be triggered when loading the map, on purpose.
  * @param {int} layersInTreeView The number of layers to find in the treeview if the map is on error.
  * @param {boolean} waitForGetLegendGraphics
+ * @deprecated Use Project page instead and migrate the test to use proper methods
  */
 export async function gotoMap(url, page, mapMustLoad = true, layersInTreeView = 0, waitForGetLegendGraphics = true) {
     // TODO keep this function synchronized with the Cypress equivalent

tests/end2end/playwright/landing_page_config.spec.js

@@ -3,7 +3,9 @@ import { test, expect } from '@playwright/test';
 import {HomePage} from "./pages/homepage";
 import {AdminPage} from "./pages/admin";
 
-test.describe('Landing page content', () => {
+test.describe('Landing page content', {
+    tag: ['@write'],
+}, () => {
 
     test('Fill form & check content', async ({ browser }) => {
 

tests/end2end/playwright/mock/README.md

@@ -0,0 +1,15 @@
+# Mock
+
+### Helper
+
+To save request response in a file for mocking, you can copy/paste this snippet temporary.
+
+Please consider using a low size PDF file, for testing, if there isn't test done on the PDF file itself, like :
+* [tests/qgis-projects/tests/media/test.pdf](../../../qgis-projects/tests/media/test.pdf)
+* [tests/end2end/playwright/mock/playwright-test.pdf](./playwright-test.pdf)
+
+```js
+const downloadPromise = page.waitForEvent('download');
+const download = await downloadPromise;
+await download.saveAs(path.join(__dirname, 'mock/print_in_project_projection/baselayer/Paysage_A4.pdf'));
+```

tests/end2end/playwright/print_in_project_projection.spec.js

@@ -3,17 +3,6 @@ import * as path from 'path';
 import { test, expect } from '@playwright/test';
 import { gotoMap, expectParametersToContain } from './globals';
 
-/**
- * To save request response in a file for mocking, you can copy/paste this snippet temporary.
- * Please consider using a low size PDF file, for testing, if there isn't test done on the PDF file itself, like :
- * tests/qgis-projects/tests/media/test.pdf
- * tests/end2end/playwright/mock/playwright-test.pdf
- *
- * const downloadPromise = page.waitForEvent('download');
- * const download = await downloadPromise;
- * await download.saveAs(path.join(__dirname, 'mock/print_in_project_projection/baselayer/Paysage_A4.pdf'));
- */
-
 test.describe('Print in project projection', () => {
 
     test.beforeEach(async ({ page }) => {