From a82e2dab0dcaf0b52b197fedc9233b78018cbcc3 Mon Sep 17 00:00:00 2001 From: Niels Date: Wed, 16 Sep 2026 16:47:36 +0200 Subject: [PATCH 1/7] Flaky test hunter on the CI --- .github/workflows/test_e2e_portal.yml | 115 +++++++++++- .../PurposelyFlakyTest/PurposelyFlaky.spec.ts | 10 + tools/package.json | 3 +- tools/select-changed-e2e-tests.mjs | 175 ++++++++++++++++++ 4 files changed, 301 insertions(+), 2 deletions(-) create mode 100644 e2e/portal/tests/PurposelyFlakyTest/PurposelyFlaky.spec.ts create mode 100644 tools/select-changed-e2e-tests.mjs diff --git a/.github/workflows/test_e2e_portal.yml b/.github/workflows/test_e2e_portal.yml index ef18e4f6c1e..91ca8893366 100644 --- a/.github/workflows/test_e2e_portal.yml +++ b/.github/workflows/test_e2e_portal.yml @@ -7,6 +7,11 @@ name: 'Test: E2E (Portal)' on: workflow_dispatch: + inputs: + repeat_count: + description: 'Number of repetitions per test when hunting flaky tests' + type: string + default: '30' merge_group: types: [checks_requested] pull_request: @@ -81,6 +86,114 @@ jobs: working-directory: ${{ env.e2eTestsPath }} run: 'npm run lint' + detect-flaky-candidates: + needs: path-filter + runs-on: ubuntu-latest + if: ${{ needs.path-filter.outputs.should_skip != 'true' && github.event_name == 'pull_request' }} + outputs: + files: ${{ steps.select.outputs.files }} + has_candidates: ${{ steps.select.outputs.has_candidates }} + steps: + - name: Check out repository code + uses: actions/checkout@v7 + with: + filter: blob:none # Reduce clone size/speed + fetch-depth: 0 # Full history is needed to diff against the PR's base commit + + - name: Set up Node.js version + uses: actions/setup-node@v7 + with: + node-version-file: '${{ env.e2eTestsPath }}/.node-version' + + - name: Select changed/new E2E spec files for flaky-test detection + id: select + run: | + node tools/select-changed-e2e-tests.mjs --base ${{ github.event.pull_request.base.sha }} --head ${{ github.sha }} + + find-flaky-changed-tests: + runs-on: ubuntu-latest + needs: + - path-filter + - lint-code + - detect-flaky-candidates + if: ${{ needs.detect-flaky-candidates.outputs.has_candidates == 'true' }} + strategy: + fail-fast: false + matrix: + file: ${{ fromJson(needs.detect-flaky-candidates.outputs.files) }} + + steps: + - name: Check out repository code + uses: actions/checkout@v7 + + - name: Set up Node.js version + uses: actions/setup-node@v7 + with: + node-version-file: '${{ env.e2eTestsPath }}/.node-version' + cache: 'npm' + cache-dependency-path: '${{ env.e2eTestsPath }}/package-lock.json' + + - name: Install E2E-Tests code-dependencies + working-directory: ${{ env.e2eTestsPath }} + # We only need to install the tests' dependencies, so we can skip the `postinstall`-step + run: | + npm ci --ignore-scripts + + - name: Set ENV-variables for test-environment + run: | + cp services/.env.example services/.env + + - name: Run Services with Docker + run: | + npm run start:services:ci:production + + - name: Run Portal + working-directory: ./interfaces/portal + env: + NG_PRODUCTION: true + NG_URL_121_SERVICE_API: http://localhost:3000/api + NG_DOWNLOAD_TRANSLATIONS_AT_BUILD: false + run: | + npm ci + cp .env.example .env + npm run start:debug-production > run-portal-logs.txt 2>&1 & + + - name: Install E2E-Tests runtime-dependencies + working-directory: ${{ env.e2eTestsPath }} + run: | + npx playwright install chromium --with-deps --no-shell + + - name: Wait for Portal + run: | + timeout 90s sh -c 'until curl http://localhost:8088 --head --silent; do echo "Waiting for Portal to be running..."; sleep 1; done' + + - name: Hunt for flaky tests in changed/new spec file + working-directory: ${{ env.e2eTestsPath }} + env: + # Passed via env (not inlined into the script) since SPEC_FILE originates from PR file paths and must never be interpreted by the shell. + SPEC_FILE: ${{ matrix.file }} + REPEAT_COUNT: ${{ github.event.inputs.repeat_count || '30' }} + run: | + npm test -- "$SPEC_FILE" --fail-on-flaky-tests --retries=0 --repeat-each "$REPEAT_COUNT" -x --trace retain-on-failure --reporter=html + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: test-result-artifacts-flaky-${{ strategy.job-index }} + path: | + ${{ env.e2eTestsPath }}/test-results/ + ${{ env.e2eTestsPath }}/playwright-report/ + ./interfaces/portal/run-portal-logs.txt + + - name: Portal logs + if: always() + shell: bash + run: cat ./interfaces/portal/run-portal-logs.txt + + - name: Docker logs + if: always() + uses: global-121/collect-container-logs@78594c6154439e32858bf3dc17c9ff36561a1d31 + test-shard-e2e: runs-on: ubuntu-latest needs: @@ -165,7 +278,7 @@ jobs: # This is a workaround for this issue: https://github.com/orgs/community/discussions/44490 test-shard-resolution-e2e: runs-on: ubuntu-slim - needs: [test-shard-e2e] + needs: [test-shard-e2e, find-flaky-changed-tests] if: always() steps: - name: Check required job results diff --git a/e2e/portal/tests/PurposelyFlakyTest/PurposelyFlaky.spec.ts b/e2e/portal/tests/PurposelyFlakyTest/PurposelyFlaky.spec.ts new file mode 100644 index 00000000000..e7261da9ee1 --- /dev/null +++ b/e2e/portal/tests/PurposelyFlakyTest/PurposelyFlaky.spec.ts @@ -0,0 +1,10 @@ +import { expect, test } from '@playwright/test'; + +// TEMPORARY — for validating the `find-flaky-changed-tests` CI job only. +// Delete this file before merging; do not commit it to `main`. +// Fails on the first attempt and passes on Playwright's automatic retry, so +// it is reported as "flaky" on every run (requires `retries >= 1`, see +// playwright.config.ts). +test('Purposely flaky test for CI-pipeline validation', () => { + expect(test.info().retry).toBeGreaterThan(0); +}); diff --git a/tools/package.json b/tools/package.json index f776bfc9b5d..75452dfd120 100644 --- a/tools/package.json +++ b/tools/package.json @@ -7,7 +7,8 @@ "check-versions": "node check-versions.mjs", "download-logs": "node --env-file-if-exists=.env download-logs.mjs", "find-flaky-tests-API": "node find-flaky-tests-API.mjs", - "find-flaky-tests-E2E": "node find-flaky-tests-E2E.mjs" + "find-flaky-tests-E2E": "node find-flaky-tests-E2E.mjs", + "select-changed-e2e-tests": "node select-changed-e2e-tests.mjs" }, "dependencies": { "@azure/identity": "^4.13.1", diff --git a/tools/select-changed-e2e-tests.mjs b/tools/select-changed-e2e-tests.mjs new file mode 100644 index 00000000000..c0d20d680aa --- /dev/null +++ b/tools/select-changed-e2e-tests.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node + +/** + * Selects Playwright E2E spec files that changed "enough" in a pull request + * to be worth running through the flaky-test hunter before merging: + * - Brand-new spec files (didn't exist in the base commit) are always selected. + * - Existing spec files are selected if the changed lines (added + deleted) + * make up at least `--threshold` percent of the file's current line count. + * + * Used by .github/workflows/test_e2e_portal.yml to decide which spec files + * (if any) to run via `find-flaky-changed-tests`. + * + * Usage: + * node select-changed-e2e-tests.mjs --base [--head HEAD] + * [--threshold 10] [--max-files 15] + */ +import { execFile } from 'node:child_process'; +import { appendFileSync, readFileSync } from 'node:fs'; +import { parseArgs } from 'node:util'; + +const execFileAsync = (command, commandArgs, options) => + new Promise((resolve, reject) => { + execFile(command, commandArgs, options, (error, stdout) => { + if (error) { + reject(error); + return; + } + resolve(stdout); + }); + }); + +const E2E_SPEC_TESTS_DIRECTORY = 'e2e/portal/tests/'; + +const { values: args } = parseArgs({ + options: { + base: { type: 'string' }, + head: { type: 'string', default: 'HEAD' }, + threshold: { type: 'string', default: '10' }, + 'max-files': { type: 'string', default: '15' }, + }, +}); + +if (!args.base) { + console.error('Missing required argument: --base '); + process.exit(1); +} + +const threshold = Number(args.threshold); +// Bounds how many parallel Docker+Portal CI jobs a single PR can trigger. +const maxFiles = Number(args['max-files']); + +async function getRepositoryRoot() { + const stdout = await execFileAsync('git', ['rev-parse', '--show-toplevel']); + return stdout.trim(); +} + +async function getChangedSpecFiles({ repositoryRoot, base, head }) { + const stdout = await execFileAsync( + 'git', + ['diff', '--no-renames', '--numstat', base, head], + { cwd: repositoryRoot }, + ); + + return stdout + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => { + const [added, deleted, path] = line.split('\t'); + return { added, deleted, path }; + }) + .filter( + ({ added, deleted, path }) => + added !== '-' && // Excludes binary files, which have no line-based diff. + deleted !== '-' && + path.startsWith(E2E_SPEC_TESTS_DIRECTORY) && + path.endsWith('.spec.ts'), + ); +} + +async function isNewFile({ repositoryRoot, base, path }) { + try { + await execFileAsync('git', ['cat-file', '-e', `${base}:${path}`], { + cwd: repositoryRoot, + }); + return false; + } catch { + return true; + } +} + +function getChangePercentage({ repositoryRoot, path, added, deleted }) { + const content = readFileSync(`${repositoryRoot}/${path}`, 'utf8'); + const totalLines = content.split('\n').length; + if (totalLines === 0) { + return 100; + } + + const changedLines = Number(added) + Number(deleted); + return (changedLines / totalLines) * 100; +} + +async function selectCandidateSpecFiles({ repositoryRoot, base, head }) { + const changedSpecFiles = await getChangedSpecFiles({ + repositoryRoot, + base, + head, + }); + + const candidates = []; + for (const { added, deleted, path } of changedSpecFiles) { + const isNew = await isNewFile({ repositoryRoot, base, path }); + if (isNew) { + candidates.push({ path, reason: 'new file' }); + continue; + } + + const percentage = getChangePercentage({ + repositoryRoot, + path, + added, + deleted, + }); + if (percentage >= threshold) { + candidates.push({ path, reason: `${percentage.toFixed(1)}% changed` }); + } + } + + return candidates; +} + +function capCandidates({ candidates }) { + if (candidates.length <= maxFiles) { + return candidates; + } + + console.warn( + `Found ${candidates.length} candidate spec files, which exceeds --max-files (${maxFiles}). Only the first ${maxFiles} will be checked for flakiness.`, + ); + return candidates.slice(0, maxFiles); +} + +function toE2eRelativePath(path) { + return path.slice('e2e/'.length); +} + +function writeGithubOutput({ candidates }) { + const files = candidates.map(({ path }) => toE2eRelativePath(path)); + const hasCandidates = files.length > 0; + + const outputPath = process.env.GITHUB_OUTPUT; + if (outputPath) { + appendFileSync(outputPath, `files=${JSON.stringify(files)}\n`); + appendFileSync(outputPath, `has_candidates=${hasCandidates}\n`); + } +} + +const repositoryRoot = await getRepositoryRoot(); +const candidates = capCandidates({ + candidates: await selectCandidateSpecFiles({ + repositoryRoot, + base: args.base, + head: args.head, + }), +}); + +if (candidates.length === 0) { + console.log('No changed/new E2E spec files meet the threshold.'); +} else { + console.log('Selected E2E spec files for flaky-test detection:'); + for (const { path, reason } of candidates) { + console.log(` - ${path} (${reason})`); + } +} + +writeGithubOutput({ candidates }); From 0b12fd3217c403a754371ca09e02749626648df4 Mon Sep 17 00:00:00 2001 From: Niels Date: Thu, 17 Sep 2026 09:51:56 +0200 Subject: [PATCH 2/7] Adding purposlyStable as well --- .../{PurposelyFlakyTest => TestingCi}/PurposelyFlaky.spec.ts | 0 e2e/portal/tests/TestingCi/PurposlyStable.spec.ts | 5 +++++ 2 files changed, 5 insertions(+) rename e2e/portal/tests/{PurposelyFlakyTest => TestingCi}/PurposelyFlaky.spec.ts (100%) create mode 100644 e2e/portal/tests/TestingCi/PurposlyStable.spec.ts diff --git a/e2e/portal/tests/PurposelyFlakyTest/PurposelyFlaky.spec.ts b/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts similarity index 100% rename from e2e/portal/tests/PurposelyFlakyTest/PurposelyFlaky.spec.ts rename to e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts diff --git a/e2e/portal/tests/TestingCi/PurposlyStable.spec.ts b/e2e/portal/tests/TestingCi/PurposlyStable.spec.ts new file mode 100644 index 00000000000..041d5b1deec --- /dev/null +++ b/e2e/portal/tests/TestingCi/PurposlyStable.spec.ts @@ -0,0 +1,5 @@ +import { expect, test } from '@playwright/test'; + +test('Stable test for CI-pipeline validation', () => { + expect(1).toBe(1); +}); From 5ed45f5b7c53c0c858852ee7c2139ae950018a29 Mon Sep 17 00:00:00 2001 From: Niels Date: Thu, 17 Sep 2026 10:59:52 +0200 Subject: [PATCH 3/7] True flakyness --- e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts b/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts index e7261da9ee1..85830979d9e 100644 --- a/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts +++ b/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts @@ -2,9 +2,9 @@ import { expect, test } from '@playwright/test'; // TEMPORARY — for validating the `find-flaky-changed-tests` CI job only. // Delete this file before merging; do not commit it to `main`. -// Fails on the first attempt and passes on Playwright's automatic retry, so -// it is reported as "flaky" on every run (requires `retries >= 1`, see -// playwright.config.ts). +// Fails ~50% of the time, so across `find-flaky`'s repeated runs it has a +// realistic chance of failing once and passing on retry (requires +// `retries >= 1`, see playwright.config.ts), instead of behaving deterministically. test('Purposely flaky test for CI-pipeline validation', () => { - expect(test.info().retry).toBeGreaterThan(0); + expect(Math.random()).toBeGreaterThan(0.5); }); From 9811c1a3b3f8e2d0f716c3b7edc33ea4e6ae883a Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 22 Sep 2026 16:14:43 +0200 Subject: [PATCH 4/7] Addressing copilot comments --- .github/workflows/test_e2e_portal.yml | 20 ++++++++++++---- tools/select-changed-e2e-tests.mjs | 34 ++++++++++++++++++--------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_e2e_portal.yml b/.github/workflows/test_e2e_portal.yml index 91ca8893366..68f6c680e19 100644 --- a/.github/workflows/test_e2e_portal.yml +++ b/.github/workflows/test_e2e_portal.yml @@ -89,7 +89,7 @@ jobs: detect-flaky-candidates: needs: path-filter runs-on: ubuntu-latest - if: ${{ needs.path-filter.outputs.should_skip != 'true' && github.event_name == 'pull_request' }} + if: ${{ needs.path-filter.outputs.should_skip != 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') }} outputs: files: ${{ steps.select.outputs.files }} has_candidates: ${{ steps.select.outputs.has_candidates }} @@ -98,17 +98,27 @@ jobs: uses: actions/checkout@v7 with: filter: blob:none # Reduce clone size/speed - fetch-depth: 0 # Full history is needed to diff against the PR's base commit + fetch-depth: 0 # Full history is needed to diff against the base commit - name: Set up Node.js version uses: actions/setup-node@v7 with: node-version-file: '${{ env.e2eTestsPath }}/.node-version' + - name: Determine base commit for flaky-candidate selection + id: base + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "sha=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_OUTPUT" + else + # Manual runs have no PR base, so diff against the default branch instead. + echo "sha=$(git merge-base "origin/${{ github.event.repository.default_branch }}" HEAD)" >> "$GITHUB_OUTPUT" + fi + - name: Select changed/new E2E spec files for flaky-test detection id: select run: | - node tools/select-changed-e2e-tests.mjs --base ${{ github.event.pull_request.base.sha }} --head ${{ github.sha }} + node tools/select-changed-e2e-tests.mjs --base ${{ steps.base.outputs.sha }} --head ${{ github.sha }} find-flaky-changed-tests: runs-on: ubuntu-latest @@ -174,7 +184,7 @@ jobs: SPEC_FILE: ${{ matrix.file }} REPEAT_COUNT: ${{ github.event.inputs.repeat_count || '30' }} run: | - npm test -- "$SPEC_FILE" --fail-on-flaky-tests --retries=0 --repeat-each "$REPEAT_COUNT" -x --trace retain-on-failure --reporter=html + npm test -- "$SPEC_FILE" --fail-on-flaky-tests --repeat-each "$REPEAT_COUNT" -x --trace retain-on-failure --reporter=html - uses: actions/upload-artifact@v7 if: always() @@ -278,7 +288,7 @@ jobs: # This is a workaround for this issue: https://github.com/orgs/community/discussions/44490 test-shard-resolution-e2e: runs-on: ubuntu-slim - needs: [test-shard-e2e, find-flaky-changed-tests] + needs: [detect-flaky-candidates, test-shard-e2e, find-flaky-changed-tests] if: always() steps: - name: Check required job results diff --git a/tools/select-changed-e2e-tests.mjs b/tools/select-changed-e2e-tests.mjs index c0d20d680aa..dd51c647b3a 100644 --- a/tools/select-changed-e2e-tests.mjs +++ b/tools/select-changed-e2e-tests.mjs @@ -18,7 +18,7 @@ import { execFile } from 'node:child_process'; import { appendFileSync, readFileSync } from 'node:fs'; import { parseArgs } from 'node:util'; -const execFileAsync = (command, commandArgs, options) => +const execFileAsync = ({ command, commandArgs, options = {} }) => new Promise((resolve, reject) => { execFile(command, commandArgs, options, (error, stdout) => { if (error) { @@ -50,16 +50,26 @@ const threshold = Number(args.threshold); const maxFiles = Number(args['max-files']); async function getRepositoryRoot() { - const stdout = await execFileAsync('git', ['rev-parse', '--show-toplevel']); + const stdout = await execFileAsync({ + command: 'git', + commandArgs: ['rev-parse', '--show-toplevel'], + }); return stdout.trim(); } async function getChangedSpecFiles({ repositoryRoot, base, head }) { - const stdout = await execFileAsync( - 'git', - ['diff', '--no-renames', '--numstat', base, head], - { cwd: repositoryRoot }, - ); + const stdout = await execFileAsync({ + command: 'git', + commandArgs: [ + 'diff', + '--no-renames', + '--diff-filter=AM', // Excludes deleted paths, which no longer exist in `head` to read. + '--numstat', + base, + head, + ], + options: { cwd: repositoryRoot }, + }); return stdout .split('\n') @@ -79,8 +89,10 @@ async function getChangedSpecFiles({ repositoryRoot, base, head }) { async function isNewFile({ repositoryRoot, base, path }) { try { - await execFileAsync('git', ['cat-file', '-e', `${base}:${path}`], { - cwd: repositoryRoot, + await execFileAsync({ + command: 'git', + commandArgs: ['cat-file', '-e', `${base}:${path}`], + options: { cwd: repositoryRoot }, }); return false; } catch { @@ -139,12 +151,12 @@ function capCandidates({ candidates }) { return candidates.slice(0, maxFiles); } -function toE2eRelativePath(path) { +function toE2eRelativePath({ path }) { return path.slice('e2e/'.length); } function writeGithubOutput({ candidates }) { - const files = candidates.map(({ path }) => toE2eRelativePath(path)); + const files = candidates.map(({ path }) => toE2eRelativePath({ path })); const hasCandidates = files.length > 0; const outputPath = process.env.GITHUB_OUTPUT; From 725216557db6fa3c4b938b4839daf0db39ab981f Mon Sep 17 00:00:00 2001 From: Niels Date: Tue, 22 Sep 2026 16:33:43 +0200 Subject: [PATCH 5/7] Only test E2E's from that PR --- .github/workflows/test_e2e_portal.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_e2e_portal.yml b/.github/workflows/test_e2e_portal.yml index 68f6c680e19..f14cfb33ae4 100644 --- a/.github/workflows/test_e2e_portal.yml +++ b/.github/workflows/test_e2e_portal.yml @@ -109,11 +109,14 @@ jobs: id: base run: | if [ "${{ github.event_name }}" = "pull_request" ]; then - echo "sha=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_OUTPUT" + base_ref="${{ github.event.pull_request.base.ref }}" else # Manual runs have no PR base, so diff against the default branch instead. - echo "sha=$(git merge-base "origin/${{ github.event.repository.default_branch }}" HEAD)" >> "$GITHUB_OUTPUT" + base_ref="${{ github.event.repository.default_branch }}" fi + # Merge-base (not the branch tip) avoids treating base-branch commits + # made after this branch diverged as changes introduced by this PR. + echo "sha=$(git merge-base "origin/$base_ref" HEAD)" >> "$GITHUB_OUTPUT" - name: Select changed/new E2E spec files for flaky-test detection id: select From 02be763244681b8a591cc4b2ec64fcbb3e266871 Mon Sep 17 00:00:00 2001 From: Niels Date: Wed, 23 Sep 2026 09:37:44 +0200 Subject: [PATCH 6/7] Addressing copilot comments --- .github/workflows/test_e2e_portal.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_e2e_portal.yml b/.github/workflows/test_e2e_portal.yml index f14cfb33ae4..c173d02295a 100644 --- a/.github/workflows/test_e2e_portal.yml +++ b/.github/workflows/test_e2e_portal.yml @@ -291,7 +291,14 @@ jobs: # This is a workaround for this issue: https://github.com/orgs/community/discussions/44490 test-shard-resolution-e2e: runs-on: ubuntu-slim - needs: [detect-flaky-candidates, test-shard-e2e, find-flaky-changed-tests] + needs: + [ + path-filter, + lint-code, + detect-flaky-candidates, + test-shard-e2e, + find-flaky-changed-tests, + ] if: always() steps: - name: Check required job results From 70550aa8700be0bc89f16925316bd417391b6e64 Mon Sep 17 00:00:00 2001 From: Niels Date: Mon, 28 Sep 2026 09:08:15 +0200 Subject: [PATCH 7/7] Removing test e2e's --- e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts | 10 ---------- e2e/portal/tests/TestingCi/PurposlyStable.spec.ts | 5 ----- 2 files changed, 15 deletions(-) delete mode 100644 e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts delete mode 100644 e2e/portal/tests/TestingCi/PurposlyStable.spec.ts diff --git a/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts b/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts deleted file mode 100644 index 85830979d9e..00000000000 --- a/e2e/portal/tests/TestingCi/PurposelyFlaky.spec.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { expect, test } from '@playwright/test'; - -// TEMPORARY — for validating the `find-flaky-changed-tests` CI job only. -// Delete this file before merging; do not commit it to `main`. -// Fails ~50% of the time, so across `find-flaky`'s repeated runs it has a -// realistic chance of failing once and passing on retry (requires -// `retries >= 1`, see playwright.config.ts), instead of behaving deterministically. -test('Purposely flaky test for CI-pipeline validation', () => { - expect(Math.random()).toBeGreaterThan(0.5); -}); diff --git a/e2e/portal/tests/TestingCi/PurposlyStable.spec.ts b/e2e/portal/tests/TestingCi/PurposlyStable.spec.ts deleted file mode 100644 index 041d5b1deec..00000000000 --- a/e2e/portal/tests/TestingCi/PurposlyStable.spec.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { expect, test } from '@playwright/test'; - -test('Stable test for CI-pipeline validation', () => { - expect(1).toBe(1); -});