diff --git a/.github/scripts/comment-on-flaky-tests.js b/.github/scripts/comment-on-flaky-tests.js new file mode 100644 index 000000000..2511f7712 --- /dev/null +++ b/.github/scripts/comment-on-flaky-tests.js @@ -0,0 +1,70 @@ +const fs = require('fs'); +const { execSync } = require('child_process'); + +// Check if flaky-tests.json exists +if (!fs.existsSync('flaky-tests.json')) { + console.log('No flaky-tests.json found'); + process.exit(0); +} + +// Get changed files in PR +let changedFiles = []; +try { + changedFiles = execSync('git diff --name-only origin/main...HEAD') + .toString() + .split('\n') + .filter(f => f.endsWith('.spec.ts')); +} catch (error) { + console.log('Could not determine changed files:', error.message); + process.exit(0); +} + +if (changedFiles.length === 0) { + console.log('No test files were modified in this PR'); + process.exit(0); +} + +// Read flaky tests +const flakyTests = JSON.parse(fs.readFileSync('flaky-tests.json', 'utf8')); + +if (flakyTests.length === 0) { + console.log('No flaky/failed tests found'); + process.exit(0); +} + +// Find modified flaky tests +const modifiedFlakyTests = flakyTests.filter(test => + changedFiles.some(file => test.file.includes(file)) +); + +if (modifiedFlakyTests.length === 0) { + console.log('No modified test files are flaky'); + process.exit(0); +} + +// Generate comment markdown +let comment = '## ⚠️ Warning: You modified flaky/failed test files\n\n'; +comment += 'The following test files you modified have reliability issues:\n\n'; + +modifiedFlakyTests.forEach(test => { + const testType = test.status === 'failed' ? '❌ Failed' : '⚠️ Flaky'; + comment += `### ${testType}: \`${test.file}\`\n`; + comment += `**Test:** ${test.testTitle}\n`; + comment += `**Status:** ${test.status}\n`; + if (test.retryAttempt > 0) { + comment += `**Retry Attempt:** ${test.retryAttempt}\n`; + } + comment += '\n**To debug locally, run:**\n'; + comment += '```bash\n'; + comment += `npx playwright test ${test.file} --repeat-each=5 --workers=1\n`; + comment += '```\n\n'; +}); + +comment += '---\n'; +comment += '**Note:** Flaky tests passed after retrying, failed tests did not pass. '; +comment += 'Please investigate and fix the root cause before merging.\n'; + +// Save comment to file for GitHub Action to post +fs.writeFileSync('pr-comment.md', comment); + +console.log(`Found ${modifiedFlakyTests.length} modified flaky tests`); diff --git a/.github/scripts/detect-flaky-tests.js b/.github/scripts/detect-flaky-tests.js new file mode 100644 index 000000000..9e3972d5c --- /dev/null +++ b/.github/scripts/detect-flaky-tests.js @@ -0,0 +1,78 @@ +const fs = require('fs'); + + +// Read Playwright JSON report +const resultsPath = 'playwright-report/results.json'; + +if (!fs.existsSync(resultsPath)) { + console.log('No Playwright results found at', resultsPath); + process.exit(0); +} + +const results = JSON.parse(fs.readFileSync(resultsPath, 'utf8')); + +// Extract flaky tests +// A test is flaky if: status === "passed" AND retry > 0 +// A test is failed if: status === "failed" +// This means it failed initially but passed on retry OR failed completely +const flakyTests = []; + +function traverseSuites(suites) { + for (const suite of suites) { + // Process specs in this suite + for (const spec of suite.specs || []) { + for (const test of spec.tests || []) { + // Check each test result + for (const result of test.results || []) { + // Track two types of problematic tests: + // 1. Flaky: passed on a retry attempt (retry > 0) + // 2. Failed: failed on all attempts + if ((result.status === 'passed' && result.retry > 0) || result.status === 'failed') { + flakyTests.push({ + file: spec.file, + title: spec.title, + testTitle: spec.title, + line: spec.line, + status: result.status, + retryAttempt: result.retry + }); + break; // Only record once per test + } + } + } + } + + // Recursively process nested suites + if (suite.suites && suite.suites.length > 0) { + traverseSuites(suite.suites); + } + } +} + +traverseSuites(results.suites || []); + +// Save flaky tests to JSON +fs.writeFileSync('flaky-tests.json', JSON.stringify(flakyTests, null, 2)); + +// Generate markdown report +let markdown = '## ⚠️ Flaky/Failed Tests Detected\n\n'; +markdown += 'The following tests are problematic:\n\n'; + +flakyTests.forEach(test => { + const testType = test.status === 'failed' ? '❌ Failed' : '⚠️ Flaky'; + markdown += `### ${testType}: \`${test.file}\`\n`; + markdown += `- **Test:** ${test.testTitle}\n`; + markdown += `- **Status:** ${test.status}\n`; + if (test.retryAttempt > 0) { + markdown += `- **Retry Attempt:** ${test.retryAttempt}\n`; + } + markdown += `- **Debug command:**\n`; + markdown += '```bash\n'; + markdown += `npx playwright test ${test.file} --repeat-each=5 --workers=1\n`; + markdown += '```\n\n'; +}); + +fs.writeFileSync('flaky-report.md', markdown); + +console.log(`Found ${flakyTests.length} flaky/failed tests`); +process.exit(flakyTests.length > 0 ? 1 : 0); diff --git a/.github/workflows/flaky-test-detector.yml b/.github/workflows/flaky-test-detector.yml new file mode 100644 index 000000000..de2120fc7 --- /dev/null +++ b/.github/workflows/flaky-test-detector.yml @@ -0,0 +1,119 @@ +name: Flaky Test Detector + +on: + pull_request: + branches: [main] + paths: + - 'tests/**/*.spec.ts' + +permissions: + contents: read + pull-requests: write + checks: write + +jobs: + detect-flaky-tests: + name: Detect Flaky Tests + runs-on: ubuntu-24.04 + timeout-minutes: 60 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 # Need full history to compare with main + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get --no-install-recommends install -y \ + libglib2.0-0 libnss3 libdbus-1-3 libatk1.0-0 libatk-bridge2.0-0 \ + libcups2 libgtk-3-0 libasound2t64 xvfb + + - name: Install npm dependencies + run: | + npm ci --legacy-peer-deps + sudo chown root /home/runner/work/bruno/bruno/node_modules/electron/dist/chrome-sandbox + sudo chmod 4755 /home/runner/work/bruno/bruno/node_modules/electron/dist/chrome-sandbox + + - name: Install test collection dependencies + run: npm ci --prefix packages/bruno-tests/collection + + - name: Build libraries + run: | + npm run build:graphql-docs + npm run build:bruno-query + npm run build:bruno-common + npm run sandbox:bundle-libraries --workspace=packages/bruno-js + npm run build:bruno-converters + npm run build:bruno-requests + npm run build:schema-types + npm run build:bruno-filestore + + - name: Run Playwright tests + run: xvfb-run npm run test:e2e + continue-on-error: true # Continue even if tests fail + + - name: Detect flaky tests + id: detect + run: node .github/scripts/detect-flaky-tests.js + continue-on-error: true # Don't fail workflow if flaky tests found + + - name: Check modified flaky tests + id: check-modified + run: node .github/scripts/comment-on-flaky-tests.js + continue-on-error: true + + - name: Post PR comment + if: hashFiles('pr-comment.md') != '' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const comment = fs.readFileSync('pr-comment.md', 'utf8'); + + // Check if we already commented + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number + }); + + const botComment = comments.find(c => + c.user.type === 'Bot' && c.body.includes('Warning: You modified flaky/failed test files') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: comment + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: comment + }); + } + + - name: Upload flaky test artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: flaky-test-results + path: | + flaky-tests.json + flaky-report.md + playwright-report/ + retention-days: 30 diff --git a/playwright.config.ts b/playwright.config.ts index d30abca86..a36b30546 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,6 +1,6 @@ import { defineConfig } from '@playwright/test'; -const reporter: any[] = [['list'], ['html']]; +const reporter: any[] = [['list'], ['html'], ['json', { outputFile: 'playwright-report/results.json' }]]; if (process.env.CI) { reporter.push(['github']); diff --git a/tests/import/insomnia/import-insomnia-v4-environments.spec.ts b/tests/import/insomnia/import-insomnia-v4-environments.spec.ts index acbc1efe9..3f8882ee6 100644 --- a/tests/import/insomnia/import-insomnia-v4-environments.spec.ts +++ b/tests/import/insomnia/import-insomnia-v4-environments.spec.ts @@ -73,6 +73,9 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { .first() .click(); + // Wait for environment variables to load - use input selector as it's more reliable + await expect(page.locator('input[value="baseUrl"]')).toBeVisible({ timeout: 10000 }); + // **Assertion 1: Basic Variables (Top-level keys)** // Verifies that simple key-value pairs from the base environment are imported correctly const v4BaseUrlInput = page.locator('input[value="baseUrl"]'); @@ -81,8 +84,10 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { await expect(v4AuthTokenInput).toBeVisible(); // Assert: Top-level string values are preserved exactly as in the source - await expect(page.getByTestId('env-var-row-baseUrl').locator('.CodeMirror-line').first()).toHaveText('https://api.example.com'); - await expect(page.getByTestId('env-var-row-authToken').locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); + const baseUrlRow = page.locator('tbody tr').filter({ has: page.locator('input[value="baseUrl"]') }); + await expect(baseUrlRow.locator('.CodeMirror-line').first()).toHaveText('https://api.example.com'); + const authTokenRow = page.locator('tbody tr').filter({ has: page.locator('input[value="authToken"]') }); + await expect(authTokenRow.locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); // **Assertion 2: Nested Object Flattening** // Verifies that nested objects are flattened to dot-notation keys (e.g., user.name, user.id) @@ -92,9 +97,11 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { await expect(v4UserIdInput).toBeVisible(); // Assert: Nested object properties are accessible via dot notation - await expect(page.getByTestId('env-var-row-user.name').locator('.CodeMirror-line').first()).toHaveText('admin'); + const userNameRow = page.locator('tbody tr').filter({ has: page.locator('input[value="user.name"]') }); + await expect(userNameRow.locator('.CodeMirror-line').first()).toHaveText('admin'); // Assert: Numeric values are converted to strings and preserved - await expect(page.getByTestId('env-var-row-user.id').locator('.CodeMirror-line').first()).toHaveText('123'); + const userIdRow = page.locator('tbody tr').filter({ has: page.locator('input[value="user.id"]') }); + await expect(userIdRow.locator('.CodeMirror-line').first()).toHaveText('123'); // **Assertion 3: Array Flattening** // Verifies that arrays are flattened using JavaScript-style square bracket notation (e.g., user.roles[0], user.roles[1]) @@ -104,8 +111,10 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { await expect(v4UserRoles1Input).toBeVisible(); // Assert: Array elements are accessible via JavaScript-style square bracket notation - await expect(page.getByTestId('env-var-row-user.roles[0]').locator('.CodeMirror-line').first()).toHaveText('admin'); - await expect(page.getByTestId('env-var-row-user.roles[1]').locator('.CodeMirror-line').first()).toHaveText('user'); + const userRoles0Row = page.locator('tbody tr').filter({ has: page.locator('input[value="user.roles[0]"]') }); + await expect(userRoles0Row.locator('.CodeMirror-line').first()).toHaveText('admin'); + const userRoles1Row = page.locator('tbody tr').filter({ has: page.locator('input[value="user.roles[1]"]') }); + await expect(userRoles1Row.locator('.CodeMirror-line').first()).toHaveText('user'); }); await test.step('Test Staging Environment - verify merging with base', async () => { @@ -120,14 +129,16 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { const v4StagingBaseUrlInput = page.locator('input[value="baseUrl"]'); await expect(v4StagingBaseUrlInput).toBeVisible(); // Assert: Staging overrides baseUrl with its own value - await expect(page.getByTestId('env-var-row-baseUrl').locator('.CodeMirror-line').first()).toHaveText('https://staging-api.example.com'); + const stagingBaseUrlRow = page.locator('tbody tr').filter({ has: page.locator('input[value="baseUrl"]') }); + await expect(stagingBaseUrlRow.locator('.CodeMirror-line').first()).toHaveText('https://staging-api.example.com'); // **Assertion 2: Top-level Variable Inheritance** // Verifies that staging environment inherits base environment values when not overridden const v4StagingAuthTokenInput = page.locator('input[value="authToken"]'); await expect(v4StagingAuthTokenInput).toBeVisible(); // Assert: Staging inherits authToken from base (not overridden in staging) - await expect(page.getByTestId('env-var-row-authToken').locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); + const stagingAuthTokenRow = page.locator('tbody tr').filter({ has: page.locator('input[value="authToken"]') }); + await expect(stagingAuthTokenRow.locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); // **Assertion 3: Nested Object Variable Override and Inheritance** // Verifies that nested object properties can be selectively overridden while inheriting others @@ -139,11 +150,14 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { await expect(v4StagingUserRoles0Input).toBeVisible(); // Assert: Staging overrides user.name with its own value - await expect(page.getByTestId('env-var-row-user.name').locator('.CodeMirror-line').first()).toHaveText('staging_admin'); + const stagingUserNameRow = page.locator('tbody tr').filter({ has: page.locator('input[value="user.name"]') }); + await expect(stagingUserNameRow.locator('.CodeMirror-line').first()).toHaveText('staging_admin'); // Assert: Staging inherits user.id from base (not overridden in staging) - await expect(page.getByTestId('env-var-row-user.id').locator('.CodeMirror-line').first()).toHaveText('123'); + const stagingUserIdRow = page.locator('tbody tr').filter({ has: page.locator('input[value="user.id"]') }); + await expect(stagingUserIdRow.locator('.CodeMirror-line').first()).toHaveText('123'); // Assert: Staging inherits user.roles[0] from base (not overridden in staging) - await expect(page.getByTestId('env-var-row-user.roles[0]').locator('.CodeMirror-line').first()).toHaveText('admin'); + const stagingUserRoles0Row = page.locator('tbody tr').filter({ has: page.locator('input[value="user.roles[0]"]') }); + await expect(stagingUserRoles0Row.locator('.CodeMirror-line').first()).toHaveText('admin'); }); await test.step('Test Development Environment - verify new variables', async () => { @@ -161,9 +175,11 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { await expect(v4DevAuthTokenInput).toBeVisible(); // Assert: Development overrides baseUrl with its own value - await expect(page.getByTestId('env-var-row-baseUrl').locator('.CodeMirror-line').first()).toHaveText('https://dev-api.example.com'); + const devBaseUrlRow = page.locator('tbody tr').filter({ has: page.locator('input[value="baseUrl"]') }); + await expect(devBaseUrlRow.locator('.CodeMirror-line').first()).toHaveText('https://dev-api.example.com'); // Assert: Development overrides authToken with its own value - await expect(page.getByTestId('env-var-row-authToken').locator('.CodeMirror-line').first()).toHaveText('dev_token_123'); + const devAuthTokenRow = page.locator('tbody tr').filter({ has: page.locator('input[value="authToken"]') }); + await expect(devAuthTokenRow.locator('.CodeMirror-line').first()).toHaveText('dev_token_123'); // **Assertion 2: New Nested Variables Addition** // Verifies that development environment can add completely new nested variables not present in base @@ -173,9 +189,11 @@ test.describe('Import Insomnia v4 Collection - Environment Import', () => { await expect(v4NewFeatureVersionInput).toBeVisible(); // Assert: New boolean variable is added and converted to string - await expect(page.getByTestId('env-var-row-newFeature.enabled').locator('.CodeMirror-line').first()).toHaveText('true'); + const newFeatureEnabledRow = page.locator('tbody tr').filter({ has: page.locator('input[value="newFeature.enabled"]') }); + await expect(newFeatureEnabledRow.locator('.CodeMirror-line').first()).toHaveText('true'); // Assert: New numeric variable is added and converted to string with full precision - await expect(page.getByTestId('env-var-row-newFeature.version').locator('.CodeMirror-line').first()).toHaveText('2.099123123'); + const newFeatureVersionRow = page.locator('tbody tr').filter({ has: page.locator('input[value="newFeature.version"]') }); + await expect(newFeatureVersionRow.locator('.CodeMirror-line').first()).toHaveText('2.099123123'); }); await test.step('Close environment tab', async () => { diff --git a/tests/import/insomnia/import-insomnia-v5-environments.spec.ts b/tests/import/insomnia/import-insomnia-v5-environments.spec.ts index 575754eb8..dda31a960 100644 --- a/tests/import/insomnia/import-insomnia-v5-environments.spec.ts +++ b/tests/import/insomnia/import-insomnia-v5-environments.spec.ts @@ -78,8 +78,10 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(authTokenInput).toBeVisible(); // Assert: Top-level string values are preserved exactly as in the source - await expect(page.getByTestId('env-var-row-base_url').locator('.CodeMirror-line').first()).toHaveText('https://api.example.com'); - await expect(page.getByTestId('env-var-row-auth_token').locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); + const baseUrlRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"base_url\"]') }); + await expect(baseUrlRow.locator('.CodeMirror-line').first()).toHaveText('https://api.example.com'); + const authTokenRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"auth_token\"]') }); + await expect(authTokenRow.locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); // **Assertion 2: Nested Object Flattening** // Verifies that nested objects are flattened to dot-notation keys (e.g., user.name, user.id) @@ -89,9 +91,11 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(userIdInput).toBeVisible(); // Assert: Nested object properties are accessible via dot notation - await expect(page.getByTestId('env-var-row-user.name').locator('.CodeMirror-line').first()).toHaveText('admin'); + const userNameRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.name\"]') }); + await expect(userNameRow.locator('.CodeMirror-line').first()).toHaveText('admin'); // Assert: Numeric values are converted to strings and preserved - await expect(page.getByTestId('env-var-row-user.id').locator('.CodeMirror-line').first()).toHaveText('123'); + const userIdRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.id\"]') }); + await expect(userIdRow.locator('.CodeMirror-line').first()).toHaveText('123'); // **Assertion 3: Array Flattening** // Verifies that arrays are flattened using JavaScript-style square bracket notation (e.g., user.roles[0], user.roles[1]) @@ -101,8 +105,10 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(userRoles1Input).toBeVisible(); // Assert: Array elements are accessible via JavaScript-style square bracket notation - await expect(page.getByTestId('env-var-row-user.roles[0]').locator('.CodeMirror-line').first()).toHaveText('admin'); - await expect(page.getByTestId('env-var-row-user.roles[1]').locator('.CodeMirror-line').first()).toHaveText('user'); + const userRoles0Row = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.roles[0]\"]') }); + await expect(userRoles0Row.locator('.CodeMirror-line').first()).toHaveText('admin'); + const userRoles1Row = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.roles[1]\"]') }); + await expect(userRoles1Row.locator('.CodeMirror-line').first()).toHaveText('user'); // **Assertion 4: Deeply Nested Config Objects** // Verifies that deeply nested objects are properly flattened (e.g., config.timeout, config.debug) @@ -112,9 +118,11 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(configDebugInput).toBeVisible(); // Assert: Numeric values in nested objects are converted to strings - await expect(page.getByTestId('env-var-row-config.timeout').locator('.CodeMirror-line').first()).toHaveText('30000'); + const configTimeoutRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"config.timeout\"]') }); + await expect(configTimeoutRow.locator('.CodeMirror-line').first()).toHaveText('30000'); // Assert: Boolean values in nested objects are converted to strings - await expect(page.getByTestId('env-var-row-config.debug').locator('.CodeMirror-line').first()).toHaveText('true'); + const configDebugRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"config.debug\"]') }); + await expect(configDebugRow.locator('.CodeMirror-line').first()).toHaveText('true'); }); await test.step('Test Staging Environment - verify merging and overrides', async () => { @@ -129,14 +137,16 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { const stagingBaseUrlInput = page.locator('input[value="base_url"]'); await expect(stagingBaseUrlInput).toBeVisible(); // Assert: Staging overrides base_url with its own value - await expect(page.getByTestId('env-var-row-base_url').locator('.CodeMirror-line').first()).toHaveText('https://staging-api.example.com'); + const baseUrlRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"base_url\"]') }); + await expect(baseUrlRow.locator('.CodeMirror-line').first()).toHaveText('https://staging-api.example.com'); // **Assertion 2: Top-level Variable Inheritance** // Verifies that staging environment inherits base environment values when not overridden const stagingAuthTokenInput = page.locator('input[value="auth_token"]'); await expect(stagingAuthTokenInput).toBeVisible(); // Assert: Staging inherits auth_token from base (not overridden in staging) - await expect(page.getByTestId('env-var-row-auth_token').locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); + const authTokenRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"auth_token\"]') }); + await expect(authTokenRow.locator('.CodeMirror-line').first()).toHaveText('your_auth_token_here'); // **Assertion 3: Nested Object Variable Override and Inheritance** // Verifies that nested object properties can be selectively overridden while inheriting others @@ -146,9 +156,11 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(stagingUserIdInput).toBeVisible(); // Assert: Staging overrides user.name with its own value - await expect(page.getByTestId('env-var-row-user.name').locator('.CodeMirror-line').first()).toHaveText('staging_admin'); + const userNameRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.name\"]') }); + await expect(userNameRow.locator('.CodeMirror-line').first()).toHaveText('staging_admin'); // Assert: Staging inherits user.id from base (not overridden in staging) - await expect(page.getByTestId('env-var-row-user.id').locator('.CodeMirror-line').first()).toHaveText('123'); + const userIdRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.id\"]') }); + await expect(userIdRow.locator('.CodeMirror-line').first()).toHaveText('123'); // **Assertion 4: Deeply Nested Config Override** // Verifies that deeply nested object properties can be overridden @@ -158,9 +170,11 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(stagingConfigDebugInput).toBeVisible(); // Assert: Staging overrides config.timeout with its own value - await expect(page.getByTestId('env-var-row-config.timeout').locator('.CodeMirror-line').first()).toHaveText('60000'); + const configTimeoutRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"config.timeout\"]') }); + await expect(configTimeoutRow.locator('.CodeMirror-line').first()).toHaveText('60000'); // Assert: Staging overrides config.debug with its own value - await expect(page.getByTestId('env-var-row-config.debug').locator('.CodeMirror-line').first()).toHaveText('false'); + const configDebugRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"config.debug\"]') }); + await expect(configDebugRow.locator('.CodeMirror-line').first()).toHaveText('false'); }); await test.step('Test Development Environment - verify new variables', async () => { @@ -178,9 +192,11 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(devAuthTokenInput).toBeVisible(); // Assert: Development overrides base_url with its own value - await expect(page.getByTestId('env-var-row-base_url').locator('.CodeMirror-line').first()).toHaveText('https://dev-api.example.com'); + const baseUrlRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"base_url\"]') }); + await expect(baseUrlRow.locator('.CodeMirror-line').first()).toHaveText('https://dev-api.example.com'); // Assert: Development overrides auth_token with its own value - await expect(page.getByTestId('env-var-row-auth_token').locator('.CodeMirror-line').first()).toHaveText('dev_token_123'); + const authTokenRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"auth_token\"]') }); + await expect(authTokenRow.locator('.CodeMirror-line').first()).toHaveText('dev_token_123'); // **Assertion 2: New Nested Variables Addition** // Verifies that development environment can add completely new nested variables not present in base @@ -190,16 +206,19 @@ test.describe('Import Insomnia v5 Collection - Environment Import', () => { await expect(newFeatureVersionInput).toBeVisible(); // Assert: New boolean variable is added and converted to string - await expect(page.getByTestId('env-var-row-new_feature.enabled').locator('.CodeMirror-line').first()).toHaveText('true'); + const newFeatureEnabledRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"new_feature.enabled\"]') }); + await expect(newFeatureEnabledRow.locator('.CodeMirror-line').first()).toHaveText('true'); // Assert: New numeric variable is added and converted to string with full precision - await expect(page.getByTestId('env-var-row-new_feature.version').locator('.CodeMirror-line').first()).toHaveText('2.099123123'); + const newFeatureVersionRow = page.locator('tbody tr').filter({ has: page.locator('input[value=\"new_feature.version\"]') }); + await expect(newFeatureVersionRow.locator('.CodeMirror-line').first()).toHaveText('2.099123123'); // **Assertion 3: Base Variable Inheritance** // Verifies that development environment still inherits base variables that are not overridden const devUserRoles0Input = page.locator('input[value="user.roles[0]"]'); await expect(devUserRoles0Input).toBeVisible(); // Assert: Development inherits user.roles[0] from base (not overridden in development) - await expect(page.getByTestId('env-var-row-user.roles[0]').locator('.CodeMirror-line').first()).toHaveText('admin'); + const userRoles0Row = page.locator('tbody tr').filter({ has: page.locator('input[value=\"user.roles[0]\"]') }); + await expect(userRoles0Row.locator('.CodeMirror-line').first()).toHaveText('admin'); }); await test.step('Close environment tab', async () => { diff --git a/tests/utils/page/actions.ts b/tests/utils/page/actions.ts index 71d73160d..36798ccf8 100644 --- a/tests/utils/page/actions.ts +++ b/tests/utils/page/actions.ts @@ -702,7 +702,10 @@ const switchResponseFormat = async (page: Page, format: string) => { await test.step(`Switch response format to ${format}`, async () => { const responseFormatTab = page.getByTestId('format-response-tab'); await responseFormatTab.click(); - await page.getByTestId('format-response-tab-dropdown').getByText(format).click(); + // Wait for dropdown to be visible before clicking the format option + const dropdown = page.getByTestId('format-response-tab-dropdown'); + await dropdown.waitFor({ state: 'visible' }); + await dropdown.getByText(format).click(); }); };