bruno/.github/scripts/detect-flaky-tests.js

const fs = require('fs');


// Read Playwright JSON report
const resultsPath = 'playwright-report/results.json';

if (!fs.existsSync(resultsPath)) {
  console.log('No Playwright results found at', resultsPath);
  process.exit(0);
}

const results = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));

// Extract flaky tests
// A test is flaky if: status === "passed" AND retry > 0
// A test is failed if: status === "failed"
// This means it failed initially but passed on retry OR failed completely
const flakyTests = [];

function traverseSuites(suites) {
  for (const suite of suites) {
    // Process specs in this suite
    for (const spec of suite.specs || []) {
      for (const test of spec.tests || []) {
        // Check each test result
        for (const result of test.results || []) {
          // Track two types of problematic tests:
          // 1. Flaky: passed on a retry attempt (retry > 0)
          // 2. Failed: failed on all attempts
          if ((result.status === 'passed' && result.retry > 0) || result.status === 'failed') {
            flakyTests.push({
              file: spec.file,
              title: spec.title,
              testTitle: spec.title,
              line: spec.line,
              status: result.status,
              retryAttempt: result.retry
            });
            break; // Only record once per test
          }
        }
      }
    }

    // Recursively process nested suites
    if (suite.suites && suite.suites.length > 0) {
      traverseSuites(suite.suites);
    }
  }
}

traverseSuites(results.suites || []);

// Save flaky tests to JSON
fs.writeFileSync('flaky-tests.json', JSON.stringify(flakyTests, null, 2));

// Generate markdown report
let markdown = '## ⚠️ Flaky/Failed Tests Detected\n\n';
markdown += 'The following tests are problematic:\n\n';

flakyTests.forEach(test => {
  const testType = test.status === 'failed' ? '❌ Failed' : '⚠️ Flaky';
  markdown += `### ${testType}: \`${test.file}\`\n`;
  markdown += `- **Test:** ${test.testTitle}\n`;
  markdown += `- **Status:** ${test.status}\n`;
  if (test.retryAttempt > 0) {
    markdown += `- **Retry Attempt:** ${test.retryAttempt}\n`;
  }
  markdown += `- **Debug command:**\n`;
  markdown += '```bash\n';
  markdown += `npx playwright test ${test.file} --repeat-each=5 --workers=1\n`;
  markdown += '```\n\n';
});

fs.writeFileSync('flaky-report.md', markdown);

console.log(`Found ${flakyTests.length} flaky/failed tests`);
process.exit(flakyTests.length > 0 ? 1 : 0);