CI: flaky test monitor (#7007)

.github/scripts/comment-on-flaky-tests.js

@@ -0,0 +1,70 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+// Check if flaky-tests.json exists
+if (!fs.existsSync('flaky-tests.json')) {
+  console.log('No flaky-tests.json found');
+  process.exit(0);
+}
+
+// Get changed files in PR
+let changedFiles = [];
+try {
+  changedFiles = execSync('git diff --name-only origin/main...HEAD')
+    .toString()
+    .split('\n')
+    .filter(f => f.endsWith('.spec.ts'));
+} catch (error) {
+  console.log('Could not determine changed files:', error.message);
+  process.exit(0);
+}
+
+if (changedFiles.length === 0) {
+  console.log('No test files were modified in this PR');
+  process.exit(0);
+}
+
+// Read flaky tests
+const flakyTests = JSON.parse(fs.readFileSync('flaky-tests.json', 'utf8'));
+
+if (flakyTests.length === 0) {
+  console.log('No flaky/failed tests found');
+  process.exit(0);
+}
+
+// Find modified flaky tests
+const modifiedFlakyTests = flakyTests.filter(test =>
+  changedFiles.some(file => test.file.includes(file))
+);
+
+if (modifiedFlakyTests.length === 0) {
+  console.log('No modified test files are flaky');
+  process.exit(0);
+}
+
+// Generate comment markdown
+let comment = '## ⚠️ Warning: You modified flaky/failed test files\n\n';
+comment += 'The following test files you modified have reliability issues:\n\n';
+
+modifiedFlakyTests.forEach(test => {
+  const testType = test.status === 'failed' ? '❌ Failed' : '⚠️ Flaky';
+  comment += `### ${testType}: \`${test.file}\`\n`;
+  comment += `**Test:** ${test.testTitle}\n`;
+  comment += `**Status:** ${test.status}\n`;
+  if (test.retryAttempt > 0) {
+    comment += `**Retry Attempt:** ${test.retryAttempt}\n`;
+  }
+  comment += '\n**To debug locally, run:**\n';
+  comment += '```bash\n';
+  comment += `npx playwright test ${test.file} --repeat-each=5 --workers=1\n`;
+  comment += '```\n\n';
+});
+
+comment += '---\n';
+comment += '**Note:** Flaky tests passed after retrying, failed tests did not pass. ';
+comment += 'Please investigate and fix the root cause before merging.\n';
+
+// Save comment to file for GitHub Action to post
+fs.writeFileSync('pr-comment.md', comment);
+
+console.log(`Found ${modifiedFlakyTests.length} modified flaky tests`);

.github/scripts/detect-flaky-tests.js

@@ -0,0 +1,78 @@
+const fs = require('fs');
+
+
+// Read Playwright JSON report
+const resultsPath = 'playwright-report/results.json';
+
+if (!fs.existsSync(resultsPath)) {
+  console.log('No Playwright results found at', resultsPath);
+  process.exit(0);
+}
+
+const results = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
+
+// Extract flaky tests
+// A test is flaky if: status === "passed" AND retry > 0
+// A test is failed if: status === "failed"
+// This means it failed initially but passed on retry OR failed completely
+const flakyTests = [];
+
+function traverseSuites(suites) {
+  for (const suite of suites) {
+    // Process specs in this suite
+    for (const spec of suite.specs || []) {
+      for (const test of spec.tests || []) {
+        // Check each test result
+        for (const result of test.results || []) {
+          // Track two types of problematic tests:
+          // 1. Flaky: passed on a retry attempt (retry > 0)
+          // 2. Failed: failed on all attempts
+          if ((result.status === 'passed' && result.retry > 0) || result.status === 'failed') {
+            flakyTests.push({
+              file: spec.file,
+              title: spec.title,
+              testTitle: spec.title,
+              line: spec.line,
+              status: result.status,
+              retryAttempt: result.retry
+            });
+            break; // Only record once per test
+          }
+        }
+      }
+    }
+
+    // Recursively process nested suites
+    if (suite.suites && suite.suites.length > 0) {
+      traverseSuites(suite.suites);
+    }
+  }
+}
+
+traverseSuites(results.suites || []);
+
+// Save flaky tests to JSON
+fs.writeFileSync('flaky-tests.json', JSON.stringify(flakyTests, null, 2));
+
+// Generate markdown report
+let markdown = '## ⚠️ Flaky/Failed Tests Detected\n\n';
+markdown += 'The following tests are problematic:\n\n';
+
+flakyTests.forEach(test => {
+  const testType = test.status === 'failed' ? '❌ Failed' : '⚠️ Flaky';
+  markdown += `### ${testType}: \`${test.file}\`\n`;
+  markdown += `- **Test:** ${test.testTitle}\n`;
+  markdown += `- **Status:** ${test.status}\n`;
+  if (test.retryAttempt > 0) {
+    markdown += `- **Retry Attempt:** ${test.retryAttempt}\n`;
+  }
+  markdown += `- **Debug command:**\n`;
+  markdown += '```bash\n';
+  markdown += `npx playwright test ${test.file} --repeat-each=5 --workers=1\n`;
+  markdown += '```\n\n';
+});
+
+fs.writeFileSync('flaky-report.md', markdown);
+
+console.log(`Found ${flakyTests.length} flaky/failed tests`);
+process.exit(flakyTests.length > 0 ? 1 : 0);