chore: fix flaky playwright tests (#7159)

This commit is contained in:
Bijin A B
2026-02-17 01:25:41 +05:30
committed by GitHub
parent 634b62642f
commit dfc3a1b78c
39 changed files with 111 additions and 160 deletions

View File

@@ -56,13 +56,13 @@ async function usePageWithTracing(
if (useChunks) {
await context.tracing.startChunk();
await use(page);
await context.tracing.stopChunk({ path: tracePath });
try { await context.tracing.stopChunk({ path: tracePath }); } catch { }
} else {
await use(page);
await context.tracing.stop({ path: tracePath });
try { await context.tracing.stop({ path: tracePath }); } catch { }
}
await testInfo.attach('trace', { path: tracePath });
try { await testInfo.attach('trace', { path: tracePath }); } catch { }
}
/**
@@ -96,13 +96,14 @@ export const test = baseTest.extend<
page: Page;
newPage: Page;
pageWithUserData: Page;
collectionFixturePath: string | null;
restartApp: (options?: { initUserDataPath?: string }) => Promise<ElectronApplication>;
},
{
createTmpDir: (tag?: string) => Promise<string>;
launchElectronApp: (options?: { initUserDataPath?: string; userDataPath?: string; dotEnv?: Record<string, string> }) => Promise<ElectronApplication>;
launchElectronApp: (options?: { initUserDataPath?: string; userDataPath?: string; dotEnv?: Record<string, string>; templateVars?: Record<string, string> }) => Promise<ElectronApplication>;
electronApp: ElectronApplication;
reuseOrLaunchElectronApp: (options?: { initUserDataPath?: string; testFile?: string; userDataPath?: string; dotEnv?: Record<string, string> }) => Promise<ElectronApplication>;
reuseOrLaunchElectronApp: (options?: { initUserDataPath?: string; testFile?: string; userDataPath?: string; dotEnv?: Record<string, string>; templateVars?: Record<string, string>; closePrevious?: boolean }) => Promise<ElectronApplication>;
}
>({
createTmpDir: [
@@ -120,10 +121,27 @@ export const test = baseTest.extend<
{ scope: 'worker' }
],
collectionFixturePath: async ({ createTmpDir }, use, testInfo) => {
const testDir = path.dirname(testInfo.file);
const fixturesDir = path.join(testDir, 'fixtures');
// fixtures/collections — multiple named collections (subdirs with bruno.json/opencollection.yml)
// fixtures/collection — single collection (single dir with bruno.json/opencollection.yml)
const srcPath = [path.join(fixturesDir, 'collections'), path.join(fixturesDir, 'collection')]
.find((p) => fs.existsSync(p));
if (srcPath) {
const tmpDir = await createTmpDir(path.basename(srcPath));
await fs.promises.cp(srcPath, tmpDir, { recursive: true });
await use(tmpDir);
} else {
await use(null);
}
},
launchElectronApp: [
async ({ playwright, createTmpDir }, use, workerInfo) => {
const apps: ElectronApplication[] = [];
await use(async ({ initUserDataPath, userDataPath: providedUserDataPath, dotEnv = {} } = {}) => {
await use(async ({ initUserDataPath, userDataPath: providedUserDataPath, dotEnv = {}, templateVars = {} } = {}) => {
const userDataPath = providedUserDataPath || (await createTmpDir('electron-userdata'));
// Ensure dir exists when caller supplies their own path
@@ -132,8 +150,9 @@ export const test = baseTest.extend<
}
if (initUserDataPath) {
const replacements = {
projectRoot: path.posix.join(__dirname, '..')
const replacements: Record<string, string> = {
projectRoot: path.posix.join(__dirname, '..'),
...templateVars
};
for (const file of await fs.promises.readdir(initUserDataPath)) {
@@ -217,12 +236,26 @@ export const test = baseTest.extend<
reuseOrLaunchElectronApp: [
async ({ launchElectronApp }, use, testInfo) => {
const apps: Record<string, ElectronApplication> = {};
await use(async ({ initUserDataPath, testFile, userDataPath, dotEnv = {} } = {}) => {
await use(async ({ initUserDataPath, testFile, userDataPath, dotEnv = {}, templateVars = {}, closePrevious = false } = {}) => {
const key = testFile || userDataPath || initUserDataPath;
if (key && apps[key]) {
return apps[key];
if (closePrevious) {
await closeElectronApp(apps[key]);
delete apps[key];
} else {
return apps[key];
}
}
const app = await launchElectronApp({ initUserDataPath, userDataPath, dotEnv });
// Close other cached apps to prevent resource accumulation across test files
for (const existingKey of Object.keys(apps)) {
if (existingKey !== key) {
await closeElectronApp(apps[existingKey]);
delete apps[existingKey];
}
}
const app = await launchElectronApp({ initUserDataPath, userDataPath, dotEnv, templateVars });
if (key) {
apps[key] = app;
}
@@ -232,32 +265,39 @@ export const test = baseTest.extend<
{ scope: 'worker' }
],
restartApp: async ({ launchElectronApp }, use, testInfo) => {
const appInstances: Array<{ app: ElectronApplication; initUserDataPath?: string }> = [];
restartApp: async ({ reuseOrLaunchElectronApp, createTmpDir, collectionFixturePath }, use, testInfo) => {
await use(async ({ initUserDataPath } = {}) => {
// Get the test directory and check for init-user-data folder
const testDir = path.dirname(testInfo.file);
const defaultInitUserDataPath = path.join(testDir, 'init-user-data');
// Use provided initUserDataPath, or check if default path exists, or use undefined
let userDataPath = initUserDataPath;
if (!userDataPath) {
let srcUserDataPath = initUserDataPath;
if (!srcUserDataPath) {
const hasInitUserData = await fs.promises.stat(defaultInitUserDataPath).catch(() => false);
userDataPath = hasInitUserData ? defaultInitUserDataPath : undefined;
srcUserDataPath = hasInitUserData ? defaultInitUserDataPath : undefined;
}
const app = await launchElectronApp({ initUserDataPath: userDataPath });
appInstances.push({ app, initUserDataPath: userDataPath });
return app;
});
// Copy init-user-data to a fresh tmp dir (same as pageWithUserData)
const tmpAppDataDir = await createTmpDir();
if (srcUserDataPath) {
await recursiveCopy(srcUserDataPath, tmpAppDataDir);
}
// Clean up all app instances
for (const { app } of appInstances) {
await closeElectronApp(app);
}
const templateVars: Record<string, string> = {};
if (collectionFixturePath) {
templateVars.collectionPath = collectionFixturePath;
}
// Close the previous app (from pageWithUserData) before launching a new one
return await reuseOrLaunchElectronApp({
initUserDataPath: tmpAppDataDir,
testFile: testInfo.file,
templateVars,
closePrevious: true
});
});
},
pageWithUserData: async ({ reuseOrLaunchElectronApp, createTmpDir }, use, testInfo) => {
pageWithUserData: async ({ reuseOrLaunchElectronApp, createTmpDir, collectionFixturePath }, use, testInfo) => {
const testDir = path.dirname(testInfo.file);
const initUserDataPath = path.join(testDir, 'init-user-data');
@@ -271,10 +311,18 @@ export const test = baseTest.extend<
throw err;
}
const app = await reuseOrLaunchElectronApp({ initUserDataPath: tmpAppDataDir, testFile: testInfo.file });
const templateVars: Record<string, string> = {};
if (collectionFixturePath) {
templateVars.collectionPath = collectionFixturePath;
}
const app = await reuseOrLaunchElectronApp({ initUserDataPath: tmpAppDataDir, testFile: testInfo.file, templateVars });
const context = await app.context();
const page = await app.firstWindow();
// Wait for app to be ready
await page.locator('[data-app-state="loaded"]').waitFor({ timeout: 30000 });
await usePageWithTracing(context, page, testInfo, use, { initTracing: true });
}
});

View File

@@ -10,19 +10,6 @@ test.describe('Create GraphQL Requests', () => {
});
test.afterAll(async ({ pageWithUserData: page }) => {
// Clean up Root GraphQL Request
await locators.sidebar.request('Root GraphQL Request').hover();
await locators.actions.collectionItemActions('Root GraphQL Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up Folder GraphQL Request
await locators.sidebar.request('Folder GraphQL Request').hover();
await locators.actions.collectionItemActions('Folder GraphQL Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up collection
await closeAllCollections(page);
});

View File

@@ -10,21 +10,6 @@ test.describe('Create gRPC Requests', () => {
});
test.afterAll(async ({ pageWithUserData: page }) => {
const locators = buildCommonLocators(page);
// Clean up Root gRPC Request
await locators.sidebar.request('Root gRPC Request').hover();
await locators.actions.collectionItemActions('Root gRPC Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up Folder gRPC Request
await locators.sidebar.request('Folder gRPC Request').hover();
await locators.actions.collectionItemActions('Folder gRPC Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up collection
await closeAllCollections(page);
});

View File

@@ -10,21 +10,6 @@ test.describe('Create HTTP Requests', () => {
});
test.afterAll(async ({ pageWithUserData: page }) => {
const locators = buildCommonLocators(page);
// Clean up Root HTTP Request
await locators.sidebar.request('Root HTTP Request').hover();
await locators.actions.collectionItemActions('Root HTTP Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up Folder HTTP Request
await locators.sidebar.request('Folder HTTP Request').hover();
await locators.actions.collectionItemActions('Folder HTTP Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up collection
await closeAllCollections(page);
});

View File

@@ -1,7 +1,7 @@
{
"collections": [
{
"path": "{{projectRoot}}/tests/collection/create-requests/fixtures/collection",
"path": "{{collectionPath}}",
"securityConfig": {
"jsSandboxMode": "safe"
}

View File

@@ -1,6 +1,6 @@
{
"maximized": false,
"lastOpenedCollections": [
"{{projectRoot}}/tests/collection/create-requests/fixtures/collection"
"{{collectionPath}}"
]
}

View File

@@ -10,21 +10,6 @@ test.describe('Create WebSocket Requests', () => {
});
test.afterAll(async ({ pageWithUserData: page }) => {
const locators = buildCommonLocators(page);
// Clean up Folder WebSocket Request
await locators.sidebar.request('Folder WebSocket Request').hover();
await locators.actions.collectionItemActions('Folder WebSocket Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up Root WebSocket Request
await locators.sidebar.request('Root WebSocket Request').hover();
await locators.actions.collectionItemActions('Root WebSocket Request').click();
await locators.dropdown.item('Delete').click();
await locators.modal.button('Delete').click();
// Clean up collection
await closeAllCollections(page);
});

View File

@@ -94,8 +94,8 @@ test.describe('Cross-Collection Drag and Drop', () => {
await expect(page.getByText(/Error: Cannot copy.*already exists/i)).toBeVisible();
// source and target collection request should remain unchanged
await expect(sourceCollectionContainer.locator('.collection-item-name').filter({ hasText: requestName })).toBeVisible();
await expect(sourceCollectionContainer.locator('.collection-item-name').filter({ hasText: requestName }).first()).toBeVisible();
await page.locator('#sidebar-collection-name').filter({ hasText: 'target-collection' }).click();
await expect(targetCollectionContainer.locator('.collection-item-name').filter({ hasText: requestName })).toBeVisible();
await expect(targetCollectionContainer.locator('.collection-item-name').filter({ hasText: requestName }).first()).toBeVisible();
});
});

View File

@@ -1,14 +1,8 @@
import { test, expect } from '../../../playwright';
import fs from 'fs';
import path from 'path';
import { sendRequest } from '../../utils/page';
test.describe.serial('bru.setEnvVar(name, value, { persist: true })', () => {
test('set env var with persist using script', async ({ pageWithUserData: page, restartApp }) => {
// Keep a copy of the original Stage.bru file
const originalStageBruPath = path.join(__dirname, 'fixtures/collection/environments/Stage.bru');
const originalStageBruContent = fs.readFileSync(originalStageBruPath, 'utf8');
// Select the collection and request
await page.locator('#sidebar-collection-name').click();
await page.getByText('api-setEnvVar-with-persist', { exact: true }).click();
@@ -61,8 +55,6 @@ test.describe.serial('bru.setEnvVar(name, value, { persist: true })', () => {
await newEnvTab.hover();
await newEnvTab.getByTestId('request-tab-close-icon').click({ force: true });
// Restore the original Stage.bru file
fs.writeFileSync(originalStageBruPath, originalStageBruContent);
await newPage.close();
});
});

View File

@@ -1,7 +1,7 @@
{
"collections": [
{
"path": "{{projectRoot}}/tests/environments/api-setEnvVar/fixtures/collection",
"path": "{{collectionPath}}",
"securityConfig": {
"jsSandboxMode": "safe"
}

View File

@@ -1,6 +1,6 @@
{
"maximized": false,
"lastOpenedCollections": [
"{{projectRoot}}/tests/environments/api-setEnvVar/fixtures/collection"
"{{collectionPath}}"
]
}

View File

@@ -35,7 +35,7 @@ test.describe.serial('bru.setEnvVar multiple persistent variables', () => {
}
});
test('should persist multiple environment variables from different requests', async ({ pageWithUserData: page }) => {
test('should persist multiple environment variables from different requests', async ({ pageWithUserData: page, collectionFixturePath }) => {
await test.step('Select collection', async () => {
await page.locator('#sidebar-collection-name').click();
// The collection name should be 'collection' based on the test setup
@@ -90,7 +90,7 @@ test.describe.serial('bru.setEnvVar multiple persistent variables', () => {
await test.step('Verify variables are persisted to file', async () => {
// Check that the variables are written to the Stage.bru file
const stageBruPath = path.join(__dirname, 'fixtures/collection/environments/Stage.bru');
const stageBruPath = path.join(collectionFixturePath!, 'environments', 'Stage.bru');
const stageBruContent = fs.readFileSync(stageBruPath, 'utf8');
// Both variables should be present in the file

View File

@@ -1,6 +1,6 @@
{
"maximized": false,
"lastOpenedCollections": [
"{{projectRoot}}/tests/environments/import-environment/env-color-import/fixtures/collection"
"{{collectionPath}}"
]
}

View File

@@ -1,7 +1,7 @@
{
"collections": [
{
"path": "{{projectRoot}}/tests/environments/multiline-variables/collection",
"path": "{{collectionPath}}",
"securityConfig": {
"jsSandboxMode": "developer"
}

View File

@@ -1,7 +1,7 @@
{
"maximized": true,
"lastOpenedCollections": [
"{{projectRoot}}/tests/environments/multiline-variables/collection"
"{{collectionPath}}"
],
"request": {
"sslVerification": false,

View File

@@ -73,18 +73,4 @@ test.describe('Multiline Variables - Write Test', () => {
= '{\n "user": {\n "name": "John Doe",\n "email": "john@example.com",\n "preferences": {\n "theme": "dark",\n "notifications": true\n }\n },\n "metadata": {\n "created": "2025-09-03",\n "version": "1.0"\n }\n}';
await expect(page.locator('.response-pane')).toContainText(`"body": ${JSON.stringify(expectedBody)}`);
});
// clean up created variable after test
test.afterEach(async () => {
const fs = require('fs');
const path = require('path');
const testBruPath = path.join(__dirname, 'collection/environments/Test.bru');
let content = fs.readFileSync(testBruPath, 'utf8');
// remove the multiline_data_json variable and its content
content = content.replace(/\s*multiline_data_json:\s*'''\s*[\s\S]*?\s*'''/g, '');
fs.writeFileSync(testBruPath, content);
});
});

View File

@@ -25,7 +25,7 @@ test.describe('make grpc requests', () => {
await test.step('select unary method', async () => {
await locators.sidebar.request('SayHello').click();
await expect(locators.method.dropdownTrigger()).toContainText('HelloService/SayHello');
await expect(locators.method.dropdownTrigger()).toContainText('HelloService/SayHello', { timeout: 30000 });
});
await test.step('verify gRPC unary request is opened successfully', async () => {

View File

@@ -11,25 +11,25 @@
"protobuf": {
"protoFiles": [
{
"path": "../protos/services/invalid-file-path.proto",
"path": "./protos/services/invalid-file-path.proto",
"type": "file"
},
{
"path": "../protos/services/product.proto",
"path": "./protos/services/product.proto",
"type": "file"
},
{
"path": "../protos/services/order.proto",
"path": "./protos/services/order.proto",
"type": "file"
}
],
"importPaths": [
{
"path": "../protos/invalid-import-path",
"path": "./protos/invalid-import-path",
"enabled": true
},
{
"path": "../protos/types",
"path": "./protos/types",
"enabled": false
},
{

View File

@@ -1,7 +1,7 @@
{
"collections": [
{
"path": "{{projectRoot}}/tests/protobuf/collection",
"path": "{{collectionPath}}",
"securityConfig": {
"jsSandboxMode": "safe"
}

View File

@@ -1,7 +1,7 @@
{
"maximized": true,
"lastOpenedCollections": [
"{{projectRoot}}/tests/protobuf/collection"
"{{collectionPath}}"
],
"preferences": {
"beta": {

View File

@@ -1,25 +1,9 @@
import path from 'path';
import { test, expect } from '../../playwright';
import { closeAllCollections } from '../utils/page';
import fs from 'fs';
const COLLECTION_PATH = path.join(__dirname, 'collection', 'bruno.json');
const BACKUP_PATH = path.join(__dirname, 'collection', 'bruno.json.backup');
import { execSync } from 'child_process';
test.describe('manage protofile', () => {
test.beforeAll(async () => {
// Backup original file
if (fs.existsSync(COLLECTION_PATH)) {
fs.copyFileSync(COLLECTION_PATH, BACKUP_PATH);
}
});
test.afterAll(async ({ pageWithUserData: page }) => {
// Close all collections
await closeAllCollections(page);
// Reset the collection request file to the original state
execSync(`git checkout -- ${path.join(__dirname, 'collection', 'bruno.json')}`);
});
test('protofiles, import paths from bruno.json are visible in the protobuf settings', async ({ pageWithUserData: page }) => {
@@ -39,7 +23,7 @@ test.describe('manage protofile', () => {
const file = page.getByRole('cell', { name: 'product.proto', exact: true });
await expect(file).toBeVisible();
const filePath = page.getByRole('cell', { name: '../protos/services/product.proto' });
const filePath = page.getByRole('cell', { name: './protos/services/product.proto' });
await expect(filePath).toBeVisible();
// Check import paths table
@@ -47,18 +31,18 @@ test.describe('manage protofile', () => {
await expect(importPathsTable).toBeVisible();
// Wait for import paths table data to load
const importPath = page.getByRole('cell', { name: '../protos/types', exact: true });
const importPath = page.getByRole('cell', { name: './protos/types', exact: true });
await expect(importPath).toBeVisible();
// Wait for invalid file path cell to appear
const invalidFilePath = page.getByRole('cell', { name: 'invalid-file-path.proto', exact: true });
await expect(invalidFilePath).toBeVisible();
const invalidImportPath = page.getByRole('cell', { name: '../protos/invalid-import-path', exact: true });
const invalidImportPath = page.getByRole('cell', { name: './protos/invalid-import-path', exact: true });
await expect(invalidImportPath).toBeVisible();
const collectionPathAsImportPath = page.getByRole('cell', { name: '.', exact: true });
const collectionPathName = page.getByRole('cell', { name: 'collection', exact: true });
const collectionPathName = page.getByRole('cell', { name: /^pw-collection-/ });
// Invalid messages using test IDs
const invalidProtoFilesMessage = page.getByTestId('protobuf-invalid-files-message');
@@ -76,9 +60,9 @@ test.describe('manage protofile', () => {
await expect(page.getByRole('cell', { name: 'invalid-file-path.proto', exact: true })).not.toBeVisible();
await expect(invalidProtoFilesMessage).not.toBeVisible();
await page.getByRole('row', { name: '../protos/invalid-import-path' }).getByTestId('protobuf-remove-import-path-button').click();
await page.getByRole('row', { name: './protos/invalid-import-path' }).getByTestId('protobuf-remove-import-path-button').click();
await expect(page.getByRole('cell', { name: '../protos/invalid-import-path', exact: true })).not.toBeVisible();
await expect(page.getByRole('cell', { name: './protos/invalid-import-path', exact: true })).not.toBeVisible();
await expect(invalidImportPathsMessage).not.toBeVisible();
// Save the changes to persist them to bruno.json
@@ -98,7 +82,7 @@ test.describe('manage protofile', () => {
await page.getByText('Proto FileReflection').click();
// Use more specific selector for proto file selection
await page.locator('div').filter({ hasText: /^order\.proto\.\.\/protos\/services\/order\.proto$/ }).first().click();
await page.locator('div').filter({ hasText: /^order\.proto\.\/protos\/services\/order\.proto$/ }).first().click();
// Use test ID for method selection
const grpcMethodsDropdown = page.getByTestId('grpc-methods-dropdown');
@@ -125,7 +109,7 @@ test.describe('manage protofile', () => {
await page.getByText('Proto FileReflection').click();
// Use more specific selector for proto file selection
await page.locator('div').filter({ hasText: /^product\.proto\.\.\/protos\/services\/product\.proto$/ }).first().click();
await page.locator('div').filter({ hasText: /^product\.proto\.\/protos\/services\/product\.proto$/ }).first().click();
// Verify the error message is visible (auto-retrying)
await expect(page.getByText('Failed to load gRPC methods: Unknown error').first()).toBeVisible();
@@ -170,7 +154,7 @@ test.describe('manage protofile', () => {
await page.getByText('Proto FileReflection').click();
// Use more specific selector for proto file selection
await page.locator('div').filter({ hasText: /^product\.proto\.\.\/protos\/services\/product\.proto$/ }).first().click();
await page.locator('div').filter({ hasText: /^product\.proto\.\/protos\/services\/product\.proto$/ }).first().click();
const grpcMethodsDropdown = page.getByTestId('grpc-methods-dropdown');
await grpcMethodsDropdown.click();
const method = page.getByTestId('grpc-methods-list').filter({ hasText: 'CreateProductunary' }).first();

View File

@@ -1,7 +1,7 @@
{
"collections": [
{
"path": "{{projectRoot}}/tests/response/json-response-formatting/fixtures/collection",
"path": "{{collectionPath}}",
"securityConfig": {
"jsSandboxMode": "safe"
}

View File

@@ -1,6 +1,6 @@
{
"maximized": false,
"lastOpenedCollections": [
"{{projectRoot}}/tests/response/json-response-formatting/fixtures/collection"
"{{collectionPath}}"
]
}

View File

@@ -1,7 +1,7 @@
{
"collections": [
{
"path": "{{projectRoot}}/tests/scripting/bru-api/isSafeMode/fixtures/collections/is-safe-mode-test",
"path": "{{collectionPath}}/is-safe-mode-test",
"securityConfig": {
"jsSandboxMode": "developer"
}

View File

@@ -1,6 +1,6 @@
{
"maximized": false,
"lastOpenedCollections": [
"{{projectRoot}}/tests/scripting/bru-api/isSafeMode/fixtures/collections/is-safe-mode-test"
"{{collectionPath}}/is-safe-mode-test"
]
}

View File

@@ -1,6 +1,6 @@
{
"maximized": false,
"lastOpenedCollections": [
"{{projectRoot}}/tests/scripting/url-helpers/fixtures/collections/url_helpers_test"
"{{collectionPath}}/url_helpers_test"
]
}

View File

@@ -114,11 +114,10 @@ export const setSandboxMode = async (page: Page, collectionName: string, mode: '
if (mode === 'developer') {
await sandboxLocators.developerModeRadio().waitFor({ state: 'visible', timeout: 5000 });
await sandboxLocators.developerModeRadio().check();
await sandboxLocators.developerModeRadio().click();
} else {
// Ensure Safe Mode radio is visible and check it
await sandboxLocators.safeModeRadio().waitFor({ state: 'visible', timeout: 5000 });
await sandboxLocators.safeModeRadio().check();
await sandboxLocators.safeModeRadio().click();
}
await page.keyboard.press('Escape');