feat: async parser workers (#3834)

This commit is contained in:
Anoop M D
2025-01-27 15:33:05 +05:30
parent 074c6be5f4
commit e34ac3de7c
12 changed files with 148 additions and 95 deletions

View File

@@ -1,41 +1,55 @@
import React from 'react';
import StyledWrapper from './StyledWrapper';
import { getTotalRequestCountInCollection } from 'utils/collections/';
import { IconFolder, IconFileOff, IconWorld, IconApi } from '@tabler/icons';
const Info = ({ collection }) => {
const totalRequestsInCollection = getTotalRequestCountInCollection(collection);
return (
<StyledWrapper className="w-full flex flex-col h-fit">
<div className="text-xs mb-4 text-muted">General information about the collection.</div>
<table className="w-full border-collapse">
<tbody>
<tr className="">
<td className="py-2 px-2 text-right">Name&nbsp;:</td>
<td className="py-2 px-2">{collection.name}</td>
</tr>
<tr className="">
<td className="py-2 px-2 text-right">Location&nbsp;:</td>
<td className="py-2 px-2 break-all">{collection.pathname}</td>
</tr>
<tr className="">
<td className="py-2 px-2 text-right">Ignored files&nbsp;:</td>
<td className="py-2 px-2 break-all">{collection.brunoConfig?.ignore?.map((x) => `'${x}'`).join(', ')}</td>
</tr>
<tr className="">
<td className="py-2 px-2 text-right">Environments&nbsp;:</td>
<td className="py-2 px-2">{collection.environments?.length || 0}</td>
</tr>
<tr className="">
<td className="py-2 px-2 text-right">Requests&nbsp;:</td>
<td className="py-2 px-2">{totalRequestsInCollection}</td>
</tr>
<tr className="">
<td className="py-2 px-2 text-right">Size&nbsp;:</td>
<td className="py-2 px-2">{collection?.brunoConfig?.size?.toFixed?.(3)} MB</td>
</tr>
</tbody>
</table>
<StyledWrapper className="w-full flex flex-col h-fit mt-2">
<div className="bg-white dark:bg-gray-800 rounded-lg py-6">
<div className="grid gap-6">
{/* Location Row */}
<div className="flex items-start">
<div className="flex-shrink-0 p-3 bg-blue-50 dark:bg-blue-900/20 rounded-lg">
<IconFolder className="w-5 h-5 text-blue-500" stroke={1.5} />
</div>
<div className="ml-4">
<div className="font-semibold text-sm text-gray-900 dark:text-gray-100">Location</div>
<div className="mt-1 text-xs text-gray-600 dark:text-gray-300 break-all">
{collection.pathname}
</div>
</div>
</div>
{/* Environments Row */}
<div className="flex items-start">
<div className="flex-shrink-0 p-3 bg-green-50 dark:bg-green-900/20 rounded-lg">
<IconWorld className="w-5 h-5 text-green-500" stroke={1.5} />
</div>
<div className="ml-4">
<div className="font-semibold text-sm text-gray-900 dark:text-gray-100">Environments</div>
<div className="mt-1 text-sm text-gray-600 dark:text-gray-300">
{collection.environments?.length || 0} environment{collection.environments?.length !== 1 ? 's' : ''} configured
</div>
</div>
</div>
{/* Requests Row */}
<div className="flex items-start">
<div className="flex-shrink-0 p-3 bg-purple-50 dark:bg-purple-900/20 rounded-lg">
<IconApi className="w-5 h-5 text-purple-500" stroke={1.5} />
</div>
<div className="ml-4">
<div className="font-semibold text-sm text-gray-900 dark:text-gray-100">Requests</div>
<div className="mt-1 text-sm text-gray-600 dark:text-gray-300">
{totalRequestsInCollection} request{totalRequestsInCollection !== 1 ? 's' : ''} in collection
</div>
</div>
</div>
</div>
</div>
</StyledWrapper>
);
};

View File

@@ -2,14 +2,19 @@ import { flattenItems } from "utils/collections/index";
import StyledWrapper from "./StyledWrapper";
import Docs from "../Docs/index";
import Info from "../Info/index";
import { IconBox, IconAlertTriangle } from '@tabler/icons';
const Overview = ({ collection }) => {
const flattenedItems = flattenItems(collection.items);
const itemsFailedLoading = flattenedItems?.filter(item => item?.partial && !item?.loading);
return (
<StyledWrapper className="flex flex-col h-full relative px-4 py-4 gap-4">
<StyledWrapper className="flex flex-col h-full relative py-2 gap-4">
<div className="flex flex-row grid grid-cols-5 w-full gap-8">
<div className="col-span-2 flex flex-col gap-12">
<div className="col-span-2 flex flex-col">
<div className="text-xl font-semibold flex items-center gap-2">
<IconBox size={24} />
{collection?.name}
</div>
<Info collection={collection} />
{
itemsFailedLoading?.length ?

View File

@@ -22,9 +22,9 @@ import SecuritySettings from 'components/SecuritySettings';
import FolderSettings from 'components/FolderSettings';
import { getGlobalEnvironmentVariables, getGlobalEnvironmentVariablesMasked } from 'utils/collections/index';
import { produce } from 'immer';
import CollectionLoadStats from 'components/CollectionSettings/Overview/index';
import RequestNotLoaded from './RequestNotLoaded/index';
import RequestIsLoading from './RequestIsLoading/index';
import CollectionOverview from 'components/CollectionSettings/Overview';
import RequestNotLoaded from './RequestNotLoaded';
import RequestIsLoading from './RequestIsLoading';
const MIN_LEFT_PANE_WIDTH = 300;
const MIN_RIGHT_PANE_WIDTH = 350;
@@ -158,7 +158,7 @@ const RequestTabPanel = () => {
}
if (focusedTab.type === 'collection-overview') {
return <CollectionLoadStats collection={collection} />;
return <CollectionOverview collection={collection} />;
}
if (focusedTab.type === 'folder-settings') {

View File

@@ -38,7 +38,9 @@ const CollectionItem = ({ item, collection, searchText }) => {
const [newRequestModalOpen, setNewRequestModalOpen] = useState(false);
const [newFolderModalOpen, setNewFolderModalOpen] = useState(false);
const [runCollectionModalOpen, setRunCollectionModalOpen] = useState(false);
const [itemIsCollapsed, setItemisCollapsed] = useState(item.collapsed);
const hasSearchText = searchText && searchText?.trim()?.length;
const itemIsCollapsed = hasSearchText ? false : item.collapsed;
const [{ isDragging }, drag] = useDrag({
type: `COLLECTION_ITEM_${collection.uid}`,
@@ -63,14 +65,6 @@ const CollectionItem = ({ item, collection, searchText }) => {
})
});
useEffect(() => {
if (searchText && searchText.length) {
setItemisCollapsed(false);
} else {
setItemisCollapsed(item.collapsed);
}
}, [searchText, item]);
const dropdownTippyRef = useRef();
const MenuIcon = forwardRef((props, ref) => {
return (

View File

@@ -5,8 +5,8 @@ import filter from 'lodash/filter';
import { useDrop } from 'react-dnd';
import { IconChevronRight, IconDots, IconLoader2 } from '@tabler/icons';
import Dropdown from 'components/Dropdown';
import { collectionClicked } from 'providers/ReduxStore/slices/collections';
import { loadCollection, moveItemToRootOfCollection } from 'providers/ReduxStore/slices/collections/actions';
import { collapseCollection } from 'providers/ReduxStore/slices/collections';
import { mountCollection, moveItemToRootOfCollection } from 'providers/ReduxStore/slices/collections/actions';
import { useDispatch } from 'react-redux';
import { addTab } from 'providers/ReduxStore/slices/tabs';
import NewRequest from 'components/Sidebar/NewRequest';
@@ -29,8 +29,6 @@ const Collection = ({ collection, searchText }) => {
const [showCloneCollectionModalOpen, setShowCloneCollectionModalOpen] = useState(false);
const [showExportCollectionModal, setShowExportCollectionModal] = useState(false);
const [showRemoveCollectionModal, setShowRemoveCollectionModal] = useState(false);
const [collectionIsCollapsed, setCollectionIsCollapsed] = useState(collection.collapsed);
const [hasCollectionLoadingBeenTriggered, setHasCollectionLoadingBeenTriggered] = useState(false);
const dispatch = useDispatch();
const isLoading = areItemsLoading(collection);
@@ -54,34 +52,38 @@ const Collection = ({ collection, searchText }) => {
);
};
useEffect(() => {
if (searchText && searchText.length) {
setCollectionIsCollapsed(false);
} else {
setCollectionIsCollapsed(collection.collapsed);
}
}, [searchText, collection]);
const hasSearchText = searchText && searchText?.trim()?.length;
const collectionIsCollapsed = hasSearchText ? false : collection.collapsed;
const iconClassName = classnames({
'rotate-90': !collectionIsCollapsed
});
const handleClick = (event) => {
dispatch(collectionClicked(collection.uid));
};
// Check if the click came from the chevron icon
const isChevronClick = event.target.closest('svg')?.classList.contains('chevron-icon');
const handleCollapseCollection = () => {
dispatch(collectionClicked(collection.uid));
setHasCollectionLoadingBeenTriggered(true);
!hasCollectionLoadingBeenTriggered && dispatch(loadCollection({ collectionUid: collection?.uid, collectionPathname: collection?.pathname, brunoConfig: collection?.brunoConfig }));
dispatch(
addTab({
uid: uuid(),
console.log('handleClick', collection.mountStatus);
if (collection.mountStatus === 'unmounted') {
dispatch(mountCollection({
collectionUid: collection.uid,
type: 'collection-settings'
})
);
}
collectionPathname: collection.pathname,
brunoConfig: collection.brunoConfig
}));
}
dispatch(collapseCollection(collection.uid));
// Only open collection settings if not clicking the chevron
if(!isChevronClick) {
dispatch(
addTab({
uid: uuid(),
collectionUid: collection.uid,
type: 'collection-settings'
})
);
}
};
const handleRightClick = (event) => {
const _menuDropdown = menuDropdownTippyRef.current;
@@ -156,15 +158,14 @@ const Collection = ({ collection, searchText }) => {
<div className="flex py-1 collection-name items-center" ref={drop}>
<div
className="flex flex-grow items-center overflow-hidden"
onClick={handleCollapseCollection}
onClick={handleClick}
onContextMenu={handleRightClick}
>
<IconChevronRight
size={16}
strokeWidth={2}
className={iconClassName}
className={`chevron-icon ${iconClassName}`}
style={{ width: 16, minWidth: 16, color: 'rgb(160 160 160)' }}
onClick={handleClick}
/>
<div className="ml-1" id="sidebar-collection-name">
{collection.name}

View File

@@ -23,6 +23,7 @@ import {
import { uuid, waitForNextTick } from 'utils/common';
import { PATH_SEPARATOR, getDirectoryName } from 'utils/common/platform';
import { cancelNetworkRequest, sendNetworkRequest } from 'utils/network';
import { callIpc } from 'utils/common/ipc';
import {
collectionAddEnvFileEvent as _collectionAddEnvFileEvent,
@@ -30,6 +31,7 @@ import {
removeCollection as _removeCollection,
selectEnvironment as _selectEnvironment,
sortCollections as _sortCollections,
updateCollectionMountStatus,
requestCancelled,
resetRunResults,
responseReceived,
@@ -42,7 +44,6 @@ import { closeAllCollectionTabs } from 'providers/ReduxStore/slices/tabs';
import { resolveRequestFilename } from 'utils/common/platform';
import { parsePathParams, parseQueryParams, splitOnFirst } from 'utils/url/index';
import { sendCollectionOauth2Request as _sendCollectionOauth2Request } from 'utils/network/index';
import { name } from 'file-loader';
import slash from 'utils/common/slash';
import { getGlobalEnvironmentVariables } from 'utils/collections/index';
import { findCollectionByPathname, findEnvironmentInCollectionByName } from 'utils/collections/index';
@@ -1208,9 +1209,15 @@ export const loadRequestSync = ({ collectionUid, pathname }) => (dispatch, getSt
});
};
export const loadCollection = ({ collectionUid, collectionPathname, brunoConfig }) => (dispatch, getState) => {
export const mountCollection = ({ collectionUid, collectionPathname, brunoConfig }) => (dispatch, getState) => {
dispatch(updateCollectionMountStatus({ collectionUid, mountStatus: 'mounting' }));
return new Promise(async (resolve, reject) => {
const { ipcRenderer } = window;
ipcRenderer.invoke('renderer:load-collection', { collectionUid, collectionPathname, brunoConfig }).then(resolve).catch(reject);
callIpc('renderer:mount-collection', { collectionUid, collectionPathname, brunoConfig })
.then(() => dispatch(updateCollectionMountStatus({ collectionUid, mountStatus: 'mounted' })))
.then(resolve)
.catch(() => {
dispatch(updateCollectionMountStatus({ collectionUid, mountStatus: 'unmounted' }));
reject();
});
});
};
};

View File

@@ -4,7 +4,7 @@ import { createSlice } from '@reduxjs/toolkit';
import {
addDepth,
areItemsTheSameExceptSeqUpdate,
collapseCollection,
collapseAllItemsInCollection,
deleteItemInCollection,
deleteItemInCollectionByPathname,
findCollectionByPathname,
@@ -35,6 +35,10 @@ export const collectionsSlice = createSlice({
collection.settingsSelectedTab = 'overview';
collection.folderLevelSettingsSelectedTab = {};
// Collection mount status is used to track the mount status of the collection
// values can be 'unmounted', 'mounting', 'mounted'
collection.mountStatus = 'unmounted';
// TODO: move this to use the nextAction approach
// last action is used to track the last action performed on the collection
// this is optional
@@ -44,12 +48,18 @@ export const collectionsSlice = createSlice({
collection.importedAt = new Date().getTime();
collection.lastAction = null;
collapseCollection(collection);
collapseAllItemsInCollection(collection);
addDepth(collection.items);
if (!collectionUids.includes(collection.uid)) {
state.collections.push(collection);
}
},
updateCollectionMountStatus: (state, action) => {
const collection = findCollectionByUid(state.collections, action.payload.collectionUid);
if (collection) {
collection.mountStatus = action.payload.mountStatus;
}
},
setCollectionSecurityConfig: (state, action) => {
const collection = findCollectionByUid(state.collections, action.payload.collectionUid);
if (collection) {
@@ -358,7 +368,7 @@ export const collectionsSlice = createSlice({
collection.items.push(item);
}
},
collectionClicked: (state, action) => {
collapseCollection: (state, action) => {
const collection = findCollectionByUid(state.collections, action.payload);
if (collection) {
@@ -1896,6 +1906,7 @@ export const collectionsSlice = createSlice({
export const {
createCollection,
updateCollectionMountStatus,
setCollectionSecurityConfig,
brunoConfigUpdateEvent,
renameCollection,
@@ -1919,7 +1930,7 @@ export const {
saveRequest,
deleteRequestDraft,
newEphemeralHttpRequest,
collectionClicked,
collapseCollection,
collectionFolderClicked,
requestUrlChanged,
updateAuth,

View File

@@ -34,7 +34,7 @@ export const addDepth = (items = []) => {
depth(items, 1);
};
export const collapseCollection = (collection) => {
export const collapseAllItemsInCollection = (collection) => {
collection.collapsed = true;
const collapseItem = (items) => {
@@ -47,7 +47,7 @@ export const collapseCollection = (collection) => {
});
};
collapseItem(collection.items, 1);
collapseItem(collection.items);
};
export const sortItems = (collection) => {

View File

@@ -0,0 +1,14 @@
/**
* Wrapper for ipcRenderer.invoke that handles error cases
* @param {string} channel - The IPC channel name
* @param {...any} args - Arguments to pass to the channel
* @returns {Promise} - Resolves with the result or rejects with error
*/
export const callIpc = (channel, ...args) => {
const { ipcRenderer } = window;
if (!ipcRenderer) {
return Promise.reject(new Error('IPC Renderer not available'));
}
return ipcRenderer.invoke(channel, ...args);
};

View File

@@ -901,6 +901,12 @@ const registerRendererEventHandlers = (mainWindow, watcher, lastOpenedCollection
return Promise.reject(error);
}
});
ipcMain.handle('renderer:mount-collection', async (event, { collectionUid, collectionPathname, brunoConfig }) => {
const { size: collectionSize, filesCount: collectionBruFilesCount, maxFileSize: maxSingleBruFileSize } = await getCollectionStats(collectionPathname);
const shouldLoadCollectionAsync = (collectionSize > MAX_COLLECTION_SIZE_IN_MB) || (collectionBruFilesCount > MAX_COLLECTION_FILES_COUNT) || (maxSingleBruFileSize > MAX_SINGLE_FILE_SIZE_IN_COLLECTION_IN_MB);
watcher.addWatcher(mainWindow, collectionPathname, collectionUid, brunoConfig, false, shouldLoadCollectionAsync);
});
};
const registerMainEventHandlers = (mainWindow, watcher, lastOpenedCollections) => {
@@ -925,12 +931,6 @@ const registerMainEventHandlers = (mainWindow, watcher, lastOpenedCollections) =
mainWindow.webContents.send('main:start-quit-flow');
});
ipcMain.handle('renderer:load-collection', async (event, { collectionUid, collectionPathname, brunoConfig }) => {
const { size: collectionSize, filesCount: collectionBruFilesCount, maxFileSize: maxSingleBruFileSize } = await getCollectionStats(collectionPathname);
const shouldLoadCollectionAsync = (collectionSize > MAX_COLLECTION_SIZE_IN_MB) || (collectionBruFilesCount > MAX_COLLECTION_FILES_COUNT) || (maxSingleBruFileSize > MAX_SINGLE_FILE_SIZE_IN_COLLECTION_IN_MB);
watcher.addWatcher(mainWindow, collectionPathname, collectionUid, brunoConfig, false, shouldLoadCollectionAsync);
});
ipcMain.handle('main:complete-quit-flow', () => {
mainWindow.destroy();
});

View File

@@ -6,7 +6,9 @@ class WorkerQueue {
this.isProcessing = false;
}
async enqueue({ priority, scriptPath, data }) {
async enqueue(task) {
const { priority, scriptPath, data } = task;
return new Promise((resolve, reject) => {
this.queue.push({ priority, scriptPath, data, resolve, reject });
this.queue?.sort((taskX, taskY) => taskX?.priority - taskY?.priority);
@@ -15,9 +17,13 @@ class WorkerQueue {
}
async processQueue() {
if (this.isProcessing || this.queue.length === 0) return;
if (this.isProcessing || this.queue.length === 0){
return;
}
this.isProcessing = true;
const { scriptPath, data, resolve, reject } = this.queue.shift();
try {
const result = await this.runWorker({ scriptPath, data });
resolve(result);
@@ -41,7 +47,6 @@ class WorkerQueue {
worker.terminate();
});
worker.on('exit', (code) => {
// if (code !== 0)
reject(new Error(`stopped with ${code} exit code`));
worker.terminate();
});

View File

@@ -1,9 +1,11 @@
{
"version": "1",
"name": "collection_oauth2",
"name": "OAuth2 Demo",
"type": "collection",
"scripts": {
"moduleWhitelist": ["crypto"],
"moduleWhitelist": [
"crypto"
],
"filesystemAccess": {
"allow": true
}
@@ -15,4 +17,4 @@
"presets": {
"requestType": "http"
}
}
}