From 31409c6206a4ac4e03ef4cd6abaa4569d7f2c3db Mon Sep 17 00:00:00 2001 From: lohit Date: Tue, 18 Feb 2025 19:58:37 +0530 Subject: [PATCH] feat: reuse worker threads for bru file parsing (#4054) --- .../CollectionSettings/Overview/Info/index.js | 16 +++++++--- .../Overview/RequestsNotLoaded/index.js | 32 ++++++++++++++++++- .../RequestTabPanel/RequestNotLoaded/index.js | 8 ++--- .../ReduxStore/slices/collections/index.js | 4 ++- .../bruno-app/src/utils/collections/index.js | 14 ++++++++ .../src/bru/workers/scripts/bru-to-json.js | 22 +++++++------ .../src/bru/workers/scripts/json-to-bru.js | 23 +++++++------ packages/bruno-electron/src/ipc/collection.js | 6 ++-- packages/bruno-electron/src/workers/index.js | 18 ++++++++--- 9 files changed, 104 insertions(+), 39 deletions(-) diff --git a/packages/bruno-app/src/components/CollectionSettings/Overview/Info/index.js b/packages/bruno-app/src/components/CollectionSettings/Overview/Info/index.js index 86bf2308f..a83850e91 100644 --- a/packages/bruno-app/src/components/CollectionSettings/Overview/Info/index.js +++ b/packages/bruno-app/src/components/CollectionSettings/Overview/Info/index.js @@ -1,10 +1,14 @@ -import React from 'react'; +import React from "react"; import { getTotalRequestCountInCollection } from 'utils/collections/'; -import { IconFolder, IconFileOff, IconWorld, IconApi } from '@tabler/icons'; +import { IconFolder, IconWorld, IconApi, IconClock } from '@tabler/icons'; +import { areItemsLoading, getItemsLoadStats } from "utils/collections/index"; const Info = ({ collection }) => { const totalRequestsInCollection = getTotalRequestCountInCollection(collection); + const isCollectionLoading = areItemsLoading(collection); + const { loading: itemsLoadingCount, total: totalItems } = getItemsLoadStats(collection); + return (
@@ -42,8 +46,10 @@ const Info = ({ collection }) => {
Requests
-
- {totalRequestsInCollection} request{totalRequestsInCollection !== 1 ? 's' : ''} in collection +
+ { + isCollectionLoading? `${totalItems - itemsLoadingCount} out of ${totalItems} requests in the collection loaded` : `${totalRequestsInCollection} request${totalRequestsInCollection !== 1 ? 's' : ''} in collection` + }
@@ -53,4 +59,4 @@ const Info = ({ collection }) => { ); }; -export default Info; +export default Info; \ No newline at end of file diff --git a/packages/bruno-app/src/components/CollectionSettings/Overview/RequestsNotLoaded/index.js b/packages/bruno-app/src/components/CollectionSettings/Overview/RequestsNotLoaded/index.js index c15b36cd8..4c7406580 100644 --- a/packages/bruno-app/src/components/CollectionSettings/Overview/RequestsNotLoaded/index.js +++ b/packages/bruno-app/src/components/CollectionSettings/Overview/RequestsNotLoaded/index.js @@ -2,8 +2,15 @@ import React from 'react'; import { flattenItems } from "utils/collections"; import { IconAlertTriangle } from '@tabler/icons'; import StyledWrapper from "./StyledWrapper"; +import { useDispatch, useSelector } from 'react-redux'; +import { isItemARequest, itemIsOpenedInTabs } from 'utils/tabs/index'; +import { getDefaultRequestPaneTab } from 'utils/collections/index'; +import { addTab, focusTab } from 'providers/ReduxStore/slices/tabs'; +import { hideHomePage } from 'providers/ReduxStore/slices/app'; const RequestsNotLoaded = ({ collection }) => { + const dispatch = useDispatch(); + const tabs = useSelector((state) => state.tabs.tabs); const flattenedItems = flattenItems(collection.items); const itemsFailedLoading = flattenedItems?.filter(item => item?.partial && !item?.loading); @@ -11,6 +18,29 @@ const RequestsNotLoaded = ({ collection }) => { return null; } + const handleRequestClick = (item) => e => { + e.preventDefault(); + if (isItemARequest(item)) { + dispatch(hideHomePage()); + if (itemIsOpenedInTabs(item, tabs)) { + dispatch( + focusTab({ + uid: item.uid + }) + ); + return; + } + dispatch( + addTab({ + uid: item.uid, + collectionUid: collection.uid, + requestPaneTab: getDefaultRequestPaneTab(item) + }) + ); + return; + } + } + return (
@@ -31,7 +61,7 @@ const RequestsNotLoaded = ({ collection }) => { {flattenedItems?.map((item, index) => ( item?.partial && !item?.loading ? ( - + {item?.pathname?.split(`${collection?.pathname}/`)?.[1]} diff --git a/packages/bruno-app/src/components/RequestTabPanel/RequestNotLoaded/index.js b/packages/bruno-app/src/components/RequestTabPanel/RequestNotLoaded/index.js index 289f3c879..7908dfc09 100644 --- a/packages/bruno-app/src/components/RequestTabPanel/RequestNotLoaded/index.js +++ b/packages/bruno-app/src/components/RequestTabPanel/RequestNotLoaded/index.js @@ -21,18 +21,18 @@ const RequestNotLoaded = ({ collection, item }) => { File Info
-
+
Name:
{item?.name}
- +
Path:
{item?.pathname}
- +
Size:
{item?.size?.toFixed?.(2)} MB
@@ -67,7 +67,7 @@ const RequestNotLoaded = ({ collection, item }) => { {item?.loading && ( <> -
+
Loading... diff --git a/packages/bruno-app/src/providers/ReduxStore/slices/collections/index.js b/packages/bruno-app/src/providers/ReduxStore/slices/collections/index.js index 5d994a606..905576a2f 100644 --- a/packages/bruno-app/src/providers/ReduxStore/slices/collections/index.js +++ b/packages/bruno-app/src/providers/ReduxStore/slices/collections/index.js @@ -59,7 +59,9 @@ export const collectionsSlice = createSlice({ updateCollectionMountStatus: (state, action) => { const collection = findCollectionByUid(state.collections, action.payload.collectionUid); if (collection) { - collection.mountStatus = action.payload.mountStatus; + if (action.payload.mountStatus) { + collection.mountStatus = action.payload.mountStatus; + } } }, setCollectionSecurityConfig: (state, action) => { diff --git a/packages/bruno-app/src/utils/collections/index.js b/packages/bruno-app/src/utils/collections/index.js index e119553e4..3ac612c62 100644 --- a/packages/bruno-app/src/utils/collections/index.js +++ b/packages/bruno-app/src/utils/collections/index.js @@ -137,6 +137,20 @@ export const areItemsLoading = (folder) => { }, false); } +export const getItemsLoadStats = (folder) => { + let loadingCount = 0; + let flattenedItems = flattenItems(folder.items); + flattenedItems?.forEach(i => { + if(i?.loading) { + loadingCount += 1; + } + }); + return { + loading: loadingCount, + total: flattenedItems?.length + }; +} + export const moveCollectionItem = (collection, draggedItem, targetItem) => { let draggedItemParent = findParentItemInCollection(collection, draggedItem.uid); diff --git a/packages/bruno-electron/src/bru/workers/scripts/bru-to-json.js b/packages/bruno-electron/src/bru/workers/scripts/bru-to-json.js index c1bbb44e7..92086c4b6 100644 --- a/packages/bruno-electron/src/bru/workers/scripts/bru-to-json.js +++ b/packages/bruno-electron/src/bru/workers/scripts/bru-to-json.js @@ -1,14 +1,16 @@ -const { workerData, parentPort } = require('worker_threads'); +const { parentPort } = require('worker_threads'); const { bruToJsonV2, } = require('@usebruno/lang'); -try { - const bru = workerData; - const json = bruToJsonV2(bru); - parentPort.postMessage(json); -} -catch(error) { - console.error(error); - parentPort.postMessage({ error: error?.message }); -} \ No newline at end of file +parentPort.on('message', (workerData) => { + try { + const bru = workerData; + const json = bruToJsonV2(bru); + parentPort.postMessage(json); + } + catch(error) { + console.error(error); + parentPort.postMessage({ error: error?.message }); + } +}); \ No newline at end of file diff --git a/packages/bruno-electron/src/bru/workers/scripts/json-to-bru.js b/packages/bruno-electron/src/bru/workers/scripts/json-to-bru.js index e08be60b9..c2a4f88e4 100644 --- a/packages/bruno-electron/src/bru/workers/scripts/json-to-bru.js +++ b/packages/bruno-electron/src/bru/workers/scripts/json-to-bru.js @@ -1,13 +1,16 @@ -const { workerData, parentPort } = require('worker_threads'); +const { parentPort } = require('worker_threads'); const { jsonToBruV2, } = require('@usebruno/lang'); -try { - const json = workerData; - const bru = jsonToBruV2(json); - parentPort.postMessage(bru); -} -catch(error) { - console.error(error); - parentPort.postMessage({ error: error?.message }); -} \ No newline at end of file + +parentPort.on('message', (workerData) => { + try { + const json = workerData; + const bru = jsonToBruV2(json); + parentPort.postMessage(bru); + } + catch(error) { + console.error(error); + parentPort.postMessage({ error: error?.message }); + } +}); \ No newline at end of file diff --git a/packages/bruno-electron/src/ipc/collection.js b/packages/bruno-electron/src/ipc/collection.js index 27a34d861..43a3a1283 100644 --- a/packages/bruno-electron/src/ipc/collection.js +++ b/packages/bruno-electron/src/ipc/collection.js @@ -40,9 +40,9 @@ const collectionSecurityStore = new CollectionSecurityStore(); const uiStateSnapshotStore = new UiStateSnapshotStore(); // size and file count limits to determine whether the bru files in the collection should be loaded asynchronously or not. -const MAX_COLLECTION_SIZE_IN_MB = 5; -const MAX_SINGLE_FILE_SIZE_IN_COLLECTION_IN_MB = 2; -const MAX_COLLECTION_FILES_COUNT = 100; +const MAX_COLLECTION_SIZE_IN_MB = 20; +const MAX_SINGLE_FILE_SIZE_IN_COLLECTION_IN_MB = 5; +const MAX_COLLECTION_FILES_COUNT = 2000; const envHasSecrets = (environment = {}) => { const secrets = _.filter(environment.variables, (v) => v.secret); diff --git a/packages/bruno-electron/src/workers/index.js b/packages/bruno-electron/src/workers/index.js index 04836e9fc..d1d1a1b74 100644 --- a/packages/bruno-electron/src/workers/index.js +++ b/packages/bruno-electron/src/workers/index.js @@ -4,8 +4,18 @@ class WorkerQueue { constructor() { this.queue = []; this.isProcessing = false; + this.workers = {}; } + async getWorkerForScriptPath(scriptPath) { + if (!this.workers) this.workers = {}; + let worker = this.workers[scriptPath]; + if (!worker || worker.threadId === -1) { + this.workers[scriptPath] = worker = new Worker(scriptPath); + } + return worker; + } + async enqueue(task) { const { priority, scriptPath, data } = task; @@ -36,22 +46,20 @@ class WorkerQueue { } async runWorker({ scriptPath, data }) { - return new Promise((resolve, reject) => { - const worker = new Worker(scriptPath, { workerData: data }); + return new Promise(async (resolve, reject) => { + let worker = await this.getWorkerForScriptPath(scriptPath); + worker.postMessage(data); worker.on('message', (data) => { if (data?.error) { reject(new Error(data?.error)); } resolve(data); - worker.terminate(); }); worker.on('error', (error) => { reject(error); - worker.terminate(); }); worker.on('exit', (code) => { reject(new Error(`stopped with ${code} exit code`)); - worker.terminate(); }); }); }