fix: file streaming for multipart bodies (#7998)

This commit is contained in:
shubh-bruno
2026-05-15 13:47:36 +05:30
committed by GitHub
parent 975c638f39
commit 1ab2368f0f
5 changed files with 334 additions and 0 deletions

View File

@@ -0,0 +1,33 @@
meta {
name: binary upload json
type: http
seq: 1
}
post {
url: {{localhost}}/api/file-binary/binary-upload-json
body: file
auth: none
}
body:file {
file: @file(file.json) @contentType(application/json)
}
assert {
res.status: eq 200
res.body.bytesReceived: eq 23
res.body.sha256: eq 3f5d648773fc4a79418378d0e75768005a8ef0fbee232a7638d643b716c14175
res.body.contentType: eq application/json
res.body.looksLikeSerializedNodeStream: eq false
}
tests {
test("file body is uploaded byte-exact, not as a serialized stream envelope", function() {
const body = res.getBody();
expect(body.bytesReceived).to.equal(23);
expect(body.sha256).to.equal("3f5d648773fc4a79418378d0e75768005a8ef0fbee232a7638d643b716c14175");
expect(body.looksLikeSerializedNodeStream).to.equal(false);
expect(body.firstBytesUtf8).to.contain('"hello": "bruno"');
});
}

View File

@@ -0,0 +1,32 @@
meta {
name: binary upload octet-stream
type: http
seq: 2
}
post {
url: {{localhost}}/api/file-binary/binary-upload-octet-stream
body: file
auth: none
}
body:file {
file: @file(file.txt) @contentType(application/octet-stream)
}
assert {
res.status: eq 200
res.body.bytesReceived: eq 23
res.body.sha256: eq ddf1d7c7f9889618e0066558caa2ab5d0a691ce4cb73fcdd6543e0e1d386d61f
res.body.contentType: eq application/octet-stream
res.body.looksLikeSerializedNodeStream: eq false
}
tests {
test("non-json file body is uploaded byte-exact", function() {
const body = res.getBody();
expect(body.bytesReceived).to.equal(23);
expect(body.sha256).to.equal("ddf1d7c7f9889618e0066558caa2ab5d0a691ce4cb73fcdd6543e0e1d386d61f");
expect(body.looksLikeSerializedNodeStream).to.equal(false);
});
}

View File

@@ -0,0 +1,70 @@
const express = require('express');
const crypto = require('crypto');
const router = express.Router();
// Capture raw bytes regardless of content-type so we can verify the upload byte-exact.
// Mounted with its own raw parser (not the global JSON/text parsers) so a
// JSON content-type with a file body still arrives as a Buffer instead of being
// pre-parsed and silently size-truncated.
router.use(express.raw({ type: '*/*', limit: '200mb' }));
// The bug we're guarding against produces a tiny JSON envelope describing a
// Node fs.ReadStream (fields like fd, flags, _readableState). Detect that
// shape so any regression flips this flag to true.
const detectSerializedNodeStream = (firstBytesUtf8) => {
try {
const trimmed = firstBytesUtf8.trim();
if (!trimmed.startsWith('{')) return false;
const parsed = JSON.parse(trimmed);
return Boolean(
parsed && typeof parsed === 'object' && '_readableState' in parsed && 'flags' in parsed
);
} catch (e) {
return false;
}
};
const buildResponse = (req) => {
const buf = Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0);
const firstBytesUtf8 = buf.slice(0, 256).toString('utf8');
return {
method: req.method,
contentType: req.headers['content-type'] || null,
contentLengthHeader: req.headers['content-length'] || null,
transferEncoding: req.headers['transfer-encoding'] || null,
bytesReceived: buf.length,
sha256: crypto.createHash('sha256').update(buf).digest('hex'),
firstBytesUtf8,
firstBytesHex: buf.slice(0, 128).toString('hex'),
looksLikeSerializedNodeStream: detectSerializedNodeStream(firstBytesUtf8)
};
};
// JSON content-type endpoint — this is the original bug repro path.
// Pre-fix, large files sent here arrived as a ~342-byte serialization of the
// Node fs.ReadStream object instead of the file bytes.
router.post('/binary-upload-json', (req, res) => {
const contentType = req.headers['content-type'] || '';
if (!contentType.toLowerCase().includes('json')) {
return res.status(415).json({
error: 'Expected a content-type containing "json"',
contentType
});
}
return res.json(buildResponse(req));
});
// Octet-stream content-type endpoint — the non-JSON branch of the
// interpolation guard. Should always have worked, kept as a control test.
router.post('/binary-upload-octet-stream', (req, res) => {
const contentType = (req.headers['content-type'] || '').toLowerCase();
if (contentType !== 'application/octet-stream') {
return res.status(415).json({
error: 'Expected content-type: application/octet-stream',
contentType
});
}
return res.json(buildResponse(req));
});
module.exports = router;

View File

@@ -11,12 +11,18 @@ const mixRouter = require('./mix');
const wsRouter = require('./ws');
const setupGraphQL = require('./graphql');
const sseRouter = require('./sse');
const fileBinaryRouter = require('./file-binary');
const app = new express();
const port = process.env.PORT || 8081;
app.use(cors());
// Mount before the global body parsers so file/binary uploads (including ones
// declared as application/json) arrive as raw bytes instead of being parsed —
// this is what lets us hash the body and verify the wire payload byte-exact.
app.use('/api/file-binary', fileBinaryRouter);
const saveRawBody = (req, res, buf) => {
req.rawBuffer = Buffer.from(buf);
req.rawBody = buf.toString();

View File

@@ -0,0 +1,193 @@
import * as crypto from 'crypto';
import * as fs from 'fs';
import * as path from 'path';
import { test, expect } from '../../../playwright';
import {
closeAllCollections,
createCollection,
createRequest,
openRequest,
selectRequestPaneTab,
sendRequest
} from '../../utils/page';
import { buildCommonLocators } from '../../utils/page/locators';
const selectAllShortcut = process.platform === 'darwin' ? 'Meta+a' : 'Control+a';
/**
* E2E test for File / Binary request body uploads.
* Regression test for the bug where a file body with content-type containing
* "json" was JSON-stringified during interpolation, so the server received the
* Node ReadStream metadata (~342 bytes) instead of the file contents.
*
* Server-side endpoints:
* POST /api/file-binary/binary-upload-json — application/json
* POST /api/file-binary/binary-upload-octet-stream — application/octet-stream
*
* Both echo back { bytesReceived, sha256, looksLikeSerializedNodeStream, ... }
* so we can assert the upload arrived byte-exact.
*/
test.describe.serial('File / Binary body upload', () => {
const collectionName = 'binary-file-upload';
const jsonRequestName = 'json-upload';
const octetRequestName = 'octet-upload';
let tmpDir: string;
let jsonFilePath: string;
let octetFilePath: string;
let jsonFileSha256: string;
let octetFileSha256: string;
let jsonFileSize: number;
let octetFileSize: number;
test.beforeAll(async ({ page, electronApp, createTmpDir }) => {
tmpDir = await createTmpDir('binary-file-upload');
// The JSON file is intentionally larger than the 20 MiB streaming
// threshold (STREAMING_FILE_SIZE_THRESHOLD in prepare-request.js) so the
// body is sent as an fs.ReadStream — this is the exact code path that
// produced the bug. Anything <= 20 MiB would go through the Buffer path,
// which was never broken.
const LARGE_JSON_BYTES = 80 * 1024 * 1024; // 25 MiB > 20 MiB threshold
const jsonBuffer = Buffer.alloc(LARGE_JSON_BYTES, 'a');
jsonFilePath = path.join(tmpDir, 'payload.json');
await fs.promises.writeFile(jsonFilePath, jsonBuffer);
jsonFileSize = LARGE_JSON_BYTES;
jsonFileSha256 = crypto.createHash('sha256').update(jsonBuffer).digest('hex');
const octetContent = 'plain octet-stream payload\n';
octetFilePath = path.join(tmpDir, 'payload.bin');
await fs.promises.writeFile(octetFilePath, octetContent);
octetFileSize = Buffer.byteLength(octetContent);
octetFileSha256 = crypto.createHash('sha256').update(octetContent).digest('hex');
// Stash and replace the native file picker dialog so FilePickerEditor's
// Browse button resolves to a path of our choosing. The currentSelection
// is updated per-test so each test picks the right file.
await electronApp.evaluate(({ dialog }) => {
(dialog as any).__originalShowOpenDialog = dialog.showOpenDialog;
(dialog as any).__currentSelection = '';
dialog.showOpenDialog = async () => ({
canceled: false,
filePaths: [(dialog as any).__currentSelection]
});
});
await test.step('Create collection and requests', async () => {
await createCollection(page, collectionName, tmpDir);
await createRequest(page, jsonRequestName, collectionName, {
url: 'http://localhost:8081/api/file-binary/binary-upload-json',
method: 'POST',
inFolder: false
});
await createRequest(page, octetRequestName, collectionName, {
url: 'http://localhost:8081/api/file-binary/binary-upload-octet-stream',
method: 'POST',
inFolder: false
});
});
});
test.afterAll(async ({ page, electronApp }) => {
await electronApp.evaluate(({ dialog }) => {
if ((dialog as any).__originalShowOpenDialog) {
dialog.showOpenDialog = (dialog as any).__originalShowOpenDialog;
delete (dialog as any).__originalShowOpenDialog;
delete (dialog as any).__currentSelection;
}
});
await closeAllCollections(page);
});
// Switches the body to File / Binary, adds a row, picks the file, and
// (optionally) overrides the content-type. Bruno's `updateFile` reducer
// auto-fills content-type from the file extension via mime.contentType
// (.json → application/json; charset=utf-8, .bin → application/octet-stream),
// so when we want a different value (e.g. plain "application/json") we
// must select-all and replace — typing into the prefilled cell would
// splice into it at the caret and produce a corrupted header.
const configureFileBody = async (
page: import('@playwright/test').Page,
overrideContentType?: string
) => {
await selectRequestPaneTab(page, 'Body');
const locators = buildCommonLocators(page);
await locators.request.bodyModeSelector().click();
await page.locator('.dropdown-item').filter({ hasText: 'File / Binary' }).click();
await test.step('Add file row and pick the file', async () => {
await page.getByRole('button', { name: /Add File/i }).click();
await page.locator('.file-picker-btn').first().click();
await expect(page.locator('.file-picker-selected').first()).toBeVisible({ timeout: 5000 });
});
if (overrideContentType) {
await test.step(`Override content-type to "${overrideContentType}"`, async () => {
// Second column in the FileBody table is the content-type editor (SingleLineEditor / CodeMirror)
const contentTypeCell = page.locator('table tbody tr').first().locator('td').nth(1);
await contentTypeCell.locator('.CodeMirror').click();
await page.keyboard.press(selectAllShortcut);
await page.keyboard.press('Backspace');
await page.keyboard.type(overrideContentType);
// Commit the value and blur the editor so the new content-type is persisted
await page.keyboard.press('Tab');
});
}
};
test('JSON content-type: file bytes are sent verbatim, not as serialized stream metadata', async ({
page,
electronApp
}) => {
await electronApp.evaluate(({ dialog }, filePath: string) => {
(dialog as any).__currentSelection = filePath;
}, jsonFilePath);
await openRequest(page, collectionName, jsonRequestName, { persist: true });
// Override the auto-detected "application/json; charset=utf-8" with plain
// "application/json" — that's the exact header the bug repro used.
await configureFileBody(page, 'application/json');
await sendRequest(page, 200, 30000);
const locators = buildCommonLocators(page);
const responseText = await locators.response.previewContainer().innerText();
expect(responseText).toContain('"contentType": "application/json"');
const jsonBytesReceivedMatch = responseText.match(/"bytesReceived":\s*(\d+)/);
expect(jsonBytesReceivedMatch).not.toBeNull();
expect(Number(jsonBytesReceivedMatch![1])).toBe(jsonFileSize);
expect(responseText).toContain(`"sha256": "${jsonFileSha256}"`);
expect(responseText).toContain('"looksLikeSerializedNodeStream": false');
// Sanity check: the bug would have produced these stream-metadata fields
expect(responseText).not.toContain('"_readableState"');
});
test('octet-stream content-type: file bytes are sent verbatim (control case)', async ({
page,
electronApp
}) => {
await electronApp.evaluate(({ dialog }, filePath: string) => {
(dialog as any).__currentSelection = filePath;
}, octetFilePath);
await openRequest(page, collectionName, octetRequestName, { persist: true });
await configureFileBody(page);
await sendRequest(page, 200, 30000);
const locators = buildCommonLocators(page);
const responseText = await locators.response.previewContainer().innerText();
expect(responseText).toContain('application/octet-stream');
const octetBytesReceivedMatch = responseText.match(/"bytesReceived":\s*(\d+)/);
expect(octetBytesReceivedMatch).not.toBeNull();
expect(Number(octetBytesReceivedMatch![1])).toBe(octetFileSize);
expect(responseText).toContain(`"sha256": "${octetFileSha256}"`);
expect(responseText).toContain('"looksLikeSerializedNodeStream": false');
});
});