fix: file streaming for multipart bodies (#7998)

This commit is contained in:
shubh-bruno
2026-05-15 13:47:36 +05:30
committed by GitHub
parent 975c638f39
commit 1ab2368f0f
5 changed files with 334 additions and 0 deletions

View File

@@ -0,0 +1,33 @@
meta {
name: binary upload json
type: http
seq: 1
}
post {
url: {{localhost}}/api/file-binary/binary-upload-json
body: file
auth: none
}
body:file {
file: @file(file.json) @contentType(application/json)
}
assert {
res.status: eq 200
res.body.bytesReceived: eq 23
res.body.sha256: eq 3f5d648773fc4a79418378d0e75768005a8ef0fbee232a7638d643b716c14175
res.body.contentType: eq application/json
res.body.looksLikeSerializedNodeStream: eq false
}
tests {
test("file body is uploaded byte-exact, not as a serialized stream envelope", function() {
const body = res.getBody();
expect(body.bytesReceived).to.equal(23);
expect(body.sha256).to.equal("3f5d648773fc4a79418378d0e75768005a8ef0fbee232a7638d643b716c14175");
expect(body.looksLikeSerializedNodeStream).to.equal(false);
expect(body.firstBytesUtf8).to.contain('"hello": "bruno"');
});
}

View File

@@ -0,0 +1,32 @@
meta {
name: binary upload octet-stream
type: http
seq: 2
}
post {
url: {{localhost}}/api/file-binary/binary-upload-octet-stream
body: file
auth: none
}
body:file {
file: @file(file.txt) @contentType(application/octet-stream)
}
assert {
res.status: eq 200
res.body.bytesReceived: eq 23
res.body.sha256: eq ddf1d7c7f9889618e0066558caa2ab5d0a691ce4cb73fcdd6543e0e1d386d61f
res.body.contentType: eq application/octet-stream
res.body.looksLikeSerializedNodeStream: eq false
}
tests {
test("non-json file body is uploaded byte-exact", function() {
const body = res.getBody();
expect(body.bytesReceived).to.equal(23);
expect(body.sha256).to.equal("ddf1d7c7f9889618e0066558caa2ab5d0a691ce4cb73fcdd6543e0e1d386d61f");
expect(body.looksLikeSerializedNodeStream).to.equal(false);
});
}

View File

@@ -0,0 +1,70 @@
const express = require('express');
const crypto = require('crypto');
const router = express.Router();
// Capture raw bytes regardless of content-type so we can verify the upload byte-exact.
// Mounted with its own raw parser (not the global JSON/text parsers) so a
// JSON content-type with a file body still arrives as a Buffer instead of being
// pre-parsed and silently size-truncated.
router.use(express.raw({ type: '*/*', limit: '200mb' }));
// The bug we're guarding against produces a tiny JSON envelope describing a
// Node fs.ReadStream (fields like fd, flags, _readableState). Detect that
// shape so any regression flips this flag to true.
const detectSerializedNodeStream = (firstBytesUtf8) => {
try {
const trimmed = firstBytesUtf8.trim();
if (!trimmed.startsWith('{')) return false;
const parsed = JSON.parse(trimmed);
return Boolean(
parsed && typeof parsed === 'object' && '_readableState' in parsed && 'flags' in parsed
);
} catch (e) {
return false;
}
};
const buildResponse = (req) => {
const buf = Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0);
const firstBytesUtf8 = buf.slice(0, 256).toString('utf8');
return {
method: req.method,
contentType: req.headers['content-type'] || null,
contentLengthHeader: req.headers['content-length'] || null,
transferEncoding: req.headers['transfer-encoding'] || null,
bytesReceived: buf.length,
sha256: crypto.createHash('sha256').update(buf).digest('hex'),
firstBytesUtf8,
firstBytesHex: buf.slice(0, 128).toString('hex'),
looksLikeSerializedNodeStream: detectSerializedNodeStream(firstBytesUtf8)
};
};
// JSON content-type endpoint — this is the original bug repro path.
// Pre-fix, large files sent here arrived as a ~342-byte serialization of the
// Node fs.ReadStream object instead of the file bytes.
router.post('/binary-upload-json', (req, res) => {
const contentType = req.headers['content-type'] || '';
if (!contentType.toLowerCase().includes('json')) {
return res.status(415).json({
error: 'Expected a content-type containing "json"',
contentType
});
}
return res.json(buildResponse(req));
});
// Octet-stream content-type endpoint — the non-JSON branch of the
// interpolation guard. Should always have worked, kept as a control test.
router.post('/binary-upload-octet-stream', (req, res) => {
const contentType = (req.headers['content-type'] || '').toLowerCase();
if (contentType !== 'application/octet-stream') {
return res.status(415).json({
error: 'Expected content-type: application/octet-stream',
contentType
});
}
return res.json(buildResponse(req));
});
module.exports = router;

View File

@@ -11,12 +11,18 @@ const mixRouter = require('./mix');
const wsRouter = require('./ws');
const setupGraphQL = require('./graphql');
const sseRouter = require('./sse');
const fileBinaryRouter = require('./file-binary');
const app = new express();
const port = process.env.PORT || 8081;
app.use(cors());
// Mount before the global body parsers so file/binary uploads (including ones
// declared as application/json) arrive as raw bytes instead of being parsed —
// this is what lets us hash the body and verify the wire payload byte-exact.
app.use('/api/file-binary', fileBinaryRouter);
const saveRawBody = (req, res, buf) => {
req.rawBuffer = Buffer.from(buf);
req.rawBody = buf.toString();