sendRequest: decode non-UTF-8 response body before parsing 🐛

lib/send-request.js

@@ -11,6 +11,7 @@ import {
 	Agent as HttpAgent,
 	request as httpRequest,
 } from 'node:http'
+import {Readable} from 'node:stream'
 import _contentType from 'content-type'
 const {parse: parseContentType} = _contentType
 import {encodeXastTree} from './encode-xast-tree.js'
@@ -174,7 +175,6 @@ const createSendRequest = (cfg, opt = {}) => {
 			...logCtx,
 			// todo: timing
 		}, 'received response')
-		// todo: trace-log response body if it is small enough
 
 		// todo: as a fallback, implicitly assume text/xml & ISO-8559-1 even if no charset is specified?
 		ok(res.headers['content-type'], 'response must have a "Content-Type" header')
@@ -186,7 +186,21 @@ const createSendRequest = (cfg, opt = {}) => {
 		} = parseContentType(res.headers['content-type'])
 
 		let resBody = res
-		// todo: decode if charset is not UTF-8
+		if (_resCharset !== null) {
+			// wrap into a Web Stream first, to be able to pipe into TextDecodeStream
+			resBody = Readable.toWeb(resBody, {
+				strategy: {
+					size: (...args) => {
+						console.error('size', ...args) // todo: remove
+						return 1
+					},
+				},
+			})
+
+			const decoder = new TextDecoderStream(_resCharset)
+			resBody.pipeTo(decoder.writable)
+			resBody = decoder.readable
+		}
 
 		if ((res.statusCode < 200 || res.statusCode >= 300) && _resCType === 'text/plain') {
 			let err = new Vdv453HttpError(
@@ -227,6 +241,8 @@ const createSendRequest = (cfg, opt = {}) => {
 			throw err
 		}
 
+		// todo: trace-log response body if it is small enough
+
 		const parseResponse = (tagsToParse) => {
 			return parseTags(resBody, tagsToParse)
 		}