From 78f0fef5db0841c89956ff6d16d3fdb66d3d0d92 Mon Sep 17 00:00:00 2001
From: Jannis R <mail@jannisr.de>
Date: Thu, 10 Oct 2024 21:19:48 +0200
Subject: [PATCH] =?UTF-8?q?sendRequest:=20decode=20non-UTF-8=20response=20?=
 =?UTF-8?q?body=20before=20parsing=20=F0=9F=90=9B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lib/send-request.js | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/lib/send-request.js b/lib/send-request.js
index f15f042..033073c 100644
--- a/lib/send-request.js
+++ b/lib/send-request.js
@@ -11,6 +11,7 @@ import {
 	Agent as HttpAgent,
 	request as httpRequest,
 } from 'node:http'
+import {Readable} from 'node:stream'
 import _contentType from 'content-type'
 const {parse: parseContentType} = _contentType
 import {encodeXastTree} from './encode-xast-tree.js'
@@ -174,7 +175,6 @@ const createSendRequest = (cfg, opt = {}) => {
 			...logCtx,
 			// todo: timing
 		}, 'received response')
-		// todo: trace-log response body if it is small enough
 
 		// todo: as a fallback, implicitly assume text/xml & ISO-8559-1 even if no charset is specified?
 		ok(res.headers['content-type'], 'response must have a "Content-Type" header')
@@ -186,7 +186,21 @@ const createSendRequest = (cfg, opt = {}) => {
 		} = parseContentType(res.headers['content-type'])
 
 		let resBody = res
-		// todo: decode if charset is not UTF-8
+		if (_resCharset !== null) {
+			// wrap into a Web Stream first, to be able to pipe into TextDecodeStream
+			resBody = Readable.toWeb(resBody, {
+				strategy: {
+					size: (...args) => {
+						console.error('size', ...args) // todo: remove
+						return 1
+					},
+				},
+			})
+
+			const decoder = new TextDecoderStream(_resCharset)
+			resBody.pipeTo(decoder.writable)
+			resBody = decoder.readable
+		}
 
 		if ((res.statusCode < 200 || res.statusCode >= 300) && _resCType === 'text/plain') {
 			let err = new Vdv453HttpError(
@@ -227,6 +241,8 @@ const createSendRequest = (cfg, opt = {}) => {
 			throw err
 		}
 
+		// todo: trace-log response body if it is small enough
+
 		const parseResponse = (tagsToParse) => {
 			return parseTags(resBody, tagsToParse)
 		}