protocole to exchange huge content in chunked payload for attributes that exceed 20Mb

[FabienArcellier]

If attributes of an event eventResponse contains more than 20Mb of data, the content of those attributes will be sent after the initial response message in messages partDownloadResponse.

The content of the attribute is encoded to read expectation data:multiparts:{part};120

{
    "messageType": "eventResponse",
    "trackingId": 114,
    "payload": {
        "result": {
            "ok": true,
            "result": null
        },
        "mutations": {
            "random_df": "data:multiparts;random_df;120" # 120 parts
        },
        "mail": [
           { "type": "pageChange", "payload": "data:multiparts;payload.mail[0].payload;5"}
        ]
    }
}

{
    "messageType": "partDownloadResponse",
    "trackingId": 114,
    "part": {
	    "part": "random_df",
	    "partId": 1,
	    "data": "xxxxxxxxxxxxxxxxxxxxxxx"
    }
}


{
    "messageType": "partDownloadResponse",
    "trackingId": 114,
    "part": {
	    "part": "random_df",
	    "partId": 2,
	    "data": "xxxxxxxxxxxxxxxxxxxxxxx"
    }
}

Backend interface

The multipart encoder is active at the lowest level just before sending the websocket message.

###
asgi_app = FastAPI(lifespan=lifespan)
multipart_manager = MultipartManager() # in core.MultipartManager

###
messages = multipart_manager.payload_chunk(response.model_dump())
for message in messages:
	await websocket.send_json(message)

# core.MultipartManager 
payload_chunk(message: dict) -> List[dict]:
	return [message]

Frontend interface

The multipart decoder is active at the lowest level immediately after receiving the websocket message.

webSocket.onmessage = (wsEvent) => {
	const rawMessage = JSON.parse(wsEvent.data);
	multipartManager.payloadAssemble(rawMessage)
	
	if multipartManager.isPayloadComplete(rawMessage) { // tracking id is used as message identifier
		const message = multipartManager.message(rawMessage)  // tracking id is used as message identifier
		if (
			message.messageType == "announcement" &&
			message.payload.announce == "codeUpdate"
		) {
			webSocket.close();
			initSession();
			return;
		} else if (message.messageType == "eventResponse" || message.messageType == "stateEnquiryResponse") {
			ingestMutations(message.payload?.mutations);
			collateMail(message.payload?.mail);
		}
	
		const mapItem = frontendMessageMap.value.get(message.trackingId);
		mapItem?.callback?.({ ok: true, payload: message.payload });
		frontendMessageMap.value.delete(message.trackingId);
	}
}