From 1c5b7604071035a5f864468b15a99630603c729d Mon Sep 17 00:00:00 2001 From: davik20 Date: Wed, 23 Aug 2023 10:53:55 +0100 Subject: [PATCH 1/7] feature/block-tosjon/ --- lib/block.ts | 15 + stun/.DS_Store | Bin 0 -> 6148 bytes stun/lib/.DS_Store | Bin 0 -> 6148 bytes stun/lib/appspace/PeerManager.js | 609 ++++++++ stun/lib/appspace/call-launch.js | 93 ++ stun/lib/appspace/call-launch.template.js | 35 + stun/lib/components/PeerManager.js | 542 +++++++ stun/lib/components/add-users.template.js | 25 + stun/lib/components/audio-box.js | 50 + stun/lib/components/audio-box.template.js | 16 + .../call-interface-audio-game.template.js | 15 + .../call-interface-audio-generic.template.js | 23 + stun/lib/components/call-interface-audio.js | 231 +++ stun/lib/components/call-interface-video.js | 490 +++++++ .../call-interface-video.template.js | 73 + stun/lib/components/call-setting.js | 247 ++++ stun/lib/components/call-setting.template.js | 32 + stun/lib/components/chat-manager-large.js | 565 ++++++++ stun/lib/components/video-box.js | 190 +++ stun/lib/components/video-box.template.js | 12 + stun/lib/overlays/effects.js | 36 + stun/lib/overlays/effects.template.js | 10 + stun/lib/overlays/switch-display.js | 27 + stun/lib/overlays/switch-display.template.js | 27 + stun/stun-old.js | 1269 +++++++++++++++++ stun/stun.js | 505 +++++++ stun/web/.DS_Store | Bin 0 -> 6148 bytes stun/web/audio/end-call.mp3 | Bin 0 -> 17505 bytes stun/web/audio/enter-call.mp3 | Bin 0 -> 12703 bytes stun/web/css/invite.css | 11 + stun/web/css/stun-appspace.css | 211 +++ stun/web/css/stun-audio-interface.css | 39 + stun/web/css/stun-base.css | 1 + stun/web/css/stun-video-interface.css | 915 ++++++++++++ stun/web/img/video-call-og.png | Bin 0 -> 110502 bytes stun/web/index.html | 55 + stun/web/style.css | 1180 +++++++++++++++ 37 files changed, 7549 insertions(+) create mode 100644 stun/.DS_Store create mode 100644 stun/lib/.DS_Store create mode 100644 stun/lib/appspace/PeerManager.js create mode 100644 stun/lib/appspace/call-launch.js create mode 100644 stun/lib/appspace/call-launch.template.js create mode 100644 stun/lib/components/PeerManager.js create mode 100644 stun/lib/components/add-users.template.js create mode 100644 stun/lib/components/audio-box.js create mode 100644 stun/lib/components/audio-box.template.js create mode 100644 stun/lib/components/call-interface-audio-game.template.js create mode 100644 stun/lib/components/call-interface-audio-generic.template.js create mode 100644 stun/lib/components/call-interface-audio.js create mode 100644 stun/lib/components/call-interface-video.js create mode 100644 stun/lib/components/call-interface-video.template.js create mode 100644 stun/lib/components/call-setting.js create mode 100644 stun/lib/components/call-setting.template.js create mode 100644 stun/lib/components/chat-manager-large.js create mode 100644 stun/lib/components/video-box.js create mode 100644 stun/lib/components/video-box.template.js create mode 100644 stun/lib/overlays/effects.js create mode 100644 stun/lib/overlays/effects.template.js create mode 100644 stun/lib/overlays/switch-display.js create mode 100644 stun/lib/overlays/switch-display.template.js create mode 100644 stun/stun-old.js create mode 100644 stun/stun.js create mode 100644 stun/web/.DS_Store create mode 100644 stun/web/audio/end-call.mp3 create mode 100644 stun/web/audio/enter-call.mp3 create mode 100644 stun/web/css/invite.css create mode 100644 stun/web/css/stun-appspace.css create mode 100644 stun/web/css/stun-audio-interface.css create mode 100644 stun/web/css/stun-base.css create mode 100644 stun/web/css/stun-video-interface.css create mode 100644 stun/web/img/video-call-og.png create mode 100644 stun/web/index.html create mode 100644 stun/web/style.css diff --git a/lib/block.ts b/lib/block.ts index 09bebaa..5c1b180 100644 --- a/lib/block.ts +++ b/lib/block.ts @@ -19,10 +19,25 @@ export default class Block extends WasmWrapper { block = new Block.Type(); } super(block!); + + } + public toJson(): string { + try { + const json = JSON.stringify(this.transactions.map((tx) => tx.toJson())); + console.log(json, "this is the json") + return json; + } catch (error) { + console.error(error); + } + return "" + } + + public get transactions(): Array { try { + this.toJson(); return this.instance.transactions.map((tx) => { return Saito.getInstance().factory.createTransaction(tx); }); diff --git a/stun/.DS_Store b/stun/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..320dd07c39f4d2d16b2c8b7d537d5578de762b34 GIT binary patch literal 6148 zcmeHKJx;?=47Srs6tQ%q15zYrD)k1T3Ik&7#)R4;ew2_(rD8(L9hf);Ct&0*jNAY| z`&l%vEsO||E%|+kpY6oolj4|&c(|Ajh`L17feI$uST>04i>^sxmK`8Vj%qS4sxj^) zoy%WfK+dj31zpkjxQNd05Ig;M6)nWPii^Q$Qk65c@xlEvzO3@x&cD+;-G5DcFOtLJ z%4+(}?(DLe^-Yp*(VT8+K{wQ=YnrdT-4i_Gx4WbB)f#wjvsY9@^7sI+7HNF5hLh(; z4Ys?iVSP8Mp@p0-&VV!E4E$CG(6dE4(gR$a0cXG&Xfhz*hZHK98P);y(}6`t0AL$t z6^-z#$y_nfccNLpZTM8F}oTertKNL}UiHlZdECL>)L|Ya2t2@pbk!YlTP`X#72LI-@a7 zXXjD6#@4`JRDkEMgIUY*o5L6LH+ESr4WGW$aWa=AqnR7=wa0_k%gw#F<9goYN4)$Z zO3RwQ^bX!7M;W?ubVXD64bA!8DtXUOs9DQf?(b;${;(zeHE_!wzvEW=G#2HdpQAhU zcTF?scfZZl5+ z0;<450scN%IAa{J5cHo841Nm$Y$NOq$Gn%|92+nWSP0?@%t$IwQcc)m7)eLm23{Pn z5R`N>zB29;Dx0uFF}^zD*3-$vf;y@Ksz6nN4LR-c`hU{A|F0(LnJS { + this.room_code = room_code; + }); + + app.connection.on("stun-event-message", (data) => { + if (data.room_code !== this.room_code) { + return; + } + + if (data.type === "peer-joined") { + let peerConnection = this.peers.get(data.public_key); + if (!peerConnection) { + this.createPeerConnection(data.public_key, "offer"); + } + } else if (data.type === "peer-left") { + this.removePeerConnection(data.public_key); + } else if (data.type === "toggle-audio") { + // console.log(data); + app.connection.emit("toggle-peer-audio-status", data); + } else if (data.type === "toggle-video") { + app.connection.emit("toggle-peer-video-status", data); + } else { + let peerConnection = this.peers.get(data.public_key); + if (!peerConnection) { + console.log("Create Peer Connection with " + data.public_key); + this.createPeerConnection(data.public_key); + peerConnection = this.peers.get(data.public_key); + } + console.log("peers consoled", peerConnection); + + if (peerConnection) { + this.handleSignalingMessage(data); + } + } + }); + + app.connection.on("stun-disconnect", () => { + this.leave(); + }); + + app.connection.on("stun-toggle-video", async () => { + if (this.videoEnabled === true) { + if (!this.localStream.getVideoTracks()[0]) return; + + this.localStream.getVideoTracks()[0].enabled = false; + this.app.connection.emit("mute", "video", "local"); + this.videoEnabled = false; + } else { + if (!this.localStream.getVideoTracks()[0]) { + const oldVideoTracks = this.localStream.getVideoTracks(); + if (oldVideoTracks.length > 0) { + oldVideoTracks.forEach((track) => { + this.localStream.removeTrack(track); + }); + } + // start a video stream; + let localStream = await navigator.mediaDevices.getUserMedia({ video: true }); + + // Add new track to the local stream + this.app.connection.emit("add-local-stream-request", this.localStream, "video"); + + let track = localStream.getVideoTracks()[0]; + this.localStream.addTrack(track); + + this.peers.forEach((peerConnection, key) => { + const videoSenders = peerConnection + .getSenders() + .filter((sender) => sender.track && sender.track.kind === "video"); + if (videoSenders.length > 0) { + videoSenders.forEach((sender) => { + sender.replaceTrack(track); + }); + } else { + peerConnection.addTrack(track); + } + + this.renegotiate(key); + }); + } else { + this.localStream.getVideoTracks()[0].enabled = true; + } + this.videoEnabled = true; + } + + let data = { + room_code: this.room_code, + type: "toggle-video", + enabled: this.videoEnabled, + }; + + this.mod.sendStunMessageToServerTransaction(data); + }); + + app.connection.on("stun-toggle-audio", async () => { + // if video is enabled + if (this.audioEnabled === true) { + this.localStream.getAudioTracks()[0].enabled = false; + this.app.connection.emit("mute", "audio", "local"); + this.audioEnabled = false; + } else { + this.localStream.getAudioTracks()[0].enabled = true; + this.audioEnabled = true; + } + + let data = { + room_code: this.room_code, + type: "toggle-audio", + enabled: this.audioEnabled, + }; + this.mod.sendStunMessageToServerTransaction(data); + }); + + app.connection.on("begin-share-screen", async () => { + try { + let stream = await navigator.mediaDevices.getDisplayMedia({ video: true }); + let videoTrack = stream.getVideoTracks()[0]; + + videoTrack.onended = () => { + console.log("Screen sharing stopped by user"); + app.connection.emit("remove-peer-box", "presentation"); + this.app.connection.emit("stun-switch-view", "focus"); + this.peers.forEach((pc, key) => { + pc.dc.send("remove-presentation-box"); + }); + }; + let remoteStream = new MediaStream(); + remoteStream.addTrack(videoTrack); + + /// emit event to make presentation be the large screen and make presentation mode on + this.app.connection.emit("add-remote-stream-request", "presentation", remoteStream); + this.peers.forEach((pc, key) => { + pc.dc.send("presentation"); + pc.addTrack(videoTrack); + + this.renegotiate(key); + //console.log("adding presentation video track to peer"); + }); + } catch (err) { + console.error("Error accessing media devices.", err); + } + // let sender = pc.addTrack(videoTrack); + }); + + //Launch the Stun call + app.connection.on("start-stun-call", async () => { + console.log("start-stun-call"); + if (this.mod.ui_type == "voice") { + this.videoEnabled = false; + } + + //Get my local media + try { + this.localStream = await navigator.mediaDevices.getUserMedia({ + video: this.videoEnabled, + audio: true, + }); + } catch (err) { + console.warn("Problem attempting to get User Media", err); + console.log("Trying without video"); + + this.videoEnabled = false; + this.localStream = await navigator.mediaDevices.getUserMedia({ + video: false, + audio: true, + }); + } + + this.localStream.getAudioTracks()[0].enabled = this.audioEnabled; + + //Render the UI component + this.app.connection.emit( + "show-call-interface", + this.room_code, + this.videoEnabled, + this.audioEnabled + ); + this.app.connection.emit("add-local-stream-request", this.localStream); + + //Send Message to peers + this.join(); + + let sound = new Audio("/videocall/audio/enter-call.mp3"); + sound.play(); + + this.analyzeAudio(this.localStream, "local"); + }); + + //Chat-Settings saves whether to enter the room with mic/camera on/off + app.connection.on("update-media-preference", (kind, state) => { + if (kind === "audio") { + this.audioEnabled = state; + } else if (kind === "video") { + this.videoEnabled = state; + } + }); + } + + handleSignalingMessage(data) { + const { type, sdp, candidate, targetPeerId, public_key } = data; + if (type === "renegotiate-offer" || type === "offer") { + // if ( + // this.getPeerConnection(public_key).connectionState === "connected" || + // this.getPeerConnection(public_key).remoteDescription !== null || + // this.getPeerConnection(public_key).connectionState === "stable" + // ) { + // return; + // } + + console.log(this.getPeerConnection(public_key), "remote description offer"); + + this.getPeerConnection(public_key) + .setRemoteDescription(new RTCSessionDescription({ type: "offer", sdp })) + .then(() => { + return this.getPeerConnection(public_key).createAnswer(); + }) + .then((answer) => { + return this.getPeerConnection(public_key).setLocalDescription(answer); + }) + .then(() => { + let data = { + room_code: this.room_code, + type: "renegotiate-answer", + sdp: this.getPeerConnection(public_key).localDescription.sdp, + targetPeerId: public_key, + }; + this.mod.sendStunMessageToServerTransaction(data); + }) + .catch((error) => { + console.error("Error handling offer:", error); + }); + this.peers.set(data.public_key, this.getPeerConnection(public_key)); + } else if (type === "renegotiate-answer" || type === "answer") { + console.log( + this.getPeerConnection(public_key), + this.getPeerConnection(public_key).connectionState, + "remote description answer" + ); + //if ( + // this.getPeerConnection(public_key).connectionState === "connected" || + // this.getPeerConnection(public_key).signalingState === "stable" + //) + // return; + this.getPeerConnection(public_key) + .setRemoteDescription(new RTCSessionDescription({ type: "answer", sdp })) + .then((answer) => {}) + .catch((error) => { + console.error("Error handling answer:", error); + }); + this.peers.set(data.public_key, this.getPeerConnection(public_key)); + } else if (type === "candidate") { + if (this.getPeerConnection(public_key).remoteDescription === null) return; + this.getPeerConnection(public_key) + .addIceCandidate(new RTCIceCandidate(candidate)) + .catch((error) => { + console.error("Error adding remote candidate:", error); + }); + } + } + + async createPeerConnection(peerId, type) { + // check if peer connection already exists + const peerConnection = new RTCPeerConnection({ + iceServers: this.mod.servers, + }); + + this.peers.set(peerId, peerConnection); + + //Make sure you have a local Stream + if (!this.localStream) { + this.localStream = await navigator.mediaDevices.getUserMedia({ + video: this.videoEnabled, + audio: true, + }); + } + + // Implement the creation of a new RTCPeerConnection and its event handlers + + // Handle ICE candidates + peerConnection.onicecandidate = (event) => { + if (event.candidate) { + let data = { + room_code: this.room_code, + type: "candidate", + candidate: event.candidate, + targetPeerId: peerId, + }; + this.mod.sendStunMessageToServerTransaction(data); + } + }; + + const remoteStream = new MediaStream(); + peerConnection.addEventListener("track", (event) => { + // console.log("trackss", event.track, "stream :", event.streams); + console.log("another remote stream added", event.track); + if (this.trackIsPresentation) { + const remoteStream = new MediaStream(); + remoteStream.addTrack(event.track); + //this.remoteStreams.set("Presentation", { remoteStream, peerConnection }); + //console.log(this.remoteStreams, "presentation stream"); + this.app.connection.emit("add-remote-stream-request", "presentation", remoteStream); + setTimeout(() => { + this.trackIsPresentation = false; + }, 1000); + } else { + if (event.streams.length === 0) { + remoteStream.addTrack(event.track); + } else { + event.streams[0].getTracks().forEach((track) => { + remoteStream.addTrack(track); + }); + } + + this.remoteStreams.set(peerId, { remoteStream, peerConnection }); + console.log(this.remoteStreams, "remote stream new"); + this.app.connection.emit("add-remote-stream-request", peerId, remoteStream); + + this.analyzeAudio(remoteStream, peerId); + } + }); + + this.localStream.getTracks().forEach((track) => { + peerConnection.addTrack(track, this.localStream); + // console.log('track local ', track) + }); + + let dc = peerConnection.createDataChannel("data-channel"); + peerConnection.dc = dc; + + dc.onmessage = (event) => { + console.log("Message from data channel:", event.data); + switch (event.data) { + case "presentation": + this.trackIsPresentation = true; + break; + case "remove-presentation-box": + this.app.connection.emit("remove-peer-box", "presentation"); + this.app.connection.emit("stun-switch-view", "focus"); + default: + break; + } + }; + + dc.onopen = (event) => { + console.log("Data channel is open"); + }; + + dc.onclose = (event) => { + console.log("Data channel is closed"); + }; + + peerConnection.addEventListener("datachannel", (event) => { + let receiveChannel = event.channel; + + peerConnection.dc = receiveChannel; + + receiveChannel.onmessage = (event) => { + console.log("Message from data channel:", event.data); + switch (event.data) { + case "presentation": + this.trackIsPresentation = true; + break; + case "remove-presentation-box": + this.app.connection.emit("remove-peer-box", "presentation"); + this.app.connection.emit("stun-switch-view", "focus"); + default: + break; + } + }; + + receiveChannel.onopen = (event) => { + console.log("Data channel is open"); + }; + + receiveChannel.onclose = (event) => { + console.log("Data channel is closed"); + }; + }); + + peerConnection.addEventListener("connectionstatechange", () => { + if ( + peerConnection.connectionState === "failed" || + peerConnection.connectionState === "disconnected" + ) { + setTimeout(() => { + // console.log('sending offer'); + this.reconnect(peerId, type); + }, 10000); + } + if (peerConnection.connectionState === "connected") { + let sound = new Audio("/videocall/audio/enter-call.mp3"); + sound.play(); + } + // if(peerConnection.connectionState === "disconnected"){ + + // } + + this.app.connection.emit( + "stun-update-connection-message", + this.room_code, + peerId, + peerConnection.connectionState + ); + }); + + if (type === "offer") { + this.renegotiate(peerId); + } + } + + reconnect(peerId, type) { + const maxRetries = 2; + const retryDelay = 10000; + + const attemptReconnect = (currentRetry) => { + const peerConnection = this.peers.get(peerId); + if (currentRetry === maxRetries) { + if (peerConnection && peerConnection.connectionState !== "connected") { + console.log("Reached maximum number of reconnection attempts, giving up"); + this.removePeerConnection(peerId); + } + return; + } + + if (peerConnection && peerConnection.connectionState === "connected") { + console.log("Reconnection successful"); + // remove connection message + return; + } + + if (peerConnection && peerConnection.connectionState !== "connected") { + this.removePeerConnection(peerId); + if (type === "offer") { + this.createPeerConnection(peerId, "offer"); + } + } + + setTimeout(() => { + console.log(`Reconnection attempt ${currentRetry + 1}/${maxRetries}`); + attemptReconnect(currentRetry + 1); + }, retryDelay); + }; + + attemptReconnect(0); + } + + //This can get double processed by PeerTransaction and onConfirmation + //So need safety checks + removePeerConnection(peerId) { + const peerConnection = this.peers.get(peerId); + if (peerConnection) { + peerConnection.close(); + this.peers.delete(peerId); + + let sound = new Audio("/videocall/audio/end-call.mp3"); + sound.play(); + console.log("peer left"); + } + + this.app.connection.emit("remove-peer-box", peerId); + } + + renegotiate(peerId, retryCount = 0) { + const maxRetries = 4; + const retryDelay = 3000; + + const peerConnection = this.peers.get(peerId); + if (!peerConnection) { + return; + } + + console.log("renegotiating with pc", peerConnection); + // console.log('signalling state, ', peerConnection.signalingState) + if (peerConnection.signalingState !== "stable") { + if (retryCount < maxRetries) { + console.log( + `Signaling state is not stable, will retry in ${retryDelay} ms (attempt ${ + retryCount + 1 + }/${maxRetries})` + ); + setTimeout(() => { + this.renegotiate(peerId, retryCount + 1); + }, retryDelay); + } else { + console.log("Reached maximum number of renegotiation attempts, giving up"); + } + return; + } + + const offerOptions = { + offerToReceiveAudio: true, + offerToReceiveVideo: true, + }; + + peerConnection + .createOffer(offerOptions) + .then((offer) => { + return peerConnection.setLocalDescription(offer); + }) + .then(() => { + let data = { + room_code: this.room_code, + type: "renegotiate-offer", + sdp: peerConnection.localDescription.sdp, + targetPeerId: peerId, + }; + this.mod.sendStunMessageToServerTransaction(data); + }) + .catch((error) => { + console.error("Error creating offer:", error); + }); + + // Implement renegotiation logic for reconnections and media stream restarts + } + + join() { + console.log("joining mesh network"); + this.mod.sendStunMessageToServerTransaction({ + type: "peer-joined", + room_code: this.room_code, + }); + } + + leave() { + this.localStream.getTracks().forEach((track) => { + track.stop(); + // console.log(track); + console.log("stopping track"); + }); + this.peers.forEach((peerConnections, key) => { + peerConnections.close(); + }); + + this.peers = new Map(); + + if (this.audioStreamAnalysis) { + clearInterval(this.audioStreamAnalysis); + } + + let data = { + room_code: this.room_code, + type: "peer-left", + }; + + this.mod.sendStunMessageToServerTransaction(data); + } + + sendSignalingMessage(data) {} + + getPeerConnection(public_key) { + return this.peers.get(public_key); + } + + analyzeAudio(stream, peer) { + let peer_manager_self = this; + + const audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + source.connect(analyser); + analyser.fftSize = 512; + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + + let has_mike = false; + const threshold = 20; + + function update() { + //console.log("Update"); + + analyser.getByteFrequencyData(dataArray); + const average = dataArray.reduce((a, b) => a + b) / bufferLength; + + if (average > threshold && !has_mike) { + this.current_speaker = peer; + + setTimeout(() => { + if (peer === this.current_speaker) { + peer_manager_self.app.connection.emit("stun-new-speaker", peer); + has_mike = true; + } + }, 1000); + } else if (average <= threshold) { + has_mike = false; + } + + //requestAnimationFrame(update); + } + this.audioStreamAnalysis = setInterval(update, 1000); + //requestAnimationFrame(update); + } +} + +module.exports = PeerManager; diff --git a/stun/lib/appspace/call-launch.js b/stun/lib/appspace/call-launch.js new file mode 100644 index 0000000..7999cff --- /dev/null +++ b/stun/lib/appspace/call-launch.js @@ -0,0 +1,93 @@ +const SaitoOverlay = require('../../../../lib/saito/ui/saito-overlay/saito-overlay.js'); +const StunLaunchTemplate = require('./call-launch.template.js'); +const CallSetting = require("../components/call-setting"); + +/** + * + * This is a splash screen for initiating a Saito Video call + * + **/ + +class CallLaunch { + + constructor(app, mod, container = "") { + this.app = app; + this.mod = mod; + this.container = container; + this.overlay = new SaitoOverlay(app, mod); + this.callSetting = new CallSetting(app, this); + + this.room_code = null; + + // close-preview-window shuts downt the streams in call-settings + app.connection.on('close-preview-window', () => { + this.overlay.remove(); + if (document.querySelector(".stun-appspace")){ + document.querySelector(".stun-appspace").remove(); + } + }); + + app.connection.on('stun-to-join-room', (room_code) => { + this.room_code = room_code; + document.querySelector('#createRoom').textContent = "Join Meeting"; + }); + + } + + render() { + if (document.querySelector('.stun-appspace')) { + return; + } + if (this.container === ".saito-overlay") { + //Should add callback to "hang up the call" if we close the overlay + this.overlay.show(StunLaunchTemplate(this.app, this.mod), () => {this.app.connection.emit("close-preview-window");}); + } else if (this.container === "body") { + this.app.browser.addElementToDom(StunLaunchTemplate(this.app, this.mod)) + } + + this.attachEvents(this.app, this.mod); + // create peer manager and initialize , send an event to stun to initialize + this.app.connection.emit('stun-init-peer-manager',"large"); + + this.callSetting.render(); + } + + attachEvents(app, mod) { + + if (document.getElementById("createRoom")){ + document.getElementById("createRoom").onclick = (e) => { + if (this.room_code) { + this.joinRoom() + } else { + this.createRoom(); + } + } + } + } + + + async createRoom() { + this.room_code = await this.mod.sendCreateRoomTransaction(); + this.joinRoom(); + } + + joinRoom() { + if (!this.room_code){ + return; + } + this.app.connection.emit('stun-peer-manager-update-room-code', this.room_code); + //For myself and call-Settings + this.app.connection.emit('close-preview-window'); + this.app.connection.emit("start-stun-call"); + } + + async createConferenceCall(app, mod, room_code) { + + } + +} + + +module.exports = CallLaunch; + + diff --git a/stun/lib/appspace/call-launch.template.js b/stun/lib/appspace/call-launch.template.js new file mode 100644 index 0000000..2e6de64 --- /dev/null +++ b/stun/lib/appspace/call-launch.template.js @@ -0,0 +1,35 @@ +module.exports = StunLaunchTemplate = () => { + + return ` +
+
+ +
Saito Video
+

peer-to-peer video chat

+
+
+
Start Call
+
+
+
+ + Blockchain-mediated peer-to-peer connections can take longer to negotiate if you are on a mobile network or behind an aggressive firewall. +
+ + + +
+
+
+ +
+
+ +
+
+
+ + `; + +} + diff --git a/stun/lib/components/PeerManager.js b/stun/lib/components/PeerManager.js new file mode 100644 index 0000000..53420ea --- /dev/null +++ b/stun/lib/components/PeerManager.js @@ -0,0 +1,542 @@ +const ChatManagerLarge = require("./chat-manager-large") + + +class PeerManager { + constructor(app, mod, ui_type = "large", config) { + // console.log(config, 'config') + this.app = app; + this.mod = mod + this.ui_type = ui_type; + this.config = config; + this.peers = new Map(); + this.localStream = null; + this.remoteStreams = new Map(); + this.servers = [ + { + urls: "stun:stun-sf.saito.io:3478" + }, + { + urls: "turn:stun-sf.saito.io:3478", + username: "guest", + credential: "somepassword", + }, + { + urls: "stun:stun-sg.saito.io:3478" + }, + { + urls: "turn:stun-sg.saito.io:3478", + username: "guest", + credential: "somepassword", + }, + { + urls: "stun:stun-de.saito.io:3478" + }, + { + urls: "turn:stun-de.saito.io:3478", + username: "guest", + credential: "somepassword", + } + ]; + this.videoEnabled = true; + this.audioEnabled = true + + + + this.app.connection.on('stun-peer-manager-update-room-code', (room_code) => { + this.room_code = room_code + }); + + + app.connection.on('stun-event-message', (data) => { + if (data.room_code !== this.room_code) { + return; + } + + if (data.type === 'peer-joined') { + let peerConnection = this.peers.get(data.public_key); + if(!peerConnection){ + this.createPeerConnection(data.public_key, 'offer'); + } + + } else if (data.type === 'peer-left') { + this.removePeerConnection(data.public_key); + } else if (data.type === "toggle-audio") { + // console.log(data); + app.connection.emit('toggle-peer-audio-status', data) + } else if (data.type === "toggle-video") { + app.connection.emit('toggle-peer-video-status', data) + } + else { + let peerConnection = this.peers.get(data.public_key); + if (!peerConnection) { + this.createPeerConnection(data.public_key); + peerConnection = this.peers.get(data.public_key); + } + + if (peerConnection) { + this.handleSignalingMessage(peerConnection, data); + } + } + }) + + app.connection.on('stun-disconnect', () => { + this.leave() + }) + + app.connection.on('stun-toggle-video', async () => { + if (this.videoEnabled === true) { + this.localStream.getVideoTracks()[0].enabled = false; + this.app.connection.emit("mute", 'video', 'local'); + this.videoEnabled = false; + document.querySelector('.video-control i').classList.remove('fa-video') + document.querySelector('.video-control i').classList.add('fa-video-slash') + } else { + + document.querySelector('.video-control i').classList.add('fa-video') + document.querySelector('.video-control i').classList.remove('fa-video-slash') + if (!this.localStream.getVideoTracks()[0]) { + + const oldVideoTracks = this.localStream.getVideoTracks(); + if (oldVideoTracks.length > 0) { + oldVideoTracks.forEach(track => { + this.localStream.removeTrack(track); + }); + } + // start a video stream; + let localStream = await navigator.mediaDevices.getUserMedia({ video: true }) + + // Add new track to the local stream + + + this.app.connection.emit('render-local-stream-request', this.localStream, 'video'); + let track = localStream.getVideoTracks()[0]; + this.localStream.addTrack(track); + + this.peers.forEach((peerConnection, key) => { + const videoSenders = peerConnection.getSenders().filter(sender => sender.track && sender.track.kind === 'video'); + if (videoSenders.length > 0) { + videoSenders.forEach(sender => { + sender.replaceTrack(track); + }) + + } else { + peerConnection.addTrack(track); + } + + this.renegotiate(key); + }) + document.querySelector('.video-control i').classList.add('fa-video') + this.videoEnabled = true; + + } else { + this.localStream.getVideoTracks()[0].enabled = true; + this.app.connection.emit("unmute", 'video', 'local'); + } + this.videoEnabled = true; + } + + let data = { + room_code: this.room_code, + type: 'toggle-video', + enabled: this.videoEnabled, + } + this.app.connection.emit('stun-send-message-to-server', data); + }) + + app.connection.on('stun-toggle-audio', async () => { + // if video is enabled + if (this.audioEnabled === true) { + this.localStream.getAudioTracks()[0].enabled = false; + this.app.connection.emit("mute", 'audio', 'local'); + this.audioEnabled = false; + document.querySelectorAll('.audio-control i').forEach(item => { + item.classList.add('fa-microphone-slash') + item.classList.remove('fa-microphone'); + }) + + } + else { + this.localStream.getAudioTracks()[0].enabled = true; + this.audioEnabled = true; + document.querySelectorAll('.audio-control i').forEach(item => { + item.classList.remove('fa-microphone-slash') + item.classList.add('fa-microphone'); + }) + + } + + let data = { + room_code: this.room_code, + type: 'toggle-audio', + enabled: this.audioEnabled, + } + this.app.connection.emit('stun-send-message-to-server', data); + }) + + app.connection.on('show-chat-manager-large', async (to_join) => { + // console.log(this, "peer") + await this.showChatManagerLarge(); + + if (to_join) { + this.join() + } + let sound = new Audio('/videocall/audio/enter-call.mp3'); + sound.play(); + }) + app.connection.on('show-chat-manager-small', async (to_join, config) => { + // console.log(this, "peer") + await this.showChatManagerSmall(this.config); + if (to_join) { + this.join(); + } + let sound = new Audio('/videocall/audio/enter-call.mp3'); + sound.play(); + }) + app.connection.on('switch-ui-type-to-large', async () => { + this.ui_type = "large"; + // remove small ui + this.removeChatManagerSmall(false); + // render large ui + + this.showChatManagerLarge(); + setTimeout(()=> { + app.connection.emit('stun-toggle-video'); + }, 500) + + setTimeout(()=> { + this.renderRemoteStreams(); + }, 1000) + + + + // loop over this.remoteStreams and render them + + + }) + app.connection.on('switch-ui-type-to-small', async () => { + this.ui_type = "small" + }) + + app.connection.on('update-media-preference', (kind, state) => { + if (kind === "audio") { + this.audioEnabled = state + } else if (kind === "video") { + this.videoEnabled = state + } + }) + } + + showSetting() { + this.app.connection.emit('show-chat-setting', this.room_code); + } + + + async showChatManagerLarge() { + // emit events to show chatmanager; + // get local stream; + this.localStream = await navigator.mediaDevices.getUserMedia({ video: this.videoEnabled, audio: true }); + this.localStream.getAudioTracks()[0].enabled = this.audioEnabled; + // this.localStream.getAudioTracks()[0].enabled = this.audioEnabled; + // console.log(this.config) + this.app.connection.emit('show-video-chat-large-request', this.app, this.mod, 'video', this.room_code, this.videoEnabled, this.audioEnabled, this.config); + this.app.connection.emit('stun-remove-loader') + this.app.connection.emit('render-local-stream-large-request', this.localStream, 'video'); + this.app.connection.emit('remove-overlay-request'); + } + + + async showChatManagerSmall() { + // emit events to show chatmanager; + // get local stream; + this.videoEnabled = false; + this.localStream = await navigator.mediaDevices.getUserMedia({ video: this.videoEnabled, audio: true }); + this.localStream.getAudioTracks()[0].enabled = this.audioEnabled; + this.app.connection.emit('show-video-chat-small-request', this.app, this.mod, this.room_code, this.videoEnabled, this.audioEnabled, this.config); + this.app.connection.emit('render-local-stream-small-request', this.localStream); + } + + removeChatManagerSmall(completely) { + this.app.connection.emit('remove-video-chat-small-request', completely); + } + + handleSignalingMessage(peerConnection, data) { + const { type, sdp, candidate, targetPeerId, public_key } = data; + if (type === 'renegotiate-offer' || type === 'offer') { + peerConnection.setRemoteDescription(new RTCSessionDescription({ type: 'offer', sdp })) + + .then(() => { + + return peerConnection.createAnswer(); + }) + .then((answer) => { + return peerConnection.setLocalDescription(answer); + }) + .then(() => { + let data = { + room_code: this.room_code, + type: 'renegotiate-answer', + sdp: peerConnection.localDescription.sdp, + targetPeerId: public_key, + } + this.app.connection.emit('stun-send-message-to-server', data); + }) + .catch((error) => { + console.error('Error handling offer:', error); + }); + } else if (type === 'renegotiate-answer' || type === 'answer') { + peerConnection.setRemoteDescription(new RTCSessionDescription({ type: 'answer', sdp })).then(answer => { + + }).catch((error) => { + console.error('Error handling answer:', error); + }); + } else if (type === 'candidate') { + peerConnection.addIceCandidate(new RTCIceCandidate(candidate)) + .catch((error) => { + console.error('Error adding remote candidate:', error); + }); + } + + } + + + createPeerConnection(peerId, type) { + const peerConnection = new RTCPeerConnection({ + iceServers: this.servers, + }); + + this.peers.set(peerId, peerConnection); + // Implement the creation of a new RTCPeerConnection and its event handlers + + // Handle ICE candidates + peerConnection.onicecandidate = (event) => { + if (event.candidate) { + let data = { + room_code: this.room_code, + type: 'candidate', + candidate: event.candidate, + targetPeerId: peerId, + } + this.app.connection.emit('stun-send-message-to-server', data); + + } + } + + + + const remoteStream = new MediaStream(); + peerConnection.addEventListener('track', (event) => { + // console.log("trackss", event.track, "stream :", event.streams); + if (event.streams.length === 0) { + remoteStream.addTrack(event.track); + } else { + event.streams[0].getTracks().forEach(track => { + remoteStream.addTrack(track); + }); + } + + this.remoteStreams.set(peerId, {remoteStream, peerConnection}); + console.log(this.remoteStreams, 'remote stream new') + if (this.ui_type === "large") { + this.app.connection.emit('render-remote-stream-large-request', peerId, remoteStream, peerConnection); + } else if (this.ui_type === "small") { + this.app.connection.emit('add-remote-stream-small-request', peerId, remoteStream, peerConnection); + } + + + }); + + + + this.localStream.getTracks().forEach((track) => { + peerConnection.addTrack(track, this.localStream); + // console.log('track local ', track) + }); + + + peerConnection.addEventListener('connectionstatechange', () => { + if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'disconnected') { + + setTimeout(() => { + // console.log('sending offer'); + this.reconnect(peerId, type); + }, 10000); + + } + if (peerConnection.connectionState === "connected") { + let sound = new Audio('/videocall/audio/enter-call.mp3'); + sound.play(); + } + // if(peerConnection.connectionState === "disconnected"){ + + // } + + + this.app.connection.emit('stun-update-connection-message', this.room_code, peerId, peerConnection.connectionState); + }); + + if (type === "offer") { + this.renegotiate(peerId); + } + + } + + reconnect(peerId, type) { + const maxRetries = 2; + const retryDelay = 10000; + + const attemptReconnect = (currentRetry) => { + const peerConnection = this.peers.get(peerId); + if (currentRetry === maxRetries) { + if (peerConnection && peerConnection.connectionState !== 'connected') { + console.log('Reached maximum number of reconnection attempts, giving up'); + this.removePeerConnection(peerId); + } + return; + } + + + if (peerConnection && peerConnection.connectionState === 'connected') { + console.log('Reconnection successful'); + // remove connection message + return; + } + + if (peerConnection && peerConnection.connectionState !== 'connected') { + this.removePeerConnection(peerId); + if (type === "offer") { + this.createPeerConnection(peerId, 'offer'); + } + } + + + setTimeout(() => { + console.log(`Reconnection attempt ${currentRetry + 1}/${maxRetries}`); + attemptReconnect(currentRetry + 1); + }, retryDelay); + }; + + attemptReconnect(0); + } + + removePeerConnection(peerId) { + const peerConnection = this.peers.get(peerId); + if (peerConnection) { + peerConnection.close(); + this.peers.delete(peerId); + } + + let sound = new Audio('/videocall/audio/end-call.mp3'); + sound.play(); + if (this.ui_type === "large") { + this.app.connection.emit('video-box-remove', peerId, 'disconnection'); + } else if (this.ui_type === "small") { + // console.log('peer left') + this.app.connection.emit('audio-box-remove', peerId); + + } + + + } + + renegotiate(peerId, retryCount = 0) { + const maxRetries = 4; + const retryDelay = 3000; + + const peerConnection = this.peers.get(peerId); + if (!peerConnection) { + return; + } + + // console.log('signalling state, ', peerConnection.signalingState) + if (peerConnection.signalingState !== 'stable') { + if (retryCount < maxRetries) { + console.log(`Signaling state is not stable, will retry in ${retryDelay} ms (attempt ${retryCount + 1}/${maxRetries})`); + setTimeout(() => { + this.renegotiate(peerId, retryCount + 1); + }, retryDelay); + } else { + console.log('Reached maximum number of renegotiation attempts, giving up'); + } + return; + } + + const offerOptions = { + offerToReceiveAudio: true, + offerToReceiveVideo: true, + }; + + peerConnection.createOffer(offerOptions) + .then((offer) => { + return peerConnection.setLocalDescription(offer); + }) + .then(() => { + let data = { + room_code: this.room_code, + type: 'renegotiate-offer', + sdp: peerConnection.localDescription.sdp, + targetPeerId: peerId + } + this.app.connection.emit('stun-send-message-to-server', data); + }) + .catch((error) => { + console.error('Error creating offer:', error); + }); + + // Implement renegotiation logic for reconnections and media stream restarts + } + + join() { + console.log('joining mesh network'); + this.app.connection.emit('stun-send-message-to-server', { type: 'peer-joined', room_code: this.room_code }); + } + + leave() { + this.localStream.getTracks().forEach(track => { + track.stop(); + // console.log(track); + console.log('stopping track'); + }) + this.peers.forEach((peerConnections, key) => { + peerConnections.close(); + }) + + this.peers = new Map(); + + + + let data = { + room_code: this.room_code, + type: 'peer-left', + } + + this.app.connection.emit('stun-send-message-to-server', data); + + } + + sendSignalingMessage(data) { + + } + + switchUITypeToLarge() { + this.ui_type = "large"; + } + switchUITypeToSmall() { + this.ui_type = "small"; + } + + renderRemoteStreams(){ + // loop over remote stream + this.remoteStreams.forEach((property, key) => { + // console.log(property, 'property', key, 'key') + // console.log(stream, 'stream') + if (this.ui_type === "large") { + this.app.connection.emit('render-remote-stream-large-request', key, property.remoteStream, property.peerConnection); + } else if (this.ui_type === "small") { + this.app.connection.emit('add-remote-stream-small-request', key, property.remoteStream, property.peerConnection); + } + }) + } +} + + +module.exports = PeerManager; \ No newline at end of file diff --git a/stun/lib/components/add-users.template.js b/stun/lib/components/add-users.template.js new file mode 100644 index 0000000..6092e15 --- /dev/null +++ b/stun/lib/components/add-users.template.js @@ -0,0 +1,25 @@ +const AddUsersTemplate = (app, mod, code) => { + return ` +
+
Copy Invitation Link
+
+
+

Share this call link with others you want in the call

+
${code.slice(0, 30)}...
+

Joined as ${app.wallet.getPublicKey()}

+
+
+
+ ` +} + +module.exports = AddUsersTemplate; + + +//
+//
+// +// +//
+//
diff --git a/stun/lib/components/audio-box.js b/stun/lib/components/audio-box.js new file mode 100644 index 0000000..37bd2c9 --- /dev/null +++ b/stun/lib/components/audio-box.js @@ -0,0 +1,50 @@ + +const AudioBoxTemplate = require("./audio-box.template"); +const { setTextRange } = require("typescript"); +// import {applyVideoBackground, } from 'virtual-bg'; + +/** + * + * Audio Box is a hook for a voice call, it adds an