From b3d4b375ed01122cbfa34de640f4ecffc8101245 Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Tue, 8 Aug 2023 17:51:52 -0700 Subject: [PATCH] Video freeze detection + Other tweaks * Color code usernames on video windows to be blue or red depending on their local cam explicit setting * Attempt to detect freezes on RTCPeerConnection videos by registering a video onmute handler. If a freeze is detected, show a cyan mountain icon by their name. Clicking the icon will re-connect their video. * Update the video buttons on the Who List to always re-connect video instead of toggling it opened and closed. The X buttons on videos are now how you close a video. --- web/static/css/chat.css | 8 +++++ web/static/js/BareRTC.js | 77 +++++++++++++++++++++++++++++++++++++++- web/templates/chat.html | 17 +++++++-- 3 files changed, 98 insertions(+), 4 deletions(-) diff --git a/web/static/css/chat.css b/web/static/css/chat.css index 3f4ed4b..4d6bef8 100644 --- a/web/static/css/chat.css +++ b/web/static/css/chat.css @@ -28,6 +28,14 @@ body { text-overflow: ellipsis; } +/* Color coded webcam usernames per camera setting */ +.has-text-camera-blue { + color: #00ccff !important; +} +.has-text-camera-red { + color: #ff9999 !important; +} + /************************ * Main CSS Grid Layout * ************************/ diff --git a/web/static/js/BareRTC.js b/web/static/js/BareRTC.js index 5bdcf15..8cb195d 100644 --- a/web/static/js/BareRTC.js +++ b/web/static/js/BareRTC.js @@ -161,6 +161,13 @@ const app = Vue.createApp({ // RTCPeerConnections per username. pc: {}, + + // Video stream freeze detection. + frozenStreamInterval: {}, // map usernames to intervals + frozenStreamDetected: {}, // map usernames to bools + + // Debounce connection attempts since now every click = try to connect. + debounceOpens: {}, // map usernames to bools }, // Chat history. @@ -398,6 +405,7 @@ const app = Vue.createApp({ myVideoFlag() { // Compute the current user's video status flags. let status = 0; + if (!this.webcam.active) return 0; // unset all flags if not active now if (this.webcam.active) status |= this.VideoFlag.Active; if (this.webcam.muted) status |= this.VideoFlag.Muted; if (this.webcam.nsfw) status |= this.VideoFlag.NSFW; @@ -507,6 +515,16 @@ const app = Vue.createApp({ return; } + // DEBUGGING: fake set the freeze indicator. + let match = this.message.match(/^\/freeze (.+?)$/i); + if (match) { + let username = match[1]; + this.WebRTC.frozenStreamDetected[username] = true; + this.ChatClient(`DEBUG: Marked ${username} stream as frozen.`); + this.message = ""; + return; + } + // console.debug("Send message: %s", this.message); this.ws.conn.send(JSON.stringify({ action: "message", @@ -987,6 +1005,28 @@ const app = Vue.createApp({ // Inform them they are being watched. this.sendWatch(username, true); + + // Set a mute video handler to detect freezes. + stream.getVideoTracks().forEach(videoTrack => { + let freezeDetected = () => { + console.log("FREEZE DETECTED:", username); + // Wait 3 seconds to see if the stream has recovered on its own + setTimeout(() => { + // Flag it as likely frozen. + if (videoTrack.muted) { + this.WebRTC.frozenStreamDetected[username] = true; + } + }, 3000); + }; + + console.log("Apply onmute handler for", username); + videoTrack.onmute = freezeDetected; + + // Double check for frozen streams on an interval. + this.WebRTC.frozenStreamInterval[username] = setInterval(() => { + if (videoTrack.muted) freezeDetected(); + }, 3000); + }) }; // If we were already broadcasting video, send our stream to @@ -1212,6 +1252,14 @@ const app = Vue.createApp({ } return username; }, + isUsernameCamNSFW(username) { + // returns true if the username is broadcasting and NSFW, false otherwise. + // (used for the color coding of their nickname on their video box - so assumed they are broadcasting) + if (this.whoMap[username] != undefined && this.whoMap[username].video & this.VideoFlag.NSFW) { + return true; + } + return false; + }, leaveDM() { // Validate we're in a DM currently. if (this.channel.indexOf("@") !== 0) return; @@ -1407,7 +1455,21 @@ const app = Vue.createApp({ }, // Begin connecting to someone else's webcam. + openVideoByUsername(username, force) { + if (this.whoMap[username] != undefined) { + this.openVideo(this.whoMap[username], force); + return; + } + this.ChatClient("Couldn't open video by username: not found."); + }, openVideo(user, force) { + // Debounce so we don't spam too much for the same user. + if (this.WebRTC.debounceOpens[user.username]) return; + this.WebRTC.debounceOpens[user.username] = true; + setTimeout(() => { + delete(this.WebRTC.debounceOpens[user.username]); + }, 5000); + if (user.username === this.username) { this.ChatClient("You can already see your own webcam."); return; @@ -1434,7 +1496,6 @@ const app = Vue.createApp({ // Camera is already open? Then disconnect the connection. if (this.WebRTC.pc[user.username] != undefined && this.WebRTC.pc[user.username].offerer != undefined) { this.closeVideo(user.username, "offerer"); - return; } // If this user requests mutual viewership... @@ -1466,6 +1527,13 @@ const app = Vue.createApp({ delete (this.WebRTC.pc[username]); } + // Clean up any lingering camera freeze states. + delete (this.WebRTC.frozenStreamDetected[username]); + if (this.WebRTC.frozenStreamInterval[username]) { + clearInterval(this.WebRTC.frozenStreamInterval); + delete(this.WebRTC.frozenStreamInterval[username]); + } + // Inform backend we have closed it. this.sendWatch(username, false); return; @@ -1493,6 +1561,13 @@ const app = Vue.createApp({ delete (this.WebRTC.poppedOut[username]); } + // Clean up any lingering camera freeze states. + delete (this.WebRTC.frozenStreamDetected[username]); + if (this.WebRTC.frozenStreamInterval[username]) { + clearInterval(this.WebRTC.frozenStreamInterval); + delete(this.WebRTC.frozenStreamInterval[username]); + } + // Inform backend we have closed it. this.sendWatch(username, false); }, diff --git a/web/templates/chat.html b/web/templates/chat.html index 16022d9..528cb7b 100644 --- a/web/templates/chat.html +++ b/web/templates/chat.html @@ -591,7 +591,9 @@ autoplay muted> -
+
[[username]] @@ -631,19 +633,28 @@ :id="'videofeed-'+username" autoplay> -
+
[[username]] + + +
+ @click.prevent="closeVideo(username, 'offerer')">