From 70d71611e9085b013b879ba46a1d7072099a7ce0 Mon Sep 17 00:00:00 2001
From: Noah Petherbridge <root@kirsle.net>
Date: Sun, 17 Nov 2024 20:32:16 -0800
Subject: [PATCH] Speaking detection with hark.js

---
 package-lock.json            | 14 ++++++++++++
 package.json                 |  1 +
 public/static/css/chat.css   |  6 ++++++
 src/App.vue                  | 33 +++++++++++++++++++++++++++-
 src/components/VideoFeed.vue | 42 +++++++++++++++++++++++-------------
 5 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 4d6a398..0736b51 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,6 +9,7 @@
       "version": "0.0.0",
       "dependencies": {
         "floating-vue": "^2.0.0-beta.24",
+        "hark": "^1.2.3",
         "interactjs": "^1.10.18",
         "qrcodejs": "github:danielgjackson/qrcodejs",
         "vue": "^3.3.4",
@@ -1260,6 +1261,14 @@
       "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
       "dev": true
     },
+    "node_modules/hark": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/hark/-/hark-1.2.3.tgz",
+      "integrity": "sha512-u68vz9SCa38ESiFJSDjqK8XbXqWzyot7Cj6Y2b6jk2NJ+II3MY2dIrLMg/kjtIAun4Y1DHF/20hfx4rq1G5GMg==",
+      "dependencies": {
+        "wildemitter": "^1.2.0"
+      }
+    },
     "node_modules/has-flag": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -2056,6 +2065,11 @@
         "node": ">= 8"
       }
     },
+    "node_modules/wildemitter": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/wildemitter/-/wildemitter-1.2.1.tgz",
+      "integrity": "sha512-UMmSUoIQSir+XbBpTxOTS53uJ8s/lVhADCkEbhfRjUGFDPme/XGOb0sBWLx5sTz7Wx/2+TlAw1eK9O5lw5PiEw=="
+    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
diff --git a/package.json b/package.json
index 9d1e26a..2021015 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "floating-vue": "^2.0.0-beta.24",
+    "hark": "^1.2.3",
     "interactjs": "^1.10.18",
     "qrcodejs": "github:danielgjackson/qrcodejs",
     "vue": "^3.3.4",
diff --git a/public/static/css/chat.css b/public/static/css/chat.css
index 450dfba..8d6af2d 100644
--- a/public/static/css/chat.css
+++ b/public/static/css/chat.css
@@ -241,11 +241,17 @@ img {
     width: 168px;
     height: 112px;
     background-color: black;
+    border: 1px solid black;
     margin: 3px;
     overflow: hidden;
     resize: both;
 }
 
+/* A speaking webcam */
+.feed.is-speaking {
+    border: 1px solid #09F;
+}
+
 /* A popped-out video feed window */
 div.feed.popped-out {
     position: absolute;
diff --git a/src/App.vue b/src/App.vue
index 9c25b56..61c3421 100644
--- a/src/App.vue
+++ b/src/App.vue
@@ -4,6 +4,7 @@ import FloatingVue from 'floating-vue';
 import 'floating-vue/dist/style.css';
 import { Mentionable } from 'vue-mention';
 import EmojiPicker from 'vue3-emoji-picker';
+import hark from 'hark';
 
 import AlertModal from './components/AlertModal.vue';
 import LoginModal from './components/LoginModal.vue';
@@ -244,6 +245,7 @@ export default {
                 muted: {}, // muted bool per username
                 booted: {}, // booted bool per username
                 poppedOut: {}, // popped-out video per username
+                speaking: {}, // speaking boolean per username
 
                 // RTCPeerConnections per username.
                 pc: {},
@@ -1760,6 +1762,9 @@ export default {
                         this.WebRTC.muted[username] = true;
                         $ref.muted = true;
                     }
+
+                    // Set up the speech detector.
+                    this.initSpeakingEvents(username, $ref);
                 });
 
                 // Inform them they are being watched.
@@ -1789,7 +1794,7 @@ export default {
                     this.WebRTC.frozenStreamInterval[username] = setInterval(() => {
                         if (videoTrack.muted) freezeDetected();
                     }, 3000);
-                })
+                });
             };
 
             // ANSWERER: add our video to the connection so that the offerer (the one who
@@ -2256,6 +2261,9 @@ export default {
 
                 // Begin dark video detection.
                 this.initDarkVideoDetection();
+
+                // Begin monitoring for speaking events.
+                this.initSpeakingEvents(this.username, this.webcam.elem);
             }).catch(err => {
                 this.ChatClient(`Webcam error: ${err}<br><br>Please see the <a href="/about#troubleshooting">troubleshooting guide</a> for help.`);
             }).finally(() => {
@@ -2943,6 +2951,27 @@ export default {
             })
         },
 
+        // Webcam "is speaking" functions.
+        initSpeakingEvents(username, element) {
+            // element is the <video> element, with the video stream
+            // (whether from getUserMedia or WebRTC) on srcObject.
+
+            let stream = element.srcObject,
+                feedElem = element.closest('div.feed'),
+                options = {},
+                speechEvents = hark(stream, options);
+
+            speechEvents.on('speaking', () => {
+                feedElem.classList.add('is-speaking');
+                this.WebRTC.speaking[username] = true;
+            });
+
+            speechEvents.on('stopped_speaking', () => {
+                feedElem.classList.remove('is-speaking');
+                this.WebRTC.speaking[username] = false;
+            });
+        },
+
         // Dark video detection.
         initDarkVideoDetection() {
             if (this.webcam.darkVideo.canvas === null) {
@@ -4749,6 +4778,7 @@ export default {
                         :is-explicit="webcam.nsfw"
                         :is-muted="webcam.muted"
                         :is-source-muted="webcam.muted"
+                        :is-speaking="WebRTC.speaking[username]"
                         :watermark-image="webcam.watermark"
                         @mute-video="muteMe()"
                         @popout="popoutVideo"
@@ -4761,6 +4791,7 @@ export default {
                         v-bind:key="username"
                         :username="username"
                         :popped-out="WebRTC.poppedOut[username]"
+                        :is-speaking="WebRTC.speaking[username]"
                         :is-explicit="isUsernameCamNSFW(username)"
                         :is-source-muted="isSourceMuted(username)"
                         :is-muted="isMuted(username)"
diff --git a/src/components/VideoFeed.vue b/src/components/VideoFeed.vue
index 33cbea8..2609940 100644
--- a/src/components/VideoFeed.vue
+++ b/src/components/VideoFeed.vue
@@ -11,6 +11,7 @@ export default {
         isSourceMuted: Boolean, // camera is muted on the broadcaster's end
         isWatchingMe: Boolean, // other video is watching us back
         isFrozen: Boolean,     // video is detected as frozen
+        isSpeaking: Boolean,   // video is registering audio
         watermarkImage: Image, // watermark image to overlay (nullable)
     },
     components: {
@@ -38,6 +39,24 @@ export default {
         textColorClass() {
             return this.isExplicit ? 'has-text-camera-red' : 'has-text-camera-blue';
         },
+        muteButtonClass() {
+            let classList = [
+                'button is-small ml-1 px-2',
+            ]
+
+            if (this.isMuted) {
+                classList.push('is-danger');
+            } else {
+                classList.push('is-success is-outlined');
+            }
+            return classList.join(' ');
+        },
+        muteIconClass() {
+            if (this.localVideo) {
+                return this.isMuted ? 'fa-microphone-slash' : 'fa-microphone';
+            }
+            return this.isMuted ? 'fa-volume-xmark' : 'fa-volume-high';
+        }
     },
     methods: {
         closeVideo() {
@@ -103,7 +122,7 @@ export default {
         'popped-out': poppedOut,
         'popped-in': !poppedOut,
     }" @mouseover="mouseOver = true" @mouseleave="mouseOver = false">
-        <video class="feed"
+        <video
             :id="videoID"
             autoplay
             disablepictureinpicture
@@ -127,6 +146,11 @@ export default {
             <!-- Frozen stream detection -->
             <a class="fa fa-mountain ml-1" href="#" v-if="!localVideo && isFrozen" style="color: #00FFFF"
                 @click.prevent="reopenVideo()" title="Frozen video detected!"></a>
+
+            <!-- Is speaking -->
+            <span v-if="isSpeaking" class="ml-1" title="Speaking">
+                <i class="fa fa-volume-high has-text-info"></i>
+            </span>
         </div>
 
         <!-- Close button (others' videos only) -->
@@ -139,21 +163,9 @@ export default {
         <!-- Controls -->
         <div class="controls">
             <!-- Mute Button -->
-            <button type="button" v-if="!isMuted" class="button is-small is-success is-outlined ml-1 px-2"
-                :class="{'seethru': !mouseOver}"
+            <button type="button" :class="muteButtonClass"
                 @click="muteVideo()">
-                <i class="fa" :class="{
-                    'fa-microphone': localVideo,
-                    'fa-volume-high': !localVideo
-                }"></i>
-            </button>
-            <button type="button" v-else class="button is-small is-danger ml-1 px-2"
-                :class="{'seethru': !mouseOver}"
-                @click="muteVideo()">
-                <i class="fa" :class="{
-                    'fa-microphone-slash': localVideo,
-                    'fa-volume-xmark': !localVideo
-                }"></i>
+                <i class="fa" :class="muteIconClass"></i>
             </button>
 
             <!-- Pop-out Video -->