Speaking detection with hark.js
This commit is contained in:
parent
98bf0d9e84
commit
70d71611e9
14
package-lock.json
generated
14
package-lock.json
generated
|
|
@ -9,6 +9,7 @@
|
|||
"version": "0.0.0",
|
||||
"dependencies": {
|
||||
"floating-vue": "^2.0.0-beta.24",
|
||||
"hark": "^1.2.3",
|
||||
"interactjs": "^1.10.18",
|
||||
"qrcodejs": "github:danielgjackson/qrcodejs",
|
||||
"vue": "^3.3.4",
|
||||
|
|
@ -1260,6 +1261,14 @@
|
|||
"integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/hark": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/hark/-/hark-1.2.3.tgz",
|
||||
"integrity": "sha512-u68vz9SCa38ESiFJSDjqK8XbXqWzyot7Cj6Y2b6jk2NJ+II3MY2dIrLMg/kjtIAun4Y1DHF/20hfx4rq1G5GMg==",
|
||||
"dependencies": {
|
||||
"wildemitter": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
|
|
@ -2056,6 +2065,11 @@
|
|||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/wildemitter": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/wildemitter/-/wildemitter-1.2.1.tgz",
|
||||
"integrity": "sha512-UMmSUoIQSir+XbBpTxOTS53uJ8s/lVhADCkEbhfRjUGFDPme/XGOb0sBWLx5sTz7Wx/2+TlAw1eK9O5lw5PiEw=="
|
||||
},
|
||||
"node_modules/wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"floating-vue": "^2.0.0-beta.24",
|
||||
"hark": "^1.2.3",
|
||||
"interactjs": "^1.10.18",
|
||||
"qrcodejs": "github:danielgjackson/qrcodejs",
|
||||
"vue": "^3.3.4",
|
||||
|
|
|
|||
|
|
@ -241,11 +241,17 @@ img {
|
|||
width: 168px;
|
||||
height: 112px;
|
||||
background-color: black;
|
||||
border: 1px solid black;
|
||||
margin: 3px;
|
||||
overflow: hidden;
|
||||
resize: both;
|
||||
}
|
||||
|
||||
/* A speaking webcam */
|
||||
.feed.is-speaking {
|
||||
border: 1px solid #09F;
|
||||
}
|
||||
|
||||
/* A popped-out video feed window */
|
||||
div.feed.popped-out {
|
||||
position: absolute;
|
||||
|
|
|
|||
33
src/App.vue
33
src/App.vue
|
|
@ -4,6 +4,7 @@ import FloatingVue from 'floating-vue';
|
|||
import 'floating-vue/dist/style.css';
|
||||
import { Mentionable } from 'vue-mention';
|
||||
import EmojiPicker from 'vue3-emoji-picker';
|
||||
import hark from 'hark';
|
||||
|
||||
import AlertModal from './components/AlertModal.vue';
|
||||
import LoginModal from './components/LoginModal.vue';
|
||||
|
|
@ -244,6 +245,7 @@ export default {
|
|||
muted: {}, // muted bool per username
|
||||
booted: {}, // booted bool per username
|
||||
poppedOut: {}, // popped-out video per username
|
||||
speaking: {}, // speaking boolean per username
|
||||
|
||||
// RTCPeerConnections per username.
|
||||
pc: {},
|
||||
|
|
@ -1760,6 +1762,9 @@ export default {
|
|||
this.WebRTC.muted[username] = true;
|
||||
$ref.muted = true;
|
||||
}
|
||||
|
||||
// Set up the speech detector.
|
||||
this.initSpeakingEvents(username, $ref);
|
||||
});
|
||||
|
||||
// Inform them they are being watched.
|
||||
|
|
@ -1789,7 +1794,7 @@ export default {
|
|||
this.WebRTC.frozenStreamInterval[username] = setInterval(() => {
|
||||
if (videoTrack.muted) freezeDetected();
|
||||
}, 3000);
|
||||
})
|
||||
});
|
||||
};
|
||||
|
||||
// ANSWERER: add our video to the connection so that the offerer (the one who
|
||||
|
|
@ -2256,6 +2261,9 @@ export default {
|
|||
|
||||
// Begin dark video detection.
|
||||
this.initDarkVideoDetection();
|
||||
|
||||
// Begin monitoring for speaking events.
|
||||
this.initSpeakingEvents(this.username, this.webcam.elem);
|
||||
}).catch(err => {
|
||||
this.ChatClient(`Webcam error: ${err}<br><br>Please see the <a href="/about#troubleshooting">troubleshooting guide</a> for help.`);
|
||||
}).finally(() => {
|
||||
|
|
@ -2943,6 +2951,27 @@ export default {
|
|||
})
|
||||
},
|
||||
|
||||
// Webcam "is speaking" functions.
|
||||
initSpeakingEvents(username, element) {
|
||||
// element is the <video> element, with the video stream
|
||||
// (whether from getUserMedia or WebRTC) on srcObject.
|
||||
|
||||
let stream = element.srcObject,
|
||||
feedElem = element.closest('div.feed'),
|
||||
options = {},
|
||||
speechEvents = hark(stream, options);
|
||||
|
||||
speechEvents.on('speaking', () => {
|
||||
feedElem.classList.add('is-speaking');
|
||||
this.WebRTC.speaking[username] = true;
|
||||
});
|
||||
|
||||
speechEvents.on('stopped_speaking', () => {
|
||||
feedElem.classList.remove('is-speaking');
|
||||
this.WebRTC.speaking[username] = false;
|
||||
});
|
||||
},
|
||||
|
||||
// Dark video detection.
|
||||
initDarkVideoDetection() {
|
||||
if (this.webcam.darkVideo.canvas === null) {
|
||||
|
|
@ -4749,6 +4778,7 @@ export default {
|
|||
:is-explicit="webcam.nsfw"
|
||||
:is-muted="webcam.muted"
|
||||
:is-source-muted="webcam.muted"
|
||||
:is-speaking="WebRTC.speaking[username]"
|
||||
:watermark-image="webcam.watermark"
|
||||
@mute-video="muteMe()"
|
||||
@popout="popoutVideo"
|
||||
|
|
@ -4761,6 +4791,7 @@ export default {
|
|||
v-bind:key="username"
|
||||
:username="username"
|
||||
:popped-out="WebRTC.poppedOut[username]"
|
||||
:is-speaking="WebRTC.speaking[username]"
|
||||
:is-explicit="isUsernameCamNSFW(username)"
|
||||
:is-source-muted="isSourceMuted(username)"
|
||||
:is-muted="isMuted(username)"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ export default {
|
|||
isSourceMuted: Boolean, // camera is muted on the broadcaster's end
|
||||
isWatchingMe: Boolean, // other video is watching us back
|
||||
isFrozen: Boolean, // video is detected as frozen
|
||||
isSpeaking: Boolean, // video is registering audio
|
||||
watermarkImage: Image, // watermark image to overlay (nullable)
|
||||
},
|
||||
components: {
|
||||
|
|
@ -38,6 +39,24 @@ export default {
|
|||
textColorClass() {
|
||||
return this.isExplicit ? 'has-text-camera-red' : 'has-text-camera-blue';
|
||||
},
|
||||
muteButtonClass() {
|
||||
let classList = [
|
||||
'button is-small ml-1 px-2',
|
||||
]
|
||||
|
||||
if (this.isMuted) {
|
||||
classList.push('is-danger');
|
||||
} else {
|
||||
classList.push('is-success is-outlined');
|
||||
}
|
||||
return classList.join(' ');
|
||||
},
|
||||
muteIconClass() {
|
||||
if (this.localVideo) {
|
||||
return this.isMuted ? 'fa-microphone-slash' : 'fa-microphone';
|
||||
}
|
||||
return this.isMuted ? 'fa-volume-xmark' : 'fa-volume-high';
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
closeVideo() {
|
||||
|
|
@ -103,7 +122,7 @@ export default {
|
|||
'popped-out': poppedOut,
|
||||
'popped-in': !poppedOut,
|
||||
}" @mouseover="mouseOver = true" @mouseleave="mouseOver = false">
|
||||
<video class="feed"
|
||||
<video
|
||||
:id="videoID"
|
||||
autoplay
|
||||
disablepictureinpicture
|
||||
|
|
@ -127,6 +146,11 @@ export default {
|
|||
<!-- Frozen stream detection -->
|
||||
<a class="fa fa-mountain ml-1" href="#" v-if="!localVideo && isFrozen" style="color: #00FFFF"
|
||||
@click.prevent="reopenVideo()" title="Frozen video detected!"></a>
|
||||
|
||||
<!-- Is speaking -->
|
||||
<span v-if="isSpeaking" class="ml-1" title="Speaking">
|
||||
<i class="fa fa-volume-high has-text-info"></i>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<!-- Close button (others' videos only) -->
|
||||
|
|
@ -139,21 +163,9 @@ export default {
|
|||
<!-- Controls -->
|
||||
<div class="controls">
|
||||
<!-- Mute Button -->
|
||||
<button type="button" v-if="!isMuted" class="button is-small is-success is-outlined ml-1 px-2"
|
||||
:class="{'seethru': !mouseOver}"
|
||||
<button type="button" :class="muteButtonClass"
|
||||
@click="muteVideo()">
|
||||
<i class="fa" :class="{
|
||||
'fa-microphone': localVideo,
|
||||
'fa-volume-high': !localVideo
|
||||
}"></i>
|
||||
</button>
|
||||
<button type="button" v-else class="button is-small is-danger ml-1 px-2"
|
||||
:class="{'seethru': !mouseOver}"
|
||||
@click="muteVideo()">
|
||||
<i class="fa" :class="{
|
||||
'fa-microphone-slash': localVideo,
|
||||
'fa-volume-xmark': !localVideo
|
||||
}"></i>
|
||||
<i class="fa" :class="muteIconClass"></i>
|
||||
</button>
|
||||
|
||||
<!-- Pop-out Video -->
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user