Camera connectivity established!

* Two users can activate their cameras locally and then connect them together
  with WebRTC with video and audio support working!
* Limitation: users need to be broadcasting video themselves before they can
  connect to someone's camera. If the offerer doesn't add tracks of their own,
  the SDP offer doesn't request video channels; so even though the answerer
  adds their tracks to the connection, they aren't used by the offerer.
* As currently implemented, the offerer's camera feed will also appear on
  screen for the answerer - every video connection opens the feed both ways.

Compared to the previous commit (where clients shared SDP messages but not
ICE candidates or anything further) the fixes and learnings were:

* The back-end was trying to relay candidate messages, but there was a JSON
  marshalling error (json object casted into a string) - changed the Message
  type to map[string]interface{} and both sides could exchange ICE candidates.
* Both sides needed to add their video tracks to the connection so that there
  was anything of value to be sent over the WebRTC channel.

Other changes:

* Server will send a ping message every minute to connected WebSockets.

pkg/config.go

@@ -0,0 +1,5 @@
+package barertc
+
+import "time"
+
+const PingInterval = 60 * time.Second

pkg/messages.go

@@ -15,8 +15,8 @@ type Message struct {
 	OpenSecret string `json:"openSecret,omitempty"`
 
 	// Parameters sent on WebRTC signaling messages.
-	Candidate   string `json:"candidate,omitempty"`   // candidate
+	Candidate   map[string]interface{} `json:"candidate,omitempty"`   // candidate
-	Description string `json:"description,omitempty"` // sdp
+	Description map[string]interface{} `json:"description,omitempty"` // sdp
 }
 
 const (
@@ -30,6 +30,7 @@ const (
 	ActionRing    = "ring"    // receiver of a WebRTC open request
 
 	// Actions sent by server only
+	ActionPing     = "ping"
 	ActionWhoList  = "who"      // server pushes the Who List
 	ActionPresence = "presence" // a user joined or left the room
 	ActionError    = "error"    // ChatServer errors

pkg/websocket.go

@@ -20,6 +20,7 @@ type Subscriber struct {
 	VideoActive bool
 	conn        *websocket.Conn
 	ctx         context.Context
+	cancel      context.CancelFunc
 	messages    chan []byte
 	closeSlow   func()
 }
@@ -30,7 +31,7 @@ func (sub *Subscriber) ReadLoop(s *Server) {
 		for {
 			msgType, data, err := sub.conn.Read(sub.ctx)
 			if err != nil {
-				log.Error("ReadLoop error: %+v", err)
+				log.Error("ReadLoop error(%s): %+v", sub.Username, err)
 				s.DeleteSubscriber(sub)
 				s.Broadcast(Message{
 					Action:   ActionPresence,
@@ -119,11 +120,12 @@ func (s *Server) WebSocket() http.HandlerFunc {
 		// CloseRead starts a goroutine that will read from the connection
 		// until it is closed.
 		// ctx := c.CloseRead(r.Context())
-		ctx, _ := context.WithCancel(r.Context())
+		ctx, cancel := context.WithCancel(r.Context())
 
 		sub := &Subscriber{
 			conn:     c,
 			ctx:      ctx,
+			cancel:   cancel,
 			messages: make(chan []byte, s.subscriberMessageBuffer),
 			closeSlow: func() {
 				c.Close(websocket.StatusPolicyViolation, "connection too slow to keep up with messages")
@@ -134,6 +136,7 @@ func (s *Server) WebSocket() http.HandlerFunc {
 		// defer s.DeleteSubscriber(sub)
 
 		go sub.ReadLoop(s)
+		pinger := time.NewTicker(PingInterval)
 		for {
 			select {
 			case msg := <-sub.messages:
@@ -141,7 +144,13 @@ func (s *Server) WebSocket() http.HandlerFunc {
 				if err != nil {
 					return
 				}
+			case timestamp := <-pinger.C:
+				sub.SendJSON(Message{
+					Action:  ActionPing,
+					Message: timestamp.Format(time.RFC3339),
+				})
 			case <-ctx.Done():
+				pinger.Stop()
 				return
 			}
 		}
@@ -179,6 +188,12 @@ func (s *Server) GetSubscriber(username string) (*Subscriber, error) {
 // DeleteSubscriber removes a subscriber from the server.
 func (s *Server) DeleteSubscriber(sub *Subscriber) {
 	log.Error("DeleteSubscriber: %s", sub.Username)
+
+	// Cancel its context to clean up the for-loop goroutine.
+	if sub.cancel != nil {
+		sub.cancel()
+	}
+
 	s.subscribersMu.Lock()
 	delete(s.subscribers, sub)
 	s.subscribersMu.Unlock()

web/static/js/BareRTC.js

@@ -125,7 +125,7 @@ const app = Vue.createApp({
             // Response for the opener to begin WebRTC connection.
             const secret = msg.openSecret;
             console.log("OPEN: connect to %s with secret %s", msg.username, secret);
-            this.ChatClient(`Connecting to stream for ${msg.username}.`);
+            this.ChatClient(`onOpen called for ${msg.username}.`);
 
             this.startWebRTC(msg.username, true);
         },
@@ -137,6 +137,11 @@ const app = Vue.createApp({
 
             this.startWebRTC(msg.username, false);
         },
+        onUserExited(msg) {
+            // A user has logged off the server. Clean up any WebRTC connections.
+            delete(this.WebRTC.streams[msg.username]);
+            delete(this.WebRTC.pc[msg.username]);
+        },
 
         // Handle messages sent in chat.
         onMessage(msg) {
@@ -149,7 +154,8 @@ const app = Vue.createApp({
         // Dial the WebSocket connection.
         dial() {
             console.log("Dialing WebSocket...");
-            const conn = new WebSocket(`ws://${location.host}/ws`);
+            const proto = location.protocol === 'https:' ? 'wss' : 'ws';
+            const conn = new WebSocket(`${proto}://${location.host}/ws`);
 
             conn.addEventListener("close", ev => {
                 this.ws.connected = false;
@@ -193,6 +199,11 @@ const app = Vue.createApp({
                         this.onMessage(msg);
                         break;
                     case "presence":
+                        // TODO: make a dedicated leave event
+                        if (msg.message.indexOf("has exited the room!") > -1) {
+                            // Clean up data about this user.
+                            this.onUserExited(msg);
+                        }
                         this.pushHistory({
                             action: msg.action,
                             username: msg.username,
@@ -235,14 +246,21 @@ const app = Vue.createApp({
             let pc = new RTCPeerConnection(configuration);
             this.WebRTC.pc[username] = pc;
 
+            // Create a data channel so we have something to connect over even if
+            // the local user is not broadcasting their own camera.
+            let dataChannel = pc.createDataChannel("data");
+            dataChannel.addEventListener("open", event => {
+                // beginTransmission(dataChannel);
+            })
+
             // 'onicecandidate' notifies us whenever an ICE agent needs to deliver a
             // message to the other peer through the signaling server.
             pc.onicecandidate = event => {
-                this.ChatClient("On ICE Candidate called");
+                console.error("WebRTC OnICECandidate called!", event);
-                console.log(event);
+                // this.ChatClient("On ICE Candidate called!");
-                console.log(event.candidate);
                 if (event.candidate) {
-                    this.ChatClient(`Send ICE candidate: ${event.candidate}`);
+                    // this.ChatClient(`Send ICE candidate: ${JSON.stringify(event.candidate)}`);
+                    console.log("Sending candidate to websockets:", event.candidate);
                     this.ws.conn.send(JSON.stringify({
                         action: "candidate",
                         username: username,
@@ -253,8 +271,9 @@ const app = Vue.createApp({
 
             // If the user is offerer let the 'negotiationneeded' event create the offer.
             if (isOfferer) {
-                this.ChatClient("Sending offer:");
+                this.ChatClient("We are the offerer - set up onNegotiationNeeded");
                 pc.onnegotiationneeded = () => {
+                    console.error("WebRTC OnNegotiationNeeded called!");
                     this.ChatClient("Negotiation Needed, creating WebRTC offer.");
                     pc.createOffer().then(this.localDescCreated(pc, username)).catch(this.ChatClient);
                 };
@@ -262,6 +281,8 @@ const app = Vue.createApp({
 
             // When a remote stream arrives.
             pc.ontrack = event => {
+                this.ChatServer("ON TRACK CALLED!!!");
+                console.error("WebRTC OnTrack called!", event);
                 const stream = event.streams[0];
 
                 // Do we already have it?
@@ -270,13 +291,28 @@ const app = Vue.createApp({
                     this.WebRTC.streams[username].id !== stream.id) {
                     this.WebRTC.streams[username] = stream;
                 }
+
+                window.requestAnimationFrame(() => {
+                    this.ChatServer("Setting <video> srcObject for "+username);
+                    let $ref = document.getElementById(`videofeed-${username}`);
+                    console.log("Video elem:", $ref);
+                    $ref.srcObject = stream;
+                    // this.$refs[`videofeed-${username}`].srcObject = stream;
+                });
             };
 
             // If we were already broadcasting video, send our stream to
             // the connecting user.
-            if (!isOfferer && this.webcam.active) {
+            // TODO: currently both users need to have video on for the connection
+            // to succeed - if offerer doesn't addTrack it won't request a video channel
+            // and so the answerer (who has video) won't actually send its
+            if (this.webcam.active) {
                 this.ChatClient(`Sharing our video stream to ${username}.`);
-                this.webcam.stream.getTracks().forEach(track => pc.addTrack(track, this.webcam.stream));
+                let stream = this.webcam.stream;
+                stream.getTracks().forEach(track => {
+                    console.error("Add stream track to WebRTC", stream, track);
+                    pc.addTrack(track, stream)
+                });
             }
 
             // If we are the offerer, begin the connection.
@@ -295,7 +331,7 @@ const app = Vue.createApp({
                         this.ws.conn.send(JSON.stringify({
                             action: "sdp",
                             username: username,
-                            description: JSON.stringify(pc.localDescription),
+                            description: pc.localDescription,
                         }));
                     },
                     console.error,
@@ -305,12 +341,16 @@ const app = Vue.createApp({
 
         // Handle inbound WebRTC signaling messages proxied by the websocket.
         onCandidate(msg) {
-            if (this.WebRTC.pc[msg.username] == undefined) return;
+            console.error("onCandidate() called:", msg);
+            if (this.WebRTC.pc[msg.username] == undefined) {
+                console.error("DID NOT FIND RTCPeerConnection for username:", msg.username);
+                return;
+            }
             let pc = this.WebRTC.pc[msg.username];
 
             // Add the new ICE candidate.
             console.log("Add ICE candidate: %s", msg.candidate);
-            this.ChatClient(`Received an ICE candidate from ${username}: ${msg.candidate}`);
+            // this.ChatClient(`Received an ICE candidate from ${msg.username}: ${JSON.stringify(msg.candidate)}`);
             pc.addIceCandidate(
                 new RTCIceCandidate(
                     msg.candidate,
@@ -320,16 +360,34 @@ const app = Vue.createApp({
             );
         },
         onSDP(msg) {
-            if (this.WebRTC.pc[msg.username] == undefined) return;
+            console.error("onSDP() called:", msg);
+            if (this.WebRTC.pc[msg.username] == undefined) {
+                console.error("DID NOT FIND RTCPeerConnection for username:", msg.username);
+                return;
+            }
             let pc = this.WebRTC.pc[msg.username];
-            let description = JSON.parse(msg.description);
+            let message = msg.description;
 
             // Add the new ICE candidate.
-            console.log("Set description: %s", description);
+            console.log("Set description:", message);
-            this.ChatClient(`Received a Remote Description from ${msg.username}: ${msg.description}.`);
+            this.ChatClient(`Received a Remote Description from ${msg.username}: ${JSON.stringify(msg.description)}.`);
-            pc.setRemoteDescription(new RTCSessionDescription(description), () => {
+            pc.setRemoteDescription(new RTCSessionDescription(message), () => {
                 // When receiving an offer let's answer it.
                 if (pc.remoteDescription.type === 'offer') {
+                    console.error("Webcam:", this.webcam);
+
+                    // Add our local video tracks to the connection.
+                    // if (this.webcam.active) {
+                    //     this.ChatClient(`Sharing our video stream to ${msg.username}.`);
+                    //     let stream = this.webcam.stream;
+                    //     stream.getTracks().forEach(track => {
+                    //         console.error("Add stream track to WebRTC", stream, track);
+                    //         pc.addTrack(track, stream)
+                    //     });
+                    // }
+
+                    this.ChatClient(`setRemoteDescription callback. Offer recieved - sending answer. Cam active? ${this.webcam.active}`);
+                    console.warn("Creating answer now");
                     pc.createAnswer().then(this.localDescCreated(pc, msg.username)).catch(this.ChatClient);
                 }
             }, console.error);
@@ -368,6 +426,14 @@ const app = Vue.createApp({
                 return;
             }
 
+            // We need to broadcast video to connect to another.
+            // TODO: because if the offerer doesn't add video tracks they
+            // won't request video support so the answerer's video isn't sent
+            if (!this.webcam.active) {
+                this.ChatServer("You will need to turn your own camera on first before you can connect to " + user.username + ".");
+                return;
+            }
+
             this.sendOpen(user.username);
         },
 

web/templates/chat.html

@@ -108,8 +108,14 @@
                         autoplay muted>
                         x
                     </video>
-                    <div class="feed">
+                    <video class="feed"
-                        y
+                        v-for="(stream, username) in WebRTC.streams"
+                        v-bind:key="username"
+                        :id="'videofeed-'+username"
+                        autoplay muted>
+                    </video>
+                    <div v-for="(stream, username) in WebRTC.streams" class="feed">
+                        [[username]] - [[stream]]
                     </div>
                     <div class="feed">
                         y