Deadlock detection, DND, and Frontend Fixes

* Deadlock detection: the chatbot handlers will spin off a background goroutine
  to ping DMs at itself and test for responsiveness. If the echoes don't return
  for a minute, issue a /api/shutdown command to the HTTP server to force a
  reboot.
* New admin API endpoint: /api/shutdown, equivalent to the operator '/shutdown'
  command sent in chat. Requires your AdminAPIKey to call it. Used by the chatbot
  as part of deadlock detection.
* Adjust some uses of mutexes to hopefully mitigate deadlocks a bit.
* Do Not Disturb: if users opt to "Ignore unsolicited DMs" they will set a DND
  status on the server which will grey-out their DM icon for other chatters.
* Bring back an option for ChatServer to notify you when somebody begins watching
  your camera (on by default).
* Automatically focus the message entry box when changing channels.
* Lower webcam resolution hints to 480p to test performance implications.

client/deadlock_watch.go

@@ -0,0 +1,95 @@
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"git.kirsle.net/apps/barertc/client/config"
+	"git.kirsle.net/apps/barertc/pkg/log"
+	"git.kirsle.net/apps/barertc/pkg/messages"
+)
+
+const deadlockTTL = time.Minute
+
+/*
+Deadlock detection for the chat server.
+
+Part of the chatbot handlers. The bot will send DMs to itself on an interval
+and test whether the server is responsive; if it goes down, it will issue the
+/api/shutdown command to reboot the server automatically.
+
+This function is a goroutine spawned in the background.
+*/
+func (h *BotHandlers) watchForDeadlock() {
+	log.Info("Deadlock monitor engaged!")
+	h.deadlockLastOK = time.Now()
+
+	for {
+		time.Sleep(15 * time.Second)
+		h.client.Send(messages.Message{
+			Action:  messages.ActionMessage,
+			Channel: "@" + h.client.Username(),
+			Message: "deadlock ping",
+		})
+
+		// Has it been a while since our last ping?
+		if time.Since(h.deadlockLastOK) > deadlockTTL {
+			log.Error("Deadlock detected! Rebooting the chat server!")
+			h.deadlockLastOK = time.Now()
+			h.rebootChatServer()
+		}
+	}
+}
+
+// onMessageFromSelf handles DMs sent to ourself, e.g. for deadlock detection.
+func (h *BotHandlers) onMessageFromSelf(msg messages.Message) {
+	// If it is our own DM channel thread, it's for deadlock detection.
+	if msg.Channel == "@"+h.client.Username() {
+		h.deadlockLastOK = time.Now()
+	}
+}
+
+// Reboot the chat server via web API, in case of deadlock.
+func (h *BotHandlers) rebootChatServer() error {
+	// API request struct for BareRTC /api/shutdown endpoint.
+	var request = struct {
+		APIKey string
+	}{
+		APIKey: config.Current.BareRTC.AdminAPIKey,
+	}
+
+	// JSON request body.
+	jsonStr, err := json.Marshal(request)
+	if err != nil {
+		return err
+	}
+
+	// Make the API request to BareRTC.
+	var url = strings.TrimSuffix(config.Current.BareRTC.URL, "/") + "/api/shutdown"
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("RebootChatServer: error posting to BareRTC: status %d body %s", resp.StatusCode, body)
+	}
+
+	return nil
+}

client/handlers.go

@@ -62,6 +62,10 @@ type BotHandlers struct {
 	// so we don't accidentally take back our own reactions.
 	reactions   map[int]map[string]interface{}
 	reactionsMu sync.Mutex
+
+	// Deadlock detection (deadlock_watch.go): record time of last successful
+	// ping to self, to detect when the server is deadlocked.
+	deadlockLastOK time.Time
 }
 
 // SetupChatbot configures a sensible set of default handlers for the BareBot application.
@@ -105,6 +109,9 @@ func (c *Client) SetupChatbot() error {
 	c.OnDisconnect = handler.OnDisconnect
 	c.OnPing = handler.OnPing
 
+	// Watch for deadlocks.
+	go handler.watchForDeadlock()
+
 	return nil
 }
 
@@ -157,6 +164,7 @@ func (h *BotHandlers) OnMessage(msg messages.Message) {
 
 	// Ignore echoed message from ourself.
 	if msg.Username == h.client.Username() {
+		h.onMessageFromSelf(msg)
 		return
 	}
 

cmd/BareBot/commands/run.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"git.kirsle.net/apps/barertc/client"
 	"git.kirsle.net/apps/barertc/client/config"
@@ -74,6 +75,11 @@ func init() {
 
 			// Run!
 			log.Info("Connecting to ChatServer")
+			err = client.Run()
+			if err != nil {
+				log.Error("Error: %s (and sleeping 5 seconds before exit)", err)
+				time.Sleep(5 * time.Second)
+			}
 			return cli.Exit(client.Run(), 1)
 		},
 	}

docs/API.md

@@ -50,6 +50,31 @@ The return schema looks like:
 }
 ```
 
+## POST /api/shutdown
+
+Shut down (and hopefully, reboot) the chat server. It is equivalent to the `/shutdown` operator command issued in chat, but callable from your web application. It is also used as part of deadlock detection on the BareBot chatbot.
+
+It requires the AdminAPIKey to post:
+
+```json
+{
+	"APIKey": "from settings.toml"
+}
+```
+
+The return schema looks like:
+
+```json
+{
+	"OK": true,
+	"Error": "error string, omitted if none"
+}
+```
+
+The HTTP server will respond OK, and then shut down a couple of seconds later, attempting to send a ChatServer broadcast first (as in the `/shutdown` command). If the chat server is deadlocked, this broadcast won't go out but the program will still exit.
+
+It is up to your process supervisor to automatically restart BareRTC when it exits.
+
 ## POST /api/blocklist
 
 Your server may pre-cache the user's blocklist for them **before** they

pkg/api.go

@@ -3,8 +3,10 @@ package barertc
 import (
 	"encoding/json"
 	"net/http"
+	"os"
 	"strings"
 	"sync"
+	"time"
 
 	"git.kirsle.net/apps/barertc/pkg/config"
 	"git.kirsle.net/apps/barertc/pkg/jwt"
@@ -174,6 +176,101 @@ func (s *Server) Authenticate() http.HandlerFunc {
 	})
 }
 
+// Shutdown (/api/shutdown) the chat server, hopefully to reboot it.
+//
+// This endpoint is equivalent to the operator '/shutdown' command but may be
+// invoked by your website, or your chatbot. It requires the AdminAPIKey.
+//
+// It is a POST request with a json body containing the following schema:
+//
+//	{
+//		"APIKey": "from settings.toml",
+//	}
+//
+// The return schema looks like:
+//
+//	{
+//		"OK": true,
+//		"Error": "error string, omitted if none",
+//	}
+func (s *Server) ShutdownAPI() http.HandlerFunc {
+	type request struct {
+		APIKey string
+		Claims jwt.Claims
+	}
+
+	type result struct {
+		OK    bool
+		Error string `json:",omitempty"`
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// JSON writer for the response.
+		w.Header().Set("Content-Type", "application/json")
+		enc := json.NewEncoder(w)
+		enc.SetIndent("", "  ")
+
+		// Parse the request.
+		if r.Method != http.MethodPost {
+			w.WriteHeader(http.StatusBadRequest)
+			enc.Encode(result{
+				Error: "Only POST methods allowed",
+			})
+			return
+		} else if r.Header.Get("Content-Type") != "application/json" {
+			w.WriteHeader(http.StatusBadRequest)
+			enc.Encode(result{
+				Error: "Only application/json content-types allowed",
+			})
+			return
+		}
+
+		defer r.Body.Close()
+
+		// Parse the request payload.
+		var (
+			params request
+			dec    = json.NewDecoder(r.Body)
+		)
+		if err := dec.Decode(&params); err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			enc.Encode(result{
+				Error: err.Error(),
+			})
+			return
+		}
+
+		// Validate the API key.
+		if params.APIKey != config.Current.AdminAPIKey {
+			w.WriteHeader(http.StatusUnauthorized)
+			enc.Encode(result{
+				Error: "Authentication denied.",
+			})
+			return
+		}
+
+		// Send the response.
+		enc.Encode(result{
+			OK: true,
+		})
+
+		// Defer a shutdown a moment later.
+		go func() {
+			time.Sleep(2 * time.Second)
+			os.Exit(1)
+		}()
+
+		// Attempt to broadcast, but if deadlocked this might not go out.
+		go func() {
+			s.Broadcast(messages.Message{
+				Action:   messages.ActionError,
+				Username: "ChatServer",
+				Message:  "The chat server is going down for a reboot NOW!",
+			})
+		}()
+	})
+}
+
 // BlockList (/api/blocklist) allows your website to pre-sync mute lists between your
 // user accounts, so that when they see each other in chat they will pre-emptively mute
 // or boot one another.

pkg/commands.go

@@ -83,6 +83,19 @@ func (s *Server) ProcessCommand(sub *Subscriber, msg messages.Message) bool {
 		case "/deop":
 			s.DeopCommand(words, sub)
 			return true
+		case "/debug-dangerous-force-deadlock":
+			// TEMPORARY debug command to willfully force a deadlock.
+			s.Broadcast(messages.Message{
+				Action:   messages.ActionError,
+				Username: "ChatServer",
+				Message:  "NOTICE: The admin is testing a force deadlock of the chat server; things may become unresponsive soon.",
+			})
+			go func() {
+				time.Sleep(2 * time.Second)
+				s.subscribersMu.Lock()
+				s.subscribersMu.Lock()
+			}()
+			return true
 		}
 
 	}

pkg/handlers.go

@@ -89,6 +89,7 @@ func (s *Server) OnLogin(sub *Subscriber, msg messages.Message) {
 	// Use their username.
 	sub.Username = msg.Username
 	sub.authenticated = true
+	sub.DND = msg.DND
 	sub.loginAt = time.Now()
 	log.Debug("OnLogin: %s joins the room", sub.Username)
 
@@ -144,9 +145,8 @@ func (s *Server) OnMessage(sub *Subscriber, msg messages.Message) {
 	markdown = s.ExpandMedia(markdown)
 
 	// Assign a message ID and own it to the sender.
-	messages.MessageID++
-	var mid = messages.MessageID
 	sub.midMu.Lock()
+	var mid = messages.NextMessageID()
 	sub.messageIDs[mid] = struct{}{}
 	sub.midMu.Unlock()
 
@@ -194,8 +194,9 @@ func (s *Server) OnTakeback(sub *Subscriber, msg messages.Message) {
 	// Permission check.
 	if sub.JWTClaims == nil || !sub.JWTClaims.IsAdmin {
 		sub.midMu.Lock()
-		defer sub.midMu.Unlock()
+		_, ok := sub.messageIDs[msg.MessageID]
-		if _, ok := sub.messageIDs[msg.MessageID]; !ok {
+		sub.midMu.Unlock()
+		if !ok {
 			sub.ChatServer("That is not your message to take back.")
 			return
 		}
@@ -249,9 +250,8 @@ func (s *Server) OnFile(sub *Subscriber, msg messages.Message) {
 	var dataURL = fmt.Sprintf("data:%s;base64,%s", filetype, base64.StdEncoding.EncodeToString(img))
 
 	// Assign a message ID and own it to the sender.
-	messages.MessageID++
-	var mid = messages.MessageID
 	sub.midMu.Lock()
+	var mid = messages.NextMessageID()
 	sub.messageIDs[mid] = struct{}{}
 	sub.midMu.Unlock()
 
@@ -329,6 +329,7 @@ func (s *Server) OnMe(sub *Subscriber, msg messages.Message) {
 
 	sub.VideoStatus = msg.VideoStatus
 	sub.ChatStatus = msg.ChatStatus
+	sub.DND = msg.DND
 
 	// Sync the WhoList to everybody.
 	s.SendWhoList()

pkg/messages/messages.go

@@ -1,7 +1,21 @@
 package messages
 
+import "sync"
+
 // Auto incrementing Message ID for anything pushed out by the server.
-var MessageID int
+var (
+	messageID int
+	mu        sync.Mutex
+)
+
+// NextMessageID atomically increments and returns a new MessageID.
+func NextMessageID() int {
+	mu.Lock()
+	defer mu.Unlock()
+	messageID++
+	var mid = messageID
+	return mid
+}
 
 /*
 Message is the basic carrier of WebSocket chat protocol actions.
@@ -25,6 +39,7 @@ type Message struct {
 	// Sent on `me` actions along with Username
 	VideoStatus int    `json:"video,omitempty"`  // user video flags
 	ChatStatus  string `json:"status,omitempty"` // online vs. away
+	DND         bool   `json:"dnd,omitempty"`    // Do Not Disturb, e.g. DMs are closed
 
 	// Message ID to support takebacks/local deletions
 	MessageID int `json:"msgID,omitempty"`
@@ -87,6 +102,7 @@ type WhoList struct {
 	Nickname string `json:"nickname,omitempty"`
 	Status   string `json:"status"`
 	Video    int    `json:"video"`
+	DND      bool   `json:"dnd,omitempty"`
 	LoginAt  int64  `json:"loginAt"`
 
 	// JWT auth extra settings.

pkg/server.go

@@ -36,6 +36,7 @@ func (s *Server) Setup() error {
 	mux.Handle("/api/statistics", s.Statistics())
 	mux.Handle("/api/blocklist", s.BlockList())
 	mux.Handle("/api/authenticate", s.Authenticate())
+	mux.Handle("/api/shutdown", s.ShutdownAPI())
 	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static"))))
 
 	s.mux = mux

pkg/websocket.go

@@ -26,6 +26,7 @@ type Subscriber struct {
 	Username      string
 	ChatStatus    string
 	VideoStatus   int
+	DND           bool // Do Not Disturb status (DMs are closed)
 	JWTClaims     *jwt.Claims
 	authenticated bool // has passed the login step
 	loginAt       time.Time
@@ -266,20 +267,16 @@ func (s *Server) DeleteSubscriber(sub *Subscriber) {
 	s.subscribersMu.Unlock()
 }
 
-// IterSubscribers loops over the subscriber list with a read lock. If the
+// IterSubscribers loops over the subscriber list with a read lock.
-// caller already holds a lock, pass the optional `true` parameter for isLocked.
+func (s *Server) IterSubscribers() []*Subscriber {
-func (s *Server) IterSubscribers(isLocked ...bool) []*Subscriber {
 	var result = []*Subscriber{}
 
-	// Has the caller already taken the read lock or do we get it?
+	// Lock for reads.
-	if locked := len(isLocked) > 0 && isLocked[0]; !locked {
 	s.subscribersMu.RLock()
-		defer s.subscribersMu.RUnlock()
-	}
-
 	for sub := range s.subscribers {
 		result = append(result, sub)
 	}
+	s.subscribersMu.RUnlock()
 
 	return result
 }
@@ -399,6 +396,7 @@ func (s *Server) SendWhoList() {
 				Username: user.Username,
 				Status:   user.ChatStatus,
 				Video:    user.VideoStatus,
+				DND:      user.DND,
 				LoginAt:  user.loginAt.Unix(),
 			}
 

web/static/js/BareRTC.js

@@ -103,6 +103,7 @@ const app = Vue.createApp({
             username: "", //"test",
             autoLogin: false,  // e.g. from JWT auth
             message: "",
+            messageBox: null, // HTML element for message entry box
             typingNotifDebounce: null,
             status: "online", // away/idle status
 
@@ -127,6 +128,7 @@ const app = Vue.createApp({
             prefs: {
                 joinMessages: true,  // show "has entered the room" in public channels
                 exitMessages: false, // hide exit messages by default in public channels
+                watchNotif: true,    // notify in chat about cameras being watched
                 closeDMs: false,     // ignore unsolicited DMs
             },
 
@@ -290,6 +292,7 @@ const app = Vue.createApp({
             $center: document.querySelector(".chat-column"),
             $right: document.querySelector(".right-column"),
         };
+        this.messageBox = document.getElementById("messageBox");
 
         // Reset CSS overrides for responsive display on any window size change. In effect,
         // making the chat panel the current screen again on phone rotation.
@@ -387,8 +390,14 @@ const app = Vue.createApp({
         "prefs.exitMessages": function() {
             localStorage.exitMessages = this.prefs.exitMessages;
         },
+        "prefs.watchNotif": function() {
+            localStorage.watchNotif = this.prefs.watchNotif;
+        },
         "prefs.closeDMs": function() {
             localStorage.closeDMs = this.prefs.closeDMs;
+
+            // Tell ChatServer if we have gone to/from DND.
+            this.sendMe();
         },
     },
     computed: {
@@ -588,6 +597,9 @@ const app = Vue.createApp({
             if (localStorage.exitMessages != undefined) {
                 this.prefs.exitMessages = localStorage.exitMessages === "true";
             }
+            if (localStorage.watchNotif != undefined) {
+                this.prefs.watchNotif = localStorage.watchNotif === "true";
+            }
             if (localStorage.closeDMs != undefined) {
                 this.prefs.closeDMs = localStorage.closeDMs === "true";
             }
@@ -686,10 +698,12 @@ const app = Vue.createApp({
         // Sync the current user state (such as video broadcasting status) to
         // the backend, which will reload everybody's Who List.
         sendMe() {
+            if (!this.ws.connected) return;
             this.ws.conn.send(JSON.stringify({
                 action: "me",
                 video: this.myVideoFlag,
                 status: this.status,
+                dnd: this.prefs.closeDMs,
             }));
         },
         onMe(msg) {
@@ -974,7 +988,13 @@ const app = Vue.createApp({
                     action: "login",
                     username: this.username,
                     jwt: this.jwt.token,
+                    dnd: this.prefs.closeDMs,
                 }));
+
+                // Focus the message entry box.
+                window.requestAnimationFrame(() => {
+                    this.messageBox.focus();
+                });
             });
 
             conn.addEventListener("message", ev => {
@@ -1252,6 +1272,14 @@ const app = Vue.createApp({
         onWatch(msg) {
             // The user has our video feed open now.
             if (this.isBootedAdmin(msg.username)) return;
+
+            // Notify in chat if this was the first watch (viewer may send multiple per each track they received)
+            if (this.prefs.watchNotif && this.webcam.watching[msg.username] != true) {
+                this.ChatServer(
+                    `<strong>${msg.username}</strong> is now watching your camera.`,
+                );
+            }
+
             this.webcam.watching[msg.username] = true;
             this.playSound("Watch");
         },
@@ -1295,6 +1323,9 @@ const app = Vue.createApp({
 
             // Edit hyperlinks to open in a new window.
             this.makeLinksExternal();
+
+            // Focus the message entry box.
+            this.messageBox.focus();
         },
         hasUnread(channel) {
             if (this.channels[channel] == undefined) {
@@ -1382,6 +1413,10 @@ const app = Vue.createApp({
             }
             return username;
         },
+        isUsernameDND(username) {
+            if (!username) return false;
+            return this.whoMap[username] != undefined && this.whoMap[username].dnd;
+        },
         isUsernameCamNSFW(username) {
             // returns true if the username is broadcasting and NSFW, false otherwise.
             // (used for the color coding of their nickname on their video box - so assumed they are broadcasting)
@@ -1490,8 +1525,8 @@ const app = Vue.createApp({
             let mediaParams = {
                 audio: true,
                 video: {
-                    width: { max: 1280 },
+                    width: { max: 640 },
-                    height: { max: 720 },
+                    height: { max: 480 },
                 },
             };
 

web/templates/chat.html

@@ -373,7 +373,7 @@
                     <div v-if="settingsModal.tab==='misc'">
 
                         <div class="field">
-                            <label class="label">Presence messages in public channels</label>
+                            <label class="label">Presence messages <small>('has joined the room')</small> in public channels</label>
                             <div class="columns is-mobile mb-0">
                                 <div class="column py-1">
                                     <label class="checkbox" title="Show 'has joined the room' messages in public channels">
@@ -393,9 +393,16 @@
                                     </label>
                                 </div>
                             </div>
-                            <p class="help mt-0">
+                        </div>
-                                Whether to show <em>'has joined the room'</em> style messages in public channels.
+
-                            </p>
+                        <div class="field">
+                            <label class="label mb-0">Server notification messages</label>
+                            <label class="checkbox" title="Show 'has joined the room' messages in public channels">
+                                <input type="checkbox"
+                                    v-model="prefs.watchNotif"
+                                    :value="true">
+                                Notify when somebody opens my camera
+                            </label>
                         </div>
 
                         <div class="field">
@@ -1114,7 +1121,8 @@
                                                     v-if="!(msg.username === username || isDM)"
                                                     class="button is-grey is-outlined is-small px-2"
                                                     @click="openDMs({username: msg.username})"
-                                                    title="Open a Direct Message (DM) thread">
+                                                    :title="isUsernameDND(msg.username) ? 'This person is not accepting new DMs' : 'Open a Direct Message (DM) thread'"
+                                                    :disabled="isUsernameDND(msg.username)">
                                                     <i class="fa fa-message"></i>
                                                 </button>
 
@@ -1250,6 +1258,7 @@
                             :class="{'pl-1': canUploadFile}">
                             <form @submit.prevent="sendMessage()">
                                 <input type="text" class="input"
+                                    id="messageBox"
                                     v-model="message"
                                     placeholder="Write a message"
                                     @keydown="sendTypingNotification()"
@@ -1424,8 +1433,8 @@
                                     <button type="button" v-else
                                         class="button is-small px-2 py-1"
                                         @click="openDMs(u)"
-                                        title="Start direct message thread"
+                                        :disabled="u.username === username || u.dnd"
-                                        :disabled="u.username === username">
+                                        :title="u.dnd ? 'This person is not accepting new DMs' : 'Send a Direct Message'">
                                         <i class="fa fa-message"></i>
                                     </button>