Spam Detection for Hyperlinks on DMs

Add spam detection in case a user copy/pastes a hyperlink to everybody
on chat via their DMs:

* If the same link is copied to many different people within a time
  window, the user can be kicked from the chat room with a warning.
* The server remembers rate limits by username, so if they log back in
  and continue to spam the same links, they instead receive a temporary
  chat ban.
* The spam threshold, time window and ban hours are configurable in the
  BareRTC settings.toml.

Other fixes:

* The front-end will send a "me" update with its current status and
  video setting in the 'onLoggedIn' handler. This should help alleviate
  rough server reboots when a ton of idle users are online, so they
  don't spam "me" updates to correct their status once the WhoLists
  begin to roll in.

pkg/banned_users.go

@@ -22,7 +22,7 @@ var (
 )
 
 // BanUser adds a user to the ban list.
-func BanUser(username string, duration time.Duration) {
+func (s *Server) BanUser(username string, duration time.Duration) {
 	banListMu.Lock()
 	defer banListMu.Unlock()
 	banList[username] = Ban{
@@ -71,3 +71,14 @@ func IsBanned(username string) bool {
 	}
 	return ok
 }
+
+// BanExpires returns the time.Duration string until the user's ban expires.
+func BanExpires(username string) string {
+	banListMu.RLock()
+	defer banListMu.RUnlock()
+	ban, ok := banList[username]
+	if ok {
+		return time.Until(ban.ExpiresAt).String()
+	}
+	return ""
+}

pkg/commands.go

@@ -208,20 +208,8 @@ func (s *Server) KickCommand(words []string, sub *Subscriber) {
 	} else if other.Username == sub.Username {
 		sub.ChatServer("/kick: did you really mean to kick yourself?")
 	} else {
-		other.ChatServer("You have been kicked from the chat room by %s", sub.Username)
+		s.KickUser(other, fmt.Sprintf("You have been kicked from the chat room by %s", sub.Username), false)
-		other.SendJSON(messages.Message{
-			Action: messages.ActionKick,
-		})
-		other.authenticated = false
-		other.Username = ""
 		sub.ChatServer("%s has been kicked from the room", username)
-
-		// Broadcast it to everyone.
-		s.Broadcast(messages.Message{
-			Action:   messages.ActionPresence,
-			Username: username,
-			Message:  messages.PresenceKicked,
-		})
 	}
 }
 
@@ -290,22 +278,11 @@ func (s *Server) BanCommand(words []string, sub *Subscriber) {
 	log.Info("Operator %s bans %s for %d hours", sub.Username, username, duration/time.Hour)
 
 	// Add them to the ban list.
-	BanUser(username, duration)
+	s.BanUser(username, duration)
 
 	// If the target user is currently online, disconnect them and broadcast the ban to everybody.
 	if other, err := s.GetSubscriber(username); err == nil {
-		s.Broadcast(messages.Message{
+		s.KickUser(other, fmt.Sprintf("You have been banned from the chat room by %s. You may come back after %d hours.", sub.Username, duration/time.Hour), true)
-			Action:   messages.ActionPresence,
-			Username: username,
-			Message:  messages.PresenceBanned,
-		})
-
-		other.ChatServer("You have been banned from the chat room by %s. You may come back after %d hours.", sub.Username, duration/time.Hour)
-		other.SendJSON(messages.Message{
-			Action: messages.ActionKick,
-		})
-		other.authenticated = false
-		other.Username = ""
 	}
 
 	sub.ChatServer("%s has been banned from the room for %d hours.", username, duration/time.Hour)

pkg/config/config.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"html/template"
 	"os"
+	"time"
 
 	"git.kirsle.net/apps/barertc/pkg/log"
 	"github.com/google/uuid"
@@ -12,7 +13,7 @@ import (
 
 // Version of the config format - when new fields are added, it will attempt
 // to write the settings.toml to disk so new defaults populate.
-var currentVersion = 16
+var currentVersion = 17
 
 // Config for your BareRTC app.
 type Config struct {
@@ -53,6 +54,7 @@ type Config struct {
 	ModerationRule []*ModerationRule
 
 	DirectMessageHistory DirectMessageHistory
+	DMLinkSpamProtection DMLinkSpamProtection
 
 	Strings Strings
 
@@ -79,6 +81,13 @@ type DirectMessageHistory struct {
 	DisclaimerMessage string
 }
 
+type DMLinkSpamProtection struct {
+	Enabled    bool          `toml:"" comment:"Spam detection in case a user pastes the same URL into several DM threads in a short timeframe,\ne.g. trying to take your entire userbase to a competing chat room."`
+	MaxThreads int           `toml:"" comment:"The max number of DM threads the user can paste the same link into"`
+	TimeLimit  time.Duration `toml:"" comment:"Rate limit (in seconds), if the user spams MaxThreads within the rate limit they are kicked or banned from chat."`
+	BanHours   time.Duration `toml:"" comment:"The user will be kicked the first time, if they log back in and spam once more they are banned for this many hours."`
+}
+
 // GetChannels returns a JavaScript safe array of the default PublicChannels.
 func (c Config) GetChannels() template.JS {
 	data, _ := json.Marshal(c.PublicChannels)
@@ -238,6 +247,12 @@ func DefaultConfig() Config {
 			RetentionDays:     90,
 			DisclaimerMessage: `<i class="fa fa-info-circle mr-1"></i> <strong>Reminder:</strong> please conduct yourself honorably in Direct Messages.`,
 		},
+		DMLinkSpamProtection: DMLinkSpamProtection{
+			Enabled:    true,
+			MaxThreads: 5,
+			TimeLimit:  60 * 60,
+			BanHours:   24,
+		},
 		Logging: Logging{
 			Directory: "./logs",
 			Channels:  []string{"lobby", "offtopic"},

pkg/handlers.go

@@ -12,6 +12,7 @@ import (
 	"git.kirsle.net/apps/barertc/pkg/log"
 	"git.kirsle.net/apps/barertc/pkg/messages"
 	"git.kirsle.net/apps/barertc/pkg/models"
+	"git.kirsle.net/apps/barertc/pkg/spam"
 	"git.kirsle.net/apps/barertc/pkg/util"
 )
 
@@ -78,8 +79,9 @@ func (s *Server) OnLogin(sub *Subscriber, msg messages.Message) {
 	// Is the username currently banned?
 	if IsBanned(msg.Username) {
 		sub.ChatServer(
-			"You are currently banned from entering the chat room. Chat room bans are temporarily and usually last for " +
+			"You are currently banned from entering the chat room. Chat room bans are temporary and usually last for "+
-				"24 hours. Please try coming back later.",
+				"24 hours.<br><br>Your ban expires in: %s",
+			BanExpires(msg.Username),
 		)
 		sub.SendJSON(messages.Message{
 			Action: messages.ActionKick,
@@ -197,6 +199,46 @@ func (s *Server) OnMessage(sub *Subscriber, msg messages.Message) {
 
 	// Is this a DM?
 	if strings.HasPrefix(msg.Channel, "@") {
+
+		// Check the message for hyperlink spam: if one user is copy/pasting a link to too many others
+		// over DMs (possibly trying to evade detection from moderators), kick or ban them from chat.
+		if err := spam.LinkSpamManager.Check(sub.Username, msg.Channel, msg.Message); err != nil {
+			log.Error("Username %s has spammed a link too often (most recently to %s): %s", sub.Username, msg.Channel, msg.Message)
+
+			var (
+				isBan = err == spam.ErrLinkSpamBanUser
+
+				// For the admin report.
+				report   = "The user was **kicked** from the room with a warning."
+				username = sub.Username
+			)
+			if isBan {
+				report = "The user was **banned** from chat."
+				s.BanUser(sub.Username, config.Current.DMLinkSpamProtection.BanHours*time.Hour)
+			}
+
+			s.KickUser(sub, err.Error(), isBan)
+
+			// Send an admin report to your main website.
+			if err := PostWebhookReport(WebhookRequestReport{
+				FromUsername:  username,
+				AboutUsername: username,
+				Channel:       msg.Channel,
+				Timestamp:     time.Now().Format(time.RFC3339),
+				Reason:        "Spam Detected",
+				Message: fmt.Sprintf(
+					"_In a DM to:_ %s<br>[%s] %s",
+					msg.Channel,
+					username,
+					msg.Message,
+				),
+				Comment: report,
+			}); err != nil {
+				log.Error("Error delivering a report to your website about the /nsfw command by %s: %s", sub.Username, err)
+			}
+			return
+		}
+
 		// Echo the message only to both parties.
 		s.SendTo(sub.Username, message)
 		message.Channel = "@" + sub.Username

pkg/spam/dmlinks.go

@@ -0,0 +1,195 @@
+package spam
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"regexp"
+	"sync"
+	"time"
+
+	"git.kirsle.net/apps/barertc/pkg/config"
+)
+
+/*
+LinkSpamMap keeps track of link spamming behavior per username.
+
+It is a map of usernames to their recent history of hyperlinks sent to other usernames
+over DMs. The intention is to detect when one user is spamming (copy/pasting) the same
+hyperlink to many many people over DMs, e.g., if they are trying to take the whole chat
+room away to a competing video conference and they are sending the link by DM to everybody
+on chat in order to hide from the moderators by not using the public channels.
+*/
+type LinkSpamMap map[string]UserLinkMap
+
+// LinkSpam holds info about possibly spammy hyperlinks that Username has sent to multiple
+// others over Direct Messages, to detect e.g. somebody spamming a link to their off-site
+// video conference to everybody on chat while hiding from moderators and public channels.
+type LinkSpam struct {
+	Username  string
+	URL       string
+	SentTo    map[string]struct{} // Usernames they have sent it to
+	FirstSent time.Time           // time of the first link
+	LastSent  time.Time           // time of the most recently sent link
+	Lock      sync.RWMutex
+	Kicked    bool // user was kicked once for this spam
+}
+
+// UserLinkMap connects usernames to the set of distinct links they have sent.
+//
+// It is a map of the URL hash to the LinkSpam data struct.
+type UserLinkMap map[string]*LinkSpam
+
+// LinkSpamManager is the singleton global instance variable that checks and tracks link
+// spam sent by users on chat. It is initialized at server startup and provides an API
+// surface area to ping and test for spammy behavior from users.
+var (
+	LinkSpamManager LinkSpamMap
+	linkSpamLock    sync.Mutex // protects the top-level map from concurrent writes.
+)
+
+var HyperlinkRegexp = regexp.MustCompile(`(?:http[s]?:\/\/.)?(?:www\.)?[-a-zA-Z0-9@%._\+~#=]{2,256}\.[a-z]{2,6}\b(?:[-a-zA-Z0-9@:%_\+.~#?&\/\/=]*)`)
+
+func init() {
+	LinkSpamManager = map[string]UserLinkMap{}
+	go LinkSpamManager.expire()
+}
+
+/*
+Check if the current user has been spamming a link to too many people over DMs.
+
+This function will parse the message for any hyperlinks, and upsert/ping the
+sourceUsername's spam detection struct.
+
+If the user has pasted the same hyperlink into too many different DM threads,
+this function may return one of two sentinel errors:
+
+- ErrLinkSpamKickUser if the user should be kicked from the room.
+- ErrLinkSpamBanUser if the user should be banned from the room.
+
+The first time they trip the spam limit they are to be kicked, but if they rejoin
+the chat and (still within the spam duration window), paste the same link to yet
+another recipient they will be banned temporarily from chat.
+*/
+func (m LinkSpamMap) Check(sourceUsername, targetUsername, message string) error {
+
+	if !config.Current.DMLinkSpamProtection.Enabled {
+		return nil
+	}
+
+	linkSpamLock.Lock()
+	defer linkSpamLock.Unlock()
+
+	// Initialize data structures.
+	if _, ok := m[sourceUsername]; !ok {
+		m[sourceUsername] = UserLinkMap{}
+	}
+
+	// Parse all URLs from their message.
+	matches := HyperlinkRegexp.FindAllStringSubmatch(message, -1)
+	for _, match := range matches {
+		var (
+			url  = match[0]
+			hash = Hash([]byte(url))
+		)
+
+		// Initialize the struct?
+		if _, ok := m[sourceUsername][hash]; !ok {
+			m[sourceUsername][hash] = &LinkSpam{
+				Username:  sourceUsername,
+				URL:       url,
+				SentTo:    map[string]struct{}{},
+				FirstSent: time.Now(),
+			}
+		}
+
+		// Check and update information.
+		spam := m[sourceUsername][hash]
+		spam.Lock.Lock()
+		defer spam.Lock.Unlock()
+
+		spam.SentTo[targetUsername] = struct{}{}
+		spam.LastSent = time.Now()
+
+		// Have they sent it to too many people?
+		if len(spam.SentTo) > config.Current.DMLinkSpamProtection.MaxThreads {
+			// Kick or ban them.
+			if spam.Kicked {
+				return ErrLinkSpamBanUser
+			}
+			spam.Kicked = true
+			return ErrLinkSpamKickUser
+		}
+	}
+
+	return nil
+}
+
+// expire cleans up link spam data after the rate limit window for them had passed.
+//
+// It runs as a background goroutine and periodically cleans up expired link spam.
+func (m LinkSpamMap) expire() {
+	for {
+		time.Sleep(5 * time.Minute)
+
+		// Lock the top-level struct for cleanup.
+		linkSpamLock.Lock()
+
+		// Iterate all users who have links stored.
+		var cleanupUsernames = []string{}
+		for username, links := range m {
+			var cleanupHashes = []string{}
+			for hash, spam := range links {
+				// Has this record expired based on its LastSent time?
+				if time.Since(spam.LastSent) > config.Current.DMLinkSpamProtection.TimeLimit*time.Second {
+					cleanupHashes = append(cleanupHashes, hash)
+				}
+			}
+
+			// Clean up the hashes.
+			if len(cleanupHashes) > 0 {
+				for _, hash := range cleanupHashes {
+					delete(links, hash)
+				}
+			}
+
+			// Are any left anymore?
+			if len(links) == 0 {
+				cleanupUsernames = append(cleanupUsernames, username)
+			}
+		}
+
+		// Clean up empty usernames?
+		if len(cleanupUsernames) > 0 {
+			for _, username := range cleanupUsernames {
+				delete(m, username)
+			}
+		}
+
+		// Unlock the struct.
+		linkSpamLock.Unlock()
+	}
+}
+
+// Sentinel errors returned from LinkSpamMan.Check().
+var (
+	ErrLinkSpamKickUser = errors.New(
+		`<strong>Spam Detected:</strong> ` +
+			`You have pasted the same URL link to too many different people in a row, and this has been flagged as spam.<br><br>` +
+			`You will now be kicked from the chat room. You may refresh and log back in, however, if you continue to spam this ` +
+			`link one more time, you <strong>will be temporarily banned from the chat room.</strong>`,
+	)
+	ErrLinkSpamBanUser = errors.New(
+		`<strong>Spam Detected:</strong> ` +
+			`You recently were kicked from the chat room because you had already pasted this link to too many different people. ` +
+			`You were warned that spamming this link one more time would result in a temporary ban from the chat room.<br><br>` +
+			`<strong>You are now (temporarily) banned from the chat room.</strong>`,
+	)
+)
+
+// Hash a byte array as SHA256 and returns the hex string.
+func Hash(input []byte) string {
+	h := sha256.New()
+	h.Write(input)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}

pkg/subscriber.go

@@ -431,6 +431,34 @@ func (s *Server) SendTo(username string, msg messages.Message) error {
 	return nil
 }
 
+// KickUser kicks a user from chat with a message.
+func (s *Server) KickUser(sub *Subscriber, message string, banned bool) {
+	// Broadcast it to everyone.
+	if banned {
+		s.Broadcast(messages.Message{
+			Action:   messages.ActionPresence,
+			Username: sub.Username,
+			Message:  messages.PresenceBanned,
+		})
+	} else {
+		s.Broadcast(messages.Message{
+			Action:   messages.ActionPresence,
+			Username: sub.Username,
+			Message:  messages.PresenceKicked,
+		})
+	}
+
+	// Tell the user they were kicked.
+	sub.ChatServer(message)
+	sub.SendJSON(messages.Message{
+		Action: messages.ActionKick,
+	})
+	sub.authenticated = false
+	sub.Username = ""
+
+	s.SendWhoList()
+}
+
 // SendWhoList broadcasts the connected members to everybody in the room.
 func (s *Server) SendWhoList() {
 

src/App.vue

@@ -1048,14 +1048,15 @@ export default {
 
         // Sync the current user state (such as video broadcasting status) to
         // the backend, which will reload everybody's Who List.
-        sendMe() {
+        sendMe(force) {
-            if (!this.connected) return;
+            if (!this.connected && !force) return;
-            this.client.send({
+            let payload = {
                 action: "me",
                 video: this.myVideoFlag,
                 status: this.status,
                 dnd: this.prefs.closeDMs,
-            });
+            };
+            this.client.send(payload);
         },
         onMe(msg) {
             // We have had settings pushed to us by the server, such as a change
@@ -1141,7 +1142,14 @@ export default {
 
             // Do we need to set our me status again?
             if (sendMe) {
-                this.sendMe();
+                // TODO: Delay a random period before sending the 'me' action, to help alleviate rough server reboots.
+                // Ideally: onLoggedIn() sends a "me" with the user's status/video setting immediately. On a server
+                // reboot, the first WhoList is sent 15 seconds after boot, and should (does!) send the correct "status"
+                // flags for each user online (so they don't need to re-send "me" to correct it, triggering an exponential
+                // flurry of WhoList updates). Randomize between 1 and 10 seconds.
+                // The bug seems fixed but this timeout is a safety net.
+                let timeout = 1000 + parseInt(Math.random() * 10000)
+                setTimeout(this.sendMe, timeout);
             }
         },
 
@@ -1448,6 +1456,9 @@ export default {
             if (this.webcam.autoshare) {
                 this.startVideo({ force: true });
             }
+
+            // Send the server our current status and video setting.
+            this.sendMe(true);
         },
 
         /**

src/lib/ChatClient.js

@@ -247,6 +247,8 @@ class ChatClient {
 
     // Dial the WebSocket.
     dial() {
+        this.firstMe = false;
+
         // Polling API?
         if (this.usePolling) {
             this.ChatClient("Connecting to the server via polling API...");