96 lines
2.3 KiB
Go
96 lines
2.3 KiB
Go
|
package client
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"net/http"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"git.kirsle.net/apps/barertc/client/config"
|
||
|
|
"git.kirsle.net/apps/barertc/pkg/log"
|
||
|
|
"git.kirsle.net/apps/barertc/pkg/messages"
|
||
|
|
)
|
||
|
|
|
||
|
|
const deadlockTTL = time.Minute
|
||
|
|
|
||
|
|
/*
|
||
|
|
Deadlock detection for the chat server.
|
||
|
|
|
||
|
|
Part of the chatbot handlers. The bot will send DMs to itself on an interval
|
||
|
|
and test whether the server is responsive; if it goes down, it will issue the
|
||
|
|
/api/shutdown command to reboot the server automatically.
|
||
|
|
|
||
|
|
This function is a goroutine spawned in the background.
|
||
|
|
*/
|
||
|
|
func (h *BotHandlers) watchForDeadlock() {
|
||
|
|
log.Info("Deadlock monitor engaged!")
|
||
|
|
h.deadlockLastOK = time.Now()
|
||
|
|
|
||
|
|
for {
|
||
|
|
time.Sleep(15 * time.Second)
|
||
|
|
h.client.Send(messages.Message{
|
||
|
|
Action: messages.ActionMessage,
|
||
|
|
Channel: "@" + h.client.Username(),
|
||
|
|
Message: "deadlock ping",
|
||
|
|
})
|
||
|
|
|
||
|
|
// Has it been a while since our last ping?
|
||
|
|
if time.Since(h.deadlockLastOK) > deadlockTTL {
|
||
|
|
log.Error("Deadlock detected! Rebooting the chat server!")
|
||
|
|
h.deadlockLastOK = time.Now()
|
||
|
|
h.rebootChatServer()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// onMessageFromSelf handles DMs sent to ourself, e.g. for deadlock detection.
|
||
|
|
func (h *BotHandlers) onMessageFromSelf(msg messages.Message) {
|
||
|
|
// If it is our own DM channel thread, it's for deadlock detection.
|
||
|
|
if msg.Channel == "@"+h.client.Username() {
|
||
|
|
h.deadlockLastOK = time.Now()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Reboot the chat server via web API, in case of deadlock.
|
||
|
|
func (h *BotHandlers) rebootChatServer() error {
|
||
|
|
// API request struct for BareRTC /api/shutdown endpoint.
|
||
|
|
var request = struct {
|
||
|
|
APIKey string
|
||
|
|
}{
|
||
|
|
APIKey: config.Current.BareRTC.AdminAPIKey,
|
||
|
|
}
|
||
|
|
|
||
|
|
// JSON request body.
|
||
|
|
jsonStr, err := json.Marshal(request)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
// Make the API request to BareRTC.
|
||
|
|
var url = strings.TrimSuffix(config.Current.BareRTC.URL, "/") + "/api/shutdown"
|
||
|
|
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
req.Header.Set("Content-Type", "application/json")
|
||
|
|
|
||
|
|
client := &http.Client{
|
||
|
|
Timeout: 10 * time.Second,
|
||
|
|
}
|
||
|
|
resp, err := client.Do(req)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
defer resp.Body.Close()
|
||
|
|
|
||
|
|
body, _ := io.ReadAll(resp.Body)
|
||
|
|
if resp.StatusCode != http.StatusOK {
|
||
|
|
return fmt.Errorf("RebootChatServer: error posting to BareRTC: status %d body %s", resp.StatusCode, body)
|
||
|
|
}
|
||
|
|
|
||
|
|
return nil
|
||
|
|
}
|