From 52df45b2e9f8514cfbb760ac0ad0a78ca96e452f Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Sun, 6 Jul 2025 11:56:20 -0700 Subject: [PATCH] GORM for the database for Postgres support * For the Direct Message History database, use gorm.io as ORM so that Postgres can be used instead of SQLite for bigger chat room instances. * In settings.toml: the new DatabaseType field defaults to 'sqlite3' but can be set to 'postgres' and use the credentials in the new PostgresDatabase field. * The DirectMessage table schema is also updated to deprecate the Timestamp int field in favor of a proper CreatedAt datetime field. Existing SQLite instances will upgrade their table in the background, converting Timestamp to CreatedAt and blanking out the legacy Timestamp column. * Fix some DB queries so when paginating your DMs history username list, sorting it by timestamp now works reliably. * For existing SQLite instances that want to switch to Postgres, use the scripts/sqlite2psql.py script to transfer your database over. --- go.mod | 64 +++++---- go.sum | 73 ++++++++++ pkg/config/config.go | 6 +- pkg/models/database.go | 51 +++++-- pkg/models/direct_messages.go | 243 +++++++++++++++------------------- pkg/server.go | 13 +- scripts/requirements.txt | 1 + scripts/sqlite2psql.py | 142 ++++++++++++++++++++ 8 files changed, 420 insertions(+), 173 deletions(-) create mode 100644 scripts/requirements.txt create mode 100644 scripts/sqlite2psql.py diff --git a/go.mod b/go.mod index 93c4b7a..8d99a07 100644 --- a/go.mod +++ b/go.mod @@ -1,41 +1,50 @@ module git.kirsle.net/apps/barertc -go 1.21.0 +go 1.23.0 -toolchain go1.22.0 +toolchain go1.24.2 require ( - git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b - github.com/BurntSushi/toml v1.3.2 + git.kirsle.net/go/log v0.0.0-20240505021515-9c747daf9e9a + github.com/BurntSushi/toml v1.5.0 github.com/aichaos/rivescript-go v0.4.0 - github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f - github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.5.0 + github.com/edwvee/exiffix v0.0.0-20240229113213-0dbb146775be + github.com/golang-jwt/jwt/v4 v4.5.2 + github.com/google/uuid v1.6.0 github.com/mattn/go-shellwords v1.0.12 - github.com/mattn/go-sqlite3 v1.14.24 - github.com/microcosm-cc/bluemonday v1.0.25 - github.com/pelletier/go-toml/v2 v2.2.3 + github.com/mattn/go-sqlite3 v1.14.28 + github.com/microcosm-cc/bluemonday v1.0.27 + github.com/pelletier/go-toml/v2 v2.2.4 github.com/shurcooL/github_flavored_markdown v0.0.0-20210228213109-c3a9aa474629 - github.com/urfave/cli/v2 v2.25.7 - golang.org/x/image v0.12.0 - nhooyr.io/websocket v1.8.7 + github.com/urfave/cli/v2 v2.27.7 + golang.org/x/image v0.28.0 + gorm.io/driver/postgres v1.6.0 + gorm.io/driver/sqlite v1.6.0 + gorm.io/gorm v1.30.0 + nhooyr.io/websocket v1.8.17 ) require ( github.com/aymerick/douceur v0.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/disintegration/imaging v1.6.2 // indirect - github.com/dlclark/regexp2 v1.10.0 // indirect - github.com/dop251/goja v0.0.0-20230919151941-fc55792775de // indirect - github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect - github.com/gorilla/css v1.0.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994 // indirect + github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect + github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/russross/blackfriday v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect - github.com/sergi/go-diff v1.3.1 // indirect + github.com/sergi/go-diff v1.4.0 // indirect github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636 // indirect github.com/shurcooL/go-goon v1.0.0 // indirect github.com/shurcooL/highlight_diff v0.0.0-20230708024848-22f825814995 // indirect @@ -45,11 +54,12 @@ require ( github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect github.com/tomnomnom/xtermcolor v0.0.0-20160428124646-b78803f00a7e // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.12.0 // indirect - golang.org/x/text v0.13.0 // indirect + github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect + golang.org/x/text v0.26.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect ) diff --git a/go.sum b/go.sum index bdf65b8..4c7b107 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b h1:TDxEEWOJqMzsu9JW8/QgmT1lgQ9WD2KWlb2lKN/Ql2o= git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b/go.mod h1:jl+Qr58W3Op7OCxIYIT+b42jq8xFncJXzPufhrvza7Y= +git.kirsle.net/go/log v0.0.0-20240505021515-9c747daf9e9a h1:IHdqfXu7oqOPPotdzTFpmjJrryNWAad7eiS5BvGwXQA= +git.kirsle.net/go/log v0.0.0-20240505021515-9c747daf9e9a/go.mod h1:1hGKQt1uiIwPKfVl/fLO32Xr42+5BZl4jEwFeRns9cM= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/aichaos/rivescript-go v0.4.0 h1:+bG6h5v6IOmfyirIm1zQTiXu/dE6uWayDI/0/6yPu/s= github.com/aichaos/rivescript-go v0.4.0/go.mod h1:mf+QIyHz1dB4hmIZ6HkTF/rHqPOP9THViNbpMfN8iNU= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -13,6 +17,8 @@ github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMn github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -23,14 +29,20 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja v0.0.0-20230919151941-fc55792775de h1:lA38Xtzr1Wo+iQdkN2E11ziKXJYRxLlzK/e2/fdxoEI= github.com/dop251/goja v0.0.0-20230919151941-fc55792775de/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994 h1:aQYWswi+hRL2zJqGacdCZx32XjKYV8ApXFGntw79XAM= +github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f h1:RMnUwTnNR070mFAEIoqMYjNirHj8i0h79VXTYyBCyVA= github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f/go.mod h1:KoE3Ti1qbQXCb3s/XGj0yApHnbnNnn1bXTtB5Auq/Vc= +github.com/edwvee/exiffix v0.0.0-20240229113213-0dbb146775be h1:FNPYI8/ifKGW7kdBdlogyGGaPXZmOXBbV1uz4Amr3s0= +github.com/edwvee/exiffix v0.0.0-20240229113213-0dbb146775be/go.mod h1:G3dK5MziX9e4jUa8PWjowCOPCcyQwxsZ5a0oYA73280= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -46,6 +58,8 @@ github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= +github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -58,6 +72,8 @@ github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= @@ -79,19 +95,39 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 h1:xhMrHhTJ6zxu3gA4enFM9MLn9AY7613teCdFnlUVbSQ= +github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -109,8 +145,12 @@ github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebG github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= @@ -125,6 +165,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY= @@ -139,6 +181,8 @@ github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shurcooL/github_flavored_markdown v0.0.0-20210228213109-c3a9aa474629 h1:86e54L0i3pH3dAIA8OxBbfLrVyhoGpnNk1iJCigAWYs= github.com/shurcooL/github_flavored_markdown v0.0.0-20210228213109-c3a9aa474629/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636 h1:aSISeOcal5irEhJd1M+IrApc0PdcN7e7Aj4yuEnOrfQ= @@ -163,6 +207,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -176,8 +221,12 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= +github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= +github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -186,9 +235,13 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ= golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk= +golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE= +golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -203,11 +256,15 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -228,11 +285,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -244,6 +305,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -284,5 +349,13 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= +gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= +nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/pkg/config/config.go b/pkg/config/config.go index 97c5e94..28bcfac 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -13,7 +13,7 @@ import ( // Version of the config format - when new fields are added, it will attempt // to write the settings.toml to disk so new defaults populate. -var currentVersion = 17 +var currentVersion = 18 // Config for your BareRTC app. type Config struct { @@ -76,7 +76,9 @@ type VIP struct { type DirectMessageHistory struct { Enabled bool + DatabaseType string `toml:"" comment:"Which database driver to use: sqlite3, postgres"` SQLiteDatabase string + PostgresDatabase string RetentionDays int DisclaimerMessage string } @@ -243,7 +245,9 @@ func DefaultConfig() Config { }, DirectMessageHistory: DirectMessageHistory{ Enabled: false, + DatabaseType: "sqlite3", SQLiteDatabase: "database.sqlite", + PostgresDatabase: "host=localhost user=barertc password=barertc dbname=barertc sslmode=disable TimeZone=America/Los_Angeles", RetentionDays: 90, DisclaimerMessage: ` Reminder: please conduct yourself honorably in Direct Messages.`, }, diff --git a/pkg/models/database.go b/pkg/models/database.go index 9763411..e6ca7ea 100644 --- a/pkg/models/database.go +++ b/pkg/models/database.go @@ -1,29 +1,58 @@ package models import ( - "database/sql" "errors" + "fmt" + "git.kirsle.net/apps/barertc/pkg/log" _ "github.com/mattn/go-sqlite3" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) var ( - DB *sql.DB + DB *gorm.DB ErrNotInitialized = errors.New("database is not initialized") ) -func Initialize(connString string) error { - db, err := sql.Open("sqlite3", connString) - if err != nil { - return err +func AutoMigrate() { + DB.AutoMigrate( + &DirectMessage{}, + ) +} + +func Initialize(dbtype, connString string) error { + + var gormcfg = &gorm.Config{ + // Logger: logger.Default.LogMode(logger.Info), } - DB = db - - // Run table migrations - if err := (DirectMessage{}).CreateTable(); err != nil { - return err + switch dbtype { + case "sqlite3": + db, err := gorm.Open(sqlite.Open(connString), gormcfg) + if err != nil { + return fmt.Errorf("error opening SQLite DB: %w", err) + } + DB = db + case "postgres": + db, err := gorm.Open(postgres.Open(connString), gormcfg) + if err != nil { + return fmt.Errorf("error opening postgres DB: %w", err) + } + DB = db + default: + return errors.New("dbtype not supported: must be sqlite3 or postgres") } + AutoMigrate() + + // Run initializers. + go func() { + if err := (&DirectMessage{}).Init(); err != nil { + log.Error("DirectMessage.Init: %s", err) + } + }() + return nil } diff --git a/pkg/models/direct_messages.go b/pkg/models/direct_messages.go index 77034fe..e226be4 100644 --- a/pkg/models/direct_messages.go +++ b/pkg/models/direct_messages.go @@ -5,56 +5,78 @@ import ( "fmt" "math" "sort" - "strings" "time" "git.kirsle.net/apps/barertc/pkg/config" "git.kirsle.net/apps/barertc/pkg/log" "git.kirsle.net/apps/barertc/pkg/messages" + "gorm.io/gorm" ) type DirectMessage struct { - MessageID int64 - ChannelID string - Username string + MessageID int64 `gorm:"primaryKey"` + ChannelID string `gorm:"index"` + Username string `gorm:"index"` Message string - Timestamp int64 + Timestamp int64 // deprecated + CreatedAt time.Time `gorm:"index"` + DeletedAt gorm.DeletedAt `gorm:"index"` } const DirectMessagePerPage = 20 -func (dm DirectMessage) CreateTable() error { +// MigrateV2 upgrades the DirectMessage table for the V2 schema, where we switched +// from using SQLite to GORM so we can optionally use Postgres instead. +// +// During this switch, we also deprecate the old Timestamp column in favor of CreatedAt. +// This function will run in a background goroutine and update legacy rows to the new format. +func (db DirectMessage) MigrateV2() { + // Find rows that need upgrading. + var page int + for { + page++ + var rows = []*DirectMessage{} + res := DB.Model(&DirectMessage{}).Where( + "timestamp > 0", + ).Limit(1000).Find(&rows) + if res.Error != nil { + log.Error("DirectMessage.MigrateV2: %s", res.Error) + return + } + + if len(rows) == 0 { + break + } + + log.Warn("DirectMessage.MigrateV2: Updating %d DMs (page %d)", len(rows), page) + for _, row := range rows { + var created = time.Unix(row.Timestamp, 0) + row.CreatedAt = created + row.Timestamp = 0 + DB.Save(row) + } + } +} + +// Init runs initialization tasks for the DMs table (migrate V2 and expire old rows). +func (dm DirectMessage) Init() error { if DB == nil { return ErrNotInitialized } - _, err := DB.Exec(` - CREATE TABLE IF NOT EXISTS direct_messages ( - message_id INTEGER PRIMARY KEY, - channel_id TEXT, - username TEXT, - message TEXT, - timestamp INTEGER - ); - - CREATE INDEX IF NOT EXISTS idx_direct_messages_channel_id ON direct_messages(channel_id); - CREATE INDEX IF NOT EXISTS idx_direct_messages_username ON direct_messages(username); - CREATE INDEX IF NOT EXISTS idx_direct_messages_timestamp ON direct_messages(timestamp); - `) - if err != nil { - return err - } + // Migrate old rows to the V2 schema. + dm.MigrateV2() // Delete old messages past the retention period. if days := config.Current.DirectMessageHistory.RetentionDays; days > 0 { cutoff := time.Now().Add(time.Duration(-days) * 24 * time.Hour) log.Info("Deleting old DM history past %d days (cutoff: %s)", days, cutoff.Format(time.RFC3339)) - _, err := DB.Exec( - "DELETE FROM direct_messages WHERE timestamp < ?", - cutoff.Unix(), + res := DB.Exec( + "DELETE FROM direct_messages WHERE created_at IS NOT NULL AND created_at < ?", + cutoff, ) - if err != nil { - log.Error("Error removing old DMs: %s", err) + if res.Error != nil { + return fmt.Errorf("deleting old DMs past the cutoff period: %s", res.Error) } } @@ -73,15 +95,15 @@ func (dm DirectMessage) LogMessage(fromUsername, toUsername string, msg messages var ( channelID = CreateChannelID(fromUsername, toUsername) - timestamp = time.Now().Unix() ) - _, err := DB.Exec(` - INSERT INTO direct_messages (message_id, channel_id, username, message, timestamp) - VALUES (?, ?, ?, ?, ?) - `, msg.MessageID, channelID, fromUsername, msg.Message, timestamp) - - return err + m := &DirectMessage{ + MessageID: msg.MessageID, + ChannelID: channelID, + Username: fromUsername, + Message: msg.Message, + } + return DB.Create(&m).Error } // ClearMessages clears all stored DMs that the username as a participant in. @@ -91,6 +113,7 @@ func (dm DirectMessage) ClearMessages(username string) (int, error) { } var placeholders = []interface{}{ + time.Now(), fmt.Sprintf("@%s:%%", username), // `@alice:%` fmt.Sprintf("%%:@%s", username), // `%:@alice` username, @@ -99,25 +122,26 @@ func (dm DirectMessage) ClearMessages(username string) (int, error) { // Count all the messages we'll delete. var ( count int - row = DB.QueryRow(` + row = DB.Raw(` SELECT COUNT(message_id) FROM direct_messages WHERE (channel_id LIKE ? OR channel_id LIKE ?) OR username = ? `, placeholders...) ) - if err := row.Scan(&count); err != nil { - return 0, err + if res := row.Scan(&count); res.Error != nil && false { + return 0, res.Error } // Delete them all. - _, err := DB.Exec(` - DELETE FROM direct_messages + res := DB.Exec(` + UPDATE direct_messages + SET deleted_at = ? WHERE (channel_id LIKE ? OR channel_id LIKE ?) OR username = ? `, placeholders...) - return count, err + return count, res.Error } // TakebackMessage removes a message by its MID from the DM history. @@ -135,7 +159,7 @@ func (dm DirectMessage) TakebackMessage(username string, messageID int64, isAdmi // Does this messageID exist as sent by the user? if !isAdmin { var ( - row = DB.QueryRow( + row = DB.Raw( "SELECT message_id FROM direct_messages WHERE username = ? AND message_id = ?", username, messageID, ) @@ -148,13 +172,13 @@ func (dm DirectMessage) TakebackMessage(username string, messageID int64, isAdmi } // Delete it. - _, err := DB.Exec( + res := DB.Exec( "DELETE FROM direct_messages WHERE message_id = ?", messageID, ) // Return that it was successfully validated and deleted. - return err == nil, err + return res.Error == nil, res.Error } // PaginateDirectMessages returns a page of messages, the count of remaining, and an error. @@ -166,6 +190,7 @@ func PaginateDirectMessages(fromUsername, toUsername string, beforeID int64) ([] var ( result = []messages.Message{} channelID = CreateChannelID(fromUsername, toUsername) + rows = []*DirectMessage{} // Compute the remaining messages after finding the final messageID this page. lastMessageID int64 @@ -176,48 +201,34 @@ func PaginateDirectMessages(fromUsername, toUsername string, beforeID int64) ([] beforeID = math.MaxInt64 } - rows, err := DB.Query(` - SELECT message_id, username, message, timestamp - FROM direct_messages - WHERE channel_id = ? - AND message_id < ? - ORDER BY message_id DESC - LIMIT ? - `, channelID, beforeID, DirectMessagePerPage) - if err != nil { - return nil, 0, err + res := DB.Model(&DirectMessage{}).Where( + "channel_id = ? AND message_id < ?", + channelID, beforeID, + ).Order("message_id DESC").Limit(DirectMessagePerPage).Find(&rows) + if res.Error != nil { + return nil, 0, res.Error } - for rows.Next() { - var row DirectMessage - if err := rows.Scan( - &row.MessageID, - &row.Username, - &row.Message, - &row.Timestamp, - ); err != nil { - return nil, 0, err - } - + for _, row := range rows { msg := messages.Message{ MessageID: row.MessageID, Username: row.Username, Message: row.Message, - Timestamp: time.Unix(row.Timestamp, 0).Format(time.RFC3339), + Timestamp: row.CreatedAt.Format(time.RFC3339), } result = append(result, msg) lastMessageID = msg.MessageID } // Get a count of the remaining messages. - row := DB.QueryRow(` + row := DB.Raw(` SELECT COUNT(message_id) FROM direct_messages WHERE channel_id = ? AND message_id < ? `, channelID, lastMessageID) - if err := row.Scan(&remaining); err != nil { - return nil, 0, err + if res := row.Scan(&remaining); res.Error != nil { + return nil, 0, res.Error } return result, remaining, nil @@ -226,7 +237,7 @@ func PaginateDirectMessages(fromUsername, toUsername string, beforeID int64) ([] // PaginateUsernames returns a page of usernames that the current user has conversations with. // // Returns the usernames, total count, pages, and error. -func PaginateUsernames(fromUsername, sort string, page, perPage int) ([]string, int, int, error) { +func PaginateUsernames(fromUsername, sortBy string, page, perPage int) ([]string, int, int, error) { if DB == nil { return nil, 0, 0, ErrNotInitialized } @@ -249,17 +260,15 @@ func PaginateUsernames(fromUsername, sort string, page, perPage int) ([]string, offset = 0 } - // Whitelist the sort strings. - switch sort { + switch sortBy { case "a-z": orderBy = "username ASC" case "z-a": orderBy = "username DESC" case "oldest": - orderBy = "timestamp ASC" + orderBy = "newest_time ASC" default: - // default = newest - orderBy = "timestamp DESC" + orderBy = "newest_time DESC" } // Get all our distinct channel IDs to filter the query down: otherwise doing an ORDER BY timestamp @@ -274,49 +283,27 @@ func PaginateUsernames(fromUsername, sort string, page, perPage int) ([]string, return nil, 0, 0, errors.New("you have no direct messages stored on this chat server") } - var ( - cidPlaceholders = "?" + strings.Repeat(",?", len(channelIDs)-1) - params = []interface{}{} - ) - for _, cid := range channelIDs { - params = append(params, cid) + type record struct { + Username string + // NewestTime time.Time + // OldestTime time.Time + } + var records []record + + // Get all usernames and their newest/oldest timestamps. + res := DB.Model(&DirectMessage{}).Select(` + username, + MAX(created_at) AS newest_time + `).Where( + "channel_id IN ? AND username <> ?", + channelIDs, fromUsername, + ).Group("username").Order(orderBy).Offset(offset).Limit(perPage).Scan(&records) + if res.Error != nil { + return nil, 0, 0, res.Error } - // Note: for some reason, the SQLite driver doesn't allow a parameterized - // query for ORDER BY (e.g. "ORDER BY ?") - so, since we have already - // whitelisted acceptable orders, use a Sprintf to interpolate that - // directly into the query. - queryStr := fmt.Sprintf(` - SELECT distinct(username) - FROM direct_messages - WHERE channel_id IN (%s) - AND username <> ? - ORDER BY %s - LIMIT ? - OFFSET ?`, - cidPlaceholders, - orderBy, - ) - params = append(params, fromUsername, perPage, offset) - // fmt.Println(queryStr) - // fmt.Printf("%v\n", params) - rows, err := DB.Query( - queryStr, - params..., - ) - if err != nil { - return nil, 0, 0, err - } - - for rows.Next() { - var username string - if err := rows.Scan( - &username, - ); err != nil { - return nil, 0, 0, err - } - - result = append(result, username) + for _, row := range records { + result = append(result, row.Username) } // The count of distinct channel IDs earlier. @@ -340,24 +327,14 @@ func GetDistinctChannelIDs(username string) ([]string, error) { } ) - rows, err := DB.Query(` - SELECT distinct(channel_id) - FROM direct_messages - WHERE ( - channel_id LIKE ? - OR channel_id LIKE ? - ) - `, channelIDs[0], channelIDs[1]) - if err != nil { - return nil, err - } - - for rows.Next() { - var channelID string - if err := rows.Scan(&channelID); err != nil { - return nil, err - } - result = append(result, channelID) + res := DB.Model(&DirectMessage{}).Select( + "DISTINCT(channel_id)", + ).Where( + "(channel_id LIKE ? OR channel_id LIKE ?)", + channelIDs[0], channelIDs[1], + ).Scan(&result) + if res.Error != nil { + return nil, res.Error } return result, nil diff --git a/pkg/server.go b/pkg/server.go index fd6934c..4c3129e 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -1,6 +1,7 @@ package barertc import ( + "fmt" "io" "net/http" "sync" @@ -42,7 +43,17 @@ func NewServer() *Server { func (s *Server) Setup() error { // Enable the SQLite database for DM history? if config.Current.DirectMessageHistory.Enabled { - if err := models.Initialize(config.Current.DirectMessageHistory.SQLiteDatabase); err != nil { + var connstr string + switch config.Current.DirectMessageHistory.DatabaseType { + case "sqlite3": + connstr = config.Current.DirectMessageHistory.SQLiteDatabase + case "postgres": + connstr = config.Current.DirectMessageHistory.PostgresDatabase + default: + return fmt.Errorf("settings.toml: DirectMessageHistory: DatabaseType must be either sqlite or postgres") + } + + if err := models.Initialize(config.Current.DirectMessageHistory.DatabaseType, connstr); err != nil { log.Error("Error initializing SQLite database: %s", err) } } diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..810ba6c --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1 @@ +psycopg2-binary \ No newline at end of file diff --git a/scripts/sqlite2psql.py b/scripts/sqlite2psql.py new file mode 100644 index 0000000..914d59a --- /dev/null +++ b/scripts/sqlite2psql.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +""" +SQLite to Postgres migration script. + +BareRTC originally used SQLite directly for the DirectMessageHistory database, +but has switched to use GORM so that you can use Postgres instead of SQLite. + +This script will migrate an existing BareRTC database.sqlite script over to +Postgres. + +Instructions: + +1. Create a Python virtualenv and install dependencies + + mkvirtualenv barertc + pip install -r scripts/requirements.txt + +2. Run this script + + python3 scripts/sqlite2psql.py --sqlite database.sqlite \ + --psql 'user=barertc password=barertc dbname=barertc sslmode=disable TimeZone=America/Los_Angeles' + +Note: this script does not create the Postgres table. Run BareRTC in Postgres +mode and have it create the table first, then run this script to backfill your +old DMs from SQLite3. +""" + +import argparse +import datetime +import sqlite3 +import psycopg2 + +def main(args): + + print("Opening SQLite DB") + sqlite = open_sqlite3(args.sqlite) + + print("Connecting to Postgres") + psql = open_postgres(args.psql) + + migrate(sqlite, psql) + + +def migrate(sqlite, psql): + print("Migrating direct messages") + + counter = 0 + for row in sqlite_paginate(sqlite): + counter += 1 + + # Migrate legacy Timestamp field. + if row['timestamp'] > 0: + dt = datetime.datetime.fromtimestamp(row['timestamp']) + row['timestamp'] = 0 + row['created_at'] = dt.isoformat() + + # No created_at? (shouldn't happen), use the message_id instead. + if not row['created_at']: + print("Missing created_at! Using message_id as timestamp!") + dt = datetime.datetime.fromtimestamp(row['message_id']) + row['created_at'] = dt.isoformat() + + # Upsert it into Postgres. + psql_upsert(psql, row) + if counter % 500 == 0: + print(f"Migrated {counter} messages...") + psql.commit() + + print("Finished!") + psql.commit() + + +def sqlite_paginate(sqlite): + """Paginate over the DMs from SQLite as a generator.""" + cur = sqlite.cursor() + after_id = 0 + + while True: + res = cur.execute(f""" + SELECT + message_id, + channel_id, + username, + message, + timestamp + FROM direct_messages + WHERE message_id > {after_id} + ORDER BY message_id ASC + LIMIT 500 + """) + page = res.fetchall() + + if len(page) == 0: + return + + for row in page: + after_id = row[0] + yield dict( + message_id=row[0], + channel_id=row[1], + username=row[2], + message=row[3], + timestamp=row[4], + created_at=None, + ) + + +def psql_upsert(psql, row): + cur = psql.cursor() + cur.execute(""" + INSERT INTO direct_messages (message_id, channel_id, username, message, timestamp, created_at) + VALUES (%s, %s, %s, %s, 0, %s) + ON CONFLICT (message_id) DO NOTHING + """, ( + row['message_id'], row['channel_id'], row['username'], row['message'], row['created_at'], + )) + psql.commit() + +def open_sqlite3(connstr): + conn = sqlite3.connect(connstr) + return conn + + +def open_postgres(connstr): + conn = psycopg2.connect(connstr) + return conn + +if __name__ == "__main__": + parser = argparse.ArgumentParser("sqlite2psql") + parser.add_argument("--sqlite", + type=str, + required=True, + help="Path to your SQLite database", + ) + parser.add_argument("--psql", + type=str, + required=True, + help="Your Postgres connection string", + ) + args = parser.parse_args() + main(args) \ No newline at end of file