Age Gate, Legacy kirsle/blog Migration Program

* Add the Age Gate middleware for NSFW sites.
* Cache thumbnail images from blog entries.
* Implement the user-root properly for loading web assets.
This commit is contained in:
Noah 2020-02-17 15:50:04 -08:00
parent c1995efb7a
commit 383b5d7591
14 changed files with 434 additions and 32 deletions

3
.gitignore vendored
View File

@ -1,5 +1,4 @@
bin/
pvt-www/static/photos/
public_html/
pkg/bundled/
public_html/.settings.json
*.sqlite

209
cmd/legacy-import/main.go Normal file
View File

@ -0,0 +1,209 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"sort"
"git.kirsle.net/apps/gophertype/pkg"
"git.kirsle.net/apps/gophertype/pkg/console"
_ "git.kirsle.net/apps/gophertype/pkg/controllers"
"git.kirsle.net/apps/gophertype/pkg/models"
_ "github.com/jinzhu/gorm/dialects/mysql"
_ "github.com/jinzhu/gorm/dialects/postgres"
_ "github.com/jinzhu/gorm/dialects/sqlite"
jsondb "github.com/kirsle/blog/jsondb"
mComments "github.com/kirsle/blog/models/comments"
mPosts "github.com/kirsle/blog/models/posts"
)
// Command-line flags.
var (
optSourceRoot string
optRoot string
// Database option flags.
optSQLite string
optPostgres string
optMySQL string
// Chosen DB options.
dbDriver string
dbPath string
)
// Other global variables
var (
JsonDB *jsondb.DB
)
func init() {
flag.StringVar(&optSourceRoot, "srcroot", "", "User root from old kirsle/blog website")
flag.StringVar(&optRoot, "root", "", "User root for GopherType")
// Database driver. Choose one.
flag.StringVar(&optSQLite, "sqlite3", "", "Use SQLite database, default 'database.sqlite'")
flag.StringVar(&optPostgres, "postgres", "",
"Use Postgres database, format: "+
"host=myhost port=myport user=gorm dbname=gorm password=mypassword")
flag.StringVar(&optMySQL, "mysql", "",
"Use MySQL database, format: "+
"user:password@/dbname?charset=utf8&parseTime=True&loc=Local")
}
func main() {
console.SetDebug(false)
flag.Parse()
// Validate the choice of database.
if optSQLite != "" {
dbDriver = "sqlite3"
dbPath = optSQLite
} else if optPostgres != "" {
dbDriver = "postgres"
dbPath = optPostgres
} else if optMySQL != "" {
dbDriver = "mysql"
dbPath = optMySQL
} else {
fmt.Print(
"Specify a DB driver for Gophertype, similar to the gophertype command.",
)
os.Exit(1)
}
// Validate the roots are given.
if optSourceRoot == "" {
fmt.Print(
"Missing -srcroot parameter: this should be the User Root from the legacy kirsle/blog site.",
)
os.Exit(1)
}
if optRoot == "" {
fmt.Print(
"Missing required -root parameter: this is the User Root for Gophertype",
)
}
// Initialize the old JsonDB.
app := gophertype.NewSite(optRoot)
app.UseDB(dbDriver, dbPath)
initJsonDB()
doMigrate()
}
// Initialize the old kirsle/blog JsonDB.
func initJsonDB() {
JsonDB = jsondb.New(filepath.Join(optSourceRoot, ".private"))
mPosts.DB = JsonDB
mComments.DB = JsonDB
}
// doMigrate is the head of the migrate functions.
func doMigrate() {
migratePosts()
migrateComments()
}
// migratePosts migrates blog posts over.
func migratePosts() {
console.Warn("BEGIN: Migrating blog posts")
idx, err := mPosts.GetIndex()
if err != nil {
panic("migratePosts: GetIndex: " + err.Error())
}
// Sort the IDs to make it pretty.
var sortedIds = []int{}
for id := range idx.Posts {
sortedIds = append(sortedIds, id)
}
sort.Ints(sortedIds)
for _, id := range sortedIds {
post, err := mPosts.Load(id)
if err != nil {
panic(fmt.Sprintf("migratePosts: error loading legacy post ID %d: %s", id, err))
}
console.Info("Post ID %d: %s", id, post.Title)
// Create the post in Gophertype.
p := models.Post{
Title: post.Title,
Fragment: post.Fragment,
ContentType: post.ContentType,
AuthorID: uint(post.AuthorID),
Body: post.Body,
Privacy: post.Privacy,
Sticky: post.Sticky,
EnableComments: post.EnableComments,
}
p.ID = uint(post.ID)
p.CreatedAt = post.Created
p.UpdatedAt = post.Updated
// Convert tags.
for _, tag := range post.Tags {
p.Tags = append(p.Tags, models.TaggedPost{
Tag: tag,
})
}
err = p.Save()
if err != nil {
console.Error("Error saving post %d: %s", p.ID, err)
}
}
console.Warn("FINISH: Migrating blog posts")
}
// migrateComments migrates comments over.
func migrateComments() {
console.Warn("BEGIN: Migrating comments")
// Find all the comment threads.
files, err := JsonDB.List("comments/threads")
if err != nil {
panic("Error listing comments: " + err.Error())
}
for _, file := range files {
document := filepath.Base(file)
thread, err := mComments.Load(document)
if err != nil {
panic(fmt.Sprintf("Error loading comment thread %s: %s", file, err))
}
for _, comment := range thread.Comments {
// Migrate to the new model.
com := models.Comment{
Thread: thread.ID,
UserID: uint(comment.UserID),
Name: comment.Name,
Email: comment.Email,
Avatar: comment.Avatar,
Body: comment.Body,
EditToken: comment.EditToken,
DeleteToken: comment.DeleteToken,
}
com.CreatedAt = comment.Created
com.UpdatedAt = comment.Updated
// Special case for guestbook page.
if thread.ID == "guestbook" {
com.OriginURL = "/guestbook"
}
if err := com.Save(); err != nil {
console.Error("Error saving comment: %s", err)
}
}
}
console.Warn("FINISH: Migrating comments")
}

View File

@ -0,0 +1,43 @@
package controllers
import (
"net/http"
"git.kirsle.net/apps/gophertype/pkg/glue"
"git.kirsle.net/apps/gophertype/pkg/responses"
"git.kirsle.net/apps/gophertype/pkg/session"
)
func init() {
glue.Register(glue.Endpoint{
Path: "/age-verify",
Methods: []string{"GET", "POST"},
Handler: AgeVerify,
})
}
// AgeVerify handles the age gate prompt page for NSFW sites.
func AgeVerify(w http.ResponseWriter, r *http.Request) {
var (
v = responses.NewTemplateVars(w, r)
next = r.FormValue("next")
confirm = r.FormValue("confirm")
)
if next == "" {
next = "/"
}
v.V["Next"] = next
if r.Method == http.MethodPost {
if confirm == "true" {
session := session.Get(r)
session.Values["age-ok"] = true
session.Save(r, w)
responses.Redirect(w, r, next)
return
}
}
responses.RenderTemplate(w, r, "_builtin/age-gate.gohtml", v)
}

View File

@ -16,6 +16,7 @@ import (
"git.kirsle.net/apps/gophertype/pkg/console"
"git.kirsle.net/apps/gophertype/pkg/glue"
"git.kirsle.net/apps/gophertype/pkg/responses"
"git.kirsle.net/apps/gophertype/pkg/settings"
"github.com/edwvee/exiffix"
"github.com/nfnt/resize"
)
@ -24,7 +25,9 @@ import (
var (
MaxImageWidth = 1280
JpegQuality = 90
ImagePath = "static/photos" // images folder for upload, relative to web root.
// images folder for upload, relative to web root.
ImagePath = filepath.Join("static", "photos")
)
func init() {
@ -99,14 +102,14 @@ func UploadHandler(w http.ResponseWriter, r *http.Request) {
console.Info("Uploaded file named: %s name=%s", header.Filename, filename)
// Write to the /static/photos directory of the user root. Ensure the path
// exists or create it if not. TODO
outputPath := filepath.Join("./pvt-www", "static", "photos")
// exists or create it if not.
outputPath := filepath.Join(settings.UserRoot, ImagePath)
if _, err := os.Stat(outputPath); os.IsNotExist(err) {
os.MkdirAll(outputPath, 0755)
}
// Ensure the filename is unique.
filename = uniqueFilename("./pvt-www/static/photos", filename)
filename = uniqueFilename(filepath.Join(settings.UserRoot, ImagePath), filename)
// Write the output file.
console.Info("Uploaded image: %s", filename)

View File

@ -0,0 +1,80 @@
package middleware
import (
"net/http"
"strings"
"git.kirsle.net/apps/gophertype/pkg/responses"
"git.kirsle.net/apps/gophertype/pkg/session"
"git.kirsle.net/apps/gophertype/pkg/settings"
)
// URL suffixes to allow to bypass the age gate middleware.
var ageGateSuffixes = []string{
"/blog.rss", // Allow public access to RSS and Atom feeds.
"/blog.atom",
"/blog.json",
".js",
".css",
".txt",
".ico",
".png",
".jpg",
".jpeg",
".gif",
".mp4",
".webm",
".ttf",
".eot",
".svg",
".woff",
".woff2",
}
// AgeGate is a middleware generator that does age verification for NSFW sites.
// Single GET requests with ?over18=1 parameter may skip the middleware check.
func AgeGate(next http.Handler) http.Handler {
middleware := func(w http.ResponseWriter, r *http.Request) {
s := settings.Current
if !s.NSFW {
next.ServeHTTP(w, r)
return
}
path := r.URL.Path
// Let the age-verify handler catch its route.
if strings.HasPrefix(path, "/age-verify") {
next.ServeHTTP(w, r)
return
}
// Allow static file requests to skip the check.
for _, suffix := range ageGateSuffixes {
if strings.HasSuffix(path, suffix) {
next.ServeHTTP(w, r)
return
}
}
// POST requests are permitted (e.g. post a comment on a /?over18=1 page)
if r.Method == http.MethodPost {
next.ServeHTTP(w, r)
return
}
// Finally, check if they've confirmed their age on the age-verify handler.
ses := session.Get(r)
if val, _ := ses.Values["age-ok"].(bool); !val {
// They haven't been verified. Redirect them to the age-verify handler.
if r.FormValue("over18") == "" {
responses.Redirect(w, r, "/age-verify?next="+path)
return
}
}
next.ServeHTTP(w, r)
}
return http.HandlerFunc(middleware)
}

View File

@ -29,6 +29,7 @@ type Post struct {
Fragment string `gorm:"unique_index"`
ContentType string `gorm:"default:html"`
AuthorID uint // foreign key to User.ID
Thumbnail string // image thumbnail for the post
Body string
Privacy string
@ -52,6 +53,9 @@ type PagedPosts struct {
PreviousPage int
}
// Regexp for matching a thumbnail image for a blog post.
var ThumbnailImageRegexp = regexp.MustCompile(`['"(]([a-zA-Z0-9-_:/?.=&]+\.(?:jpe?g|png|gif))['")]`)
// New creates a new Post model.
func (m postMan) New() Post {
return Post{
@ -299,6 +303,11 @@ func (p *Post) Save() error {
}
}
// Cache the post thumbnail from the body.
if thumbnail, ok := p.ExtractThumbnail(); ok {
p.Thumbnail = thumbnail
}
// Empty tags list.
if len(p.Tags) == 1 && p.Tags[0].Tag == "" {
p.Tags = []TaggedPost{}
@ -347,6 +356,16 @@ func (p *Post) ParseForm(form *forms.Data) {
}
}
// ExtractThumbnail searches and returns a thumbnail image to represent the post.
// It will be the first image embedded in the post body, or nothing.
func (p Post) ExtractThumbnail() (string, bool) {
result := ThumbnailImageRegexp.FindStringSubmatch(p.Body)
if len(result) < 2 {
return "", false
}
return result[1], true
}
// TagsString turns the post tags into a comma separated string.
func (p Post) TagsString() string {
console.Error("TagsString: %+v", p.Tags)

View File

@ -2,28 +2,31 @@ package responses
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"git.kirsle.net/apps/gophertype/pkg/bundled"
"git.kirsle.net/apps/gophertype/pkg/settings"
)
// GetFile returns the template file's data, wherever it is.
// Checks the embedded bindata, then the user root on disk, then error.
// If it can be found, returns the contents or error.
func GetFile(path string) ([]byte, error) {
// Check bindata.
// Check the user root first.
if b, err := ioutil.ReadFile(filepath.Join(settings.UserRoot, path)); err == nil {
return b, nil
}
// Fall back on embedded bindata.
if b, err := bundled.Asset(path); err == nil {
return b, nil
}
// Check the filesystem. TODO
if b, err := ioutil.ReadFile("./pvt-www/" + path); err == nil {
return b, nil
} else {
return []byte{}, err
}
return []byte{}, fmt.Errorf("GetFile(%s): not found in user root or bindata", path)
}
// GetFileExists checks if the file exists but doesn't return its data.
@ -33,8 +36,8 @@ func GetFileExists(path string) bool {
return true
}
// Check the filesystem. TODO
if _, err := os.Stat(path); err == nil {
// Check the user root.
if stat, err := os.Stat(filepath.Join(settings.UserRoot, path)); err == nil && !stat.IsDir() {
return true
}

View File

@ -6,6 +6,8 @@ import (
"path/filepath"
"git.kirsle.net/apps/gophertype/pkg/bundled"
"git.kirsle.net/apps/gophertype/pkg/console"
"git.kirsle.net/apps/gophertype/pkg/settings"
)
// SendFile sends a file from bindata or the user root.
@ -22,5 +24,6 @@ func SendFile(w http.ResponseWriter, r *http.Request, path string) {
return
}
http.ServeFile(w, r, "pvt-www/"+path)
console.Debug("SendFile: http.ServeFile(%s)", filepath.Join(settings.UserRoot, path))
http.ServeFile(w, r, filepath.Join(settings.UserRoot, path))
}

View File

@ -19,6 +19,8 @@ func NewTemplateVars(w io.Writer, r *http.Request) TemplateValues {
var s = settings.Current
user, _ := authentication.CurrentUser(r)
ses := session.Get(r)
v := TemplateValues{
SetupNeeded: !s.Initialized,
@ -28,6 +30,7 @@ func NewTemplateVars(w io.Writer, r *http.Request) TemplateValues {
Request: r,
RequestTime: time.Now(),
RequestDuration: time.Duration(0),
Session: ses,
Path: r.URL.Path,
LoggedIn: authentication.LoggedIn(r),

View File

@ -19,6 +19,7 @@ func (s *Site) SetupRouter() error {
router.Use(session.Middleware)
router.Use(authentication.Middleware)
router.Use(middleware.CSRF)
router.Use(middleware.AgeGate)
console.Debug("Setting up HTTP Router")
for _, route := range glue.GetControllers() {

View File

@ -19,6 +19,9 @@ import (
// key. The config is not saved to DB until you call Save() on it.
var Current = Load()
// UserRoot is the folder path to the user web files.
var UserRoot string
// Spec singleton holds the app configuration.
type Spec struct {
// Sets to `true` when the site's initial setup has run and an admin created.
@ -88,6 +91,7 @@ func SetFilename(userRoot string) error {
}
Current = spec
UserRoot = userRoot
session.SetSecretKey([]byte(Current.SecretKey))
return nil
}

View File

@ -0,0 +1,34 @@
{{ define "title" }}Age Verification{{ end }}
{{ define "content" }}
<div class="card">
<div class="card-body">
<form action="/age-verify" method="POST">
{{ CSRF }}
<input type="hidden" name="next" value="{{ .V.Next }}">
<input type="hidden" name="confirm" value="true">
<h1>Restricted Content</h1>
<p>
This website has been marked <abbr title="Not Safe For Work">NSFW</abbr>
by its owner. It may contain nudity or content not suited for users
under the age of 18.
</p>
<p>
To proceed, you must verify you are at least 18 years or older.
</p>
<button type="submit"
class="btn btn-danger">
I am 18 years or older
</button>
<a class="btn btn-primary"
href="https://duckduckgo.com/">
Get me out of here!
</a>
</form>
</div>
</div>
{{ end }}

View File

@ -43,6 +43,7 @@
</div>
</div>
{{ .CurrentUser }}
{{ if .CurrentUser.IsAdmin }}
<div class="alert alert-secondary">

View File

@ -4,7 +4,21 @@
<h1>Recent Comments</h1>
{{ with .V.PagedComments }}
<p>
{{ range .Comments }}
{{ if gt .PostID 0 }}
<div class="alert alert-info">
<strong>In post <a href="{{ .Post.Fragment }}">{{ or .Post.Title "Untitled" }}</a>:</strong>
</div>
{{ else if .OriginURL }}
<div class="alert alert-info">
<strong>On page <a href="{{ .OriginURL }}">{{ .OriginURL }}</a>:</strong>
</div>
{{ end }}
{{ RenderComment $.ResponseWriter $.Request . "/comments" false }}
{{ end }}
<div class="alert alert-success">
Page {{ .Page }} of {{ .Pages }} ({{ .Total }} total)
{{ if or (gt .PreviousPage 0) (gt .NextPage 0) }}
[
@ -17,21 +31,7 @@
{{ end }}
]
{{ end }}
</p>
{{ range .Comments }}
{{ if gt .PostID 0 }}
<p>
<strong>In post <a href="{{ .Post.Fragment }}">{{ or .Post.Title "Untitled" }}</a>:</strong>
</p>
{{ else if .OriginURL }}
<p>
<strong>On page <a href="{{ .OriginURL }}">{{ .OriginURL }}</a>:</strong>
</p>
{{ end }}
{{ RenderComment $.ResponseWriter $.Request . "/comments" false }}
{{ end }}
</div>
{{ end }}
{{ end }}