From 383b5d7591d47eb5140e312e4ade8a5b868152f1 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 17 Feb 2020 15:50:04 -0800 Subject: [PATCH] Age Gate, Legacy kirsle/blog Migration Program * Add the Age Gate middleware for NSFW sites. * Cache thumbnail images from blog entries. * Implement the user-root properly for loading web assets. --- .gitignore | 3 +- cmd/legacy-import/main.go | 209 ++++++++++++++++++++++++ pkg/controllers/age_gate.go | 43 +++++ pkg/controllers/uploads.go | 11 +- pkg/middleware/age_gate.go | 80 +++++++++ pkg/models/posts.go | 19 +++ pkg/responses/filesystem.go | 21 ++- pkg/responses/static_file.go | 5 +- pkg/responses/template_vars.go | 3 + pkg/routes.go | 1 + pkg/settings/settings.go | 4 + pvt-www/_builtin/age-gate.gohtml | 34 ++++ pvt-www/_builtin/blog/view-post.gohtml | 1 + pvt-www/_builtin/comments/recent.gohtml | 32 ++-- 14 files changed, 434 insertions(+), 32 deletions(-) create mode 100644 cmd/legacy-import/main.go create mode 100644 pkg/controllers/age_gate.go create mode 100644 pkg/middleware/age_gate.go create mode 100644 pvt-www/_builtin/age-gate.gohtml diff --git a/.gitignore b/.gitignore index 751ea64..c3ed9eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ bin/ -pvt-www/static/photos/ +public_html/ pkg/bundled/ -public_html/.settings.json *.sqlite diff --git a/cmd/legacy-import/main.go b/cmd/legacy-import/main.go new file mode 100644 index 0000000..bddfd59 --- /dev/null +++ b/cmd/legacy-import/main.go @@ -0,0 +1,209 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "sort" + + "git.kirsle.net/apps/gophertype/pkg" + "git.kirsle.net/apps/gophertype/pkg/console" + _ "git.kirsle.net/apps/gophertype/pkg/controllers" + "git.kirsle.net/apps/gophertype/pkg/models" + _ "github.com/jinzhu/gorm/dialects/mysql" + _ "github.com/jinzhu/gorm/dialects/postgres" + _ "github.com/jinzhu/gorm/dialects/sqlite" + jsondb "github.com/kirsle/blog/jsondb" + mComments "github.com/kirsle/blog/models/comments" + mPosts "github.com/kirsle/blog/models/posts" +) + +// Command-line flags. +var ( + optSourceRoot string + optRoot string + + // Database option flags. + optSQLite string + optPostgres string + optMySQL string + + // Chosen DB options. + dbDriver string + dbPath string +) + +// Other global variables +var ( + JsonDB *jsondb.DB +) + +func init() { + flag.StringVar(&optSourceRoot, "srcroot", "", "User root from old kirsle/blog website") + flag.StringVar(&optRoot, "root", "", "User root for GopherType") + + // Database driver. Choose one. + flag.StringVar(&optSQLite, "sqlite3", "", "Use SQLite database, default 'database.sqlite'") + flag.StringVar(&optPostgres, "postgres", "", + "Use Postgres database, format: "+ + "host=myhost port=myport user=gorm dbname=gorm password=mypassword") + flag.StringVar(&optMySQL, "mysql", "", + "Use MySQL database, format: "+ + "user:password@/dbname?charset=utf8&parseTime=True&loc=Local") +} + +func main() { + console.SetDebug(false) + flag.Parse() + + // Validate the choice of database. + if optSQLite != "" { + dbDriver = "sqlite3" + dbPath = optSQLite + } else if optPostgres != "" { + dbDriver = "postgres" + dbPath = optPostgres + } else if optMySQL != "" { + dbDriver = "mysql" + dbPath = optMySQL + } else { + fmt.Print( + "Specify a DB driver for Gophertype, similar to the gophertype command.", + ) + os.Exit(1) + } + + // Validate the roots are given. + if optSourceRoot == "" { + fmt.Print( + "Missing -srcroot parameter: this should be the User Root from the legacy kirsle/blog site.", + ) + os.Exit(1) + } + if optRoot == "" { + fmt.Print( + "Missing required -root parameter: this is the User Root for Gophertype", + ) + } + + // Initialize the old JsonDB. + + app := gophertype.NewSite(optRoot) + app.UseDB(dbDriver, dbPath) + + initJsonDB() + doMigrate() +} + +// Initialize the old kirsle/blog JsonDB. +func initJsonDB() { + JsonDB = jsondb.New(filepath.Join(optSourceRoot, ".private")) + mPosts.DB = JsonDB + mComments.DB = JsonDB +} + +// doMigrate is the head of the migrate functions. +func doMigrate() { + migratePosts() + migrateComments() +} + +// migratePosts migrates blog posts over. +func migratePosts() { + console.Warn("BEGIN: Migrating blog posts") + idx, err := mPosts.GetIndex() + if err != nil { + panic("migratePosts: GetIndex: " + err.Error()) + } + + // Sort the IDs to make it pretty. + var sortedIds = []int{} + for id := range idx.Posts { + sortedIds = append(sortedIds, id) + } + sort.Ints(sortedIds) + + for _, id := range sortedIds { + post, err := mPosts.Load(id) + if err != nil { + panic(fmt.Sprintf("migratePosts: error loading legacy post ID %d: %s", id, err)) + } + + console.Info("Post ID %d: %s", id, post.Title) + + // Create the post in Gophertype. + p := models.Post{ + Title: post.Title, + Fragment: post.Fragment, + ContentType: post.ContentType, + AuthorID: uint(post.AuthorID), + Body: post.Body, + Privacy: post.Privacy, + Sticky: post.Sticky, + EnableComments: post.EnableComments, + } + p.ID = uint(post.ID) + p.CreatedAt = post.Created + p.UpdatedAt = post.Updated + + // Convert tags. + for _, tag := range post.Tags { + p.Tags = append(p.Tags, models.TaggedPost{ + Tag: tag, + }) + } + + err = p.Save() + if err != nil { + console.Error("Error saving post %d: %s", p.ID, err) + } + } + + console.Warn("FINISH: Migrating blog posts") +} + +// migrateComments migrates comments over. +func migrateComments() { + console.Warn("BEGIN: Migrating comments") + + // Find all the comment threads. + files, err := JsonDB.List("comments/threads") + if err != nil { + panic("Error listing comments: " + err.Error()) + } + for _, file := range files { + document := filepath.Base(file) + thread, err := mComments.Load(document) + if err != nil { + panic(fmt.Sprintf("Error loading comment thread %s: %s", file, err)) + } + + for _, comment := range thread.Comments { + // Migrate to the new model. + com := models.Comment{ + Thread: thread.ID, + UserID: uint(comment.UserID), + Name: comment.Name, + Email: comment.Email, + Avatar: comment.Avatar, + Body: comment.Body, + EditToken: comment.EditToken, + DeleteToken: comment.DeleteToken, + } + com.CreatedAt = comment.Created + com.UpdatedAt = comment.Updated + + // Special case for guestbook page. + if thread.ID == "guestbook" { + com.OriginURL = "/guestbook" + } + + if err := com.Save(); err != nil { + console.Error("Error saving comment: %s", err) + } + } + } + + console.Warn("FINISH: Migrating comments") +} diff --git a/pkg/controllers/age_gate.go b/pkg/controllers/age_gate.go new file mode 100644 index 0000000..c986fd0 --- /dev/null +++ b/pkg/controllers/age_gate.go @@ -0,0 +1,43 @@ +package controllers + +import ( + "net/http" + + "git.kirsle.net/apps/gophertype/pkg/glue" + "git.kirsle.net/apps/gophertype/pkg/responses" + "git.kirsle.net/apps/gophertype/pkg/session" +) + +func init() { + glue.Register(glue.Endpoint{ + Path: "/age-verify", + Methods: []string{"GET", "POST"}, + Handler: AgeVerify, + }) +} + +// AgeVerify handles the age gate prompt page for NSFW sites. +func AgeVerify(w http.ResponseWriter, r *http.Request) { + var ( + v = responses.NewTemplateVars(w, r) + next = r.FormValue("next") + confirm = r.FormValue("confirm") + ) + + if next == "" { + next = "/" + } + v.V["Next"] = next + + if r.Method == http.MethodPost { + if confirm == "true" { + session := session.Get(r) + session.Values["age-ok"] = true + session.Save(r, w) + responses.Redirect(w, r, next) + return + } + } + + responses.RenderTemplate(w, r, "_builtin/age-gate.gohtml", v) +} diff --git a/pkg/controllers/uploads.go b/pkg/controllers/uploads.go index 271cb5c..6acc86f 100644 --- a/pkg/controllers/uploads.go +++ b/pkg/controllers/uploads.go @@ -16,6 +16,7 @@ import ( "git.kirsle.net/apps/gophertype/pkg/console" "git.kirsle.net/apps/gophertype/pkg/glue" "git.kirsle.net/apps/gophertype/pkg/responses" + "git.kirsle.net/apps/gophertype/pkg/settings" "github.com/edwvee/exiffix" "github.com/nfnt/resize" ) @@ -24,7 +25,9 @@ import ( var ( MaxImageWidth = 1280 JpegQuality = 90 - ImagePath = "static/photos" // images folder for upload, relative to web root. + + // images folder for upload, relative to web root. + ImagePath = filepath.Join("static", "photos") ) func init() { @@ -99,14 +102,14 @@ func UploadHandler(w http.ResponseWriter, r *http.Request) { console.Info("Uploaded file named: %s name=%s", header.Filename, filename) // Write to the /static/photos directory of the user root. Ensure the path - // exists or create it if not. TODO - outputPath := filepath.Join("./pvt-www", "static", "photos") + // exists or create it if not. + outputPath := filepath.Join(settings.UserRoot, ImagePath) if _, err := os.Stat(outputPath); os.IsNotExist(err) { os.MkdirAll(outputPath, 0755) } // Ensure the filename is unique. - filename = uniqueFilename("./pvt-www/static/photos", filename) + filename = uniqueFilename(filepath.Join(settings.UserRoot, ImagePath), filename) // Write the output file. console.Info("Uploaded image: %s", filename) diff --git a/pkg/middleware/age_gate.go b/pkg/middleware/age_gate.go new file mode 100644 index 0000000..835fe97 --- /dev/null +++ b/pkg/middleware/age_gate.go @@ -0,0 +1,80 @@ +package middleware + +import ( + "net/http" + "strings" + + "git.kirsle.net/apps/gophertype/pkg/responses" + "git.kirsle.net/apps/gophertype/pkg/session" + "git.kirsle.net/apps/gophertype/pkg/settings" +) + +// URL suffixes to allow to bypass the age gate middleware. +var ageGateSuffixes = []string{ + "/blog.rss", // Allow public access to RSS and Atom feeds. + "/blog.atom", + "/blog.json", + ".js", + ".css", + ".txt", + ".ico", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".mp4", + ".webm", + ".ttf", + ".eot", + ".svg", + ".woff", + ".woff2", +} + +// AgeGate is a middleware generator that does age verification for NSFW sites. +// Single GET requests with ?over18=1 parameter may skip the middleware check. +func AgeGate(next http.Handler) http.Handler { + middleware := func(w http.ResponseWriter, r *http.Request) { + s := settings.Current + if !s.NSFW { + next.ServeHTTP(w, r) + return + } + + path := r.URL.Path + + // Let the age-verify handler catch its route. + if strings.HasPrefix(path, "/age-verify") { + next.ServeHTTP(w, r) + return + } + + // Allow static file requests to skip the check. + for _, suffix := range ageGateSuffixes { + if strings.HasSuffix(path, suffix) { + next.ServeHTTP(w, r) + return + } + } + + // POST requests are permitted (e.g. post a comment on a /?over18=1 page) + if r.Method == http.MethodPost { + next.ServeHTTP(w, r) + return + } + + // Finally, check if they've confirmed their age on the age-verify handler. + ses := session.Get(r) + if val, _ := ses.Values["age-ok"].(bool); !val { + // They haven't been verified. Redirect them to the age-verify handler. + if r.FormValue("over18") == "" { + responses.Redirect(w, r, "/age-verify?next="+path) + return + } + } + + next.ServeHTTP(w, r) + } + + return http.HandlerFunc(middleware) +} diff --git a/pkg/models/posts.go b/pkg/models/posts.go index 18c6910..2358664 100644 --- a/pkg/models/posts.go +++ b/pkg/models/posts.go @@ -29,6 +29,7 @@ type Post struct { Fragment string `gorm:"unique_index"` ContentType string `gorm:"default:html"` AuthorID uint // foreign key to User.ID + Thumbnail string // image thumbnail for the post Body string Privacy string @@ -52,6 +53,9 @@ type PagedPosts struct { PreviousPage int } +// Regexp for matching a thumbnail image for a blog post. +var ThumbnailImageRegexp = regexp.MustCompile(`['"(]([a-zA-Z0-9-_:/?.=&]+\.(?:jpe?g|png|gif))['")]`) + // New creates a new Post model. func (m postMan) New() Post { return Post{ @@ -299,6 +303,11 @@ func (p *Post) Save() error { } } + // Cache the post thumbnail from the body. + if thumbnail, ok := p.ExtractThumbnail(); ok { + p.Thumbnail = thumbnail + } + // Empty tags list. if len(p.Tags) == 1 && p.Tags[0].Tag == "" { p.Tags = []TaggedPost{} @@ -347,6 +356,16 @@ func (p *Post) ParseForm(form *forms.Data) { } } +// ExtractThumbnail searches and returns a thumbnail image to represent the post. +// It will be the first image embedded in the post body, or nothing. +func (p Post) ExtractThumbnail() (string, bool) { + result := ThumbnailImageRegexp.FindStringSubmatch(p.Body) + if len(result) < 2 { + return "", false + } + return result[1], true +} + // TagsString turns the post tags into a comma separated string. func (p Post) TagsString() string { console.Error("TagsString: %+v", p.Tags) diff --git a/pkg/responses/filesystem.go b/pkg/responses/filesystem.go index e944c5c..b4d2d33 100644 --- a/pkg/responses/filesystem.go +++ b/pkg/responses/filesystem.go @@ -2,28 +2,31 @@ package responses import ( "errors" + "fmt" "io/ioutil" "os" + "path/filepath" "strings" "git.kirsle.net/apps/gophertype/pkg/bundled" + "git.kirsle.net/apps/gophertype/pkg/settings" ) // GetFile returns the template file's data, wherever it is. // Checks the embedded bindata, then the user root on disk, then error. // If it can be found, returns the contents or error. func GetFile(path string) ([]byte, error) { - // Check bindata. + // Check the user root first. + if b, err := ioutil.ReadFile(filepath.Join(settings.UserRoot, path)); err == nil { + return b, nil + } + + // Fall back on embedded bindata. if b, err := bundled.Asset(path); err == nil { return b, nil } - // Check the filesystem. TODO - if b, err := ioutil.ReadFile("./pvt-www/" + path); err == nil { - return b, nil - } else { - return []byte{}, err - } + return []byte{}, fmt.Errorf("GetFile(%s): not found in user root or bindata", path) } // GetFileExists checks if the file exists but doesn't return its data. @@ -33,8 +36,8 @@ func GetFileExists(path string) bool { return true } - // Check the filesystem. TODO - if _, err := os.Stat(path); err == nil { + // Check the user root. + if stat, err := os.Stat(filepath.Join(settings.UserRoot, path)); err == nil && !stat.IsDir() { return true } diff --git a/pkg/responses/static_file.go b/pkg/responses/static_file.go index ed85b7f..0c96a32 100644 --- a/pkg/responses/static_file.go +++ b/pkg/responses/static_file.go @@ -6,6 +6,8 @@ import ( "path/filepath" "git.kirsle.net/apps/gophertype/pkg/bundled" + "git.kirsle.net/apps/gophertype/pkg/console" + "git.kirsle.net/apps/gophertype/pkg/settings" ) // SendFile sends a file from bindata or the user root. @@ -22,5 +24,6 @@ func SendFile(w http.ResponseWriter, r *http.Request, path string) { return } - http.ServeFile(w, r, "pvt-www/"+path) + console.Debug("SendFile: http.ServeFile(%s)", filepath.Join(settings.UserRoot, path)) + http.ServeFile(w, r, filepath.Join(settings.UserRoot, path)) } diff --git a/pkg/responses/template_vars.go b/pkg/responses/template_vars.go index c665b17..8c20d4e 100644 --- a/pkg/responses/template_vars.go +++ b/pkg/responses/template_vars.go @@ -19,6 +19,8 @@ func NewTemplateVars(w io.Writer, r *http.Request) TemplateValues { var s = settings.Current user, _ := authentication.CurrentUser(r) + ses := session.Get(r) + v := TemplateValues{ SetupNeeded: !s.Initialized, @@ -28,6 +30,7 @@ func NewTemplateVars(w io.Writer, r *http.Request) TemplateValues { Request: r, RequestTime: time.Now(), RequestDuration: time.Duration(0), + Session: ses, Path: r.URL.Path, LoggedIn: authentication.LoggedIn(r), diff --git a/pkg/routes.go b/pkg/routes.go index 395a65d..0c70fdd 100644 --- a/pkg/routes.go +++ b/pkg/routes.go @@ -19,6 +19,7 @@ func (s *Site) SetupRouter() error { router.Use(session.Middleware) router.Use(authentication.Middleware) router.Use(middleware.CSRF) + router.Use(middleware.AgeGate) console.Debug("Setting up HTTP Router") for _, route := range glue.GetControllers() { diff --git a/pkg/settings/settings.go b/pkg/settings/settings.go index 141ee99..fca59d6 100644 --- a/pkg/settings/settings.go +++ b/pkg/settings/settings.go @@ -19,6 +19,9 @@ import ( // key. The config is not saved to DB until you call Save() on it. var Current = Load() +// UserRoot is the folder path to the user web files. +var UserRoot string + // Spec singleton holds the app configuration. type Spec struct { // Sets to `true` when the site's initial setup has run and an admin created. @@ -88,6 +91,7 @@ func SetFilename(userRoot string) error { } Current = spec + UserRoot = userRoot session.SetSecretKey([]byte(Current.SecretKey)) return nil } diff --git a/pvt-www/_builtin/age-gate.gohtml b/pvt-www/_builtin/age-gate.gohtml new file mode 100644 index 0000000..fb03713 --- /dev/null +++ b/pvt-www/_builtin/age-gate.gohtml @@ -0,0 +1,34 @@ +{{ define "title" }}Age Verification{{ end }} +{{ define "content" }} +
+
+
+ {{ CSRF }} + + + +

Restricted Content

+ +

+ This website has been marked NSFW + by its owner. It may contain nudity or content not suited for users + under the age of 18. +

+ +

+ To proceed, you must verify you are at least 18 years or older. +

+ + + + Get me out of here! + + +
+
+
+{{ end }} diff --git a/pvt-www/_builtin/blog/view-post.gohtml b/pvt-www/_builtin/blog/view-post.gohtml index 687b48d..227e29c 100644 --- a/pvt-www/_builtin/blog/view-post.gohtml +++ b/pvt-www/_builtin/blog/view-post.gohtml @@ -43,6 +43,7 @@ +{{ .CurrentUser }} {{ if .CurrentUser.IsAdmin }}
diff --git a/pvt-www/_builtin/comments/recent.gohtml b/pvt-www/_builtin/comments/recent.gohtml index 01b418a..fcdfeba 100644 --- a/pvt-www/_builtin/comments/recent.gohtml +++ b/pvt-www/_builtin/comments/recent.gohtml @@ -4,7 +4,21 @@

Recent Comments

{{ with .V.PagedComments }} -

+ {{ range .Comments }} + {{ if gt .PostID 0 }} +

+ In post {{ or .Post.Title "Untitled" }}: +
+ {{ else if .OriginURL }} +
+ On page {{ .OriginURL }}: +
+ {{ end }} + + {{ RenderComment $.ResponseWriter $.Request . "/comments" false }} + {{ end }} + +
Page {{ .Page }} of {{ .Pages }} ({{ .Total }} total) {{ if or (gt .PreviousPage 0) (gt .NextPage 0) }} [ @@ -17,21 +31,7 @@ {{ end }} ] {{ end }} -

- - {{ range .Comments }} - {{ if gt .PostID 0 }} -

- In post {{ or .Post.Title "Untitled" }}: -

- {{ else if .OriginURL }} -

- On page {{ .OriginURL }}: -

- {{ end }} - - {{ RenderComment $.ResponseWriter $.Request . "/comments" false }} - {{ end }} +
{{ end }} {{ end }}