Age Gate, Legacy kirsle/blog Migration Program

* Add the Age Gate middleware for NSFW sites.
* Cache thumbnail images from blog entries.
* Implement the user-root properly for loading web assets.

.gitignore

@@ -1,5 +1,4 @@
 bin/
-pvt-www/static/photos/
+public_html/
 pkg/bundled/
-public_html/.settings.json
 *.sqlite

cmd/legacy-import/main.go

@@ -0,0 +1,209 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"git.kirsle.net/apps/gophertype/pkg"
+	"git.kirsle.net/apps/gophertype/pkg/console"
+	_ "git.kirsle.net/apps/gophertype/pkg/controllers"
+	"git.kirsle.net/apps/gophertype/pkg/models"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	jsondb "github.com/kirsle/blog/jsondb"
+	mComments "github.com/kirsle/blog/models/comments"
+	mPosts "github.com/kirsle/blog/models/posts"
+)
+
+// Command-line flags.
+var (
+	optSourceRoot string
+	optRoot       string
+
+	// Database option flags.
+	optSQLite   string
+	optPostgres string
+	optMySQL    string
+
+	// Chosen DB options.
+	dbDriver string
+	dbPath   string
+)
+
+// Other global variables
+var (
+	JsonDB *jsondb.DB
+)
+
+func init() {
+	flag.StringVar(&optSourceRoot, "srcroot", "", "User root from old kirsle/blog website")
+	flag.StringVar(&optRoot, "root", "", "User root for GopherType")
+
+	// Database driver. Choose one.
+	flag.StringVar(&optSQLite, "sqlite3", "", "Use SQLite database, default 'database.sqlite'")
+	flag.StringVar(&optPostgres, "postgres", "",
+		"Use Postgres database, format: "+
+			"host=myhost port=myport user=gorm dbname=gorm password=mypassword")
+	flag.StringVar(&optMySQL, "mysql", "",
+		"Use MySQL database, format: "+
+			"user:password@/dbname?charset=utf8&parseTime=True&loc=Local")
+}
+
+func main() {
+	console.SetDebug(false)
+	flag.Parse()
+
+	// Validate the choice of database.
+	if optSQLite != "" {
+		dbDriver = "sqlite3"
+		dbPath = optSQLite
+	} else if optPostgres != "" {
+		dbDriver = "postgres"
+		dbPath = optPostgres
+	} else if optMySQL != "" {
+		dbDriver = "mysql"
+		dbPath = optMySQL
+	} else {
+		fmt.Print(
+			"Specify a DB driver for Gophertype, similar to the gophertype command.",
+		)
+		os.Exit(1)
+	}
+
+	// Validate the roots are given.
+	if optSourceRoot == "" {
+		fmt.Print(
+			"Missing -srcroot parameter: this should be the User Root from the legacy kirsle/blog site.",
+		)
+		os.Exit(1)
+	}
+	if optRoot == "" {
+		fmt.Print(
+			"Missing required -root parameter: this is the User Root for Gophertype",
+		)
+	}
+
+	// Initialize the old JsonDB.
+
+	app := gophertype.NewSite(optRoot)
+	app.UseDB(dbDriver, dbPath)
+
+	initJsonDB()
+	doMigrate()
+}
+
+// Initialize the old kirsle/blog JsonDB.
+func initJsonDB() {
+	JsonDB = jsondb.New(filepath.Join(optSourceRoot, ".private"))
+	mPosts.DB = JsonDB
+	mComments.DB = JsonDB
+}
+
+// doMigrate is the head of the migrate functions.
+func doMigrate() {
+	migratePosts()
+	migrateComments()
+}
+
+// migratePosts migrates blog posts over.
+func migratePosts() {
+	console.Warn("BEGIN: Migrating blog posts")
+	idx, err := mPosts.GetIndex()
+	if err != nil {
+		panic("migratePosts: GetIndex: " + err.Error())
+	}
+
+	// Sort the IDs to make it pretty.
+	var sortedIds = []int{}
+	for id := range idx.Posts {
+		sortedIds = append(sortedIds, id)
+	}
+	sort.Ints(sortedIds)
+
+	for _, id := range sortedIds {
+		post, err := mPosts.Load(id)
+		if err != nil {
+			panic(fmt.Sprintf("migratePosts: error loading legacy post ID %d: %s", id, err))
+		}
+
+		console.Info("Post ID %d: %s", id, post.Title)
+
+		// Create the post in Gophertype.
+		p := models.Post{
+			Title:          post.Title,
+			Fragment:       post.Fragment,
+			ContentType:    post.ContentType,
+			AuthorID:       uint(post.AuthorID),
+			Body:           post.Body,
+			Privacy:        post.Privacy,
+			Sticky:         post.Sticky,
+			EnableComments: post.EnableComments,
+		}
+		p.ID = uint(post.ID)
+		p.CreatedAt = post.Created
+		p.UpdatedAt = post.Updated
+
+		// Convert tags.
+		for _, tag := range post.Tags {
+			p.Tags = append(p.Tags, models.TaggedPost{
+				Tag: tag,
+			})
+		}
+
+		err = p.Save()
+		if err != nil {
+			console.Error("Error saving post %d: %s", p.ID, err)
+		}
+	}
+
+	console.Warn("FINISH: Migrating blog posts")
+}
+
+// migrateComments migrates comments over.
+func migrateComments() {
+	console.Warn("BEGIN: Migrating comments")
+
+	// Find all the comment threads.
+	files, err := JsonDB.List("comments/threads")
+	if err != nil {
+		panic("Error listing comments: " + err.Error())
+	}
+	for _, file := range files {
+		document := filepath.Base(file)
+		thread, err := mComments.Load(document)
+		if err != nil {
+			panic(fmt.Sprintf("Error loading comment thread %s: %s", file, err))
+		}
+
+		for _, comment := range thread.Comments {
+			// Migrate to the new model.
+			com := models.Comment{
+				Thread:      thread.ID,
+				UserID:      uint(comment.UserID),
+				Name:        comment.Name,
+				Email:       comment.Email,
+				Avatar:      comment.Avatar,
+				Body:        comment.Body,
+				EditToken:   comment.EditToken,
+				DeleteToken: comment.DeleteToken,
+			}
+			com.CreatedAt = comment.Created
+			com.UpdatedAt = comment.Updated
+
+			// Special case for guestbook page.
+			if thread.ID == "guestbook" {
+				com.OriginURL = "/guestbook"
+			}
+
+			if err := com.Save(); err != nil {
+				console.Error("Error saving comment: %s", err)
+			}
+		}
+	}
+
+	console.Warn("FINISH: Migrating comments")
+}

pkg/controllers/age_gate.go

@@ -0,0 +1,43 @@
+package controllers
+
+import (
+	"net/http"
+
+	"git.kirsle.net/apps/gophertype/pkg/glue"
+	"git.kirsle.net/apps/gophertype/pkg/responses"
+	"git.kirsle.net/apps/gophertype/pkg/session"
+)
+
+func init() {
+	glue.Register(glue.Endpoint{
+		Path:    "/age-verify",
+		Methods: []string{"GET", "POST"},
+		Handler: AgeVerify,
+	})
+}
+
+// AgeVerify handles the age gate prompt page for NSFW sites.
+func AgeVerify(w http.ResponseWriter, r *http.Request) {
+	var (
+		v       = responses.NewTemplateVars(w, r)
+		next    = r.FormValue("next")
+		confirm = r.FormValue("confirm")
+	)
+
+	if next == "" {
+		next = "/"
+	}
+	v.V["Next"] = next
+
+	if r.Method == http.MethodPost {
+		if confirm == "true" {
+			session := session.Get(r)
+			session.Values["age-ok"] = true
+			session.Save(r, w)
+			responses.Redirect(w, r, next)
+			return
+		}
+	}
+
+	responses.RenderTemplate(w, r, "_builtin/age-gate.gohtml", v)
+}

pkg/controllers/uploads.go

@@ -16,6 +16,7 @@ import (
 	"git.kirsle.net/apps/gophertype/pkg/console"
 	"git.kirsle.net/apps/gophertype/pkg/glue"
 	"git.kirsle.net/apps/gophertype/pkg/responses"
+	"git.kirsle.net/apps/gophertype/pkg/settings"
 	"github.com/edwvee/exiffix"
 	"github.com/nfnt/resize"
 )
@@ -24,7 +25,9 @@ import (
 var (
 	MaxImageWidth = 1280
 	JpegQuality   = 90
-	ImagePath     = "static/photos" // images folder for upload, relative to web root.
+
+	// images folder for upload, relative to web root.
+	ImagePath = filepath.Join("static", "photos")
 )
 
 func init() {
@@ -99,14 +102,14 @@ func UploadHandler(w http.ResponseWriter, r *http.Request) {
 	console.Info("Uploaded file named: %s  name=%s", header.Filename, filename)
 
 	// Write to the /static/photos directory of the user root. Ensure the path
-	// exists or create it if not. TODO
-	outputPath := filepath.Join("./pvt-www", "static", "photos")
+	// exists or create it if not.
+	outputPath := filepath.Join(settings.UserRoot, ImagePath)
 	if _, err := os.Stat(outputPath); os.IsNotExist(err) {
 		os.MkdirAll(outputPath, 0755)
 	}
 
 	// Ensure the filename is unique.
-	filename = uniqueFilename("./pvt-www/static/photos", filename)
+	filename = uniqueFilename(filepath.Join(settings.UserRoot, ImagePath), filename)
 
 	// Write the output file.
 	console.Info("Uploaded image: %s", filename)

pkg/middleware/age_gate.go

@@ -0,0 +1,80 @@
+package middleware
+
+import (
+	"net/http"
+	"strings"
+
+	"git.kirsle.net/apps/gophertype/pkg/responses"
+	"git.kirsle.net/apps/gophertype/pkg/session"
+	"git.kirsle.net/apps/gophertype/pkg/settings"
+)
+
+// URL suffixes to allow to bypass the age gate middleware.
+var ageGateSuffixes = []string{
+	"/blog.rss", // Allow public access to RSS and Atom feeds.
+	"/blog.atom",
+	"/blog.json",
+	".js",
+	".css",
+	".txt",
+	".ico",
+	".png",
+	".jpg",
+	".jpeg",
+	".gif",
+	".mp4",
+	".webm",
+	".ttf",
+	".eot",
+	".svg",
+	".woff",
+	".woff2",
+}
+
+// AgeGate is a middleware generator that does age verification for NSFW sites.
+// Single GET requests with ?over18=1 parameter may skip the middleware check.
+func AgeGate(next http.Handler) http.Handler {
+	middleware := func(w http.ResponseWriter, r *http.Request) {
+		s := settings.Current
+		if !s.NSFW {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		path := r.URL.Path
+
+		// Let the age-verify handler catch its route.
+		if strings.HasPrefix(path, "/age-verify") {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		// Allow static file requests to skip the check.
+		for _, suffix := range ageGateSuffixes {
+			if strings.HasSuffix(path, suffix) {
+				next.ServeHTTP(w, r)
+				return
+			}
+		}
+
+		// POST requests are permitted (e.g. post a comment on a /?over18=1 page)
+		if r.Method == http.MethodPost {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		// Finally, check if they've confirmed their age on the age-verify handler.
+		ses := session.Get(r)
+		if val, _ := ses.Values["age-ok"].(bool); !val {
+			// They haven't been verified. Redirect them to the age-verify handler.
+			if r.FormValue("over18") == "" {
+				responses.Redirect(w, r, "/age-verify?next="+path)
+				return
+			}
+		}
+
+		next.ServeHTTP(w, r)
+	}
+
+	return http.HandlerFunc(middleware)
+}

pkg/models/posts.go

@@ -29,6 +29,7 @@ type Post struct {
 	Fragment    string `gorm:"unique_index"`
 	ContentType string `gorm:"default:html"`
 	AuthorID    uint   // foreign key to User.ID
+	Thumbnail   string // image thumbnail for the post
 	Body        string
 	Privacy     string
 
@@ -52,6 +53,9 @@ type PagedPosts struct {
 	PreviousPage int
 }
 
+// Regexp for matching a thumbnail image for a blog post.
+var ThumbnailImageRegexp = regexp.MustCompile(`['"(]([a-zA-Z0-9-_:/?.=&]+\.(?:jpe?g|png|gif))['")]`)
+
 // New creates a new Post model.
 func (m postMan) New() Post {
 	return Post{
@@ -299,6 +303,11 @@ func (p *Post) Save() error {
 		}
 	}
 
+	// Cache the post thumbnail from the body.
+	if thumbnail, ok := p.ExtractThumbnail(); ok {
+		p.Thumbnail = thumbnail
+	}
+
 	// Empty tags list.
 	if len(p.Tags) == 1 && p.Tags[0].Tag == "" {
 		p.Tags = []TaggedPost{}
@@ -347,6 +356,16 @@ func (p *Post) ParseForm(form *forms.Data) {
 	}
 }
 
+// ExtractThumbnail searches and returns a thumbnail image to represent the post.
+// It will be the first image embedded in the post body, or nothing.
+func (p Post) ExtractThumbnail() (string, bool) {
+	result := ThumbnailImageRegexp.FindStringSubmatch(p.Body)
+	if len(result) < 2 {
+		return "", false
+	}
+	return result[1], true
+}
+
 // TagsString turns the post tags into a comma separated string.
 func (p Post) TagsString() string {
 	console.Error("TagsString: %+v", p.Tags)

pkg/responses/filesystem.go

@@ -2,28 +2,31 @@ package responses
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"git.kirsle.net/apps/gophertype/pkg/bundled"
+	"git.kirsle.net/apps/gophertype/pkg/settings"
 )
 
 // GetFile returns the template file's data, wherever it is.
 // Checks the embedded bindata, then the user root on disk, then error.
 // If it can be found, returns the contents or error.
 func GetFile(path string) ([]byte, error) {
-	// Check bindata.
-	if b, err := bundled.Asset(path); err == nil {
+	// Check the user root first.
+	if b, err := ioutil.ReadFile(filepath.Join(settings.UserRoot, path)); err == nil {
 		return b, nil
 	}
 
-	// Check the filesystem. TODO
-	if b, err := ioutil.ReadFile("./pvt-www/" + path); err == nil {
+	// Fall back on embedded bindata.
+	if b, err := bundled.Asset(path); err == nil {
 		return b, nil
-	} else {
-		return []byte{}, err
 	}
+
+	return []byte{}, fmt.Errorf("GetFile(%s): not found in user root or bindata", path)
 }
 
 // GetFileExists checks if the file exists but doesn't return its data.
@@ -33,8 +36,8 @@ func GetFileExists(path string) bool {
 		return true
 	}
 
-	// Check the filesystem. TODO
-	if _, err := os.Stat(path); err == nil {
+	// Check the user root.
+	if stat, err := os.Stat(filepath.Join(settings.UserRoot, path)); err == nil && !stat.IsDir() {
 		return true
 	}
 

pkg/responses/static_file.go

@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 
 	"git.kirsle.net/apps/gophertype/pkg/bundled"
+	"git.kirsle.net/apps/gophertype/pkg/console"
+	"git.kirsle.net/apps/gophertype/pkg/settings"
 )
 
 // SendFile sends a file from bindata or the user root.
@@ -22,5 +24,6 @@ func SendFile(w http.ResponseWriter, r *http.Request, path string) {
 		return
 	}
 
-	http.ServeFile(w, r, "pvt-www/"+path)
+	console.Debug("SendFile: http.ServeFile(%s)", filepath.Join(settings.UserRoot, path))
+	http.ServeFile(w, r, filepath.Join(settings.UserRoot, path))
 }

pkg/responses/template_vars.go

@@ -19,6 +19,8 @@ func NewTemplateVars(w io.Writer, r *http.Request) TemplateValues {
 	var s = settings.Current
 	user, _ := authentication.CurrentUser(r)
 
+	ses := session.Get(r)
+
 	v := TemplateValues{
 		SetupNeeded: !s.Initialized,
 
@@ -28,6 +30,7 @@ func NewTemplateVars(w io.Writer, r *http.Request) TemplateValues {
 		Request:         r,
 		RequestTime:     time.Now(),
 		RequestDuration: time.Duration(0),
+		Session:         ses,
 		Path:            r.URL.Path,
 
 		LoggedIn:    authentication.LoggedIn(r),

pkg/routes.go

@@ -19,6 +19,7 @@ func (s *Site) SetupRouter() error {
 	router.Use(session.Middleware)
 	router.Use(authentication.Middleware)
 	router.Use(middleware.CSRF)
+	router.Use(middleware.AgeGate)
 
 	console.Debug("Setting up HTTP Router")
 	for _, route := range glue.GetControllers() {

pkg/settings/settings.go

@@ -19,6 +19,9 @@ import (
 // key. The config is not saved to DB until you call Save() on it.
 var Current = Load()
 
+// UserRoot is the folder path to the user web files.
+var UserRoot string
+
 // Spec singleton holds the app configuration.
 type Spec struct {
 	// Sets to `true` when the site's initial setup has run and an admin created.
@@ -88,6 +91,7 @@ func SetFilename(userRoot string) error {
 	}
 
 	Current = spec
+	UserRoot = userRoot
 	session.SetSecretKey([]byte(Current.SecretKey))
 	return nil
 }

pvt-www/_builtin/age-gate.gohtml

@@ -0,0 +1,34 @@
+{{ define "title" }}Age Verification{{ end }}
+{{ define "content" }}
+<div class="card">
+    <div class="card-body">
+        <form action="/age-verify" method="POST">
+        {{ CSRF }}
+        <input type="hidden" name="next" value="{{ .V.Next }}">
+        <input type="hidden" name="confirm" value="true">
+
+        <h1>Restricted Content</h1>
+
+        <p>
+            This website has been marked <abbr title="Not Safe For Work">NSFW</abbr>
+            by its owner. It may contain nudity or content not suited for users
+            under the age of 18.
+        </p>
+
+        <p>
+            To proceed, you must verify you are at least 18 years or older.
+        </p>
+
+        <button type="submit"
+            class="btn btn-danger">
+            I am 18 years or older
+        </button>
+        <a class="btn btn-primary"
+            href="https://duckduckgo.com/">
+            Get me out of here!
+        </a>
+
+        </form>
+    </div>
+</div>
+{{ end }}

pvt-www/_builtin/blog/view-post.gohtml

@@ -43,6 +43,7 @@
 	</div>
 </div>
 
+{{ .CurrentUser }}
 
 {{ if .CurrentUser.IsAdmin }}
 <div class="alert alert-secondary">

pvt-www/_builtin/comments/recent.gohtml

@@ -4,7 +4,21 @@
 <h1>Recent Comments</h1>
 
 {{ with .V.PagedComments }}
-	<p>
+	{{ range .Comments }}
+		{{ if gt .PostID 0 }}
+			<div class="alert alert-info">
+				<strong>In post <a href="{{ .Post.Fragment }}">{{ or .Post.Title "Untitled" }}</a>:</strong>
+			</div>
+		{{ else if .OriginURL }}
+			<div class="alert alert-info">
+				<strong>On page <a href="{{ .OriginURL }}">{{ .OriginURL }}</a>:</strong>
+			</div>
+		{{ end }}
+
+		{{ RenderComment $.ResponseWriter $.Request . "/comments" false }}
+	{{ end }}
+
+	<div class="alert alert-success">
 		Page {{ .Page }} of {{ .Pages }} ({{ .Total }} total)
 		{{ if or (gt .PreviousPage 0) (gt .NextPage 0) }}
 		[
@@ -17,21 +31,7 @@
 			{{ end }}
 		]
 		{{ end }}
-	</p>
-
-	{{ range .Comments }}
-		{{ if gt .PostID 0 }}
-			<p>
-				<strong>In post <a href="{{ .Post.Fragment }}">{{ or .Post.Title "Untitled" }}</a>:</strong>
-			</p>
-		{{ else if .OriginURL }}
-			<p>
-				<strong>On page <a href="{{ .OriginURL }}">{{ .OriginURL }}</a>:</strong>
-			</p>
-		{{ end }}
-
-		{{ RenderComment $.ResponseWriter $.Request . "/comments" false }}
-	{{ end }}
+	</div>
 {{ end }}
 
 {{ end }}