Redis caches for JsonDB and Pygments

core/app.go

@@ -1,11 +1,15 @@
 package core
 
 import (
+	"fmt"
 	"net/http"
 	"path/filepath"
 
 	"github.com/gorilla/mux"
 	"github.com/gorilla/sessions"
+	"github.com/kirsle/blog/core/caches"
+	"github.com/kirsle/blog/core/caches/null"
+	"github.com/kirsle/blog/core/caches/redis"
 	"github.com/kirsle/blog/core/jsondb"
 	"github.com/kirsle/blog/core/models/comments"
 	"github.com/kirsle/blog/core/models/posts"
@@ -24,7 +28,8 @@ type Blog struct {
 	DocumentRoot string
 	UserRoot     string
 
-	DB *jsondb.DB
+	DB    *jsondb.DB
+	Cache caches.Cacher
 
 	// Web app objects.
 	n     *negroni.Negroni // Negroni middleware manager
@@ -38,6 +43,7 @@ func New(documentRoot, userRoot string) *Blog {
 		DocumentRoot: documentRoot,
 		UserRoot:     userRoot,
 		DB:           jsondb.New(filepath.Join(userRoot, ".private")),
+		Cache:        null.New(),
 	}
 
 	// Load the site config, or start with defaults if not found.
@@ -56,6 +62,23 @@ func New(documentRoot, userRoot string) *Blog {
 	users.DB = blog.DB
 	comments.DB = blog.DB
 
+	// Redis cache?
+	if config.Redis.Enabled {
+		addr := fmt.Sprintf("%s:%d", config.Redis.Host, config.Redis.Port)
+		log.Info("Connecting to Redis at %s/%d", addr, config.Redis.DB)
+		cache, err := redis.New(
+			addr,
+			config.Redis.DB,
+			config.Redis.Prefix,
+		)
+		if err != nil {
+			log.Error("Redis init error: %s", err.Error())
+		} else {
+			blog.Cache = cache
+			blog.DB.Cache = cache
+		}
+	}
+
 	// Initialize the router.
 	r := mux.NewRouter()
 	r.HandleFunc("/initial-setup", blog.SetupHandler)

core/caches/caches.go

@@ -0,0 +1,11 @@
+package caches
+
+// Cacher is an interface for a key/value cacher.
+type Cacher interface {
+	Get(key string) ([]byte, error)
+	Set(key string, v []byte, expires int) error
+	Delete(key ...string)
+	Keys(pattern string) ([]string, error)
+	Lock(key, value string, expires int) bool
+	Unlock(key string)
+}

core/caches/null/null.go

@@ -0,0 +1,35 @@
+package null
+
+// Null is a cache that doesn't do anything.
+type Null struct{}
+
+// New Null cache backend.
+func New() *Null {
+	return &Null{}
+}
+
+// Get a key from Null.
+func (r *Null) Get(key string) ([]byte, error) {
+	return []byte{}, nil
+}
+
+// Set a key in Null.
+func (r *Null) Set(key string, v []byte, expires int) error {
+	return nil
+}
+
+// Delete keys from Null.
+func (r *Null) Delete(key ...string) {}
+
+// Keys returns a list of Null keys matching a pattern.
+func (r *Null) Keys(pattern string) ([]string, error) {
+	return []string{}, nil
+}
+
+// Lock a mutex.
+func (r *Null) Lock(key string, value string, expires int) bool {
+	return true
+}
+
+// Unlock a mutex.
+func (r *Null) Unlock(key string) {}

core/caches/redis/redis.go

@@ -0,0 +1,88 @@
+package redis
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/garyburd/redigo/redis"
+)
+
+// Redis is a cache backend.
+type Redis struct {
+	pool   *redis.Pool
+	prefix string
+}
+
+// New Redis backend.
+func New(address string, db int, prefix string) (*Redis, error) {
+	r := &Redis{
+		prefix: prefix,
+		pool: &redis.Pool{
+			MaxIdle:     3,
+			IdleTimeout: 240 * time.Second,
+			Dial: func() (redis.Conn, error) {
+				return redis.Dial("tcp", address,
+					redis.DialConnectTimeout(10*time.Second),
+					redis.DialDatabase(db),
+					redis.DialKeepAlive(30*time.Second),
+				)
+			},
+		},
+	}
+
+	return r, nil
+}
+
+// Get a key from Redis.
+func (r *Redis) Get(key string) ([]byte, error) {
+	conn := r.pool.Get()
+
+	n, err := redis.Bytes(conn.Do("GET", r.prefix+key))
+	if err != nil {
+		return nil, fmt.Errorf("Redis SET error: %s (conn error: %s)", err, conn.Err())
+	}
+	return n, err
+}
+
+// Set a key in Redis.
+func (r *Redis) Set(key string, v []byte, expires int) error {
+	conn := r.pool.Get()
+
+	_, err := conn.Do("SETEX", r.prefix+key, expires, v)
+	if err != nil {
+		return fmt.Errorf("Redis SET error: %s (conn error: %s)", err, conn.Err())
+	}
+	return nil
+}
+
+// Delete keys from Redis.
+func (r *Redis) Delete(key ...string) {
+	conn := r.pool.Get()
+
+	for _, v := range key {
+		conn.Send("DEL", v)
+	}
+	conn.Flush()
+	conn.Receive()
+}
+
+// Keys returns a list of Redis keys matching a pattern.
+func (r *Redis) Keys(pattern string) ([]string, error) {
+	conn := r.pool.Get()
+
+	n, err := redis.Strings(conn.Do("KEYS", pattern))
+	return n, err
+}
+
+// Lock a mutex.
+func (r *Redis) Lock(key, value string, expires int) bool {
+	conn := r.pool.Get()
+
+	n, err := redis.Int(conn.Do("SETNX", r.prefix+key, value))
+	return err == nil && n == 1
+}
+
+// Unlock a mutex.
+func (r *Redis) Unlock(key string) {
+	r.Delete(key)
+}

core/jsondb/cache.go

@@ -0,0 +1,86 @@
+package jsondb
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand"
+	"time"
+)
+
+var errCacheDisabled = errors.New("cache disabled")
+
+// SetCache sets a cache key.
+func (db *DB) SetCache(key, value string, expires int) error {
+	if db.Cache == nil {
+		return errCacheDisabled
+	}
+	return db.Cache.Set(key, []byte(value), expires)
+}
+
+// SetJSONCache caches a JSON object.
+func (db *DB) SetJSONCache(key string, v interface{}, expires int) error {
+	if db.Cache == nil {
+		return errCacheDisabled
+	}
+	bytes, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return db.SetCache(key, string(bytes), expires)
+}
+
+// GetCache gets a cache key.
+func (db *DB) GetCache(key string) (string, error) {
+	if db.Cache == nil {
+		return "", errCacheDisabled
+	}
+	v, err := db.Cache.Get(key)
+	return string(v), err
+}
+
+// DeleteCache deletes a cache key.
+func (db *DB) DeleteCache(key string) error {
+	if db.Cache == nil {
+		return errCacheDisabled
+	}
+	db.Cache.Delete(key)
+	return nil
+}
+
+// LockCache implements 'file locking' in your cache.
+func (db *DB) LockCache(key string) bool {
+	if db.Cache == nil {
+		return true
+	}
+
+	log.Info("LockCache(%s)", key)
+
+	var (
+		// In seconds
+		timeout = 5 * time.Second
+		expire  = 20
+	)
+
+	identifier := fmt.Sprintf("%d", rand.Uint64())
+	log.Info("id: %s", identifier)
+
+	end := time.Now().Add(timeout)
+	for time.Now().Before(end) {
+		if ok := db.Cache.Lock("lock:"+key, identifier, expire); ok {
+			log.Info("JsonDB: Acquired lock for %s", key)
+			return true
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	log.Error("JsonDB: lock timeout for %s", key)
+	return false
+}
+
+// UnlockCache releases the lock on a cache key.
+func (db *DB) UnlockCache(key string) {
+	if db.Cache == nil {
+		return
+	}
+	db.Cache.Unlock("lock:" + key)
+}

core/jsondb/jsondb.go

@@ -9,15 +9,22 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
+
+	"github.com/kirsle/blog/core/caches"
+)
+
+var (
+	// CacheTimeout is how long the Redis cache keys live for in seconds, default 2 hours.
+	CacheTimeout = 60 * 60 * 2
+	CacheLock    sync.RWMutex
 )
 
 // DB is the database manager.
 type DB struct {
-	Root string // The root directory of the database
+	Root  string        // The root directory of the database
-
+	Cache caches.Cacher // A cacher for the JSON documents, i.e. Redis
-	// Use Redis to cache filesystem reads of the database.
-	EnableRedis bool
-	RedisURL    string
 }
 
 // Error codes returned.
@@ -33,6 +40,12 @@ func New(root string) *DB {
 	}
 }
 
+// WithCache configures a memory cacher for the JSON documents.
+func (db *DB) WithCache(cache caches.Cacher) *DB {
+	db.Cache = cache
+	return db
+}
+
 // Get a document by path and load it into the object `v`.
 func (db *DB) Get(document string, v interface{}) error {
 	log.Debug("[JsonDB] GET %s", document)
@@ -42,17 +55,45 @@ func (db *DB) Get(document string, v interface{}) error {
 
 	// Get the file path and stats.
 	path := db.toPath(document)
-	_, err := os.Stat(path) // TODO: mtime for caching
+	stat, err := os.Stat(path)
 	if err != nil {
 		return err
 	}
 
+	// Do we have it cached?
+	data, err := db.GetCache(document)
+	if err == nil {
+		// Check if the cache is fresh.
+		cachedTime, err2 := db.GetCache(document + "_mtime")
+		if err2 == nil {
+			modTime := stat.ModTime()
+			mtime, _ := time.Parse(time.RFC3339Nano, cachedTime)
+			if modTime.After(mtime) && !modTime.Equal(mtime) {
+				log.Debug("[JsonDB] %s: On-disk file is newer than cache", document)
+				db.DeleteCache(document)
+				db.DeleteCache(document + "_mtime")
+			} else {
+				log.Debug("[JsonDB] %s: Returning cached copy", document)
+				return json.Unmarshal([]byte(data), v)
+			}
+		}
+	}
+
+	// Get a lock for reading.
+	CacheLock.RLock()
+
 	// Read the JSON.
 	err = db.readJSON(path, &v)
 	if err != nil {
+		CacheLock.RUnlock()
 		return err
 	}
 
+	// Unlock & cache it.
+	db.SetJSONCache(document, v, CacheTimeout)
+	db.SetCache(document+"_mtime", stat.ModTime().Format(time.RFC3339Nano), CacheTimeout)
+	CacheLock.RUnlock()
+
 	return nil
 }
 
@@ -61,18 +102,28 @@ func (db *DB) Commit(document string, v interface{}) error {
 	log.Debug("[JsonDB] COMMIT %s", document)
 	path := db.toPath(document)
 
+	// Get a write lock for the cache.
+	CacheLock.Lock()
+
 	// Ensure the directory tree is ready.
 	err := db.makePath(path)
 	if err != nil {
+		CacheLock.Unlock()
 		return err
 	}
 
 	// Write the document.
 	err = db.writeJSON(path, v)
 	if err != nil {
+		CacheLock.Unlock()
 		return fmt.Errorf("failed to write JSON to path %s: %s", path, err.Error())
 	}
 
+	// Unlock & cache it.
+	db.SetJSONCache(document, v, CacheTimeout)
+	db.SetCache(document+"_mtime", time.Now().Format(time.RFC3339Nano), CacheTimeout)
+	CacheLock.Unlock()
+
 	return nil
 }
 
@@ -86,6 +137,7 @@ func (db *DB) Delete(document string) error {
 		return nil
 	}
 
+	db.DeleteCache(document)
 	return os.Remove(path)
 }
 

core/markdown.go

@@ -2,8 +2,10 @@ package core
 
 import (
 	"bytes"
+	"crypto/md5"
 	"errors"
 	"fmt"
+	"io"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -85,7 +87,7 @@ func (b *Blog) RenderTrustedMarkdown(input string) string {
 
 	// Substitute fenced codes back in.
 	for _, block := range codeBlocks {
-		highlighted, _ := Pygmentize(block.language, block.source)
+		highlighted, _ := b.Pygmentize(block.language, block.source)
 		html = strings.Replace(html,
 			fmt.Sprintf("[?FENCED_CODE_%d_BLOCK?]", block.placeholder),
 			highlighted,
@@ -101,10 +103,24 @@ func (b *Blog) RenderTrustedMarkdown(input string) string {
 //
 // On error the original given source is returned back.
 //
-// TODO: this takes ~0.6s per go, we need something faster.
+// The rendered result is cached in Redis if available, because the CLI
-func Pygmentize(language, source string) (string, error) {
+// call takes ~0.6s which is slow if you're rendering a lot of code blocks.
-	bin := "pygmentize"
+func (b *Blog) Pygmentize(language, source string) (string, error) {
+	var result string
 
+	// Hash the source for the cache key.
+	h := md5.New()
+	io.WriteString(h, language+source)
+	hash := fmt.Sprintf("%x", h.Sum(nil))
+	cacheKey := "pygmentize:" + hash
+
+	// Do we have it cached?
+	if cached, err := b.Cache.Get(cacheKey); err == nil {
+		return string(cached), nil
+	}
+
+	// Defer to the `pygmentize` command
+	bin := "pygmentize"
 	if _, err := exec.LookPath(bin); err != nil {
 		return source, errors.New("pygmentize not installed")
 	}
@@ -123,5 +139,11 @@ func Pygmentize(language, source string) (string, error) {
 		return source, err
 	}
 
-	return out.String(), nil
+	result = out.String()
+	err := b.Cache.Set(cacheKey, []byte(result), 60*60*24) // cool md5's don't change
+	if err != nil {
+		log.Error("Couldn't cache Pygmentize output: %s", err)
+	}
+
+	return result, nil
 }

core/middleware.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/sessions"
@@ -15,13 +16,23 @@ type key int
 const (
 	sessionKey key = iota
 	userKey
+	requestTimeKey
 )
 
 // SessionLoader gets the Gorilla session store and makes it available on the
 // Request context.
+//
+// SessionLoader is the first custom middleware applied, so it takes the current
+// datetime to make available later in the request and stores it on the request
+// context.
 func (b *Blog) SessionLoader(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
+	// Store the current datetime on the request context.
+	ctx := context.WithValue(r.Context(), requestTimeKey, time.Now())
+
+	// Get the Gorilla session and make it available in the request context.
 	session, _ := b.store.Get(r, "session")
-	ctx := context.WithValue(r.Context(), sessionKey, session)
+	ctx = context.WithValue(ctx, sessionKey, session)
+
 	next(w, r.WithContext(ctx))
 }
 

core/templates.go

@@ -17,13 +17,15 @@ import (
 // when the template is rendered.
 type Vars struct {
 	// Global, "constant" template variables.
-	SetupNeeded bool
+	SetupNeeded     bool
-	Title       string
+	Title           string
-	Path        string
+	Path            string
-	LoggedIn    bool
+	LoggedIn        bool
-	CurrentUser *users.User
+	CurrentUser     *users.User
-	CSRF        string
+	CSRF            string
-	Request     *http.Request
+	Request         *http.Request
+	RequestTime     time.Time
+	RequestDuration time.Duration
 
 	// Configuration variables
 	NoLayout bool // don't wrap in .layout.html, just render the template
@@ -62,6 +64,7 @@ func (v *Vars) LoadDefaults(b *Blog, r *http.Request) {
 		v.SetupNeeded = true
 	}
 	v.Request = r
+	v.RequestTime = r.Context().Value(requestTimeKey).(time.Time)
 	v.Title = s.Site.Title
 	v.Path = r.URL.Path
 
@@ -176,6 +179,7 @@ func (b *Blog) RenderTemplate(w http.ResponseWriter, r *http.Request, path strin
 		session.Save(r, w)
 	}
 
+	vars.RequestDuration = time.Now().Sub(vars.RequestTime)
 	vars.CSRF = b.GenerateCSRFToken(w, r, session)
 
 	w.Header().Set("Content-Type", "text/html; encoding=UTF-8")