From c03a0106962a05a615de8d658d8226576be4c9c2 Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Sat, 23 Dec 2017 13:22:51 -0800 Subject: [PATCH] Redis caches for JsonDB and Pygments --- core/app.go | 25 ++++++++++- core/caches/caches.go | 11 +++++ core/caches/null/null.go | 35 +++++++++++++++ core/caches/redis/redis.go | 88 ++++++++++++++++++++++++++++++++++++++ core/jsondb/cache.go | 86 +++++++++++++++++++++++++++++++++++++ core/jsondb/jsondb.go | 64 ++++++++++++++++++++++++--- core/markdown.go | 32 +++++++++++--- core/middleware.go | 13 +++++- core/templates.go | 18 +++++--- 9 files changed, 352 insertions(+), 20 deletions(-) create mode 100644 core/caches/caches.go create mode 100644 core/caches/null/null.go create mode 100644 core/caches/redis/redis.go create mode 100644 core/jsondb/cache.go diff --git a/core/app.go b/core/app.go index 757706d..256c307 100644 --- a/core/app.go +++ b/core/app.go @@ -1,11 +1,15 @@ package core import ( + "fmt" "net/http" "path/filepath" "github.com/gorilla/mux" "github.com/gorilla/sessions" + "github.com/kirsle/blog/core/caches" + "github.com/kirsle/blog/core/caches/null" + "github.com/kirsle/blog/core/caches/redis" "github.com/kirsle/blog/core/jsondb" "github.com/kirsle/blog/core/models/comments" "github.com/kirsle/blog/core/models/posts" @@ -24,7 +28,8 @@ type Blog struct { DocumentRoot string UserRoot string - DB *jsondb.DB + DB *jsondb.DB + Cache caches.Cacher // Web app objects. n *negroni.Negroni // Negroni middleware manager @@ -38,6 +43,7 @@ func New(documentRoot, userRoot string) *Blog { DocumentRoot: documentRoot, UserRoot: userRoot, DB: jsondb.New(filepath.Join(userRoot, ".private")), + Cache: null.New(), } // Load the site config, or start with defaults if not found. @@ -56,6 +62,23 @@ func New(documentRoot, userRoot string) *Blog { users.DB = blog.DB comments.DB = blog.DB + // Redis cache? + if config.Redis.Enabled { + addr := fmt.Sprintf("%s:%d", config.Redis.Host, config.Redis.Port) + log.Info("Connecting to Redis at %s/%d", addr, config.Redis.DB) + cache, err := redis.New( + addr, + config.Redis.DB, + config.Redis.Prefix, + ) + if err != nil { + log.Error("Redis init error: %s", err.Error()) + } else { + blog.Cache = cache + blog.DB.Cache = cache + } + } + // Initialize the router. r := mux.NewRouter() r.HandleFunc("/initial-setup", blog.SetupHandler) diff --git a/core/caches/caches.go b/core/caches/caches.go new file mode 100644 index 0000000..d2c8fcb --- /dev/null +++ b/core/caches/caches.go @@ -0,0 +1,11 @@ +package caches + +// Cacher is an interface for a key/value cacher. +type Cacher interface { + Get(key string) ([]byte, error) + Set(key string, v []byte, expires int) error + Delete(key ...string) + Keys(pattern string) ([]string, error) + Lock(key, value string, expires int) bool + Unlock(key string) +} diff --git a/core/caches/null/null.go b/core/caches/null/null.go new file mode 100644 index 0000000..8271223 --- /dev/null +++ b/core/caches/null/null.go @@ -0,0 +1,35 @@ +package null + +// Null is a cache that doesn't do anything. +type Null struct{} + +// New Null cache backend. +func New() *Null { + return &Null{} +} + +// Get a key from Null. +func (r *Null) Get(key string) ([]byte, error) { + return []byte{}, nil +} + +// Set a key in Null. +func (r *Null) Set(key string, v []byte, expires int) error { + return nil +} + +// Delete keys from Null. +func (r *Null) Delete(key ...string) {} + +// Keys returns a list of Null keys matching a pattern. +func (r *Null) Keys(pattern string) ([]string, error) { + return []string{}, nil +} + +// Lock a mutex. +func (r *Null) Lock(key string, value string, expires int) bool { + return true +} + +// Unlock a mutex. +func (r *Null) Unlock(key string) {} diff --git a/core/caches/redis/redis.go b/core/caches/redis/redis.go new file mode 100644 index 0000000..59a4757 --- /dev/null +++ b/core/caches/redis/redis.go @@ -0,0 +1,88 @@ +package redis + +import ( + "fmt" + "time" + + "github.com/garyburd/redigo/redis" +) + +// Redis is a cache backend. +type Redis struct { + pool *redis.Pool + prefix string +} + +// New Redis backend. +func New(address string, db int, prefix string) (*Redis, error) { + r := &Redis{ + prefix: prefix, + pool: &redis.Pool{ + MaxIdle: 3, + IdleTimeout: 240 * time.Second, + Dial: func() (redis.Conn, error) { + return redis.Dial("tcp", address, + redis.DialConnectTimeout(10*time.Second), + redis.DialDatabase(db), + redis.DialKeepAlive(30*time.Second), + ) + }, + }, + } + + return r, nil +} + +// Get a key from Redis. +func (r *Redis) Get(key string) ([]byte, error) { + conn := r.pool.Get() + + n, err := redis.Bytes(conn.Do("GET", r.prefix+key)) + if err != nil { + return nil, fmt.Errorf("Redis SET error: %s (conn error: %s)", err, conn.Err()) + } + return n, err +} + +// Set a key in Redis. +func (r *Redis) Set(key string, v []byte, expires int) error { + conn := r.pool.Get() + + _, err := conn.Do("SETEX", r.prefix+key, expires, v) + if err != nil { + return fmt.Errorf("Redis SET error: %s (conn error: %s)", err, conn.Err()) + } + return nil +} + +// Delete keys from Redis. +func (r *Redis) Delete(key ...string) { + conn := r.pool.Get() + + for _, v := range key { + conn.Send("DEL", v) + } + conn.Flush() + conn.Receive() +} + +// Keys returns a list of Redis keys matching a pattern. +func (r *Redis) Keys(pattern string) ([]string, error) { + conn := r.pool.Get() + + n, err := redis.Strings(conn.Do("KEYS", pattern)) + return n, err +} + +// Lock a mutex. +func (r *Redis) Lock(key, value string, expires int) bool { + conn := r.pool.Get() + + n, err := redis.Int(conn.Do("SETNX", r.prefix+key, value)) + return err == nil && n == 1 +} + +// Unlock a mutex. +func (r *Redis) Unlock(key string) { + r.Delete(key) +} diff --git a/core/jsondb/cache.go b/core/jsondb/cache.go new file mode 100644 index 0000000..f1d16cd --- /dev/null +++ b/core/jsondb/cache.go @@ -0,0 +1,86 @@ +package jsondb + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand" + "time" +) + +var errCacheDisabled = errors.New("cache disabled") + +// SetCache sets a cache key. +func (db *DB) SetCache(key, value string, expires int) error { + if db.Cache == nil { + return errCacheDisabled + } + return db.Cache.Set(key, []byte(value), expires) +} + +// SetJSONCache caches a JSON object. +func (db *DB) SetJSONCache(key string, v interface{}, expires int) error { + if db.Cache == nil { + return errCacheDisabled + } + bytes, err := json.Marshal(v) + if err != nil { + return err + } + return db.SetCache(key, string(bytes), expires) +} + +// GetCache gets a cache key. +func (db *DB) GetCache(key string) (string, error) { + if db.Cache == nil { + return "", errCacheDisabled + } + v, err := db.Cache.Get(key) + return string(v), err +} + +// DeleteCache deletes a cache key. +func (db *DB) DeleteCache(key string) error { + if db.Cache == nil { + return errCacheDisabled + } + db.Cache.Delete(key) + return nil +} + +// LockCache implements 'file locking' in your cache. +func (db *DB) LockCache(key string) bool { + if db.Cache == nil { + return true + } + + log.Info("LockCache(%s)", key) + + var ( + // In seconds + timeout = 5 * time.Second + expire = 20 + ) + + identifier := fmt.Sprintf("%d", rand.Uint64()) + log.Info("id: %s", identifier) + + end := time.Now().Add(timeout) + for time.Now().Before(end) { + if ok := db.Cache.Lock("lock:"+key, identifier, expire); ok { + log.Info("JsonDB: Acquired lock for %s", key) + return true + } + time.Sleep(1 * time.Millisecond) + } + log.Error("JsonDB: lock timeout for %s", key) + return false +} + +// UnlockCache releases the lock on a cache key. +func (db *DB) UnlockCache(key string) { + if db.Cache == nil { + return + } + db.Cache.Unlock("lock:" + key) +} diff --git a/core/jsondb/jsondb.go b/core/jsondb/jsondb.go index 445dd97..27167e0 100644 --- a/core/jsondb/jsondb.go +++ b/core/jsondb/jsondb.go @@ -9,15 +9,22 @@ import ( "os" "path/filepath" "strings" + "sync" + "time" + + "github.com/kirsle/blog/core/caches" +) + +var ( + // CacheTimeout is how long the Redis cache keys live for in seconds, default 2 hours. + CacheTimeout = 60 * 60 * 2 + CacheLock sync.RWMutex ) // DB is the database manager. type DB struct { - Root string // The root directory of the database - - // Use Redis to cache filesystem reads of the database. - EnableRedis bool - RedisURL string + Root string // The root directory of the database + Cache caches.Cacher // A cacher for the JSON documents, i.e. Redis } // Error codes returned. @@ -33,6 +40,12 @@ func New(root string) *DB { } } +// WithCache configures a memory cacher for the JSON documents. +func (db *DB) WithCache(cache caches.Cacher) *DB { + db.Cache = cache + return db +} + // Get a document by path and load it into the object `v`. func (db *DB) Get(document string, v interface{}) error { log.Debug("[JsonDB] GET %s", document) @@ -42,17 +55,45 @@ func (db *DB) Get(document string, v interface{}) error { // Get the file path and stats. path := db.toPath(document) - _, err := os.Stat(path) // TODO: mtime for caching + stat, err := os.Stat(path) if err != nil { return err } + // Do we have it cached? + data, err := db.GetCache(document) + if err == nil { + // Check if the cache is fresh. + cachedTime, err2 := db.GetCache(document + "_mtime") + if err2 == nil { + modTime := stat.ModTime() + mtime, _ := time.Parse(time.RFC3339Nano, cachedTime) + if modTime.After(mtime) && !modTime.Equal(mtime) { + log.Debug("[JsonDB] %s: On-disk file is newer than cache", document) + db.DeleteCache(document) + db.DeleteCache(document + "_mtime") + } else { + log.Debug("[JsonDB] %s: Returning cached copy", document) + return json.Unmarshal([]byte(data), v) + } + } + } + + // Get a lock for reading. + CacheLock.RLock() + // Read the JSON. err = db.readJSON(path, &v) if err != nil { + CacheLock.RUnlock() return err } + // Unlock & cache it. + db.SetJSONCache(document, v, CacheTimeout) + db.SetCache(document+"_mtime", stat.ModTime().Format(time.RFC3339Nano), CacheTimeout) + CacheLock.RUnlock() + return nil } @@ -61,18 +102,28 @@ func (db *DB) Commit(document string, v interface{}) error { log.Debug("[JsonDB] COMMIT %s", document) path := db.toPath(document) + // Get a write lock for the cache. + CacheLock.Lock() + // Ensure the directory tree is ready. err := db.makePath(path) if err != nil { + CacheLock.Unlock() return err } // Write the document. err = db.writeJSON(path, v) if err != nil { + CacheLock.Unlock() return fmt.Errorf("failed to write JSON to path %s: %s", path, err.Error()) } + // Unlock & cache it. + db.SetJSONCache(document, v, CacheTimeout) + db.SetCache(document+"_mtime", time.Now().Format(time.RFC3339Nano), CacheTimeout) + CacheLock.Unlock() + return nil } @@ -86,6 +137,7 @@ func (db *DB) Delete(document string) error { return nil } + db.DeleteCache(document) return os.Remove(path) } diff --git a/core/markdown.go b/core/markdown.go index e064475..be1448b 100644 --- a/core/markdown.go +++ b/core/markdown.go @@ -2,8 +2,10 @@ package core import ( "bytes" + "crypto/md5" "errors" "fmt" + "io" "os/exec" "regexp" "strings" @@ -85,7 +87,7 @@ func (b *Blog) RenderTrustedMarkdown(input string) string { // Substitute fenced codes back in. for _, block := range codeBlocks { - highlighted, _ := Pygmentize(block.language, block.source) + highlighted, _ := b.Pygmentize(block.language, block.source) html = strings.Replace(html, fmt.Sprintf("[?FENCED_CODE_%d_BLOCK?]", block.placeholder), highlighted, @@ -101,10 +103,24 @@ func (b *Blog) RenderTrustedMarkdown(input string) string { // // On error the original given source is returned back. // -// TODO: this takes ~0.6s per go, we need something faster. -func Pygmentize(language, source string) (string, error) { - bin := "pygmentize" +// The rendered result is cached in Redis if available, because the CLI +// call takes ~0.6s which is slow if you're rendering a lot of code blocks. +func (b *Blog) Pygmentize(language, source string) (string, error) { + var result string + // Hash the source for the cache key. + h := md5.New() + io.WriteString(h, language+source) + hash := fmt.Sprintf("%x", h.Sum(nil)) + cacheKey := "pygmentize:" + hash + + // Do we have it cached? + if cached, err := b.Cache.Get(cacheKey); err == nil { + return string(cached), nil + } + + // Defer to the `pygmentize` command + bin := "pygmentize" if _, err := exec.LookPath(bin); err != nil { return source, errors.New("pygmentize not installed") } @@ -123,5 +139,11 @@ func Pygmentize(language, source string) (string, error) { return source, err } - return out.String(), nil + result = out.String() + err := b.Cache.Set(cacheKey, []byte(result), 60*60*24) // cool md5's don't change + if err != nil { + log.Error("Couldn't cache Pygmentize output: %s", err) + } + + return result, nil } diff --git a/core/middleware.go b/core/middleware.go index 84aabc4..0fc8d8f 100644 --- a/core/middleware.go +++ b/core/middleware.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "time" "github.com/google/uuid" "github.com/gorilla/sessions" @@ -15,13 +16,23 @@ type key int const ( sessionKey key = iota userKey + requestTimeKey ) // SessionLoader gets the Gorilla session store and makes it available on the // Request context. +// +// SessionLoader is the first custom middleware applied, so it takes the current +// datetime to make available later in the request and stores it on the request +// context. func (b *Blog) SessionLoader(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + // Store the current datetime on the request context. + ctx := context.WithValue(r.Context(), requestTimeKey, time.Now()) + + // Get the Gorilla session and make it available in the request context. session, _ := b.store.Get(r, "session") - ctx := context.WithValue(r.Context(), sessionKey, session) + ctx = context.WithValue(ctx, sessionKey, session) + next(w, r.WithContext(ctx)) } diff --git a/core/templates.go b/core/templates.go index a3a5017..b490ff9 100644 --- a/core/templates.go +++ b/core/templates.go @@ -17,13 +17,15 @@ import ( // when the template is rendered. type Vars struct { // Global, "constant" template variables. - SetupNeeded bool - Title string - Path string - LoggedIn bool - CurrentUser *users.User - CSRF string - Request *http.Request + SetupNeeded bool + Title string + Path string + LoggedIn bool + CurrentUser *users.User + CSRF string + Request *http.Request + RequestTime time.Time + RequestDuration time.Duration // Configuration variables NoLayout bool // don't wrap in .layout.html, just render the template @@ -62,6 +64,7 @@ func (v *Vars) LoadDefaults(b *Blog, r *http.Request) { v.SetupNeeded = true } v.Request = r + v.RequestTime = r.Context().Value(requestTimeKey).(time.Time) v.Title = s.Site.Title v.Path = r.URL.Path @@ -176,6 +179,7 @@ func (b *Blog) RenderTemplate(w http.ResponseWriter, r *http.Request, path strin session.Save(r, w) } + vars.RequestDuration = time.Now().Sub(vars.RequestTime) vars.CSRF = b.GenerateCSRFToken(w, r, session) w.Header().Set("Content-Type", "text/html; encoding=UTF-8")