kirsle
/
blog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Redis caches for JsonDB and Pygments

pull/4/head
Noah 2017-12-23 13:22:51 -08:00
parent 94cdc916ac
commit c03a010696
9 changed files with 352 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
core/app.go
View File

@ -1,11 +1,15 @@
package core
import (
"fmt"
"net/http"
"path/filepath"
"github.com/gorilla/mux"
"github.com/gorilla/sessions"
"github.com/kirsle/blog/core/caches"
"github.com/kirsle/blog/core/caches/null"
"github.com/kirsle/blog/core/caches/redis"
"github.com/kirsle/blog/core/jsondb"
"github.com/kirsle/blog/core/models/comments"
"github.com/kirsle/blog/core/models/posts"
@ -24,7 +28,8 @@ type Blog struct {
DocumentRoot string
UserRoot string
DB *jsondb.DB
DB *jsondb.DB
Cache caches.Cacher
// Web app objects.
n *negroni.Negroni // Negroni middleware manager
@ -38,6 +43,7 @@ func New(documentRoot, userRoot string) *Blog {
DocumentRoot: documentRoot,
UserRoot: userRoot,
DB: jsondb.New(filepath.Join(userRoot, ".private")),
Cache: null.New(),
}
// Load the site config, or start with defaults if not found.
@ -56,6 +62,23 @@ func New(documentRoot, userRoot string) *Blog {
users.DB = blog.DB
comments.DB = blog.DB
// Redis cache?
if config.Redis.Enabled {
addr := fmt.Sprintf("%s:%d", config.Redis.Host, config.Redis.Port)
log.Info("Connecting to Redis at %s/%d", addr, config.Redis.DB)
cache, err := redis.New(
addr,
config.Redis.DB,
config.Redis.Prefix,
)
if err != nil {
log.Error("Redis init error: %s", err.Error())
} else {
blog.Cache = cache
blog.DB.Cache = cache
}
}
// Initialize the router.
r := mux.NewRouter()
r.HandleFunc("/initial-setup", blog.SetupHandler)

11
core/caches/caches.go Normal file
View File

@ -0,0 +1,11 @@
package caches
// Cacher is an interface for a key/value cacher.
type Cacher interface {
Get(key string) ([]byte, error)
Set(key string, v []byte, expires int) error
Delete(key ...string)
Keys(pattern string) ([]string, error)
Lock(key, value string, expires int) bool
Unlock(key string)
}

35
core/caches/null/null.go Normal file
View File

@ -0,0 +1,35 @@
package null
// Null is a cache that doesn't do anything.
type Null struct{}
// New Null cache backend.
func New() *Null {
return &Null{}
}
// Get a key from Null.
func (r *Null) Get(key string) ([]byte, error) {
return []byte{}, nil
}
// Set a key in Null.
func (r *Null) Set(key string, v []byte, expires int) error {
return nil
}
// Delete keys from Null.
func (r *Null) Delete(key ...string) {}
// Keys returns a list of Null keys matching a pattern.
func (r *Null) Keys(pattern string) ([]string, error) {
return []string{}, nil
}
// Lock a mutex.
func (r *Null) Lock(key string, value string, expires int) bool {
return true
}
// Unlock a mutex.
func (r *Null) Unlock(key string) {}

88
core/caches/redis/redis.go Normal file
View File

@ -0,0 +1,88 @@
package redis
import (
"fmt"
"time"
"github.com/garyburd/redigo/redis"
)
// Redis is a cache backend.
type Redis struct {
pool *redis.Pool
prefix string
}
// New Redis backend.
func New(address string, db int, prefix string) (*Redis, error) {
r := &Redis{
prefix: prefix,
pool: &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
return redis.Dial("tcp", address,
redis.DialConnectTimeout(10*time.Second),
redis.DialDatabase(db),
redis.DialKeepAlive(30*time.Second),
)
},
},
}
return r, nil
}
// Get a key from Redis.
func (r *Redis) Get(key string) ([]byte, error) {
conn := r.pool.Get()
n, err := redis.Bytes(conn.Do("GET", r.prefix+key))
if err != nil {
return nil, fmt.Errorf("Redis SET error: %s (conn error: %s)", err, conn.Err())
}
return n, err
}
// Set a key in Redis.
func (r *Redis) Set(key string, v []byte, expires int) error {
conn := r.pool.Get()
_, err := conn.Do("SETEX", r.prefix+key, expires, v)
if err != nil {
return fmt.Errorf("Redis SET error: %s (conn error: %s)", err, conn.Err())
}
return nil
}
// Delete keys from Redis.
func (r *Redis) Delete(key ...string) {
conn := r.pool.Get()
for _, v := range key {
conn.Send("DEL", v)
}
conn.Flush()
conn.Receive()
}
// Keys returns a list of Redis keys matching a pattern.
func (r *Redis) Keys(pattern string) ([]string, error) {
conn := r.pool.Get()
n, err := redis.Strings(conn.Do("KEYS", pattern))
return n, err
}
// Lock a mutex.
func (r *Redis) Lock(key, value string, expires int) bool {
conn := r.pool.Get()
n, err := redis.Int(conn.Do("SETNX", r.prefix+key, value))
return err == nil && n == 1
}
// Unlock a mutex.
func (r *Redis) Unlock(key string) {
r.Delete(key)
}

86
core/jsondb/cache.go Normal file
View File

@ -0,0 +1,86 @@
package jsondb
import (
"encoding/json"
"errors"
"fmt"
"math/rand"
"time"
)
var errCacheDisabled = errors.New("cache disabled")
// SetCache sets a cache key.
func (db *DB) SetCache(key, value string, expires int) error {
if db.Cache == nil {
return errCacheDisabled
}
return db.Cache.Set(key, []byte(value), expires)
}
// SetJSONCache caches a JSON object.
func (db *DB) SetJSONCache(key string, v interface{}, expires int) error {
if db.Cache == nil {
return errCacheDisabled
}
bytes, err := json.Marshal(v)
if err != nil {
return err
}
return db.SetCache(key, string(bytes), expires)
}
// GetCache gets a cache key.
func (db *DB) GetCache(key string) (string, error) {
if db.Cache == nil {
return "", errCacheDisabled
}
v, err := db.Cache.Get(key)
return string(v), err
}
// DeleteCache deletes a cache key.
func (db *DB) DeleteCache(key string) error {
if db.Cache == nil {
return errCacheDisabled
}
db.Cache.Delete(key)
return nil
}
// LockCache implements 'file locking' in your cache.
func (db *DB) LockCache(key string) bool {
if db.Cache == nil {
return true
}
log.Info("LockCache(%s)", key)
var (
// In seconds
timeout = 5 * time.Second
expire = 20
)
identifier := fmt.Sprintf("%d", rand.Uint64())
log.Info("id: %s", identifier)
end := time.Now().Add(timeout)
for time.Now().Before(end) {
if ok := db.Cache.Lock("lock:"+key, identifier, expire); ok {
log.Info("JsonDB: Acquired lock for %s", key)
return true
}
time.Sleep(1 * time.Millisecond)
}
log.Error("JsonDB: lock timeout for %s", key)
return false
}
// UnlockCache releases the lock on a cache key.
func (db *DB) UnlockCache(key string) {
if db.Cache == nil {
return
}
db.Cache.Unlock("lock:" + key)
}

64
core/jsondb/jsondb.go
View File

@ -9,15 +9,22 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/kirsle/blog/core/caches"
)
var (
// CacheTimeout is how long the Redis cache keys live for in seconds, default 2 hours.
CacheTimeout = 60 * 60 * 2
CacheLock sync.RWMutex
)
// DB is the database manager.
type DB struct {
Root string // The root directory of the database
// Use Redis to cache filesystem reads of the database.
EnableRedis bool
RedisURL string
Root string // The root directory of the database
Cache caches.Cacher // A cacher for the JSON documents, i.e. Redis
}
// Error codes returned.
@ -33,6 +40,12 @@ func New(root string) *DB {
}
}
// WithCache configures a memory cacher for the JSON documents.
func (db *DB) WithCache(cache caches.Cacher) *DB {
db.Cache = cache
return db
}
// Get a document by path and load it into the object `v`.
func (db *DB) Get(document string, v interface{}) error {
log.Debug("[JsonDB] GET %s", document)
@ -42,17 +55,45 @@ func (db *DB) Get(document string, v interface{}) error {
// Get the file path and stats.
path := db.toPath(document)
_, err := os.Stat(path) // TODO: mtime for caching
stat, err := os.Stat(path)
if err != nil {
return err
}
// Do we have it cached?
data, err := db.GetCache(document)
if err == nil {
// Check if the cache is fresh.
cachedTime, err2 := db.GetCache(document + "_mtime")
if err2 == nil {
modTime := stat.ModTime()
mtime, _ := time.Parse(time.RFC3339Nano, cachedTime)
if modTime.After(mtime) && !modTime.Equal(mtime) {
log.Debug("[JsonDB] %s: On-disk file is newer than cache", document)
db.DeleteCache(document)
db.DeleteCache(document + "_mtime")
} else {
log.Debug("[JsonDB] %s: Returning cached copy", document)
return json.Unmarshal([]byte(data), v)
}
}
}
// Get a lock for reading.
CacheLock.RLock()
// Read the JSON.
err = db.readJSON(path, &v)
if err != nil {
CacheLock.RUnlock()
return err
}
// Unlock & cache it.
db.SetJSONCache(document, v, CacheTimeout)
db.SetCache(document+"_mtime", stat.ModTime().Format(time.RFC3339Nano), CacheTimeout)
CacheLock.RUnlock()
return nil
}
@ -61,18 +102,28 @@ func (db *DB) Commit(document string, v interface{}) error {
log.Debug("[JsonDB] COMMIT %s", document)
path := db.toPath(document)
// Get a write lock for the cache.
CacheLock.Lock()
// Ensure the directory tree is ready.
err := db.makePath(path)
if err != nil {
CacheLock.Unlock()
return err
}
// Write the document.
err = db.writeJSON(path, v)
if err != nil {
CacheLock.Unlock()
return fmt.Errorf("failed to write JSON to path %s: %s", path, err.Error())
}
// Unlock & cache it.
db.SetJSONCache(document, v, CacheTimeout)
db.SetCache(document+"_mtime", time.Now().Format(time.RFC3339Nano), CacheTimeout)
CacheLock.Unlock()
return nil
}
@ -86,6 +137,7 @@ func (db *DB) Delete(document string) error {
return nil
}
db.DeleteCache(document)
return os.Remove(path)
}

32
core/markdown.go
View File

@ -2,8 +2,10 @@ package core
import (
"bytes"
"crypto/md5"
"errors"
"fmt"
"io"
"os/exec"
"regexp"
"strings"
@ -85,7 +87,7 @@ func (b *Blog) RenderTrustedMarkdown(input string) string {
// Substitute fenced codes back in.
for _, block := range codeBlocks {
highlighted, _ := Pygmentize(block.language, block.source)
highlighted, _ := b.Pygmentize(block.language, block.source)
html = strings.Replace(html,
fmt.Sprintf("[?FENCED_CODE_%d_BLOCK?]", block.placeholder),
highlighted,
@ -101,10 +103,24 @@ func (b *Blog) RenderTrustedMarkdown(input string) string {
//
// On error the original given source is returned back.
//
// TODO: this takes ~0.6s per go, we need something faster.
func Pygmentize(language, source string) (string, error) {
bin := "pygmentize"
// The rendered result is cached in Redis if available, because the CLI
// call takes ~0.6s which is slow if you're rendering a lot of code blocks.
func (b *Blog) Pygmentize(language, source string) (string, error) {
var result string
// Hash the source for the cache key.
h := md5.New()
io.WriteString(h, language+source)
hash := fmt.Sprintf("%x", h.Sum(nil))
cacheKey := "pygmentize:" + hash
// Do we have it cached?
if cached, err := b.Cache.Get(cacheKey); err == nil {
return string(cached), nil
}
// Defer to the `pygmentize` command
bin := "pygmentize"
if _, err := exec.LookPath(bin); err != nil {
return source, errors.New("pygmentize not installed")
}
@ -123,5 +139,11 @@ func Pygmentize(language, source string) (string, error) {
return source, err
}
return out.String(), nil
result = out.String()
err := b.Cache.Set(cacheKey, []byte(result), 60*60*24) // cool md5's don't change
if err != nil {
log.Error("Couldn't cache Pygmentize output: %s", err)
}
return result, nil
}

13
core/middleware.go
View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"net/http"
"time"
"github.com/google/uuid"
"github.com/gorilla/sessions"
@ -15,13 +16,23 @@ type key int
const (
sessionKey key = iota
userKey
requestTimeKey
)
// SessionLoader gets the Gorilla session store and makes it available on the
// Request context.
//
// SessionLoader is the first custom middleware applied, so it takes the current
// datetime to make available later in the request and stores it on the request
// context.
func (b *Blog) SessionLoader(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
// Store the current datetime on the request context.
ctx := context.WithValue(r.Context(), requestTimeKey, time.Now())
// Get the Gorilla session and make it available in the request context.
session, _ := b.store.Get(r, "session")
ctx := context.WithValue(r.Context(), sessionKey, session)
ctx = context.WithValue(ctx, sessionKey, session)
next(w, r.WithContext(ctx))
}

18
core/templates.go
View File

@ -17,13 +17,15 @@ import (
// when the template is rendered.
type Vars struct {
// Global, "constant" template variables.
SetupNeeded bool
Title string
Path string
LoggedIn bool
CurrentUser *users.User
CSRF string
Request *http.Request
SetupNeeded bool
Title string
Path string
LoggedIn bool
CurrentUser *users.User
CSRF string
Request *http.Request
RequestTime time.Time
RequestDuration time.Duration
// Configuration variables
NoLayout bool // don't wrap in .layout.html, just render the template
@ -62,6 +64,7 @@ func (v *Vars) LoadDefaults(b *Blog, r *http.Request) {
v.SetupNeeded = true
}
v.Request = r
v.RequestTime = r.Context().Value(requestTimeKey).(time.Time)
v.Title = s.Site.Title
v.Path = r.URL.Path
@ -176,6 +179,7 @@ func (b *Blog) RenderTemplate(w http.ResponseWriter, r *http.Request, path strin
session.Save(r, w)
}
vars.RequestDuration = time.Now().Sub(vars.RequestTime)
vars.CSRF = b.GenerateCSRFToken(w, r, session)
w.Header().Set("Content-Type", "text/html; encoding=UTF-8")