rophako/rophako/jsondb.py

224 lines
5.7 KiB
Python

# -*- coding: utf-8 -*-
"""JSON flat file database system."""
import codecs
import os
import os.path
import glob
import re
from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN
import redis
import json
import time
from rophako.settings import Config
from rophako.log import logger
redis_client = None
cache_lifetime = 60*60 # 1 hour
def get(document, cache=True):
"""Get a specific document from the DB."""
logger.debug("JsonDB: GET {}".format(document))
# Exists?
if not exists(document):
logger.debug("Requested document doesn't exist")
return None
path = mkpath(document)
stat = os.stat(path)
# Do we have it cached?
data = get_cache(document) if cache else None
if data:
# Check if the cache is fresh.
if stat.st_mtime > get_cache(document+"_mtime"):
del_cache(document)
del_cache(document+"_mtime")
else:
return data
# Get the JSON data.
data = read_json(path)
# Cache and return it.
if cache:
set_cache(document, data, expires=cache_lifetime)
set_cache(document+"_mtime", stat.st_mtime, expires=cache_lifetime)
return data
def commit(document, data, cache=True):
"""Insert/update a document in the DB."""
# Need to create the file?
path = mkpath(document)
if not os.path.isfile(path):
parts = path.split("/")
parts.pop() # Remove the file part
directory = list()
# Create all the folders.
for part in parts:
directory.append(part)
segment = "/".join(directory)
if len(segment) > 0 and not os.path.isdir(segment):
logger.debug("JsonDB: mkdir {}".format(segment))
os.mkdir(segment, 0o755)
# Update the cached document.
if cache:
set_cache(document, data, expires=cache_lifetime)
set_cache(document+"_mtime", time.time(), expires=cache_lifetime)
# Write the JSON.
write_json(path, data)
def delete(document):
"""Delete a document from the DB."""
path = mkpath(document)
if os.path.isfile(path):
logger.info("Delete DB document: {}".format(path))
os.unlink(path)
del_cache(document)
def exists(document):
"""Query whether a document exists."""
path = mkpath(document)
return os.path.isfile(path)
def list_docs(path):
"""List all the documents at the path."""
path = mkpath("{}/*".format(path))
docs = list()
for item in glob.glob(path):
name = re.sub(r'\.json$', '', item)
name = name.split("/")[-1]
docs.append(name)
return docs
def mkpath(document):
"""Turn a DB path into a JSON file path."""
if document.endswith(".json"):
# Let's not do that.
raise Exception("mkpath: document path already includes .json extension!")
return "{}/{}.json".format(Config.db.db_root, str(document))
def read_json(path):
"""Slurp, decode and return the data from a JSON document."""
path = str(path)
if not os.path.isfile(path):
raise Exception("Can't read JSON file {}: file not found!".format(path))
# Don't allow any fishy looking paths.
if ".." in path:
logger.error("ERROR: JsonDB tried to read a path with two dots: {}".format(path))
raise Exception()
# Open and lock the file.
fh = codecs.open(path, 'r', 'utf-8')
flock(fh, LOCK_SH)
text = fh.read()
flock(fh, LOCK_UN)
fh.close()
# Decode.
try:
data = json.loads(text)
except:
logger.error("Couldn't decode JSON data from {}".format(path))
logger.error(text)
data = None
return data
def write_json(path, data):
"""Write a JSON document."""
path = str(path)
# Don't allow any fishy looking paths.
if ".." in path:
logger.error("ERROR: JsonDB tried to write a path with two dots: {}".format(path))
raise Exception()
logger.debug("JsonDB: WRITE > {}".format(path))
# Open and lock the file.
fh = None
if os.path.isfile(path):
fh = codecs.open(path, 'r+', 'utf-8')
else:
fh = codecs.open(path, 'w', 'utf-8')
flock(fh, LOCK_EX)
# Write it.
fh.truncate(0)
fh.write(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')))
# Unlock and close.
flock(fh, LOCK_UN)
fh.close()
############################################################################
# Redis Caching Functions #
############################################################################
def get_redis():
"""Connect to Redis or return the existing connection."""
global redis_client
if not redis_client:
redis_client = redis.StrictRedis(
host = Config.db.redis_host,
port = Config.db.redis_port,
db = Config.db.redis_db,
)
return redis_client
def set_cache(key, value, expires=None):
"""Set a key in the Redis cache."""
key = Config.db.redis_prefix + key
try:
client = get_redis()
client.set(key, json.dumps(value))
# Expiration date?
if expires:
client.expire(key, expires)
except:
logger.error("Redis exception: couldn't set_cache {}".format(key))
def get_cache(key):
"""Get a cached item."""
key = Config.db.redis_prefix + key
value = None
try:
client = get_redis()
value = client.get(key)
if value:
value = json.loads(value)
except:
logger.warning("Redis exception: couldn't get_cache {}".format(key))
value = None
return value
def del_cache(key):
"""Delete a cached item."""
key = Config.db.redis_prefix + key
client = get_redis()
client.delete(key)