A Python content management system designed for kirsle.net featuring a blog, comments and photo albums. https://rophako.kirsle.net/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

228 lines
5.8KB

  1. # -*- coding: utf-8 -*-
  2. """JSON flat file database system."""
  3. import codecs
  4. import os
  5. import os.path
  6. import glob
  7. import re
  8. from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN
  9. import redis
  10. import json
  11. import time
  12. from rophako.settings import Config
  13. from rophako.utils import handle_exception
  14. from rophako.log import logger
  15. redis_client = None
  16. cache_lifetime = 60*60 # 1 hour
  17. def get(document, cache=True):
  18. """Get a specific document from the DB."""
  19. logger.debug("JsonDB: GET {}".format(document))
  20. # Exists?
  21. if not exists(document):
  22. logger.debug("Requested document doesn't exist")
  23. return None
  24. path = mkpath(document)
  25. stat = os.stat(path)
  26. # Do we have it cached?
  27. data = get_cache(document) if cache else None
  28. if data:
  29. # Check if the cache is fresh.
  30. if stat.st_mtime > get_cache(document+"_mtime"):
  31. del_cache(document)
  32. del_cache(document+"_mtime")
  33. else:
  34. return data
  35. # Get the JSON data.
  36. data = read_json(path)
  37. # Cache and return it.
  38. if cache:
  39. set_cache(document, data, expires=cache_lifetime)
  40. set_cache(document+"_mtime", stat.st_mtime, expires=cache_lifetime)
  41. return data
  42. def commit(document, data, cache=True):
  43. """Insert/update a document in the DB."""
  44. # Need to create the file?
  45. path = mkpath(document)
  46. if not os.path.isfile(path):
  47. parts = path.split("/")
  48. parts.pop() # Remove the file part
  49. directory = list()
  50. # Create all the folders.
  51. for part in parts:
  52. directory.append(part)
  53. segment = "/".join(directory)
  54. if len(segment) > 0 and not os.path.isdir(segment):
  55. logger.debug("JsonDB: mkdir {}".format(segment))
  56. os.mkdir(segment, 0o755)
  57. # Update the cached document.
  58. if cache:
  59. set_cache(document, data, expires=cache_lifetime)
  60. set_cache(document+"_mtime", time.time(), expires=cache_lifetime)
  61. # Write the JSON.
  62. write_json(path, data)
  63. def delete(document):
  64. """Delete a document from the DB."""
  65. path = mkpath(document)
  66. if os.path.isfile(path):
  67. logger.info("Delete DB document: {}".format(path))
  68. os.unlink(path)
  69. del_cache(document)
  70. def exists(document):
  71. """Query whether a document exists."""
  72. path = mkpath(document)
  73. return os.path.isfile(path)
  74. def list_docs(path):
  75. """List all the documents at the path."""
  76. path = mkpath("{}/*".format(path))
  77. docs = list()
  78. for item in glob.glob(path):
  79. name = re.sub(r'\.json$', '', item)
  80. name = name.split("/")[-1]
  81. docs.append(name)
  82. return docs
  83. def mkpath(document):
  84. """Turn a DB path into a JSON file path."""
  85. if document.endswith(".json"):
  86. # Let's not do that.
  87. raise Exception("mkpath: document path already includes .json extension!")
  88. return "{}/{}.json".format(Config.db.db_root, str(document))
  89. def read_json(path):
  90. """Slurp, decode and return the data from a JSON document."""
  91. path = str(path)
  92. if not os.path.isfile(path):
  93. raise Exception("Can't read JSON file {}: file not found!".format(path))
  94. # Don't allow any fishy looking paths.
  95. if ".." in path:
  96. logger.error("ERROR: JsonDB tried to read a path with two dots: {}".format(path))
  97. raise Exception()
  98. # Open and lock the file.
  99. fh = codecs.open(path, 'r', 'utf-8')
  100. flock(fh, LOCK_SH)
  101. text = fh.read()
  102. flock(fh, LOCK_UN)
  103. fh.close()
  104. # Decode.
  105. try:
  106. data = json.loads(text)
  107. except:
  108. logger.error("Couldn't decode JSON data from {}".format(path))
  109. handle_exception(Exception("Couldn't decode JSON from {}\n{}".format(
  110. path,
  111. text,
  112. )))
  113. data = None
  114. return data
  115. def write_json(path, data):
  116. """Write a JSON document."""
  117. path = str(path)
  118. # Don't allow any fishy looking paths.
  119. if ".." in path:
  120. logger.error("ERROR: JsonDB tried to write a path with two dots: {}".format(path))
  121. raise Exception()
  122. logger.debug("JsonDB: WRITE > {}".format(path))
  123. # Open and lock the file.
  124. fh = None
  125. if os.path.isfile(path):
  126. fh = codecs.open(path, 'r+', 'utf-8')
  127. else:
  128. fh = codecs.open(path, 'w', 'utf-8')
  129. flock(fh, LOCK_EX)
  130. # Write it.
  131. fh.truncate(0)
  132. fh.write(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')))
  133. # Unlock and close.
  134. flock(fh, LOCK_UN)
  135. fh.close()
  136. ############################################################################
  137. # Redis Caching Functions #
  138. ############################################################################
  139. def get_redis():
  140. """Connect to Redis or return the existing connection."""
  141. global redis_client
  142. if not redis_client:
  143. redis_client = redis.StrictRedis(
  144. host = Config.db.redis_host,
  145. port = Config.db.redis_port,
  146. db = Config.db.redis_db,
  147. )
  148. return redis_client
  149. def set_cache(key, value, expires=None):
  150. """Set a key in the Redis cache."""
  151. key = Config.db.redis_prefix + key
  152. try:
  153. client = get_redis()
  154. client.set(key, json.dumps(value))
  155. # Expiration date?
  156. if expires:
  157. client.expire(key, expires)
  158. except:
  159. logger.error("Redis exception: couldn't set_cache {}".format(key))
  160. def get_cache(key):
  161. """Get a cached item."""
  162. key = Config.db.redis_prefix + key
  163. value = None
  164. try:
  165. client = get_redis()
  166. value = client.get(key)
  167. if value:
  168. value = json.loads(value)
  169. except:
  170. logger.warning("Redis exception: couldn't get_cache {}".format(key))
  171. value = None
  172. return value
  173. def del_cache(key):
  174. """Delete a cached item."""
  175. key = Config.db.redis_prefix + key
  176. client = get_redis()
  177. client.delete(key)