A Python content management system designed for kirsle.net featuring a blog, comments and photo albums. https://rophako.kirsle.net/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

224 lines
5.7 KiB

  1. # -*- coding: utf-8 -*-
  2. """JSON flat file database system."""
  3. import codecs
  4. import os
  5. import os.path
  6. import glob
  7. import re
  8. from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN
  9. import redis
  10. import json
  11. import time
  12. from rophako.settings import Config
  13. from rophako.log import logger
  14. redis_client = None
  15. cache_lifetime = 60*60 # 1 hour
  16. def get(document, cache=True):
  17. """Get a specific document from the DB."""
  18. logger.debug("JsonDB: GET {}".format(document))
  19. # Exists?
  20. if not exists(document):
  21. logger.debug("Requested document doesn't exist")
  22. return None
  23. path = mkpath(document)
  24. stat = os.stat(path)
  25. # Do we have it cached?
  26. data = get_cache(document) if cache else None
  27. if data:
  28. # Check if the cache is fresh.
  29. if stat.st_mtime > get_cache(document+"_mtime"):
  30. del_cache(document)
  31. del_cache(document+"_mtime")
  32. else:
  33. return data
  34. # Get the JSON data.
  35. data = read_json(path)
  36. # Cache and return it.
  37. if cache:
  38. set_cache(document, data, expires=cache_lifetime)
  39. set_cache(document+"_mtime", stat.st_mtime, expires=cache_lifetime)
  40. return data
  41. def commit(document, data, cache=True):
  42. """Insert/update a document in the DB."""
  43. # Need to create the file?
  44. path = mkpath(document)
  45. if not os.path.isfile(path):
  46. parts = path.split("/")
  47. parts.pop() # Remove the file part
  48. directory = list()
  49. # Create all the folders.
  50. for part in parts:
  51. directory.append(part)
  52. segment = "/".join(directory)
  53. if len(segment) > 0 and not os.path.isdir(segment):
  54. logger.debug("JsonDB: mkdir {}".format(segment))
  55. os.mkdir(segment, 0o755)
  56. # Update the cached document.
  57. if cache:
  58. set_cache(document, data, expires=cache_lifetime)
  59. set_cache(document+"_mtime", time.time(), expires=cache_lifetime)
  60. # Write the JSON.
  61. write_json(path, data)
  62. def delete(document):
  63. """Delete a document from the DB."""
  64. path = mkpath(document)
  65. if os.path.isfile(path):
  66. logger.info("Delete DB document: {}".format(path))
  67. os.unlink(path)
  68. del_cache(document)
  69. def exists(document):
  70. """Query whether a document exists."""
  71. path = mkpath(document)
  72. return os.path.isfile(path)
  73. def list_docs(path):
  74. """List all the documents at the path."""
  75. path = mkpath("{}/*".format(path))
  76. docs = list()
  77. for item in glob.glob(path):
  78. name = re.sub(r'\.json$', '', item)
  79. name = name.split("/")[-1]
  80. docs.append(name)
  81. return docs
  82. def mkpath(document):
  83. """Turn a DB path into a JSON file path."""
  84. if document.endswith(".json"):
  85. # Let's not do that.
  86. raise Exception("mkpath: document path already includes .json extension!")
  87. return "{}/{}.json".format(Config.db.db_root, str(document))
  88. def read_json(path):
  89. """Slurp, decode and return the data from a JSON document."""
  90. path = str(path)
  91. if not os.path.isfile(path):
  92. raise Exception("Can't read JSON file {}: file not found!".format(path))
  93. # Don't allow any fishy looking paths.
  94. if ".." in path:
  95. logger.error("ERROR: JsonDB tried to read a path with two dots: {}".format(path))
  96. raise Exception()
  97. # Open and lock the file.
  98. fh = codecs.open(path, 'r', 'utf-8')
  99. flock(fh, LOCK_SH)
  100. text = fh.read()
  101. flock(fh, LOCK_UN)
  102. fh.close()
  103. # Decode.
  104. try:
  105. data = json.loads(text)
  106. except:
  107. logger.error("Couldn't decode JSON data from {}".format(path))
  108. logger.error(text)
  109. data = None
  110. return data
  111. def write_json(path, data):
  112. """Write a JSON document."""
  113. path = str(path)
  114. # Don't allow any fishy looking paths.
  115. if ".." in path:
  116. logger.error("ERROR: JsonDB tried to write a path with two dots: {}".format(path))
  117. raise Exception()
  118. logger.debug("JsonDB: WRITE > {}".format(path))
  119. # Open and lock the file.
  120. fh = None
  121. if os.path.isfile(path):
  122. fh = codecs.open(path, 'r+', 'utf-8')
  123. else:
  124. fh = codecs.open(path, 'w', 'utf-8')
  125. flock(fh, LOCK_EX)
  126. # Write it.
  127. fh.truncate(0)
  128. fh.write(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')))
  129. # Unlock and close.
  130. flock(fh, LOCK_UN)
  131. fh.close()
  132. ############################################################################
  133. # Redis Caching Functions #
  134. ############################################################################
  135. def get_redis():
  136. """Connect to Redis or return the existing connection."""
  137. global redis_client
  138. if not redis_client:
  139. redis_client = redis.StrictRedis(
  140. host = Config.db.redis_host,
  141. port = Config.db.redis_port,
  142. db = Config.db.redis_db,
  143. )
  144. return redis_client
  145. def set_cache(key, value, expires=None):
  146. """Set a key in the Redis cache."""
  147. key = Config.db.redis_prefix + key
  148. try:
  149. client = get_redis()
  150. client.set(key, json.dumps(value))
  151. # Expiration date?
  152. if expires:
  153. client.expire(key, expires)
  154. except:
  155. logger.error("Redis exception: couldn't set_cache {}".format(key))
  156. def get_cache(key):
  157. """Get a cached item."""
  158. key = Config.db.redis_prefix + key
  159. value = None
  160. try:
  161. client = get_redis()
  162. value = client.get(key)
  163. if value:
  164. value = json.loads(value)
  165. except:
  166. logger.warning("Redis exception: couldn't get_cache {}".format(key))
  167. value = None
  168. return value
  169. def del_cache(key):
  170. """Delete a cached item."""
  171. key = Config.db.redis_prefix + key
  172. client = get_redis()
  173. client.delete(key)