A Python content management system designed for kirsle.net featuring a blog, comments and photo albums. https://rophako.kirsle.net/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

209 lines
5.2KB

  1. # -*- coding: utf-8 -*-
  2. """JSON flat file database system."""
  3. import codecs
  4. import os
  5. import os.path
  6. import glob
  7. import re
  8. from fcntl import flock, LOCK_EX, LOCK_SH, LOCK_UN
  9. import redis
  10. import json
  11. import time
  12. import config
  13. from rophako.log import logger
  14. redis_client = None
  15. cache_lifetime = 60*60 # 1 hour
  16. def get(document):
  17. """Get a specific document from the DB."""
  18. logger.debug("JsonDB: GET {}".format(document))
  19. # Exists?
  20. if not exists(document):
  21. logger.debug("Requested document doesn't exist")
  22. return None
  23. path = mkpath(document)
  24. stat = os.stat(path)
  25. # Do we have it cached?
  26. data = get_cache(document)
  27. if data:
  28. # Check if the cache is fresh.
  29. if stat.st_mtime > get_cache(document+"_mtime"):
  30. del_cache(document)
  31. del_cache(document+"_mtime")
  32. else:
  33. return data
  34. # Get the JSON data.
  35. data = read_json(path)
  36. # Cache and return it.
  37. set_cache(document, data, expires=cache_lifetime)
  38. set_cache(document+"_mtime", stat.st_mtime, expires=cache_lifetime)
  39. return data
  40. def commit(document, data):
  41. """Insert/update a document in the DB."""
  42. # Need to create the file?
  43. path = mkpath(document)
  44. if not os.path.isfile(path):
  45. parts = path.split("/")
  46. parts.pop() # Remove the file part
  47. directory = list()
  48. # Create all the folders.
  49. for part in parts:
  50. directory.append(part)
  51. segment = "/".join(directory)
  52. if len(segment) > 0 and not os.path.isdir(segment):
  53. logger.debug("JsonDB: mkdir {}".format(segment))
  54. os.mkdir(segment, 0755)
  55. # Update the cached document.
  56. set_cache(document, data, expires=cache_lifetime)
  57. set_cache(document+"_mtime", time.time(), expires=cache_lifetime)
  58. # Write the JSON.
  59. write_json(path, data)
  60. def delete(document):
  61. """Delete a document from the DB."""
  62. path = mkpath(document)
  63. if os.path.isfile(path):
  64. logger.info("Delete DB document: {}".format(path))
  65. os.unlink(path)
  66. def exists(document):
  67. """Query whether a document exists."""
  68. path = mkpath(document)
  69. return os.path.isfile(path)
  70. def list_docs(path):
  71. """List all the documents at the path."""
  72. path = mkpath("{}/*".format(path))
  73. docs = list()
  74. for item in glob.glob(path):
  75. name = re.sub(r'\.json$', '', item)
  76. name = name.split("/")[-1]
  77. docs.append(name)
  78. return docs
  79. def mkpath(document):
  80. """Turn a DB path into a JSON file path."""
  81. if document.endswith(".json"):
  82. # Let's not do that.
  83. raise Exception("mkpath: document path already includes .json extension!")
  84. return "{}/{}.json".format(config.DB_ROOT, str(document))
  85. def read_json(path):
  86. """Slurp, decode and return the data from a JSON document."""
  87. path = str(path)
  88. if not os.path.isfile(path):
  89. raise Exception("Can't read JSON file {}: file not found!".format(path))
  90. # Open and lock the file.
  91. fh = codecs.open(path, 'r', 'utf-8')
  92. flock(fh, LOCK_SH)
  93. text = fh.read()
  94. flock(fh, LOCK_UN)
  95. fh.close()
  96. # Decode.
  97. try:
  98. data = json.loads(text)
  99. except:
  100. logger.error("Couldn't decode JSON data from {}".format(path))
  101. data = None
  102. return data
  103. def write_json(path, data):
  104. """Write a JSON document."""
  105. path = str(path)
  106. logger.debug("JsonDB: WRITE > {}".format(path))
  107. # Open and lock the file.
  108. fh = None
  109. if os.path.isfile(path):
  110. fh = codecs.open(path, 'r+', 'utf-8')
  111. else:
  112. fh = codecs.open(path, 'w', 'utf-8')
  113. flock(fh, LOCK_EX)
  114. # Write it.
  115. fh.truncate(0)
  116. fh.write(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')))
  117. # Unlock and close.
  118. flock(fh, LOCK_UN)
  119. fh.close()
  120. ############################################################################
  121. # Redis Caching Functions #
  122. ############################################################################
  123. def get_redis():
  124. """Connect to Redis or return the existing connection."""
  125. global redis_client
  126. if not redis_client:
  127. redis_client = redis.StrictRedis(
  128. host = config.REDIS_HOST,
  129. port = config.REDIS_PORT,
  130. db = config.REDIS_DB,
  131. )
  132. return redis_client
  133. def set_cache(key, value, expires=None):
  134. """Set a key in the Redis cache."""
  135. key = config.REDIS_PREFIX + key
  136. try:
  137. client = get_redis()
  138. client.set(key, json.dumps(value))
  139. # Expiration date?
  140. if expires:
  141. client.expire(key, expires)
  142. except:
  143. logger.error("Redis exception: couldn't set_cache {}".format(key))
  144. def get_cache(key):
  145. """Get a cached item."""
  146. key = config.REDIS_PREFIX + key
  147. value = None
  148. try:
  149. client = get_redis()
  150. value = client.get(key)
  151. if value:
  152. value = json.loads(value)
  153. except:
  154. logger.warning("Redis exception: couldn't get_cache {}".format(key))
  155. value = None
  156. return value
  157. def del_cache(key):
  158. """Delete a cached item."""
  159. key = config.REDIS_PREFIX + key
  160. client = get_redis()
  161. client.delete(key)