Python Library Caching
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

__init__.py 9.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. #
  4. """
  5. caching (Caching Module)
  6. ========================
  7. **Author:**
  8. * Dirk Alders <sudo-dirk@mount-mockery.de>
  9. **Description:**
  10. This Module supports functions and classes for caching e.g. properties of other instances.
  11. **Submodules:**
  12. * :class:`caching.property_cache_json`
  13. * :class:`caching.property_cache_pickle`
  14. **Unittest:**
  15. See also the :download:`unittest <../../caching/_testresults_/unittest.pdf>` documentation.
  16. """
  17. __DEPENDENCIES__ = []
  18. import hashlib
  19. import hmac
  20. import json
  21. import logging
  22. import os
  23. import pickle
  24. import sys
  25. logger_name = 'CACHING'
  26. logger = logging.getLogger(logger_name)
  27. __DESCRIPTION__ = """The Module {\\tt %s} is designed to store information in {\\tt json} or {\\tt pickle} files to support them much faster then generating them from the original source file.
  28. For more Information read the documentation.""" % __name__.replace('_', '\_')
  29. """The Module Description"""
  30. __INTERPRETER__ = (2, 3)
  31. """The Tested Interpreter-Versions"""
  32. class property_cache_pickle(object):
  33. """
  34. Class to cache properties, which take longer on initialising than reading a file in pickle format.
  35. :param source_instance: The source instance holding the data
  36. :type source_instance: instance
  37. :param cache_filename: File name, where the properties are stored as cache
  38. :type cache_filename: str
  39. :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
  40. :raises: ?
  41. .. note:: This module uses logging. So logging should be initialised at least by executing logging.basicConfig(...)
  42. .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
  43. * uid(): returns the unique id of the source.
  44. * keys(): returns a list of all available keys.
  45. * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
  46. * get(key, default): returns the property for a key. If key does not exists, default will be returned.
  47. Reasons for updating the complete data set:
  48. * UID of source_instance has changed (in comparison to the cached value).
  49. * data_version is increased
  50. **Example:**
  51. .. literalinclude:: ../../caching/_examples_/property_cache_pickle.py
  52. Will result on the first execution to the following output (with a long execution time):
  53. .. literalinclude:: ../../caching/_examples_/property_cache_pickle_1.log
  54. With every following execution (slow for getting "two" which is not cached - see implementation):
  55. .. literalinclude:: ../../caching/_examples_/property_cache_pickle_2.log
  56. """
  57. LOG_PREFIX = 'PickCache:'
  58. DATA_VERSION_TAG = '_property_cache_data_version_'
  59. UID_TAG = '_property_cache_uid_'
  60. def __init__(self, source_instance, cache_filename, load_all_on_init=False, callback_on_data_storage=None):
  61. self._source_instance = source_instance
  62. self._cache_filename = cache_filename
  63. self._load_all_on_init = load_all_on_init
  64. self._callback_on_data_storage = callback_on_data_storage
  65. self._cached_props = None
  66. def get(self, key, default=None):
  67. """
  68. Method to get the cached property. If key does not exists in cache, the property will be loaded from source_instance and stored in cache (file).
  69. :param key: key for value to get.
  70. :param default: value to be returned, if key does not exists.
  71. :returns: value for a given key or default value.
  72. """
  73. if key in self.keys():
  74. if self._cached_props is None:
  75. self._init_cache()
  76. if self._key_filter(key) not in self._cached_props:
  77. val = self._source_instance.get(key, None)
  78. logger.debug("%s Loading property for '%s' from source instance (%s)", self.LOG_PREFIX, key, repr(val))
  79. self._cached_props[self._key_filter(key)] = val
  80. self._save_cache()
  81. else:
  82. logger.debug("%s Providing property for '%s' from cache (%s)", self.LOG_PREFIX, key, repr(self._cached_props.get(self._key_filter(key), default)))
  83. return self._cached_props.get(self._key_filter(key), default)
  84. else:
  85. logger.info("%s Key '%s' is not in cached_keys. Uncached data will be returned.", self.LOG_PREFIX, key)
  86. return self._source_instance.get(key, default)
  87. def keys(self):
  88. return self._source_instance.keys()
  89. def _data_version(self):
  90. if self._cached_props is None:
  91. return None
  92. else:
  93. return self._cached_props.get(self.DATA_VERSION_TAG, None)
  94. def _init_cache(self):
  95. if not self._load_cache() or self._source_instance.uid() != self._uid() or self._source_instance.data_version() > self._data_version():
  96. if self._uid() is not None and self._source_instance.uid() != self._uid():
  97. logger.debug("%s Source uid changed, ignoring previous cache data", self.LOG_PREFIX)
  98. if self._data_version() is not None and self._source_instance.data_version() > self._data_version():
  99. logger.debug("%s Data version increased, ignoring previous cache data", self.LOG_PREFIX)
  100. self._cached_props = dict()
  101. if self._load_all_on_init:
  102. self._load_source()
  103. self._cached_props[self.UID_TAG] = self._source_instance.uid()
  104. self._cached_props[self.DATA_VERSION_TAG] = self._source_instance.data_version()
  105. self._save_cache()
  106. def _load_cache(self):
  107. if os.path.exists(self._cache_filename):
  108. with open(self._cache_filename, 'rb') as fh:
  109. self._cached_props = pickle.load(fh)
  110. logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
  111. return True
  112. else:
  113. logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
  114. return False
  115. def _key_filter(self, key):
  116. if sys.version_info >= (3, 0):
  117. tps = [str]
  118. else:
  119. tps = [str, unicode]
  120. if type(key) in tps:
  121. if key.endswith(self.DATA_VERSION_TAG) or key.endswith(self.UID_TAG):
  122. return '_' + key
  123. return key
  124. def _load_source(self):
  125. logger.debug('%s Loading all data from source - %s', self.LOG_PREFIX, repr(self._source_instance.keys()))
  126. for key in self._source_instance.keys():
  127. val = self._source_instance.get(key)
  128. self._cached_props[self._key_filter(key)] = val
  129. def _save_cache(self):
  130. with open(self._cache_filename, 'wb') as fh:
  131. pickle.dump(self._cached_props, fh)
  132. logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
  133. if self._callback_on_data_storage is not None:
  134. self._callback_on_data_storage()
  135. def _uid(self):
  136. if self._cached_props is None:
  137. return None
  138. else:
  139. return self._cached_props.get(self.UID_TAG, None)
  140. class property_cache_json(property_cache_pickle):
  141. """
  142. Class to cache properties, which take longer on initialising than reading a file in json format. See also parent :py:class:`property_cache_pickle`
  143. :param source_instance: The source instance holding the data
  144. :type source_instance: instance
  145. :param cache_filename: File name, where the properties are stored as cache
  146. :type cache_filename: str
  147. :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
  148. :raises: ?
  149. .. warning::
  150. * This class uses json. You should **only** use keys of type string!
  151. * Unicode types are transfered to strings
  152. .. note:: This module uses logging. So logging should be initialised at least by executing logging.basicConfig(...)
  153. .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
  154. * uid(): returns the unique id of the source.
  155. * keys(): returns a list of all available keys.
  156. * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
  157. * get(key, default): returns the property for a key. If key does not exists, default will be returned.
  158. Reasons for updating the complete data set:
  159. * UID of source_instance has changed (in comparison to the cached value).
  160. * data_version is increased
  161. **Example:**
  162. .. literalinclude:: ../../caching/_examples_/property_cache_json.py
  163. Will result on the first execution to the following output (with a long execution time):
  164. .. literalinclude:: ../../caching/_examples_/property_cache_json_1.log
  165. With every following execution (slow for getting "two" which is not cached - see implementation):
  166. .. literalinclude:: ../../caching/_examples_/property_cache_json_2.log
  167. """
  168. LOG_PREFIX = 'JsonCache:'
  169. def _load_cache(self):
  170. if os.path.exists(self._cache_filename):
  171. with open(self._cache_filename, 'r') as fh:
  172. self._cached_props = json.load(fh)
  173. logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
  174. return True
  175. else:
  176. logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
  177. return False
  178. def _save_cache(self):
  179. with open(self._cache_filename, 'w') as fh:
  180. json.dump(self._cached_props, fh, sort_keys=True, indent=4)
  181. logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
  182. if self._callback_on_data_storage is not None:
  183. self._callback_on_data_storage()