|
@@ -0,0 +1,236 @@
|
|
1
|
+#!/usr/bin/env python
|
|
2
|
+# -*- coding: utf-8 -*-
|
|
3
|
+#
|
|
4
|
+"""
|
|
5
|
+caching (Caching Module)
|
|
6
|
+========================
|
|
7
|
+
|
|
8
|
+**Author:**
|
|
9
|
+
|
|
10
|
+* Dirk Alders <sudo-dirk@mount-mockery.de>
|
|
11
|
+
|
|
12
|
+**Description:**
|
|
13
|
+
|
|
14
|
+ This Module supports functions and classes for caching e.g. properties of other instances.
|
|
15
|
+
|
|
16
|
+**Submodules:**
|
|
17
|
+
|
|
18
|
+* :class:`caching.property_cache_json`
|
|
19
|
+* :class:`caching.property_cache_pickle`
|
|
20
|
+
|
|
21
|
+**Unittest:**
|
|
22
|
+
|
|
23
|
+ See also the :download:`unittest <../../caching/_testresults_/unittest.pdf>` documentation.
|
|
24
|
+"""
|
|
25
|
+__DEPENDENCIES__ = []
|
|
26
|
+
|
|
27
|
+import hashlib
|
|
28
|
+import hmac
|
|
29
|
+import json
|
|
30
|
+import logging
|
|
31
|
+import os
|
|
32
|
+import pickle
|
|
33
|
+import sys
|
|
34
|
+
|
|
35
|
+logger_name = 'CACHING'
|
|
36
|
+logger = logging.getLogger(logger_name)
|
|
37
|
+
|
|
38
|
+__DESCRIPTION__ = """The Module {\\tt %s} is designed to store information in {\\tt json} or {\\tt pickle} files to support them much faster then generating them from the original source file.
|
|
39
|
+For more Information read the documentation.""" % __name__.replace('_', '\_')
|
|
40
|
+"""The Module Description"""
|
|
41
|
+__INTERPRETER__ = (2, 3)
|
|
42
|
+"""The Tested Interpreter-Versions"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+class property_cache_pickle(object):
|
|
46
|
+ """
|
|
47
|
+ Class to cache properties, which take longer on initialising than reading a file in pickle format.
|
|
48
|
+
|
|
49
|
+ :param source_instance: The source instance holding the data
|
|
50
|
+ :type source_instance: instance
|
|
51
|
+ :param cache_filename: File name, where the properties are stored as cache
|
|
52
|
+ :type cache_filename: str
|
|
53
|
+ :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
|
|
54
|
+ :raises: ?
|
|
55
|
+
|
|
56
|
+ .. note:: This module uses logging. So logging should be initialised at least by executing logging.basicConfig(...)
|
|
57
|
+
|
|
58
|
+ .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
|
|
59
|
+
|
|
60
|
+ * uid(): returns the unique id of the source.
|
|
61
|
+ * keys(): returns a list of all available keys.
|
|
62
|
+ * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
|
|
63
|
+ * get(key, default): returns the property for a key. If key does not exists, default will be returned.
|
|
64
|
+
|
|
65
|
+ Reasons for updating the complete data set:
|
|
66
|
+
|
|
67
|
+ * UID of source_instance has changed (in comparison to the cached value).
|
|
68
|
+ * data_version is increased
|
|
69
|
+
|
|
70
|
+ **Example:**
|
|
71
|
+
|
|
72
|
+ .. literalinclude:: ../../caching/_examples_/property_cache_pickle.py
|
|
73
|
+
|
|
74
|
+ Will result on the first execution to the following output (with a long execution time):
|
|
75
|
+
|
|
76
|
+ .. literalinclude:: ../../caching/_examples_/property_cache_pickle_1.log
|
|
77
|
+
|
|
78
|
+ With every following execution (slow for getting "two" which is not cached - see implementation):
|
|
79
|
+
|
|
80
|
+ .. literalinclude:: ../../caching/_examples_/property_cache_pickle_2.log
|
|
81
|
+ """
|
|
82
|
+ LOG_PREFIX = 'PickCache:'
|
|
83
|
+ DATA_VERSION_TAG = '_property_cache_data_version_'
|
|
84
|
+ UID_TAG = '_property_cache_uid_'
|
|
85
|
+
|
|
86
|
+ def __init__(self, source_instance, cache_filename, load_all_on_init=False, callback_on_data_storage=None):
|
|
87
|
+ self._source_instance = source_instance
|
|
88
|
+ self._cache_filename = cache_filename
|
|
89
|
+ self._load_all_on_init = load_all_on_init
|
|
90
|
+ self._callback_on_data_storage = callback_on_data_storage
|
|
91
|
+ self._cached_props = None
|
|
92
|
+
|
|
93
|
+ def get(self, key, default=None):
|
|
94
|
+ """
|
|
95
|
+ Method to get the cached property. If key does not exists in cache, the property will be loaded from source_instance and stored in cache (file).
|
|
96
|
+
|
|
97
|
+ :param key: key for value to get.
|
|
98
|
+ :param default: value to be returned, if key does not exists.
|
|
99
|
+ :returns: value for a given key or default value.
|
|
100
|
+ """
|
|
101
|
+ if key in self.keys():
|
|
102
|
+ if self._cached_props is None:
|
|
103
|
+ self._init_cache()
|
|
104
|
+ if self._key_filter(key) not in self._cached_props:
|
|
105
|
+ val = self._source_instance.get(key, None)
|
|
106
|
+ logger.debug("%s Loading property for '%s' from source instance (%s)", self.LOG_PREFIX, key, repr(val))
|
|
107
|
+ self._cached_props[self._key_filter(key)] = val
|
|
108
|
+ self._save_cache()
|
|
109
|
+ else:
|
|
110
|
+ logger.debug("%s Providing property for '%s' from cache (%s)", self.LOG_PREFIX, key, repr(self._cached_props.get(self._key_filter(key), default)))
|
|
111
|
+ return self._cached_props.get(self._key_filter(key), default)
|
|
112
|
+ else:
|
|
113
|
+ logger.info("%s Key '%s' is not in cached_keys. Uncached data will be returned.", self.LOG_PREFIX, key)
|
|
114
|
+ return self._source_instance.get(key, default)
|
|
115
|
+
|
|
116
|
+ def keys(self):
|
|
117
|
+ return self._source_instance.keys()
|
|
118
|
+
|
|
119
|
+ def _data_version(self):
|
|
120
|
+ if self._cached_props is None:
|
|
121
|
+ return None
|
|
122
|
+ else:
|
|
123
|
+ return self._cached_props.get(self.DATA_VERSION_TAG, None)
|
|
124
|
+
|
|
125
|
+ def _init_cache(self):
|
|
126
|
+ if not self._load_cache() or self._source_instance.uid() != self._uid() or self._source_instance.data_version() > self._data_version():
|
|
127
|
+ if self._uid() is not None and self._source_instance.uid() != self._uid():
|
|
128
|
+ logger.debug("%s Source uid changed, ignoring previous cache data", self.LOG_PREFIX)
|
|
129
|
+ if self._data_version() is not None and self._source_instance.data_version() > self._data_version():
|
|
130
|
+ logger.debug("%s Data version increased, ignoring previous cache data", self.LOG_PREFIX)
|
|
131
|
+ self._cached_props = dict()
|
|
132
|
+ if self._load_all_on_init:
|
|
133
|
+ self._load_source()
|
|
134
|
+ self._cached_props[self.UID_TAG] = self._source_instance.uid()
|
|
135
|
+ self._cached_props[self.DATA_VERSION_TAG] = self._source_instance.data_version()
|
|
136
|
+ self._save_cache()
|
|
137
|
+
|
|
138
|
+ def _load_cache(self):
|
|
139
|
+ if os.path.exists(self._cache_filename):
|
|
140
|
+ with open(self._cache_filename, 'rb') as fh:
|
|
141
|
+ self._cached_props = pickle.load(fh)
|
|
142
|
+ logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
|
|
143
|
+ return True
|
|
144
|
+ else:
|
|
145
|
+ logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
|
|
146
|
+ return False
|
|
147
|
+
|
|
148
|
+ def _key_filter(self, key):
|
|
149
|
+ if sys.version_info >= (3, 0):
|
|
150
|
+ tps = [str]
|
|
151
|
+ else:
|
|
152
|
+ tps = [str, unicode]
|
|
153
|
+ if type(key) in tps:
|
|
154
|
+ if key.endswith(self.DATA_VERSION_TAG) or key.endswith(self.UID_TAG):
|
|
155
|
+ return '_' + key
|
|
156
|
+ return key
|
|
157
|
+
|
|
158
|
+ def _load_source(self):
|
|
159
|
+ logger.debug('%s Loading all data from source - %s', self.LOG_PREFIX, repr(self._source_instance.keys()))
|
|
160
|
+ for key in self._source_instance.keys():
|
|
161
|
+ val = self._source_instance.get(key)
|
|
162
|
+ self._cached_props[self._key_filter(key)] = val
|
|
163
|
+
|
|
164
|
+ def _save_cache(self):
|
|
165
|
+ with open(self._cache_filename, 'wb') as fh:
|
|
166
|
+ pickle.dump(self._cached_props, fh)
|
|
167
|
+ logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
|
|
168
|
+ if self._callback_on_data_storage is not None:
|
|
169
|
+ self._callback_on_data_storage()
|
|
170
|
+
|
|
171
|
+ def _uid(self):
|
|
172
|
+ if self._cached_props is None:
|
|
173
|
+ return None
|
|
174
|
+ else:
|
|
175
|
+ return self._cached_props.get(self.UID_TAG, None)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+class property_cache_json(property_cache_pickle):
|
|
179
|
+ """
|
|
180
|
+ Class to cache properties, which take longer on initialising than reading a file in json format. See also parent :py:class:`property_cache_pickle`
|
|
181
|
+
|
|
182
|
+ :param source_instance: The source instance holding the data
|
|
183
|
+ :type source_instance: instance
|
|
184
|
+ :param cache_filename: File name, where the properties are stored as cache
|
|
185
|
+ :type cache_filename: str
|
|
186
|
+ :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
|
|
187
|
+ :raises: ?
|
|
188
|
+
|
|
189
|
+ .. warning::
|
|
190
|
+ * This class uses json. You should **only** use keys of type string!
|
|
191
|
+ * Unicode types are transfered to strings
|
|
192
|
+
|
|
193
|
+ .. note:: This module uses logging. So logging should be initialised at least by executing logging.basicConfig(...)
|
|
194
|
+
|
|
195
|
+ .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
|
|
196
|
+
|
|
197
|
+ * uid(): returns the unique id of the source.
|
|
198
|
+ * keys(): returns a list of all available keys.
|
|
199
|
+ * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
|
|
200
|
+ * get(key, default): returns the property for a key. If key does not exists, default will be returned.
|
|
201
|
+
|
|
202
|
+ Reasons for updating the complete data set:
|
|
203
|
+
|
|
204
|
+ * UID of source_instance has changed (in comparison to the cached value).
|
|
205
|
+ * data_version is increased
|
|
206
|
+
|
|
207
|
+ **Example:**
|
|
208
|
+
|
|
209
|
+ .. literalinclude:: ../../caching/_examples_/property_cache_json.py
|
|
210
|
+
|
|
211
|
+ Will result on the first execution to the following output (with a long execution time):
|
|
212
|
+
|
|
213
|
+ .. literalinclude:: ../../caching/_examples_/property_cache_json_1.log
|
|
214
|
+
|
|
215
|
+ With every following execution (slow for getting "two" which is not cached - see implementation):
|
|
216
|
+
|
|
217
|
+ .. literalinclude:: ../../caching/_examples_/property_cache_json_2.log
|
|
218
|
+ """
|
|
219
|
+ LOG_PREFIX = 'JsonCache:'
|
|
220
|
+
|
|
221
|
+ def _load_cache(self):
|
|
222
|
+ if os.path.exists(self._cache_filename):
|
|
223
|
+ with open(self._cache_filename, 'r') as fh:
|
|
224
|
+ self._cached_props = json.load(fh)
|
|
225
|
+ logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
|
|
226
|
+ return True
|
|
227
|
+ else:
|
|
228
|
+ logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
|
|
229
|
+ return False
|
|
230
|
+
|
|
231
|
+ def _save_cache(self):
|
|
232
|
+ with open(self._cache_filename, 'w') as fh:
|
|
233
|
+ json.dump(self._cached_props, fh, sort_keys=True, indent=4)
|
|
234
|
+ logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
|
|
235
|
+ if self._callback_on_data_storage is not None:
|
|
236
|
+ self._callback_on_data_storage()
|