302 lines
13 KiB
Python
302 lines
13 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
"""
|
|
caching (Caching Module)
|
|
========================
|
|
|
|
**Author:**
|
|
|
|
* Dirk Alders <sudo-dirk@mount-mockery.de>
|
|
|
|
**Description:**
|
|
|
|
This Module supports functions and classes for caching e.g. properties of other instances.
|
|
|
|
**Submodules:**
|
|
|
|
* :class:`caching.property_cache_json`
|
|
* :class:`caching.property_cache_pickle`
|
|
|
|
**Unittest:**
|
|
|
|
See also the :download:`unittest <caching/_testresults_/unittest.pdf>` documentation.
|
|
"""
|
|
__DEPENDENCIES__ = []
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import pickle
|
|
import time
|
|
|
|
try:
|
|
from config import APP_NAME as ROOT_LOGGER_NAME
|
|
except ImportError:
|
|
ROOT_LOGGER_NAME = 'root'
|
|
logger = logging.getLogger(ROOT_LOGGER_NAME).getChild(__name__)
|
|
|
|
__DESCRIPTION__ = """The Module {\\tt %s} is designed to store information in {\\tt json} or {\\tt pickle} files to support them much faster then generating them from the original source file.
|
|
For more Information read the documentation.""" % __name__.replace('_', '\_')
|
|
"""The Module Description"""
|
|
__INTERPRETER__ = (3, )
|
|
"""The Tested Interpreter-Versions"""
|
|
|
|
|
|
class property_cache_pickle(object):
|
|
"""
|
|
This class caches the data from a given `source_instance`. It takes the data from the cache instead of generating the data from the `source_instance`,
|
|
if the conditions for the cache usage are given.
|
|
|
|
.. admonition:: Required properties for the `source_instance`
|
|
|
|
* **uid():** returns the unique id of the source's source or None, if you don't want to use the unique id.
|
|
* **keys():** returns a list of all available keys.
|
|
* **data_version():** returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
|
|
* **get(key, default):** returns the property for a key. If key does not exists, default will be returned.
|
|
|
|
:param source_instance: The source instance holding the data
|
|
:type source_instance: instance
|
|
:param cache_filename: File name, where the properties are stored as cache
|
|
:type cache_filename: str
|
|
:param load_all_on_init: True will load all data from the source instance, when the cache will be initialised the first time.
|
|
:type load_all_on_init: bool
|
|
:param callback_on_data_storage: The callback will be executed every time when the cache file is stored. It will be executed with the instance of this class as first argument.
|
|
:type callback_on_data_storage: method
|
|
:param max_age: The maximum age of the cached data in seconds or None for no maximum age.
|
|
:type max_age: int or None
|
|
:param store_on_get: False will prevent cache storage with execution of the `.get(key, default)` method. You need to store the cache somewhere else.
|
|
:type store_on_get: bool
|
|
|
|
.. admonition:: The cache will be used, if all following conditions are given
|
|
|
|
* The key is in the list returned by `.keys()` method of the `source_instance`
|
|
* The key is not in the list of keys added by the `.add_source_get_keys()` method.
|
|
* The cache age is less then the given max_age parameter or the given max_age is None.
|
|
* The uid of the source instance (e.g. a checksum or unique id of the source) is identically to to uid stored in the cache.
|
|
* The data version of the `source_instance` is <= the data version stored in the cache.
|
|
* The value is available in the previous stored information
|
|
|
|
**Example:**
|
|
|
|
.. literalinclude:: caching/_examples_/property_cache_pickle.py
|
|
|
|
Will result on the first execution to the following output (with a long execution time):
|
|
|
|
.. literalinclude:: caching/_examples_/property_cache_pickle_1.log
|
|
|
|
With every following execution the time cosumption my by much smaller:
|
|
|
|
.. literalinclude:: caching/_examples_/property_cache_pickle_2.log
|
|
"""
|
|
DATA_VERSION_TAG = '_property_cache_data_version_'
|
|
STORAGE_VERSION_TAG = '_storage_version_'
|
|
UID_TAG = '_property_cache_uid_'
|
|
DATA_TAG = '_data_'
|
|
AGE_TAG = '_age_'
|
|
#
|
|
STORAGE_VERSION = 1
|
|
|
|
def __init__(self, source_instance, cache_filename, load_all_on_init=False, callback_on_data_storage=None, max_age=None, store_on_get=True, return_source_on_none=False):
|
|
self._source_instance = source_instance
|
|
self._cache_filename = cache_filename
|
|
self._load_all_on_init = load_all_on_init
|
|
self._callback_on_data_storage = callback_on_data_storage
|
|
self._max_age = max_age
|
|
self._store_on_get = store_on_get
|
|
self._return_source_on_none = return_source_on_none
|
|
#
|
|
self._source_get_keys = []
|
|
self._cached_props = None
|
|
|
|
def add_source_get_keys(self, keys):
|
|
"""
|
|
This will add one or more keys to a list of keys which will always be provided by the `source_instance` instead of the cache.
|
|
|
|
:param keys: The key or keys to be added
|
|
:type keys: list, tuple, str
|
|
"""
|
|
if type(keys) in [list, tuple]:
|
|
self._source_get_keys.extend(keys)
|
|
else:
|
|
self._source_get_keys.append(keys)
|
|
|
|
def full_update(self, sleep_between_keys=0):
|
|
"""
|
|
With the execution of this method, the complete source data which needs to be cached, will be read from the source instance
|
|
and the resulting cache will be stored to the given file.
|
|
|
|
:param sleep_between_keys: Time to sleep between each source data generation
|
|
:type sleep_between_keys: float, int
|
|
|
|
.. hint:: Use this method, if you initiallised the class with `store_on_get=False`
|
|
"""
|
|
self._load_source(sleep_between_keys=sleep_between_keys)
|
|
self._save_cache()
|
|
|
|
def get(self, key, default=None):
|
|
"""
|
|
Method to get the cached property. If the key does not exists in the cache or `source_instance`, `default` will be returned.
|
|
|
|
:param key: key for value to get.
|
|
:param default: value to be returned, if key does not exists.
|
|
:returns: value for a given key or default value.
|
|
"""
|
|
# Init cache
|
|
if self._cached_props is None:
|
|
self._init_cache()
|
|
# Identify old cache
|
|
if self._max_age is None:
|
|
cache_old = False
|
|
else:
|
|
cache_old = time.time() - self._cached_props[self.AGE_TAG].get(self._key_filter(key), 0) > self._max_age
|
|
if cache_old:
|
|
logger.debug("The cached value is old, cached value will be ignored")
|
|
# Return cached value
|
|
if not cache_old and key not in self._source_get_keys and self._key_filter(key) in self._cached_props[self.DATA_TAG]:
|
|
logger.debug("Providing property for '%s' from cache", key)
|
|
rv = self._cached_props[self.DATA_TAG].get(self._key_filter(key), default)
|
|
if rv is not None or not self._return_source_on_none:
|
|
return rv
|
|
# Create cache and return value
|
|
if key in self._source_instance.keys():
|
|
logger.debug("Loading property for key='%s' from source instance", key)
|
|
val = self._source_instance.get(key, None)
|
|
if self._store_on_get:
|
|
tm = int(time.time())
|
|
logger.debug("Adding key=%s, value=%s with timestamp=%d to chache", key, val, tm)
|
|
self._cached_props[self.DATA_TAG][self._key_filter(key)] = val
|
|
self._cached_props[self.AGE_TAG][self._key_filter(key)] = tm
|
|
self._save_cache()
|
|
else:
|
|
return val
|
|
cached_data = self._cached_props[self.DATA_TAG].get(self._key_filter(key), default)
|
|
if cached_data is None and self._return_source_on_none:
|
|
return self._source_instance.get(key, default)
|
|
return cached_data
|
|
else:
|
|
if key not in self._source_instance.keys():
|
|
logger.debug("Key '%s' is not in cached_keys. Uncached data will be returned.", key)
|
|
else:
|
|
logger.debug("Key '%s' is excluded by .add_source_get_keys(). Uncached data will be returned.", key)
|
|
return self._source_instance.get(key, default)
|
|
|
|
def _data_version(self):
|
|
if self._cached_props is None:
|
|
return None
|
|
else:
|
|
return self._cached_props.get(self.DATA_VERSION_TAG, None)
|
|
|
|
def _storage_version(self):
|
|
if self._cached_props is None:
|
|
return None
|
|
else:
|
|
return self._cached_props.get(self.STORAGE_VERSION_TAG, None)
|
|
|
|
def _init_cache(self):
|
|
load_cache = self._load_cache()
|
|
uid = self._source_instance.uid() != self._uid()
|
|
try:
|
|
data_version = self._source_instance.data_version() > self._data_version()
|
|
except TypeError:
|
|
data_version = True
|
|
try:
|
|
storage_version = self._storage_version() != self.STORAGE_VERSION
|
|
except TypeError:
|
|
storage_version = True
|
|
#
|
|
if not load_cache or uid or data_version or storage_version:
|
|
if load_cache:
|
|
if self._uid() is not None and uid:
|
|
logger.debug("Source uid changed, ignoring previous cache data")
|
|
if self._data_version() is not None and data_version:
|
|
logger.debug("Data version increased, ignoring previous cache data")
|
|
if storage_version:
|
|
logger.debug("Storage version changed, ignoring previous cache data")
|
|
self._cached_props = {self.AGE_TAG: {}, self.DATA_TAG: {}}
|
|
if self._load_all_on_init:
|
|
self._load_source()
|
|
self._cached_props[self.UID_TAG] = self._source_instance.uid()
|
|
self._cached_props[self.DATA_VERSION_TAG] = self._source_instance.data_version()
|
|
self._cached_props[self.STORAGE_VERSION_TAG] = self.STORAGE_VERSION
|
|
|
|
def _load_only(self):
|
|
with open(self._cache_filename, 'rb') as fh:
|
|
self._cached_props = pickle.load(fh)
|
|
logger.debug('Loading properties from cache (%s)', self._cache_filename)
|
|
|
|
def _load_cache(self):
|
|
if os.path.exists(self._cache_filename):
|
|
try:
|
|
self._load_only()
|
|
except:
|
|
logger.exception("Exception while loading cache file %s", self._cache_filename)
|
|
else:
|
|
return True
|
|
else:
|
|
logger.debug('Cache file does not exists (yet).')
|
|
return False
|
|
|
|
def _key_filter(self, key):
|
|
return key
|
|
|
|
def _load_source(self, sleep_between_keys=0):
|
|
if self._cached_props is None:
|
|
self._init_cache()
|
|
logger.debug('Loading all data from source - %s', repr(self._source_instance.keys()))
|
|
for key in self._source_instance.keys():
|
|
if key not in self._source_get_keys:
|
|
self._cached_props[self.DATA_TAG][self._key_filter(key)] = self._source_instance.get(key)
|
|
self._cached_props[self.AGE_TAG][self._key_filter(key)] = int(time.time())
|
|
time.sleep(sleep_between_keys)
|
|
|
|
def _save_only(self):
|
|
with open(self._cache_filename, 'wb') as fh:
|
|
pickle.dump(self._cached_props, fh)
|
|
logger.debug('cache-file stored (%s)', self._cache_filename)
|
|
|
|
def _save_cache(self):
|
|
self._save_only()
|
|
if self._callback_on_data_storage is not None:
|
|
self._callback_on_data_storage(self)
|
|
|
|
def _uid(self):
|
|
if self._cached_props is None:
|
|
return None
|
|
else:
|
|
return self._cached_props.get(self.UID_TAG, None)
|
|
|
|
|
|
class property_cache_json(property_cache_pickle):
|
|
"""
|
|
See also parent :py:class:`property_cache_pickle` for detailed information.
|
|
|
|
.. important::
|
|
* This class uses json. You should **only** use keys of type string!
|
|
* Unicode types are transfered to strings
|
|
|
|
See limitations of json.
|
|
|
|
**Example:**
|
|
|
|
.. literalinclude:: caching/_examples_/property_cache_json.py
|
|
|
|
Will result on the first execution to the following output (with a long execution time):
|
|
|
|
.. literalinclude:: caching/_examples_/property_cache_json_1.log
|
|
|
|
With every following execution the time cosumption my by much smaller:
|
|
|
|
.. literalinclude:: caching/_examples_/property_cache_json_2.log
|
|
"""
|
|
|
|
def _load_only(self):
|
|
with open(self._cache_filename, 'r') as fh:
|
|
self._cached_props = json.load(fh)
|
|
logger.debug('Loading properties from cache (%s)', self._cache_filename)
|
|
|
|
def _save_only(self):
|
|
with open(self._cache_filename, 'w') as fh:
|
|
json.dump(self._cached_props, fh, sort_keys=True, indent=4)
|
|
logger.debug('cache-file stored (%s)', self._cache_filename)
|