Release: e10fa482d1

__init__.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+"""
+caching (Caching Module)
+========================
+
+**Author:**
+
+* Dirk Alders <sudo-dirk@mount-mockery.de>
+
+**Description:**
+
+    This Module supports functions and classes for caching e.g. properties of other instances.
+
+**Submodules:**
+
+* :class:`caching.property_cache_json`
+* :class:`caching.property_cache_pickle`
+
+**Unittest:**
+
+    See also the :download:`unittest <../../caching/_testresults_/unittest.pdf>` documentation.
+"""
+__DEPENDENCIES__ = []
+
+import hashlib
+import hmac
+import json
+import logging
+import os
+import pickle
+import sys
+
+logger_name = 'CACHING'
+logger = logging.getLogger(logger_name)
+
+__DESCRIPTION__ = """The Module {\\tt %s} is designed to store information in {\\tt json} or {\\tt pickle} files to support them much faster then generating them from the original source file.
+For more Information read the documentation.""" % __name__.replace('_', '\_')
+"""The Module Description"""
+__INTERPRETER__ = (2, 3)
+"""The Tested Interpreter-Versions"""
+
+
+class property_cache_pickle(object):
+    """
+    Class to cache properties, which take longer on initialising than reading a file in pickle format.
+
+    :param source_instance: The source instance holding the data
+    :type source_instance: instance
+    :param cache_filename: File name, where the properties are stored as cache
+    :type cache_filename: str
+    :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
+    :raises: ?
+
+    .. note:: This module uses logging. So logging should be initialised at least by executing logging.basicConfig(...)
+
+    .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
+
+                * uid(): returns the unique id of the source.
+                * keys(): returns a list of all available keys.
+                * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
+                * get(key, default): returns the property for a key. If key does not exists, default will be returned.
+
+    Reasons for updating the complete data set:
+
+    * UID of source_instance has changed (in comparison to the cached value).
+    * data_version is increased
+
+    **Example:**
+
+    .. literalinclude:: ../../caching/_examples_/property_cache_pickle.py
+
+    Will result on the first execution to the following output (with a long execution time):
+
+    .. literalinclude:: ../../caching/_examples_/property_cache_pickle_1.log
+
+    With every following execution (slow for getting "two" which is not cached - see implementation):
+
+    .. literalinclude:: ../../caching/_examples_/property_cache_pickle_2.log
+    """
+    LOG_PREFIX = 'PickCache:'
+    DATA_VERSION_TAG = '_property_cache_data_version_'
+    UID_TAG = '_property_cache_uid_'
+
+    def __init__(self, source_instance, cache_filename, load_all_on_init=False, callback_on_data_storage=None):
+        self._source_instance = source_instance
+        self._cache_filename = cache_filename
+        self._load_all_on_init = load_all_on_init
+        self._callback_on_data_storage = callback_on_data_storage
+        self._cached_props = None
+
+    def get(self, key, default=None):
+        """
+        Method to get the cached property. If key does not exists in cache, the property will be loaded from source_instance and stored in cache (file).
+
+        :param key: key for value to get.
+        :param default: value to be returned, if key does not exists.
+        :returns: value for a given key or default value.
+        """
+        if key in self.keys():
+            if self._cached_props is None:
+                self._init_cache()
+            if self._key_filter(key) not in self._cached_props:
+                val = self._source_instance.get(key, None)
+                logger.debug("%s Loading property for '%s' from source instance (%s)", self.LOG_PREFIX, key, repr(val))
+                self._cached_props[self._key_filter(key)] = val
+                self._save_cache()
+            else:
+                logger.debug("%s Providing property for '%s' from cache (%s)", self.LOG_PREFIX, key, repr(self._cached_props.get(self._key_filter(key), default)))
+            return self._cached_props.get(self._key_filter(key), default)
+        else:
+            logger.info("%s Key '%s' is not in cached_keys. Uncached data will be returned.", self.LOG_PREFIX, key)
+            return self._source_instance.get(key, default)
+
+    def keys(self):
+        return self._source_instance.keys()
+
+    def _data_version(self):
+        if self._cached_props is None:
+            return None
+        else:
+            return self._cached_props.get(self.DATA_VERSION_TAG, None)
+
+    def _init_cache(self):
+        if not self._load_cache() or self._source_instance.uid() != self._uid() or self._source_instance.data_version() > self._data_version():
+            if self._uid() is not None and self._source_instance.uid() != self._uid():
+                logger.debug("%s Source uid changed, ignoring previous cache data", self.LOG_PREFIX)
+            if self._data_version() is not None and self._source_instance.data_version() > self._data_version():
+                logger.debug("%s Data version increased, ignoring previous cache data", self.LOG_PREFIX)
+            self._cached_props = dict()
+            if self._load_all_on_init:
+                self._load_source()
+            self._cached_props[self.UID_TAG] = self._source_instance.uid()
+            self._cached_props[self.DATA_VERSION_TAG] = self._source_instance.data_version()
+            self._save_cache()
+
+    def _load_cache(self):
+        if os.path.exists(self._cache_filename):
+            with open(self._cache_filename, 'rb') as fh:
+                self._cached_props = pickle.load(fh)
+            logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
+            return True
+        else:
+            logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
+        return False
+
+    def _key_filter(self, key):
+        if sys.version_info >= (3, 0):
+            tps = [str]
+        else:
+            tps = [str, unicode]
+        if type(key) in tps:
+            if key.endswith(self.DATA_VERSION_TAG) or key.endswith(self.UID_TAG):
+                return '_' + key
+        return key
+
+    def _load_source(self):
+        logger.debug('%s Loading all data from source - %s', self.LOG_PREFIX, repr(self._source_instance.keys()))
+        for key in self._source_instance.keys():
+            val = self._source_instance.get(key)
+            self._cached_props[self._key_filter(key)] = val
+
+    def _save_cache(self):
+        with open(self._cache_filename, 'wb') as fh:
+            pickle.dump(self._cached_props, fh)
+            logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
+        if self._callback_on_data_storage is not None:
+            self._callback_on_data_storage()
+
+    def _uid(self):
+        if self._cached_props is None:
+            return None
+        else:
+            return self._cached_props.get(self.UID_TAG, None)
+
+
+class property_cache_json(property_cache_pickle):
+    """
+    Class to cache properties, which take longer on initialising than reading a file in json format. See also parent :py:class:`property_cache_pickle`
+
+    :param source_instance: The source instance holding the data
+    :type source_instance: instance
+    :param cache_filename: File name, where the properties are stored as cache
+    :type cache_filename: str
+    :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
+    :raises: ?
+
+    .. warning::
+        * This class uses json. You should **only** use keys of type string!
+        * Unicode types are transfered to strings
+
+    .. note:: This module uses logging. So logging should be initialised at least by executing logging.basicConfig(...)
+
+    .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
+
+                * uid(): returns the unique id of the source.
+                * keys(): returns a list of all available keys.
+                * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
+                * get(key, default): returns the property for a key. If key does not exists, default will be returned.
+
+    Reasons for updating the complete data set:
+
+    * UID of source_instance has changed (in comparison to the cached value).
+    * data_version is increased
+
+    **Example:**
+
+    .. literalinclude:: ../../caching/_examples_/property_cache_json.py
+
+    Will result on the first execution to the following output (with a long execution time):
+
+    .. literalinclude:: ../../caching/_examples_/property_cache_json_1.log
+
+    With every following execution (slow for getting "two" which is not cached - see implementation):
+
+    .. literalinclude:: ../../caching/_examples_/property_cache_json_2.log
+    """
+    LOG_PREFIX = 'JsonCache:'
+
+    def _load_cache(self):
+        if os.path.exists(self._cache_filename):
+            with open(self._cache_filename, 'r') as fh:
+                self._cached_props = json.load(fh)
+            logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
+            return True
+        else:
+            logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
+        return False
+
+    def _save_cache(self):
+        with open(self._cache_filename, 'w') as fh:
+            json.dump(self._cached_props, fh, sort_keys=True, indent=4)
+            logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
+        if self._callback_on_data_storage is not None:
+            self._callback_on_data_storage()