caching/__init__.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
"""
caching (Caching Module)
========================

**Author:**

* Dirk Alders <sudo-dirk@mount-mockery.de>

**Description:**

    This Module supports functions and classes for caching e.g. properties of other instances.

**Submodules:**

* :class:`caching.property_cache_json`
* :class:`caching.property_cache_pickle`

**Unittest:**

    See also the :download:`unittest <caching/_testresults_/unittest.pdf>` documentation.
"""
__DEPENDENCIES__ = []

import json
import logging
import os
import pickle
import sys
import time

try:
    from config import APP_NAME as ROOT_LOGGER_NAME
except ImportError:
    ROOT_LOGGER_NAME = 'root'
logger = logging.getLogger(ROOT_LOGGER_NAME).getChild(__name__)

__DESCRIPTION__ = """The Module {\\tt %s} is designed to store information in {\\tt json} or {\\tt pickle} files to support them much faster then generating them from the original source file.
For more Information read the documentation.""" % __name__.replace('_', '\_')
"""The Module Description"""
__INTERPRETER__ = (3, )
"""The Tested Interpreter-Versions"""


class property_cache_pickle(object):
    """
    Class to cache properties, which take longer on initialising than reading a file in pickle format.

    :param source_instance: The source instance holding the data
    :type source_instance: instance
    :param cache_filename: File name, where the properties are stored as cache
    :type cache_filename: str
    :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.

    .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()

                * uid(): returns the unique id of the source.
                * keys(): returns a list of all available keys.
                * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
                * get(key, default): returns the property for a key. If key does not exists, default will be returned.

    Reasons for updating the complete data set:

    * UID of source_instance has changed (in comparison to the cached value).
    * data_version is increased

    **Example:**

    .. literalinclude:: caching/_examples_/property_cache_pickle.py

    Will result on the first execution to the following output (with a long execution time):

    .. literalinclude:: caching/_examples_/property_cache_pickle_1.log

    With every following execution (slow for getting "two" which is not cached - see implementation):

    .. literalinclude:: caching/_examples_/property_cache_pickle_2.log
    """
    LOG_PREFIX = 'PickCache:'
    DATA_VERSION_TAG = '_property_cache_data_version_'
    STORAGE_VERSION_TAG = '_storage_version_'
    UID_TAG = '_property_cache_uid_'
    DATA_TAG = '_data_'
    AGE_TAG = '_age_'
    #
    STORAGE_VERSION = 1

    def __init__(self, source_instance, cache_filename, load_all_on_init=False, callback_on_data_storage=None, max_age=None, store_none_value=False):
        self._source_instance = source_instance
        self._cache_filename = cache_filename
        self._load_all_on_init = load_all_on_init
        self._callback_on_data_storage = callback_on_data_storage
        self._max_age = max_age
        self._store_none_value = store_none_value
        self._cached_props = None

    def get(self, key, default=None):
        """
        Method to get the cached property. If key does not exists in cache, the property will be loaded from source_instance and stored in cache (file).

        :param key: key for value to get.
        :param default: value to be returned, if key does not exists.
        :returns: value for a given key or default value.
        """
        if key in self.keys():
            if self._cached_props is None:
                self._init_cache()
            if self._max_age is None:
                cache_old = False
            else:
                cache_old = time.time() - self._cached_props[self.AGE_TAG].get(self._key_filter(key), 0) > self._max_age
                if cache_old:
                    logger.debug("The cached value is old, cached value will be ignored")
            if self._key_filter(key) not in self._cached_props[self.DATA_TAG] or cache_old:
                val = self._source_instance.get(key, None)
                logger.debug("%s Loading property for '%s' from source instance (%s)", self.LOG_PREFIX, key, repr(val))
                if val or self._store_none_value:
                    tm = int(time.time())
                    logger.debug("Storing value=%s with timestamp=%d to chache", val, tm)
                    self._cached_props[self.DATA_TAG][self._key_filter(key)] = val
                    self._cached_props[self.AGE_TAG][self._key_filter(key)] = tm
                    self._save_cache()
                else:
                    return val
            else:
                logger.debug("%s Providing property for '%s' from cache (%s)", self.LOG_PREFIX,
                             key, repr(self._cached_props[self.DATA_TAG].get(self._key_filter(key), default)))
            return self._cached_props[self.DATA_TAG].get(self._key_filter(key), default)
        else:
            logger.info("%s Key '%s' is not in cached_keys. Uncached data will be returned.", self.LOG_PREFIX, key)
            return self._source_instance.get(key, default)

    def keys(self):
        """
        Method to get the available keys (from :data:`source_instance`).
        """
        return self._source_instance.keys()

    def _data_version(self):
        if self._cached_props is None:
            return None
        else:
            return self._cached_props.get(self.DATA_VERSION_TAG, None)

    def _storage_version(self):
        if self._cached_props is None:
            return None
        else:
            return self._cached_props.get(self.STORAGE_VERSION_TAG, None)

    def _init_cache(self):
        load_cache = self._load_cache()
        uid = self._source_instance.uid() != self._uid()
        try:
            data_version = self._source_instance.data_version() > self._data_version()
        except TypeError:
            data_version = True
        try:
            storage_version = self._storage_version() != self.STORAGE_VERSION
        except TypeError:
            storage_version = True
        #
        if not load_cache or uid or data_version or storage_version:
            if self._uid() is not None and uid:
                logger.debug("%s Source uid changed, ignoring previous cache data", self.LOG_PREFIX)
            if self._data_version() is not None and data_version:
                logger.debug("%s Data version increased, ignoring previous cache data", self.LOG_PREFIX)
            if storage_version:
                logger.debug("%s Storage version changed, ignoring previous cache data", self.LOG_PREFIX)
            self._cached_props = {self.AGE_TAG: {}, self.DATA_TAG: {}}
            if self._load_all_on_init:
                self._load_source()
            self._cached_props[self.UID_TAG] = self._source_instance.uid()
            self._cached_props[self.DATA_VERSION_TAG] = self._source_instance.data_version()
            self._cached_props[self.STORAGE_VERSION_TAG] = self.STORAGE_VERSION
            self._save_cache()

    def _load_cache(self):
        if os.path.exists(self._cache_filename):
            with open(self._cache_filename, 'rb') as fh:
                self._cached_props = pickle.load(fh)
            logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
            return True
        else:
            logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
        return False

    def _key_filter(self, key):
        return key

    def _load_source(self):
        logger.debug('%s Loading all data from source - %s', self.LOG_PREFIX, repr(self._source_instance.keys()))
        for key in self._source_instance.keys():
            val = self._source_instance.get(key)
            self._cached_props[self.DATA_TAG][self._key_filter(key)] = val
            self._cached_props[self.AGE_TAG][self._key_filter(key)] = int(time.time())

    def _save_cache(self):
        with open(self._cache_filename, 'wb') as fh:
            pickle.dump(self._cached_props, fh)
            logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
        if self._callback_on_data_storage is not None:
            self._callback_on_data_storage()

    def _uid(self):
        if self._cached_props is None:
            return None
        else:
            return self._cached_props.get(self.UID_TAG, None)


class property_cache_json(property_cache_pickle):
    """
    Class to cache properties, which take longer on initialising than reading a file in json format. See also parent :py:class:`property_cache_pickle`

    :param source_instance: The source instance holding the data
    :type source_instance: instance
    :param cache_filename: File name, where the properties are stored as cache
    :type cache_filename: str
    :param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.

    .. warning::
        * This class uses json. You should **only** use keys of type string!
        * Unicode types are transfered to strings

    .. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()

                * uid(): returns the unique id of the source.
                * keys(): returns a list of all available keys.
                * data_version(): returns a version number of the current data (it should be increased, if the get method of the source instance returns improved values or the data structure had been changed).
                * get(key, default): returns the property for a key. If key does not exists, default will be returned.

    Reasons for updating the complete data set:

    * UID of source_instance has changed (in comparison to the cached value).
    * data_version is increased

    **Example:**

    .. literalinclude:: caching/_examples_/property_cache_json.py

    Will result on the first execution to the following output (with a long execution time):

    .. literalinclude:: caching/_examples_/property_cache_json_1.log

    With every following execution (slow for getting "two" which is not cached - see implementation):

    .. literalinclude:: caching/_examples_/property_cache_json_2.log
    """
    LOG_PREFIX = 'JsonCache:'

    def _load_cache(self):
        if os.path.exists(self._cache_filename):
            with open(self._cache_filename, 'r') as fh:
                self._cached_props = json.load(fh)
            logger.info('%s Loading properties from cache (%s)', self.LOG_PREFIX, self._cache_filename)
            return True
        else:
            logger.debug('%s Cache file does not exists (yet).', self.LOG_PREFIX)
        return False

    def _save_cache(self):
        with open(self._cache_filename, 'w') as fh:
            json.dump(self._cached_props, fh, sort_keys=True, indent=4)
            logger.info('%s cache-file stored (%s)', self.LOG_PREFIX, self._cache_filename)
        if self._callback_on_data_storage is not None:
            self._callback_on_data_storage()