pylib/caching
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Interface improvement for cached object (inkl. example adaption)

Browse Source
This commit is contained in:
Dirk Alders 2024-09-15 18:58:42 +02:00
parent 8eeb7657be
commit 529f4100b6
3 changed files with 108 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
__init__.py
View File

@ -28,7 +28,6 @@ import json
import logging
import os
import pickle
import sys
import time
try:
@ -53,6 +52,9 @@ class property_cache_pickle(object):
:param cache_filename: File name, where the properties are stored as cache
:type cache_filename: str
:param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
:param max_age: The maximum age of the cache object, after that time the source will be used for getting information.
:type max_age: int
:param store_on_get: Parameter to enable / disable cache storage on get. If you disble data storage, you need to update and store the cache manually
.. note:: source_instance needs to have at least the following methods: uid(), keys(), data_version(), get()
@ -94,6 +96,8 @@ class property_cache_pickle(object):
self._callback_on_data_storage = callback_on_data_storage
self._max_age = max_age
self._store_on_get = store_on_get
#
self._source_get_keys = []
self._cached_props = None
def get(self, key, default=None):
@ -104,7 +108,7 @@ class property_cache_pickle(object):
:param default: value to be returned, if key does not exists.
:returns: value for a given key or default value.
"""
if key in self.keys():
if key in self.keys() and key not in self._source_get_keys:
if self._cached_props is None:
self._init_cache()
if self._max_age is None:
@ -198,8 +202,9 @@ class property_cache_pickle(object):
def _load_source(self):
if self._cached_props is None:
self._init_cache()
logger.debug('%s Loading all data from source - %s', self.LOG_PREFIX, repr(self._source_instance.keys()))
for key in self._source_instance.keys():
logger.debug('%s Loading all data from source - %s', self.LOG_PREFIX, repr(self.keys()))
for key in self.keys():
if key not in self._source_get_keys:
self._cached_props[self.DATA_TAG][self._key_filter(key)] = self._source_instance.get(key)
self._cached_props[self.AGE_TAG][self._key_filter(key)] = int(time.time())
@ -216,6 +221,18 @@ class property_cache_pickle(object):
else:
return self._cached_props.get(self.UID_TAG, None)
def add_source_get_keys(self, keys):
if type(keys) in [list, tuple]:
self._source_get_keys.extend(keys)
else:
self._source_get_keys.append(keys)
def __getattribute__(self, name):
try:
return super().__getattribute__(name)
except AttributeError:
return getattr(self._source_instance, name)
class property_cache_json(property_cache_pickle):
"""
@ -226,6 +243,9 @@ class property_cache_json(property_cache_pickle):
:param cache_filename: File name, where the properties are stored as cache
:type cache_filename: str
:param load_all_on_init: Optionally init behaviour control parameter. True will load all available properties from source on init, False not.
:param max_age: The maximum age of the cache object, after that time the source will be used for getting information.
:type max_age: int
:param store_on_get: Parameter to enable / disable cache storage on get. If you disble data storage, you need to update and store the cache manually
.. warning::
* This class uses json. You should **only** use keys of type string!

99
_examples_/property_cache_json.py
View File

@ -2,62 +2,31 @@
# -*- coding: UTF-8 -*-
import sys
import time
sys.path.append('../..')
import caching
import report
import caching
import time
report.stdoutLoggingConfigure(log_name_lvl=[('root', 'DEBUG'), ])
class test_slow_data(object):
_ONE = '1'
_TWO = '2'
_THREE = '_property_cache_data_version_'
_FOUR = '_property_cache_uid_'
_FIVE = '__property_cache_uid_'
KEYS = [_ONE, _TWO, _THREE, _FOUR, _FIVE]
VERS = 0.1
DATA_VERSION = 0.1
KEY_ONE = '1'
KEY_TWO = '2'
KEY_THREE = 'three'
KEY_FOUR = 'four'
KEY_FIVE = 'five'
KEYS = [KEY_ONE, KEY_TWO, KEY_THREE, KEY_FOUR, KEY_FIVE]
def data_version(self):
return self.VERS
def one(self):
return self.get(self._ONE)
def two(self):
return self.get(self._TWO)
def three(self):
return self.get(self._THREE)
def four(self):
return self.get(self._FOUR)
def five(self):
return self.get(self._FIVE)
return self.DATA_VERSION
def get(self, key, default=None):
def print_n_sleep(k):
sys.stdout.write('slow get executed for %s\n' % k)
time.sleep(3)
if key == self._ONE:
print_n_sleep(key)
return 'one'
if key == self._TWO:
print_n_sleep(key)
return 'two'
if key == self._THREE:
print_n_sleep(key)
return 'three'
if key == self._FOUR:
print_n_sleep(key)
return 'four'
if key == self._FIVE:
print_n_sleep(key)
return 'five'
try:
return getattr(self, f'__{key}__')()
except AttributeError:
return default
def keys(self):
@ -66,23 +35,35 @@ class test_slow_data(object):
def uid(self):
return None
def print_n_sleep(self, k):
sys.stdout.write('slow get executed for %s\n' % k)
time.sleep(3)
class tsd_cache_json(test_slow_data):
def __init__(self, *args, **kwargs):
test_slow_data.__init__(self, *args, **kwargs)
self._cached_data = caching.property_cache_json(test_slow_data(*args, **kwargs), 'cache.json', load_all_on_init=False)
def __1__(self):
self.print_n_sleep("__1__")
return 'one'
def two(self):
return test_slow_data.get(self, self._TWO)
def __2__(self):
self.print_n_sleep("__2__")
return 'two'
def get(self, key, default=None):
return self._cached_data.get(key, default)
def __three__(self):
self.print_n_sleep("__three__")
return 'three'
def __four__(self):
self.print_n_sleep("__four__")
return 'four'
def __five__(self):
self.print_n_sleep("__five__")
return 'five'
data = tsd_cache_json()
print('Testing property_cache (json):\\n--------------------------------')
print(data.one())
print(data.two())
print(data.three())
print(data.four())
print(data.five())
if __name__ == "__main__":
data = caching.property_cache_json(test_slow_data(), 'cache.json')
data.add_source_get_keys(data.KEY_THREE)
print('Testing property_cache (json):\\n--------------------------------')
for key in data.keys():
print(data.get(key))

99
_examples_/property_cache_pickle.py
View File

@ -2,62 +2,31 @@
# -*- coding: UTF-8 -*-
import sys
import time
sys.path.append('../..')
import caching
import report
import caching
import time
report.stdoutLoggingConfigure(log_name_lvl=[('root', 'DEBUG'), ])
class test_slow_data(object):
_ONE = '1'
_TWO = '2'
_THREE = '_property_cache_data_version_'
_FOUR = '_property_cache_uid_'
_FIVE = '__property_cache_uid_'
KEYS = [_ONE, _TWO, _THREE, _FOUR, _FIVE]
VERS = 0.1
DATA_VERSION = 0.1
KEY_ONE = '1'
KEY_TWO = '2'
KEY_THREE = 'three'
KEY_FOUR = 'four'
KEY_FIVE = 'five'
KEYS = [KEY_ONE, KEY_TWO, KEY_THREE, KEY_FOUR, KEY_FIVE]
def data_version(self):
return self.VERS
def one(self):
return self.get(self._ONE)
def two(self):
return self.get(self._TWO)
def three(self):
return self.get(self._THREE)
def four(self):
return self.get(self._FOUR)
def five(self):
return self.get(self._FIVE)
return self.DATA_VERSION
def get(self, key, default=None):
def print_n_sleep(k):
sys.stdout.write('slow get executed for %s\n' % k)
time.sleep(3)
if key == self._ONE:
print_n_sleep(key)
return 'one'
if key == self._TWO:
print_n_sleep(key)
return 'two'
if key == self._THREE:
print_n_sleep(key)
return 'three'
if key == self._FOUR:
print_n_sleep(key)
return 'four'
if key == self._FIVE:
print_n_sleep(key)
return 'five'
try:
return getattr(self, f'__{key}__')()
except AttributeError:
return default
def keys(self):
@ -66,23 +35,35 @@ class test_slow_data(object):
def uid(self):
return None
def print_n_sleep(self, k):
sys.stdout.write('slow get executed for %s\n' % k)
time.sleep(3)
class tsd_cache_pickle(test_slow_data):
def __init__(self, *args, **kwargs):
test_slow_data.__init__(self, *args, **kwargs)
self._cached_data = caching.property_cache_pickle(test_slow_data(*args, **kwargs), 'cache.pickle', load_all_on_init=False)
def __1__(self):
self.print_n_sleep("__1__")
return 'one'
def two(self):
return test_slow_data.get(self, self._TWO)
def __2__(self):
self.print_n_sleep("__2__")
return 'two'
def get(self, key, default=None):
return self._cached_data.get(key, default)
def __three__(self):
self.print_n_sleep("__three__")
return 'three'
def __four__(self):
self.print_n_sleep("__four__")
return 'four'
def __five__(self):
self.print_n_sleep("__five__")
return 'five'
data = tsd_cache_pickle()
print('Testing property_cache (pickle):\\n--------------------------------')
print(data.one())
print(data.two())
print(data.three())
print(data.four())
print(data.five())
if __name__ == "__main__":
data = caching.property_cache_pickle(test_slow_data(), 'cache.pickle')
data.add_source_get_keys(data.KEY_THREE)
print('Testing property_cache (pickle):\\n--------------------------------')
for key in data.keys():
print(data.get(key))