[checkmk-commits] CMK helpers: In verbose mode and when memory usage increased also dump out the estimated cache sizes

Module: check_mk Branch: master Commit: becdad93cc719c387aa92f5783584a0caff0eee2 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=becdad93cc719c… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu May 18 11:16:04 2017 +0200 CMK helpers: In verbose mode and when memory usage increased also dump out the estimated cache sizes Change-Id: I0be0aafe018df6934f445f93e23270f5a4d4bd70 --- cmk_base/caching.py | 8 ++++++++ cmk_base/utils.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/cmk_base/caching.py b/cmk_base/caching.py index 3f98b89..de679e0 100644 --- a/cmk_base/caching.py +++ b/cmk_base/caching.py @@ -28,6 +28,7 @@ from cmk.exceptions import MKGeneralException +import cmk_base.utils class CacheManager(object): def __init__(self): @@ -66,6 +67,13 @@ class CacheManager(object): cache.clear() + def dump_sizes(self): + sizes = {} + for name, cache in self._caches.items(): + sizes[name] = cmk_base.utils.total_size(cache) + return sizes + + class Cache(object): def is_empty(self): diff --git a/cmk_base/utils.py b/cmk_base/utils.py index 36ecde2..d11deb7 100644 --- a/cmk_base/utils.py +++ b/cmk_base/utils.py @@ -29,6 +29,9 @@ Check_MK modules and/or cmk_base modules code.""" # TODO: Try to find a better place for them. +import sys +import itertools + # Works with Check_MK version (without tailing .cee and/or .demo) def is_daily_build_version(v): @@ -87,3 +90,44 @@ def parse_check_mk_version(v): val = 20000 + num*100 return int('%02d%02d%02d%05d' % (int(major), int(minor), sub, val)) + + +def total_size(o, handlers=None): + """ Returns the approximate memory footprint an object and all of its contents. + + Automatically finds the contents of the following builtin containers and + their subclasses: tuple, list, dict, set and frozenset. + To search other containers, add handlers to iterate over their contents: + + handlers = {SomeContainerClass: iter, + OtherContainerClass: OtherContainerClass.get_elements} + + """ + if handlers == None: + handlers = {} + + dict_handler = lambda d: itertools.chain.from_iterable(d.items()) + all_handlers = {tuple: iter, + list: iter, + dict: dict_handler, + set: iter, + frozenset: iter, + } + all_handlers.update(handlers) # user handlers take precedence + seen = set() # track which object id's have already been seen + default_size = sys.getsizeof(0) # estimate sizeof object without __sizeof__ + + def sizeof(o): + if id(o) in seen: # do not double count the same object + return 0 + seen.add(id(o)) + s = sys.getsizeof(o, default_size) + + + for typ, handler in all_handlers.items(): + if isinstance(o, typ): + s += sum(map(sizeof, handler(o))) + break + return s + + return sizeof(o)