[checkmk-commits] Another performance improvement to check loading

Module: check_mk Branch: master Commit: 82310aab29e5dfc9ab56dee14dd3218b20fc86e3 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=82310aab29e5df… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu May 3 08:44:36 2018 +0200 Another performance improvement to check loading This change improves the execution time of different Check_MK calls, especially command line calls for discovery and checking by additional 50%. Change-Id: If52a5e209ee8bffe5f191b79dd982511e4707dd4 --- cmk/paths.py | 1 + cmk_base/checks.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/cmk/paths.py b/cmk/paths.py index 3cb0395..226eaad 100644 --- a/cmk/paths.py +++ b/cmk/paths.py @@ -131,6 +131,7 @@ def _set_paths(): "tcp_cache_dir" : os.path.join(omd_root, "tmp/check_mk/cache"), "data_source_cache_dir" : os.path.join(omd_root, "tmp/check_mk/data_source_cache"), "snmp_scan_cache_dir" : os.path.join(omd_root, "tmp/check_mk/snmp_scan_cache"), + "include_cache_dir" : os.path.join(omd_root, "tmp/check_mk/check_includes"), "tmp_dir" : os.path.join(omd_root, "tmp/check_mk"), "logwatch_dir" : os.path.join(omd_root, "var/check_mk/logwatch"), "nagios_startscript" : os.path.join(omd_root, "etc/init.d/core"), diff --git a/cmk_base/checks.py b/cmk_base/checks.py index d639ea0..4750b0e 100644 --- a/cmk_base/checks.py +++ b/cmk_base/checks.py @@ -247,7 +247,7 @@ def new_check_context(): # Working with imports when specifying the includes would be much cleaner, # sure. But we need to deal with the current check API. def load_check_includes(check_file_path, check_context): - for include_file_name in includes_of_plugin(check_file_path): + for include_file_name in cached_includes_of_plugin(check_file_path): include_file_path = check_include_file_path(include_file_name) try: load_precompiled_plugin(include_file_path, check_context) @@ -272,6 +272,47 @@ def check_include_file_path(include_file_name): return include_file_path +def cached_includes_of_plugin(check_file_path): + cache_file_path = _include_cache_file_path(check_file_path) + try: + return _get_cached_check_includes(check_file_path, cache_file_path) + except OSError, e: + pass # No usable cache. Terminate + + includes = includes_of_plugin(check_file_path) + _write_check_include_cache(cache_file_path, includes) + return includes + + +def _get_cached_check_includes(check_file_path, cache_file_path): + check_stat = os.stat(check_file_path) + cache_stat = os.stat(cache_file_path) + + if check_stat.st_mtime >= cache_stat.st_mtime: + raise OSError("Cache is too old") + + if cache_stat.st_size == 0: + return [] # No includes + + x = open(cache_file_path).read().strip() + if not x: + return [] + return x.split("|") + + +def _write_check_include_cache(cache_file_path, includes): + if not os.path.exists(os.path.dirname(cache_file_path)): + os.makedirs(os.path.dirname(cache_file_path)) + store.save_file(cache_file_path, "|".join(includes)) + + +def _include_cache_file_path(path): + is_local = path.startswith(cmk.paths.local_checks_dir) + return os.path.join(cmk.paths.include_cache_dir, + "local" if is_local else "builtin", + os.path.basename(path)) + + # Parse the check file without executing the code to find the check include # files the check uses. The following statements are extracted: # check_info[...] = { "includes": [...] }