[checkmk-commits] 6046 Improved execution time of e.g. service discovery

Module: check_mk Branch: master Commit: e496fcedabb41b208a58a013dfa28cc3aab9a01f URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e496fcedabb41b… Author: Lars Michelsen &lt;lm(a)mathias-kettner.de&gt; Date: Sun Apr 29 19:43:50 2018 +0200 6046 Improved execution time of e.g. service discovery The overall execution time of the service discovery has been reduced by introducing some precompiling of the check plugins and their includes before loading them. The mechanism is similar to the compilation of python modules from (.py to .pyc files). This mechanism reduces the load time of the plugins by 50%. Change-Id: I97bc4f72db4fa782d183fd936863d3bb87c25c90 --- .werks/6046 | 15 +++++++++++++++ cmk/paths.py | 2 ++ cmk_base/checks.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/.werks/6046 b/.werks/6046 new file mode 100644 index 0000000..d28772a --- /dev/null +++ b/.werks/6046 @@ -0,0 +1,15 @@ +Title: Improved execution time of e.g. service discovery +Level: 1 +Component: core +Class: feature +Compatible: compat +Edition: cre +State: unknown +Version: 1.6.0i1 +Date: 1525023719 + +The overall execution time of the service discovery has been reduced by +introducing some precompiling of the check plugins and their includes +before loading them. The mechanism is similar to the compilation of +python modules from (.py to .pyc files). This mechanism reduces the +load time of the plugins by 50%. diff --git a/cmk/paths.py b/cmk/paths.py index 1d7e623..3cb0395 100644 --- a/cmk/paths.py +++ b/cmk/paths.py @@ -54,6 +54,7 @@ check_mk_config_dir = None modules_dir = None var_dir = None log_dir = None +precompiled_checks_dir = None autochecks_dir = None precompiled_hostchecks_dir= None snmpwalks_dir = None @@ -122,6 +123,7 @@ def _set_paths(): "modules_dir" : os.path.join(omd_root, "share/check_mk/modules"), "var_dir" : os.path.join(omd_root, "var/check_mk"), "log_dir" : os.path.join(omd_root, "var/log"), + "precompiled_checks_dir" : os.path.join(omd_root, "var/check_mk/precompiled_checks"), "autochecks_dir" : os.path.join(omd_root, "var/check_mk/autochecks"), "precompiled_hostchecks_dir" : os.path.join(omd_root, "var/check_mk/precompiled"), "snmpwalks_dir" : os.path.join(omd_root, "var/check_mk/snmpwalks"), diff --git a/cmk_base/checks.py b/cmk_base/checks.py index 134d6ec..de9de88 100644 --- a/cmk_base/checks.py +++ b/cmk_base/checks.py @@ -28,6 +28,7 @@ import os import math import copy import ast +import marshal from collections import OrderedDict import cmk.paths @@ -149,7 +150,7 @@ def load_checks(filelist): load_check_includes(f, check_context) - execfile(f, check_context) + load_precompiled_plugin(f, check_context) loaded_files.add(file_name) except MKTerminate: @@ -248,7 +249,7 @@ def load_check_includes(check_file_path, check_context): for include_file_name in includes_of_plugin(check_file_path): include_file_path = check_include_file_path(include_file_name) try: - execfile(include_file_path, check_context) + load_precompiled_plugin(include_file_path, check_context) except MKTerminate: raise @@ -325,6 +326,46 @@ def _plugin_pathnames_in_directory(path): return [] +def load_precompiled_plugin(path, check_context): + """Loads the given check or check include plugin into the given + check context. + + To improve loading speed the files are not read directly. The files are + python byte-code compiled before in case it has not been done before. In + case there is already a compiled file that is newer than the current one, + then the precompiled file is loaded.""" + + precompiled_path = _precompiled_plugin_path(path) + + if not _is_plugin_precompiled(path, precompiled_path): + console.vverbose("Precompile %s to %s\n" % (path, precompiled_path)) + _precompile_plugin(path, precompiled_path) + + exec(marshal.load(open(precompiled_path)), check_context) + + +def _is_plugin_precompiled(path, precompiled_path): + return os.path.exists(precompiled_path) \ + and os.stat(path).st_mtime < os.stat(precompiled_path).st_mtime + + +def _precompile_plugin(path, precompiled_path): + code = compile(open(path).read(), path, "exec") + + if not os.path.exists(os.path.dirname(precompiled_path)): + os.makedirs(os.path.dirname(precompiled_path)) + + with open(precompiled_path, "w") as compiled_file: + marshal.dump(code, compiled_file) + + +def _precompiled_plugin_path(path): + is_local = path.startswith(cmk.paths.local_checks_dir) + return os.path.join(cmk.paths.precompiled_checks_dir, + "local" if is_local else "", + os.path.basename(path)) + + def check_variable_names(): return _check_variables.keys()