Module: check_mk
Branch: master
Commit: 293e096ccfb2cb00ecce8119b0fb22a14ff34f09
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=293e096ccfb2cb…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Sat May 19 12:05:04 2018 +0200
6106 FIX Raw Edition: Fixed checking of some hosts when check speicific config variables are used
When check specific configuration variables are set in the Check_MK configuration,
they could make the Check_MK checks of hosts that are not using these checks fail.
This resulted in Check_MK services showing an empty output. When executing the
precompiled host checks on the command line in debug mode, errors are shown. Like
this:
python /omd/sites/beta/var/check_mk/precompiled/[hostname] -v
Cannot read in configuration file /omd/sites/[site]/etc/check_mk/conf.d/wato/rules.mk: name 'fileinfo_groups' is not defined
Change-Id: Ib9fab2c91819ff01562b6998d62afe9cf90258e5
---
.werks/6106 | 19 ++++++
cmk_base/config.py | 169 ++++++++++++++++++++++++++++++++++++++++++++++--
cmk_base/core_nagios.py | 12 ++--
3 files changed, 187 insertions(+), 13 deletions(-)
diff --git a/.werks/6106 b/.werks/6106
new file mode 100644
index 0000000..e6a34ca
--- /dev/null
+++ b/.werks/6106
@@ -0,0 +1,19 @@
+Title: Raw Edition: Fixed checking of some hosts when check speicific config variables are used
+Level: 2
+Component: core
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1526723631
+
+When check specific configuration variables are set in the Check_MK configuration,
+they could make the Check_MK checks of hosts that are not using these checks fail.
+
+This resulted in Check_MK services showing an empty output. When executing the
+precompiled host checks on the command line in debug mode, errors are shown. Like
+this:
+
+python /omd/sites/beta/var/check_mk/precompiled/[hostname] -v
+Cannot read in configuration file /omd/sites/[site]/etc/check_mk/conf.d/wato/rules.mk: name 'fileinfo_groups' is not defined
diff --git a/cmk_base/config.py b/cmk_base/config.py
index 54f02dc..47f5e60 100644
--- a/cmk_base/config.py
+++ b/cmk_base/config.py
@@ -31,6 +31,7 @@ import marshal
import cmk.paths
import cmk.translations
+import cmk.store as store
from cmk.exceptions import MKGeneralException
import cmk_base
@@ -142,19 +143,14 @@ def load(with_conf_d=True, validate_hosts=True, exclude_parents_mk=False):
def load_packed_config():
"""Load the configuration for the CMK helpers of CMC
- These files are written by cmk_base.cee.core_cmc.pack_config().
+ These files are written by PackedConfig().
Should have a result similar to the load() above. With the exception that the
check helpers would only need check related config variables.
The validations which are performed during load() also don't need to be performed.
"""
- _initialize_config()
-
- filepath = cmk.paths.var_dir + "/core/helper_config.mk"
- exec(marshal.load(open(filepath)), globals())
-
- _perform_post_config_loading_actions()
+ PackedConfig().load()
def _initialize_config():
@@ -413,6 +409,165 @@ def _cmp_config_paths(a, b):
cmp(pa, pb)
+class PackedConfig(object):
+ """The precompiled host checks and the CMC Check_MK helpers use a
+ "precompiled" part of the Check_MK configuration during runtime.
+
+ a) They must not use the live config from etc/check_mk during
+ startup. They are only allowed to load the config activated by
+ the user.
+
+ b) They must not load the whole Check_MK config. Because they only
+ need the options needed for checking
+ """
+
+ # These variables are part of the Check_MK configuration, but are not needed
+ # by the Check_MK keepalive mode, so exclude them from the packed config
+ _skipped_config_variable_names = [
+ "define_contactgroups",
+ "define_hostgroups",
+ "define_servicegroups",
+ "service_contactgroups",
+ "host_contactgroups",
+ "service_groups",
+ "host_groups",
+ "contacts",
+ "host_paths",
+ "timeperiods",
+ "extra_service_conf",
+ "extra_host_conf",
+ "extra_nagios_conf",
+ ]
+
+ def __init__(self):
+ super(PackedConfig, self).__init__()
+ self._path = os.path.join(cmk.paths.var_dir, "base", "precompiled_check_config.mk")
+
+
+ def save(self):
+ self._write(self._pack())
+
+
+ def _pack(self):
+ import cmk_base.checks
+
+ helper_config = (
+ "#!/usr/bin/env python\n"
+ "# encoding: utf-8\n"
+ "# Created by Check_MK. Dump of the currently active configuration\n\n"
+ )
+
+ # These functions purpose is to filter out hosts which are monitored on different sites
+ active_hosts = all_active_hosts()
+ active_clusters = all_active_clusters()
+ def filter_all_hosts(all_hosts):
+ all_hosts_red = []
+ for host_entry in all_hosts:
+ hostname = host_entry.split("|", 1)[0]
+ if hostname in active_hosts:
+ all_hosts_red.append(host_entry)
+ return all_hosts_red
+
+ def filter_clusters(clusters):
+ clusters_red = {}
+ for cluster_entry, cluster_nodes in clusters.items():
+ clustername = cluster_entry.split("|", 1)[0]
+ if clustername in active_clusters:
+ clusters_red[cluster_entry] = cluster_nodes
+ return clusters_red
+
+ def filter_hostname_in_dict(values):
+ values_red = {}
+ for hostname, attributes in values.items():
+ if hostname in active_hosts:
+ values_red[hostname] = attributes
+ return values_red
+
+ filter_var_functions = {
+ "all_hosts" : filter_all_hosts,
+ "clusters" : filter_clusters,
+ "host_attributes" : filter_hostname_in_dict,
+ "ipaddresses" : filter_hostname_in_dict,
+ "ipv6addresses" : filter_hostname_in_dict,
+ "explicit_snmp_communities": filter_hostname_in_dict,
+ "hosttags" : filter_hostname_in_dict
+ }
+
+ #
+ # Add modified Check_MK base settings
+ #
+
+ variable_defaults = get_default_config()
+ derived_config_variable_names = get_derived_config_variable_names()
+
+ global_variables = globals()
+
+ for varname in get_variable_names() + list(derived_config_variable_names):
+ if varname in self._skipped_config_variable_names:
+ continue
+
+ val = global_variables[varname]
+
+ if varname not in derived_config_variable_names and val == variable_defaults[varname]:
+ continue
+
+ if not self._packable(varname, val):
+ continue
+
+ if varname in filter_var_functions:
+ val = filter_var_functions[varname](val)
+
+ helper_config += "\n%s = %r\n" % (varname, val)
+
+ #
+ # Add modified check specific Check_MK base settings
+ #
+
+ check_variable_defaults = cmk_base.checks.get_check_variable_defaults()
+
+ for varname, val in cmk_base.checks.get_check_variables().items():
+ if val == check_variable_defaults[varname]:
+ continue
+
+ if not self._packable(varname, val):
+ continue
+
+ helper_config += "\n%s = %r\n" % (varname, val)
+
+ return helper_config
+
+
+ def _packable(self, varname, val):
+ """Checks whether or not a variable can be written to the config.mk
+ and read again from it."""
+ if type(val) in [ int, str, unicode, bool ] or not val:
+ return True
+
+ try:
+ eval(repr(val))
+ return True
+ except:
+ return False
+
+
+ def _write(self, helper_config):
+ store.makedirs(os.path.dirname(self._path))
+
+ store.save_file(self._path + ".orig", helper_config + "\n")
+
+ import marshal
+ code = compile(helper_config, '<string>', 'exec')
+ with open(self._path + ".compiled", "w") as compiled_file:
+ marshal.dump(code, compiled_file)
+
+ os.rename(self._path + ".compiled", self._path)
+
+
+ def load(self):
+ _initialize_config()
+ exec(marshal.load(open(self._path)), globals())
+ _perform_post_config_loading_actions()
+
#.
# .--Host tags-----------------------------------------------------------.
diff --git a/cmk_base/core_nagios.py b/cmk_base/core_nagios.py
index d4ef18b..f5e9f0c 100644
--- a/cmk_base/core_nagios.py
+++ b/cmk_base/core_nagios.py
@@ -911,8 +911,13 @@ def _find_check_plugins(checktype):
def precompile_hostchecks():
+ console.verbose("Creating precompiled host check config...\n")
+ config.PackedConfig().save()
+
if not os.path.exists(cmk.paths.precompiled_hostchecks_dir):
os.makedirs(cmk.paths.precompiled_hostchecks_dir)
+
+ console.verbose("Precompiling host checks...\n")
for host in config.all_active_hosts():
try:
_precompile_hostcheck(host)
@@ -1062,12 +1067,7 @@ if '-d' in sys.argv:
for check_plugin_name in sorted(needed_check_plugin_names):
console.verbose(" %s%s%s", tty.green, check_plugin_name, tty.normal, stream=sys.stderr)
- # Disable this check. We don't have all check plugins loaded and have not all possible
- # check config variables known. These issues don't have to be validated by the single
- # precompiled checks.
- output.write("config.verify_non_invalid_variables = lambda x: None\n")
-
- output.write("config.load(validate_hosts=False)\n")
+ output.write("config.load_packed_config()\n")
# handling of clusters
if config.is_cluster(hostname):
Module: check_mk
Branch: master
Commit: 398fd118f48cd8137e9b76e88567d519118122e7
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=398fd118f48cd8…
Author: Sven Panne <sp(a)mathias-kettner.de>
Date: Tue May 22 08:27:12 2018 +0200
Fixed tracking of active history period.
The EC mixed the active periods of its history and its messages, which could
lead to strange effects. Nevertheless, the bug is quite harmless because to
make it happen, you have to enable logging of messages explicitly.
Change-Id: I5b6adc02032bc170b076892265c5ed1ecfa4a5e2
---
cmk/ec/main.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/cmk/ec/main.py b/cmk/ec/main.py
index db9dfb6..8ab61d5 100644
--- a/cmk/ec/main.py
+++ b/cmk/ec/main.py
@@ -679,6 +679,7 @@ class EventServer(ECServerThread):
self._active_history_period = active_history_period
self._event_status = event_status
self._event_columns = event_columns
+ self._message_period = cmk.ec.history.ActiveHistoryPeriod()
self._time_periods = TimePeriods(self._logger)
self.create_pipe()
@@ -2188,7 +2189,7 @@ class EventServer(ECServerThread):
def log_message(self, event):
try:
- with cmk.ec.history.get_logfile(self._config, self.settings.paths.messages_dir.value, self._active_history_period).open(mode='ab') as f:
+ with cmk.ec.history.get_logfile(self._config, self.settings.paths.messages_dir.value, self._message_period).open(mode='ab') as f:
f.write("%s %s %s%s: %s\n" % (
time.strftime("%b %d %H:%M:%S", time.localtime(event["time"])),
event["host"],
Module: check_mk
Branch: master
Commit: 3751c300ea1c8c47d5a7a58fa69d503c7655686f
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=3751c300ea1c8c…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Fri May 18 19:35:17 2018 +0200
6105 FIX CMC check helper: Fixed issues related to check includes not being loaded
When using the Check_MK Microcore several random checks could not be executed
because the helper were not loading the check includes in all cases. This was
some kind of race condition which could happen during check helper initialization.
This regression was introduced with 1.5.0b3.
Change-Id: I1d8222740ced014be938265e798ed20f5d150745
---
.werks/6105 | 15 +++++++++++++++
cmk_base/checks.py | 16 +++++++++++++---
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/.werks/6105 b/.werks/6105
new file mode 100644
index 0000000..36fd5a4
--- /dev/null
+++ b/.werks/6105
@@ -0,0 +1,15 @@
+Title: CMC check helper: Fixed issues related to check includes not being loaded
+Level: 2
+Component: core
+Class: fix
+Compatible: compat
+Edition: cee
+State: unknown
+Version: 1.6.0i1
+Date: 1526664697
+
+When using the Check_MK Microcore several random checks could not be executed
+because the helper were not loading the check includes in all cases. This was
+some kind of race condition which could happen during check helper initialization.
+
+This regression was introduced with 1.5.0b3.
diff --git a/cmk_base/checks.py b/cmk_base/checks.py
index 6316ffc..288483a 100644
--- a/cmk_base/checks.py
+++ b/cmk_base/checks.py
@@ -306,18 +306,28 @@ def _get_cached_check_includes(check_file_path, cache_file_path):
if check_stat.st_mtime >= cache_stat.st_mtime:
raise OSError("Cache is too old")
- if cache_stat.st_size == 0:
+ # There are no includes (just the newline at the end)
+ if cache_stat.st_size == 1:
return [] # No includes
+ # store.save_file() creates file empty for locking (in case it does not exists).
+ # Skip loading the file.
+ # Note: When raising here this process will also write the file. This means it
+ # will write it another time after it was written by the other process. This
+ # could be optimized. Since the whole caching here is a temporary(tm) soltion,
+ # we leave it as it is.
+ if cache_stat.st_size == 0:
+ raise OSError("Cache generation in progress (file is locked)")
+
x = open(cache_file_path).read().strip()
if not x:
- return []
+ return [] # Shouldn't happen. Empty files are handled above
return x.split("|")
def _write_check_include_cache(cache_file_path, includes):
store.makedirs(os.path.dirname(cache_file_path))
- store.save_file(cache_file_path, "|".join(includes))
+ store.save_file(cache_file_path, "%s\n" % "|".join(includes))
def _include_cache_file_path(path):
Module: check_mk
Branch: master
Commit: 044252e4051e58bcb9c5646e879e67063b270bbf
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=044252e4051e58…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Fri May 18 17:52:31 2018 +0200
6104 FIX Fixed not available global settings page when check specific variable configured
The global settings page was only showing an error message like "General error:
Invalid configuration variable 'lnx_thermal_default_levels'" in case a check
specific configuration option has been set in Check_MK configuration.
Change-Id: I6a7133b3d7f886dbafae177b65d97c7b1495af88
---
.werks/6104 | 13 +++++++++++++
cmk_base/checks.py | 15 ++++++++++++++-
cmk_base/config.py | 9 +++++----
3 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/.werks/6104 b/.werks/6104
new file mode 100644
index 0000000..e86fd73
--- /dev/null
+++ b/.werks/6104
@@ -0,0 +1,13 @@
+Title: Fixed not available global settings page when check specific variable configured
+Level: 1
+Component: wato
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1526658656
+
+The global settings page was only showing an error message like "General error:
+Invalid configuration variable 'lnx_thermal_default_levels'" in case a check
+specific configuration option has been set in Check_MK configuration.
diff --git a/cmk_base/checks.py b/cmk_base/checks.py
index e956c91..c48d3c7 100644
--- a/cmk_base/checks.py
+++ b/cmk_base/checks.py
@@ -74,6 +74,7 @@ special_agent_info = {}
_check_variables = {}
# keeps the default values of all the check variables
_check_variable_defaults = {}
+_all_checks_loaded = False
# workaround: set of check-groups that are to be treated as service-checks even if
# the item is None
@@ -94,15 +95,22 @@ service_rule_groups = set([
# | Loading of check plugins |
# '----------------------------------------------------------------------'
-# Load all checks and includes
def load():
+ """Load all checks and includes"""
+ global _all_checks_loaded
+
_initialize_data_structures()
filelist = get_plugin_paths(cmk.paths.local_checks_dir, cmk.paths.checks_dir)
load_checks(filelist)
+ _all_checks_loaded = True
+
def _initialize_data_structures():
"""Initialize some data structures which are populated while loading the checks"""
+ global _all_checks_loaded
+ _all_checks_loaded = False
+
_check_variables.clear()
_check_variable_defaults.clear()
@@ -219,6 +227,11 @@ def load_checks(filelist):
initialize_check_type_caches()
+def all_checks_loaded():
+ """Whether or not all(!) checks have been loaded into the current process"""
+ return _all_checks_loaded
+
+
# Constructs a new check context dictionary. It contains the whole check API.
def new_check_context():
# Add the data structures where the checks register with Check_MK
diff --git a/cmk_base/config.py b/cmk_base/config.py
index 7dc762b..54f02dc 100644
--- a/cmk_base/config.py
+++ b/cmk_base/config.py
@@ -132,7 +132,10 @@ def load(with_conf_d=True, validate_hosts=True, exclude_parents_mk=False):
if validate_hosts:
_verify_non_duplicate_hosts()
- verify_non_invalid_variables(vars_before_config)
+ # Such validation only makes sense when all checks have been loaded
+ if cmk_base.checks.all_checks_loaded():
+ verify_non_invalid_variables(vars_before_config)
+
verify_snmp_communities_type()
@@ -169,9 +172,7 @@ def _perform_post_config_loading_actions():
# In case the checks are not loaded yet it seems the current mode
# is not working with the checks. In this case also don't load the
# static checks into the configuration.
- # TODO: Clean this up. Shouldn't we move the "checks" stuff to the
- # checks module?
- if cmk_base.checks.check_info:
+ if cmk_base.checks.all_checks_loaded():
add_wato_static_checks_to_checks()
initialize_check_caches()
set_check_variables_for_checks()