[checkmk-commits] Check_MK Git: check_mk: Speedup generation of configuration

Module: check_mk Branch: master Commit: e9abe4a2c7b1a83169a320b1d9a2370e66d24add URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e9abe4a2c7b1a8… Author: Mathias Kettner <mk(a)mathias-kettner.de> Date: Fri Jan 17 12:54:08 2014 +0100 Speedup generation of configuration The configuration generation for the core (Nagios/CMC) in large setups has been sped up. This hold especially for situations with many rules and/or many cluster definitions. --- .werks/452 | 11 ++++ ChangeLog | 1 + modules/check_mk.py | 142 +++++++++++++++++++++++++++++++++------------- modules/check_mk_base.py | 8 +++ 4 files changed, 121 insertions(+), 41 deletions(-) diff --git a/.werks/452 b/.werks/452 new file mode 100644 index 0000000..3ce24ba --- /dev/null +++ b/.werks/452 @@ -0,0 +1,11 @@ +Title: Speedup generation of configuration +Level: 2 +Component: core +Class: feature +State: unknown +Version: 1.2.5i1 +Date: 1389959599 + +The configuration generation for the core (Nagios/CMC) in large setups has been +sped up. This hold especially for situations with many rules and/or many +cluster definitions. diff --git a/ChangeLog b/ChangeLog index d127589..fc4488c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ 1.2.5i1: Core & Setup: * 0386 Added all active checks to check_mk -L output... + * 0452 Speedup generation of configuration... * 0087 FIX: Fixed possible locking issue when using datasource program with long output... * 0313 FIX: Avoid duplicate reading of configuration file on --create-rrd... * 0379 FIX: check_mk -c: Now also rewrites the location of conf.d directory diff --git a/modules/check_mk.py b/modules/check_mk.py index defccfe..0c03c6a 100755 --- a/modules/check_mk.py +++ b/modules/check_mk.py @@ -1692,29 +1692,116 @@ def get_rule_options(entry): return entry, {} -# Compute list of service_groups or contact_groups of service -# conf is either service_groups or service_contactgroups -def service_extra_conf(hostname, service, conf): - entries = [] - for entry in conf: - entry, rule_options = get_rule_options(entry) +def all_matching_hosts(tags, hostlist): + matching = set([]) + for taggedhost in all_hosts + clusters.keys(): + parts = taggedhost.split("|") + hostname = parts[0] + hosttags = parts[1:] + + if hosttags_match_taglist(hosttags, tags) and \ + in_extraconf_hostlist(hostlist, hostname): + matching.add(hostname) + return matching + + +def convert_service_ruleset(ruleset): + new_rules = [] + for rule in ruleset: + rule, rule_options = get_rule_options(rule) # Das könnte man einmal umbauen und so lassen (8 sec von 137) if rule_options.get("disabled"): continue - if len(entry) == 3: - item, hostlist, servlist = entry + if len(rule) == 3: + item, hostlist, servlist = rule tags = [] - elif len(entry) == 4: - item, tags, hostlist, servlist = entry + elif len(rule) == 4: + item, tags, hostlist, servlist = rule else: - raise MKGeneralException("Invalid entry '%r' in service configuration list: must have 3 or 4 elements" % (entry,)) + raise MKGeneralException("Invalid rule '%r' in service configuration list: must have 3 or 4 elements" % (rule,)) - if hosttags_match_taglist(tags_of_host(hostname), tags) and \ - in_extraconf_hostlist(hostlist, hostname) and \ - in_extraconf_servicelist(servlist, service): + # Directly compute set of all matching hosts here, this + # will avoid recomputation later + hosts = all_matching_hosts(tags, hostlist) + new_rules.append((item, hosts, servlist)) + + # Replace rules inplace. This finally modifies it, so we + # need this conversion only once + ruleset[:] = new_rules + +def serviceruleset_is_converted(ruleset): + if not ruleset: + return True # empty rulesets are converted in a trivial way + + if type(ruleset[0]) != tuple: + return False + + return type(ruleset[0][1]) == set + + +# Compute outcome of a service rule set that has an item +def service_extra_conf(hostname, service, ruleset): + if not serviceruleset_is_converted(ruleset): + convert_service_ruleset(ruleset) + + entries = [] + for item, hosts, servlist in ruleset: + if hostname in hosts and in_extraconf_servicelist(servlist, service): entries.append(item) return entries +def convert_boolean_service_ruleset(ruleset): + new_rules = [] + for rule in ruleset: + entry, rule_options = get_rule_options(rule) + if rule_options.get("disabled"): + continue + + if entry[0] == NEGATE: # this entry is logically negated + negate = True + entry = entry[1:] + else: + negate = False + + if len(entry) == 2: + hostlist, servlist = entry + tags = [] + elif len(entry) == 3: + tags, hostlist, servlist = entry + else: + raise MKGeneralException("Invalid entry '%r' in configuration: must have 2 or 3 elements" % (entry,)) + + # Directly compute set of all matching hosts here, this + # will avoid recomputation later + hosts = all_matching_hosts(tags, hostlist) + new_rules.append((negate, hosts, servlist)) + + # Replace rules inplace. This finally modifies it, so we + # need this conversion only once + ruleset[:] = new_rules + + +def boolean_serviceruleset_is_converted(ruleset): + if not ruleset: + return True # empty rulesets are converted in a trivial way + + if type(ruleset[0]) != tuple: + return False + + return type(ruleset[0][1]) == set + + +# Compute outcome of a service rule set that just say yes/no +def in_boolean_serviceconf_list(hostname, service_description, ruleset): + if not boolean_serviceruleset_is_converted(ruleset): + convert_boolean_service_ruleset(ruleset) + + for negate, hosts, servlist in ruleset: + if hostname in hosts and \ + in_extraconf_servicelist(servlist, service_description): + return not negate + return False # no match. Do not ignore + # Entries in list are (tagged) hostnames that must equal the @@ -2874,33 +2961,6 @@ def service_ignored(hostname, checktype, service_description): return False -def in_boolean_serviceconf_list(hostname, service_description, conflist): - for entry in conflist: - entry, rule_options = get_rule_options(entry) - if rule_options.get("disabled"): - continue - - if entry[0] == NEGATE: # this entry is logically negated - negate = True - entry = entry[1:] - else: - negate = False - - if len(entry) == 2: - hostlist, servlist = entry - tags = [] - elif len(entry) == 3: - tags, hostlist, servlist = entry - else: - raise MKGeneralException("Invalid entry '%r' in configuration: must have 2 or 3 elements" % (entry,)) - - if hosttags_match_taglist(tags_of_host(hostname), tags) and \ - in_extraconf_hostlist(hostlist, hostname) and \ - in_extraconf_servicelist(servlist, service_description): - return not negate - return False # no match. Do not ignore - - # Remove all autochecks of certain types of a certain host def remove_autochecks_of(hostname, checktypes = None): # None = all removed = 0 diff --git a/modules/check_mk_base.py b/modules/check_mk_base.py index 196371a..c2522d0 100644 --- a/modules/check_mk_base.py +++ b/modules/check_mk_base.py @@ -1347,10 +1347,18 @@ def i_am_root(): # Returns the nodes of a cluster, or None if hostname is # not a cluster +g_nodesof_cache = {} def nodes_of(hostname): + nodes = g_nodesof_cache.get(hostname, False) + if nodes != False: + return nodes + for tagged_hostname, nodes in clusters.items(): if hostname == tagged_hostname.split("|")[0]: + g_nodesof_cache[hostname] = nodes return nodes + + g_nodesof_cache[hostname] = None return None def pnp_cleanup(s):