[checkmk-commits] Check_MK Git: check_mk: Optimized the memory footprint of the Check_MK check helper processes

Module: check_mk Branch: master Commit: 3b15ced4b1155d67e0051b3f0be23e1b9d617a81 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=3b15ced4b1155d… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Feb 5 14:24:15 2015 +0100 Optimized the memory footprint of the Check_MK check helper processes which are used when using the CMC. Conflicts: ChangeLog modules/check_mk_base.py --- .werks/1977 | 11 +++++++ ChangeLog | 1 + modules/check_mk.py | 75 ++++++++++++++++++++++++++-------------------- modules/check_mk_base.py | 2 ++ 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/.werks/1977 b/.werks/1977 new file mode 100644 index 0000000..aa29d5e --- /dev/null +++ b/.werks/1977 @@ -0,0 +1,11 @@ +Title: Dramatically reduced size of Check_MK check helper processes +Level: 2 +Component: core +Class: fix +Compatible: compat +State: unknown +Version: 1.2.7i1 +Date: 1423142432 + +Optimized the memory footprint of the Check_MK check helper processes +which are used when using the CMC. diff --git a/ChangeLog b/ChangeLog index c17c21e..26cbd65 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ * 1919 FIX: cmk --snmpwalk: continue if one of the OIDs to walk fails * 1880 FIX: inventory_processes rules can now be configured without setting levels... * 1882 FIX: Fixed exception "filesystem_levels" not defined when compiling config for nagios + * 1977 FIX: Dramatically reduced size of Check_MK check helper processes... Checks & Agents: * 1665 agent_netapp: New special agent for NetApp monitoring via Web-API... diff --git a/modules/check_mk.py b/modules/check_mk.py index 9743053..578129a 100755 --- a/modules/check_mk.py +++ b/modules/check_mk.py @@ -1572,6 +1572,7 @@ def in_binary_hostlist(hostname, conf): return False + # Pick out the last element of an entry if it is a dictionary. # This is a new feature (1.2.0p3) that allows to add options # to rules. Currently only the option "disabled" is being @@ -1595,6 +1596,7 @@ def all_matching_hosts(tags, hostlist): matching.add(hostname) return matching +g_converted_rulesets_cache = {} def convert_service_ruleset(ruleset): new_rules = [] @@ -1616,18 +1618,11 @@ def convert_service_ruleset(ruleset): hosts = all_matching_hosts(tags, hostlist) new_rules.append((item, hosts, servlist)) - # Replace rules inplace. This finally modifies it, so we - # need this conversion only once - ruleset[:] = new_rules - -def serviceruleset_is_converted(ruleset): - if not ruleset: - return True # empty rulesets are converted in a trivial way + g_converted_rulesets_cache[id(ruleset)] = new_rules - if type(ruleset[0]) != tuple: - return False - return type(ruleset[0][1]) == set +def serviceruleset_is_converted(ruleset): + return id(ruleset) in g_converted_rulesets_cache # Compute outcome of a service rule set that has an item @@ -1636,11 +1631,12 @@ def service_extra_conf(hostname, service, ruleset): convert_service_ruleset(ruleset) entries = [] - for item, hosts, servlist in ruleset: + for item, hosts, servlist in g_converted_rulesets_cache[id(ruleset)]: if hostname in hosts and in_extraconf_servicelist(servlist, service): entries.append(item) return entries + def convert_boolean_service_ruleset(ruleset): new_rules = [] for rule in ruleset: @@ -1667,27 +1663,15 @@ def convert_boolean_service_ruleset(ruleset): hosts = all_matching_hosts(tags, hostlist) new_rules.append((negate, hosts, servlist)) - # Replace rules inplace. This finally modifies it, so we - # need this conversion only once - ruleset[:] = new_rules - - -def boolean_serviceruleset_is_converted(ruleset): - if not ruleset: - return True # empty rulesets are converted in a trivial way - - if type(ruleset[0]) != tuple: - return False - - return type(ruleset[0][1]) == set + g_converted_rulesets_cache[id(ruleset)] = new_rules # Compute outcome of a service rule set that just say yes/no def in_boolean_serviceconf_list(hostname, service_description, ruleset): - if not boolean_serviceruleset_is_converted(ruleset): + if not serviceruleset_is_converted(ruleset): convert_boolean_service_ruleset(ruleset) - for negate, hosts, servlist in ruleset: + for negate, hosts, servlist in g_converted_rulesets_cache[id(ruleset)]: if hostname in hosts and \ in_extraconf_servicelist(servlist, service_description): return not negate @@ -1743,6 +1727,7 @@ def in_extraconf_hostlist(hostlist, hostname): return False + def in_extraconf_servicelist(list, item): for pattern in list: # Allow negation of pattern with prefix '!' @@ -2962,7 +2947,29 @@ no_inventory_possible = None # of all autochecks files. The check helpers of the running core just # use those files, so that changes in the actual config do not harm # the running system. -derived_config_variable_names = [ "hosttags", "all_hosts_untagged" ] + +derived_config_variable_names = [ "hosttags" ] + +# These variables are part of the Check_MK configuration, but are not needed +# by the Check_MK keepalive mode, so exclude them from the packed config +skipped_config_variable_names = [ + "define_contactgroups", + "define_hostgroups", + "define_servicegroups", + "service_contactgroups", + "host_contactgroups", + "service_groups", + "host_groups", + "contacts", + "host_paths", + "timeperiods", + "host_attributes", + "all_hosts_untagged", + "extra_service_conf", + "extra_host_conf", + "extra_nagios_conf", +] + def pack_config(): # Checks whether or not a variable can be written to the config.mk # and read again from it. @@ -2978,11 +2985,15 @@ def pack_config(): filepath = var_dir + "/core/config.mk" out = file(filepath + ".new", "w") - out.write("#!/usr/bin/python\n# encoding: utf-8\n# Created by Check_MK. Dump of the currently active configuration\n\n") + out.write("#!/usr/bin/python\n" + "# encoding: utf-8\n" + "# Created by Check_MK. Dump of the currently active configuration\n\n") for varname in list(config_variable_names) + derived_config_variable_names: - val = globals()[varname] - if packable(varname, val): - out.write("\n%s = %r\n" % (varname, val)) + if varname not in skipped_config_variable_names: + val = globals()[varname] + if packable(varname, val): + out.write("\n%s = %r\n" % (varname, val)) + for varname, factory_setting in factory_settings.items(): if varname in globals(): out.write("\n%s = %r\n" % (varname, globals()[varname])) @@ -4895,7 +4906,7 @@ def copy_globals(): "g_check_table_cache", "g_singlehost_checks", "g_nodesof_cache", "g_compiled_regexes", "vars_before_config", "g_initial_times", "g_keepalive_initial_memusage", - "g_dns_cache", "g_ip_lookup_cache" ] \ + "g_dns_cache", "g_ip_lookup_cache", "g_converted_rulesets_cache" ] \ and type(value).__name__ not in [ "function", "module", "SRE_Pattern" ]: global_saved[varname] = copy.copy(value) return global_saved diff --git a/modules/check_mk_base.py b/modules/check_mk_base.py index 0ea9407..7d502a0 100644 --- a/modules/check_mk_base.py +++ b/modules/check_mk_base.py @@ -91,6 +91,8 @@ def reset_global_caches(): g_dns_cache = {} global g_ip_lookup_cache g_ip_lookup_cache = None # permanently cached ipaddresses from ipaddresses.cache + global g_converted_rulesets_cache + g_converted_rulesets_cache = {} reset_global_caches()