Module: check_mk
Branch: master
Commit: 21e37cc30a2558b23a5f97248b018bf3fc14d8e8
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=21e37cc30a2558…
Author: Mathias Kettner <mk(a)mathias-kettner.de>
Date: Fri Oct 19 15:48:20 2012 +0200
Updated bug entries #0796
---
.bugs/796 | 16 ++++++++++++++++
1 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/.bugs/796 b/.bugs/796
new file mode 100644
index 0000000..3de38dc
--- /dev/null
+++ b/.bugs/796
@@ -0,0 +1,16 @@
+Title: Better Check-Output if Check_MK check crashes
+Component: core
+State: open
+Date: 2012-10-19 15:46:26
+Targetversion: future
+Class: nastiness
+
+In the very very rare and unbelievable case that the Check_MK check
+itself crashes - the output of the Check in Nagios is currently '(null)'.
+We should try to wrap the whole code in some general exception handling
+(at least the precompiled code). If any exception appears, we should
+
+1. Make the check result with 3 (UNKNOWN)
+2. Make the str(exception) be the plugin output
+3. Optionally put the exception trace into the long output
+4. If the debug log is active, then also log into that something
Module: check_mk
Branch: master
Commit: 687b044bd89e541540a1fab38a805c2c738d16e6
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=687b044bd89e54…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Fri Oct 19 11:07:07 2012 +0200
BI: supressing exceptions in filtered single host aggregation views
---
web/htdocs/bi.py | 39 +++++++++++++++++++++++++++------------
1 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/web/htdocs/bi.py b/web/htdocs/bi.py
index 33e6100..c7731cd 100644
--- a/web/htdocs/bi.py
+++ b/web/htdocs/bi.py
@@ -195,6 +195,10 @@ def aggregation_groups():
return sorted(group_names, cmp = lambda a,b: cmp(a.lower(), b.lower()))
+def log(s):
+ if compile_logging():
+ file(config.bi_compile_log, "a").write(s)
+
# Precompile the forest of BI rules. Forest? A collection of trees.
# The compiled forest does not contain any regular expressions anymore.
# Everything is resolved. Sites, hosts and services are hardcoded. The
@@ -203,10 +207,6 @@ def aggregation_groups():
def compile_forest(user, only_hosts = None, only_groups = None):
global g_cache, g_user_cache, g_compiled_everything
- def log(s):
- if compile_logging():
- file(config.bi_compile_log, "a").write(s)
-
new_config_information = cache_needs_update()
if new_config_information:
# config changed or monitoring daemon restarted, clear cache
@@ -269,7 +269,14 @@ def compile_forest(user, only_hosts = None, only_groups = None):
global did_compilation
did_compilation = True
+ log("This request: User: %s, Only-Groups: %r, Only-Hosts: %s PID: %d\n"
+ % (user, only_groups, only_hosts, os.getpid()))
+
# Load all (needed) services
+ # The only_hosts variable is only set in "precompile on demand" mode to filter out
+ # the needed hosts/services if possible. It is used in the load_services() function
+ # to reduce the amount of hosts/services. Reducing the host/services leads to faster
+ # compilation.
load_services(cache, only_hosts)
if compile_logging():
@@ -277,6 +284,8 @@ def compile_forest(user, only_hosts = None, only_groups = None):
num_new_host_aggrs = 0
num_new_multi_aggrs = 0
+ # When only_hosts is given only use the single host aggregations for further processing.
+ # The only_hosts variable is only populated for single host tables.
if only_hosts:
aggr_list = [(AGGR_HOST, config.host_aggregations)]
else:
@@ -364,6 +373,10 @@ def compile_forest(user, only_hosts = None, only_groups = None):
for grp, aggrs in cache['forest'].iteritems():
num_total_aggr += len(aggrs)
+ num_host_aggr = 0
+ for grp, aggrs in cache['host_aggregations'].iteritems():
+ num_host_aggr += len(aggrs)
+
num_services = 0
for key, val in g_services.iteritems():
num_services += len(val[1])
@@ -391,8 +404,8 @@ def compile_forest(user, only_hosts = None, only_groups = None):
only_groups and len(only_groups) or 0,
g_compiled_everything,
- num_total_aggr - len(cache['compiled_hosts']),
- len(cache['compiled_hosts']),
+ num_total_aggr - num_host_aggr,
+ num_host_aggr,
len(cache['compiled_groups']),
len(config.aggregations),
len(config.host_aggregations),
@@ -463,7 +476,8 @@ def find_matching_services(aggr_type, what, calllist):
host_re = "(.*)"
elif not honor_site and not '*' in host_re and not '$' in host_re and not '|' in host_re:
# Exact host match
- entries = [ ((e[0], host_re), e[1]) for e in g_services_by_hostname[host_re] ]
+ entries = [ ((e[0], host_re), e[1]) for e in g_services_by_hostname.get(host_re, []) ]
+
else:
# All services
entries = g_services.items()
@@ -709,6 +723,7 @@ def compile_aggregation_rule(aggr_type, rule, args, lvl):
else:
# This is a plain leaf node with just host/service
new_elements = compile_leaf_node(subst_vars(node[0], arginfo), subst_vars(node[1], arginfo))
+
else:
# substitute our arguments in rule arguments
# rule_args:
@@ -798,8 +813,8 @@ def compile_leaf_node(host_re, service_re = config.HOST_STATE):
found = []
honor_site = SITE_SEP in host_re
if not honor_site and not '*' in host_re and not '$' in host_re and not '|' in host_re:
- # This is an exact host match, only use the services of this host
- entries = [ ((e[0], host_re), e[1]) for e in g_services_by_hostname[host_re] ]
+ entries = [ ((e[0], host_re), e[1]) for e in g_services_by_hostname.get(host_re, []) ]
+
else:
entries = g_services.items()
@@ -1331,9 +1346,9 @@ def singlehost_table(columns, add_headers, only_sites, limit, filters, joinbynam
# if limit:
# views.check_limit(hostrows, limit)
- # Apply group filter. This is important for performance. We
- # must not compute any aggregations from other groups and filter
- # later out again.
+ # Apply aggregation group filter. This is important for performance. We
+ # must not compute any aggregations from other aggregation groups and filter
+ # them later out again.
only_groups = None
for filt in filters:
if filt.name == "aggr_group":
Module: check_mk
Branch: master
Commit: c1fb846b2e850374399f1a3213ed9db80d70dad1
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c1fb846b2e8503…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Fri Oct 19 12:28:46 2012 +0200
BI: aggregations in host_aggregations structure might now also include data of their parents
---
web/htdocs/bi.py | 79 +++++++++++++++++++++++++++++++++++++++---------------
1 files changed, 57 insertions(+), 22 deletions(-)
diff --git a/web/htdocs/bi.py b/web/htdocs/bi.py
index c7731cd..639a188 100644
--- a/web/htdocs/bi.py
+++ b/web/htdocs/bi.py
@@ -126,18 +126,17 @@ g_compiled_everything = False # Is set to true if all aggregations have been com
# Load the static configuration of all services and hosts (including tags)
# without state.
def load_services(cache, only_hosts):
+ global g_services, g_services_by_hostname
+ g_services = {}
+ g_services_by_hostname = {}
+
# TODO: At the moment the data is always refetched. This could really
# be optimized. Maybe create a cache which fetches data for the given
# list of hosts, puts it to a cache and then only fetch the additionally
# needed information which are not cached yet in future requests
- global g_services, g_services_by_hostname
- g_services = {}
- g_services_by_hostname = {}
- html.live.set_prepend_site(True)
- html.live.set_auth_domain('bi')
# Create optional host filter
- filter_txt = ''
+ filter_txt = 'Filter: custom_variable_names < _REALNAME\n"' # drop summary hosts
if only_hosts:
# Only fetch the requested hosts
host_filter = []
@@ -146,8 +145,9 @@ def load_services(cache, only_hosts):
filter_txt = ''.join(host_filter)
filter_txt += "Or: %d\n" % len(host_filter)
+ html.live.set_prepend_site(True)
+ html.live.set_auth_domain('bi')
data = html.live.query("GET hosts\n"
- "Filter: custom_variable_names < _REALNAME\n" # drop summary hosts
+filter_txt+
"Columns: name custom_variable_names custom_variable_values services childs parents\n")
html.live.set_prepend_site(False)
@@ -269,9 +269,6 @@ def compile_forest(user, only_hosts = None, only_groups = None):
global did_compilation
did_compilation = True
- log("This request: User: %s, Only-Groups: %r, Only-Hosts: %s PID: %d\n"
- % (user, only_groups, only_hosts, os.getpid()))
-
# Load all (needed) services
# The only_hosts variable is only set in "precompile on demand" mode to filter out
# the needed hosts/services if possible. It is used in the load_services() function
@@ -279,6 +276,9 @@ def compile_forest(user, only_hosts = None, only_groups = None):
# compilation.
load_services(cache, only_hosts)
+ log("This request: User: %s, Only-Groups: %r, Only-Hosts: %s PID: %d\n"
+ % (user, only_groups, only_hosts, os.getpid()))
+
if compile_logging():
before = time.time()
num_new_host_aggrs = 0
@@ -332,13 +332,17 @@ def compile_forest(user, only_hosts = None, only_groups = None):
cache["aggregations_by_hostname"].setdefault(name, []).append((group, aggr))
# All single-host aggregations looked up per host
- if len(req_hosts) == 1:
- host = req_hosts[0] # pair of (site, host)
- cache["host_aggregations"].setdefault(host, []).append((group, aggr))
-
- # In case of only_groups requests construct a list of compiled
- # single-host aggregations for cached registration
- if only_groups:
+ # Only process the aggregations of hosts which are mentioned in only_hosts
+ if aggr_type == AGGR_HOST:
+ # In normal cases a host aggregation has only one req_hosts item, we could use
+ # index 0 here. But clusters (which are also allowed now) have all their nodes
+ # in the list of required nodes. The cluster node is always the last one in
+ # this list.
+ host = req_hosts[-1] # pair of (site, host)
+ if req_hosts[0] in only_hosts:
+ cache["host_aggregations"].setdefault(host, []).append((group, aggr))
+
+ # construct a list of compiled single-host aggregations for cached registration
single_affected_hosts.append(host)
# All aggregations containing a specific host
@@ -352,7 +356,7 @@ def compile_forest(user, only_hosts = None, only_groups = None):
# Register compiled objects
if only_hosts:
- cache['compiled_hosts'].update(only_hosts)
+ cache['compiled_hosts'].update(single_affected_hosts)
elif only_groups:
cache['compiled_groups'].update(only_groups)
@@ -1021,7 +1025,6 @@ def execute_rule_node(node, status_info):
# Get all status information we need for the aggregation from
# a known lists of lists (list of site/host pairs)
def get_status_info(required_hosts):
-
# Query each site only for hosts that that site provides
site_hosts = {}
for site, host in required_hosts:
@@ -1049,8 +1052,8 @@ def get_status_info(required_hosts):
# This variant of the function is configured not with a list of
# hosts but with a livestatus filter header and a list of columns
# that need to be fetched in any case
-def get_status_info_filtered(filter_header, only_sites, limit, add_columns):
- columns = [ "name", "state", "plugin_output", "services_with_info" ] + add_columns
+def get_status_info_filtered(filter_header, only_sites, limit, add_columns, fetch_parents = True):
+ columns = [ "name", "state", "plugin_output", "services_with_info", "parents" ] + add_columns
html.live.set_only_sites(only_sites)
html.live.set_prepend_site(True)
@@ -1062,7 +1065,22 @@ def get_status_info_filtered(filter_header, only_sites, limit, add_columns):
html.live.set_only_sites(None)
headers = [ "site" ] + columns
+ hostnames = [ row[1] for row in data ]
rows = [ dict(zip(headers, row)) for row in data]
+
+ # on demand compile: if parents have been found, also fetch data of the parents.
+ # This is needed to allow cluster hosts (which have the nodes as parents) in the
+ # host_aggregation construct.
+ if fetch_parents:
+ parent_filter = []
+ for row in data:
+ parent_filter += [ 'Filter: name = %s\n' % p for p in row[5] ]
+ parent_filter_txt = ''.join(parent_filter)
+ parent_filter_txt += 'Or: %d\n' % len(parent_filter)
+ for row in get_status_info_filtered(filter_header, only_sites, limit, add_columns, False):
+ if row['name'] not in hostnames:
+ rows.append(row)
+
return rows
# _ _____ _ _
@@ -1342,7 +1360,7 @@ def singlehost_table(columns, add_headers, only_sites, limit, filters, joinbynam
filter_code += header
host_columns = filter(lambda c: c.startswith("host_"), columns)
- hostrows = get_status_info_filtered(filter_code, only_sites, limit, host_columns)
+ hostrows = get_status_info_filtered(filter_code, only_sites, limit, host_columns, config.bi_precompile_on_demand)
# if limit:
# views.check_limit(hostrows, limit)
@@ -1362,6 +1380,10 @@ def singlehost_table(columns, add_headers, only_sites, limit, filters, joinbynam
else:
compile_forest(config.user_id)
+ # rows by site/host - needed for later cluster state gathering
+ if config.bi_precompile_on_demand and not joinbyname:
+ row_dict = dict([ ((r['site'], r['name']), r) for r in hostrows])
+
rows = []
# Now compute aggregations of these hosts
for hostrow in hostrows:
@@ -1379,6 +1401,19 @@ def singlehost_table(columns, add_headers, only_sites, limit, filters, joinbynam
for group, aggregation in aggrs:
row = hostrow.copy()
+
+ # on demand compile: host aggregations of clusters need data of several hosts.
+ # It is not enough to only process the hostrow. The status_info construct must
+ # also include the data of the other required hosts.
+ if config.bi_precompile_on_demand and not joinbyname and len(aggregation['reqhosts']) > 1:
+ status_info = {}
+ for site, host in aggregation['reqhosts']:
+ status_info[(site, host)] = [
+ row_dict[(site, host)]['state'],
+ row_dict[(site, host)]['plugin_output'],
+ row_dict[(site, host)]['services_with_info'],
+ ]
+
row.update(create_aggregation_row(aggregation, status_info))
row["aggr_group"] = group
rows.append(row)
Module: check_mk
Branch: master
Commit: 29a91a5ce544ddc635380857e03d121d37787aec
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=29a91a5ce544dd…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Fri Oct 19 13:35:43 2012 +0200
Aggregations can now be part of more than one aggregation group
(just configure a list of group names instead of a group name string)
---
ChangeLog | 4 +-
web/htdocs/bi.py | 113 ++++++++++++++++++++++++++++++++----------------------
2 files changed, 70 insertions(+), 47 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index d63b008..8a7ea43 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -31,13 +31,15 @@
BI:
* Added missing localizations
- * Added option bit_precompile_on_demand to split compilations of
+ * Added option bi_precompile_on_demand to split compilations of
the aggregations in several fragments. If possible only the needed
aggregations are compiled to reduce the time a user has to wait for
BI based view. This optimizes BI related views which display
information for a specific list of hosts or aggregation groups.
* Added new config option bi_compile_log to collect statistics about
aggregation compilations
+ * Aggregations can now be part of more than one aggregation group
+ (just configure a list of group names instead of a group name string)
* Correct representation of (!), (!!) and (?) markers in check output
* Corrected representation of assumed state in box layout
diff --git a/web/htdocs/bi.py b/web/htdocs/bi.py
index 639a188..b4fae74 100644
--- a/web/htdocs/bi.py
+++ b/web/htdocs/bi.py
@@ -136,7 +136,7 @@ def load_services(cache, only_hosts):
# needed information which are not cached yet in future requests
# Create optional host filter
- filter_txt = 'Filter: custom_variable_names < _REALNAME\n"' # drop summary hosts
+ filter_txt = 'Filter: custom_variable_names < _REALNAME\n' # drop summary hosts
if only_hosts:
# Only fetch the requested hosts
host_filter = []
@@ -186,7 +186,13 @@ def reused_compilation():
def aggregation_groups():
if config.bi_precompile_on_demand:
# on demand: show all configured groups
- group_names = list(set([ a[0] for a in config.aggregations + config.host_aggregations ]))
+ group_names = set([])
+ for a in config.aggregations + config.host_aggregations:
+ if type(a[0]) == list:
+ group_names.update(a[0])
+ else:
+ group_names.add(a[0])
+ group_names = list(group_names)
else:
# classic mode: precompile all and display only groups with members
@@ -298,61 +304,74 @@ def compile_forest(user, only_hosts = None, only_groups = None):
raise MKConfigError(_("<h1>Invalid aggregation <tt>%s</tt>'</h1>"
"Must have at least 3 entries (has %d)") % (entry, len(entry)))
- group = entry[0]
+ if type(entry[0]) == list:
+ groups = entry[0]
+ else:
+ groups = [ entry[0] ]
+ groups_set = set(groups)
- if only_groups and group not in only_groups:
+ if only_groups and not groups_set.intersection(only_groups):
+ log('Skip aggr (No group of the aggr has been requested: %r)\n' % groups)
continue # skip not requested groups if filtered by groups
- if group in cache['compiled_groups']:
- continue # skip already compiled groups
+ if len(groups_set) == len(groups_set.intersection(cache['compiled_groups'])):
+ log('Skip aggr (All groups have already been compiled\n')
+ continue # skip if all groups have already been compiled
new_entries = compile_rule_node(aggr_type, entry[1:], 0)
- for entry in new_entries:
- remove_empty_nodes(entry)
+ for this_entry in new_entries:
+ remove_empty_nodes(this_entry)
new_entries = [ e for e in new_entries if len(e["nodes"]) > 0 ]
- # enter new aggregations into dictionary for that group
- entries = cache["forest"].setdefault(group, [])
- entries += new_entries
-
if compile_logging():
if aggr_type == AGGR_HOST:
num_new_host_aggrs += len(new_entries)
else:
num_new_multi_aggrs += len(new_entries)
- # Update several global speed-up indices
- for aggr in new_entries:
- req_hosts = aggr["reqhosts"]
+ # enter new aggregations into dictionary for these groups
+ for group in groups:
+ if group in cache['compiled_groups']:
+ log('Skip aggr (group %s already compiled)\n' % group)
+ continue # the group has already been compiled completely
- # Aggregations by last part of title (assumed to be host name)
- name = aggr["title"].split()[-1]
- cache["aggregations_by_hostname"].setdefault(name, []).append((group, aggr))
-
- # All single-host aggregations looked up per host
- # Only process the aggregations of hosts which are mentioned in only_hosts
- if aggr_type == AGGR_HOST:
- # In normal cases a host aggregation has only one req_hosts item, we could use
- # index 0 here. But clusters (which are also allowed now) have all their nodes
- # in the list of required nodes. The cluster node is always the last one in
- # this list.
- host = req_hosts[-1] # pair of (site, host)
- if req_hosts[0] in only_hosts:
- cache["host_aggregations"].setdefault(host, []).append((group, aggr))
-
- # construct a list of compiled single-host aggregations for cached registration
- single_affected_hosts.append(host)
-
- # All aggregations containing a specific host
- for h in req_hosts:
- cache["affected_hosts"].setdefault(h, []).append((group, aggr))
-
- # All aggregations containing a specific service
- services = find_all_leaves(aggr)
- for s in services: # triples of site, host, service
- cache["affected_services"].setdefault(s, []).append((group, aggr))
+ if group not in cache['forest']:
+ cache['forest'][group] = new_entries
+ else:
+ cache['forest'][group] += new_entries
+
+ # Update several global speed-up indices
+ for aggr in new_entries:
+ req_hosts = aggr["reqhosts"]
+
+ # Aggregations by last part of title (assumed to be host name)
+ name = aggr["title"].split()[-1]
+ cache["aggregations_by_hostname"].setdefault(name, []).append((group, aggr))
+
+ # All single-host aggregations looked up per host
+ # Only process the aggregations of hosts which are mentioned in only_hosts
+ if aggr_type == AGGR_HOST:
+ # In normal cases a host aggregation has only one req_hosts item, we could use
+ # index 0 here. But clusters (which are also allowed now) have all their nodes
+ # in the list of required nodes. The cluster node is always the last one in
+ # this list.
+ host = req_hosts[-1] # pair of (site, host)
+ if not only_hosts or host in only_hosts:
+ cache["host_aggregations"].setdefault(host, []).append((group, aggr))
+
+ # construct a list of compiled single-host aggregations for cached registration
+ single_affected_hosts.append(host)
+
+ # All aggregations containing a specific host
+ for h in req_hosts:
+ cache["affected_hosts"].setdefault(h, []).append((group, aggr))
+
+ # All aggregations containing a specific service
+ services = find_all_leaves(aggr)
+ for s in services: # triples of site, host, service
+ cache["affected_services"].setdefault(s, []).append((group, aggr))
# Register compiled objects
if only_hosts:
@@ -1408,11 +1427,13 @@ def singlehost_table(columns, add_headers, only_sites, limit, filters, joinbynam
if config.bi_precompile_on_demand and not joinbyname and len(aggregation['reqhosts']) > 1:
status_info = {}
for site, host in aggregation['reqhosts']:
- status_info[(site, host)] = [
- row_dict[(site, host)]['state'],
- row_dict[(site, host)]['plugin_output'],
- row_dict[(site, host)]['services_with_info'],
- ]
+ this_row = row_dict.get((site, host))
+ if this_row:
+ status_info[(site, host)] = [
+ this_row['state'],
+ this_row['plugin_output'],
+ this_row['services_with_info'],
+ ]
row.update(create_aggregation_row(aggregation, status_info))
row["aggr_group"] = group