Module: check_mk
Branch: master
Commit: e496fcedabb41b208a58a013dfa28cc3aab9a01f
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e496fcedabb41b…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Sun Apr 29 19:43:50 2018 +0200
6046 Improved execution time of e.g. service discovery
The overall execution time of the service discovery has been reduced by
introducing some precompiling of the check plugins and their includes
before loading them. The mechanism is similar to the compilation of
python modules from (.py to .pyc files). This mechanism reduces the
load time of the plugins by 50%.
Change-Id: I97bc4f72db4fa782d183fd936863d3bb87c25c90
---
.werks/6046 | 15 +++++++++++++++
cmk/paths.py | 2 ++
cmk_base/checks.py | 45 +++++++++++++++++++++++++++++++++++++++++++--
3 files changed, 60 insertions(+), 2 deletions(-)
diff --git a/.werks/6046 b/.werks/6046
new file mode 100644
index 0000000..d28772a
--- /dev/null
+++ b/.werks/6046
@@ -0,0 +1,15 @@
+Title: Improved execution time of e.g. service discovery
+Level: 1
+Component: core
+Class: feature
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1525023719
+
+The overall execution time of the service discovery has been reduced by
+introducing some precompiling of the check plugins and their includes
+before loading them. The mechanism is similar to the compilation of
+python modules from (.py to .pyc files). This mechanism reduces the
+load time of the plugins by 50%.
diff --git a/cmk/paths.py b/cmk/paths.py
index 1d7e623..3cb0395 100644
--- a/cmk/paths.py
+++ b/cmk/paths.py
@@ -54,6 +54,7 @@ check_mk_config_dir = None
modules_dir = None
var_dir = None
log_dir = None
+precompiled_checks_dir = None
autochecks_dir = None
precompiled_hostchecks_dir= None
snmpwalks_dir = None
@@ -122,6 +123,7 @@ def _set_paths():
"modules_dir" : os.path.join(omd_root, "share/check_mk/modules"),
"var_dir" : os.path.join(omd_root, "var/check_mk"),
"log_dir" : os.path.join(omd_root, "var/log"),
+ "precompiled_checks_dir" : os.path.join(omd_root, "var/check_mk/precompiled_checks"),
"autochecks_dir" : os.path.join(omd_root, "var/check_mk/autochecks"),
"precompiled_hostchecks_dir" : os.path.join(omd_root, "var/check_mk/precompiled"),
"snmpwalks_dir" : os.path.join(omd_root, "var/check_mk/snmpwalks"),
diff --git a/cmk_base/checks.py b/cmk_base/checks.py
index 134d6ec..de9de88 100644
--- a/cmk_base/checks.py
+++ b/cmk_base/checks.py
@@ -28,6 +28,7 @@ import os
import math
import copy
import ast
+import marshal
from collections import OrderedDict
import cmk.paths
@@ -149,7 +150,7 @@ def load_checks(filelist):
load_check_includes(f, check_context)
- execfile(f, check_context)
+ load_precompiled_plugin(f, check_context)
loaded_files.add(file_name)
except MKTerminate:
@@ -248,7 +249,7 @@ def load_check_includes(check_file_path, check_context):
for include_file_name in includes_of_plugin(check_file_path):
include_file_path = check_include_file_path(include_file_name)
try:
- execfile(include_file_path, check_context)
+ load_precompiled_plugin(include_file_path, check_context)
except MKTerminate:
raise
@@ -325,6 +326,46 @@ def _plugin_pathnames_in_directory(path):
return []
+def load_precompiled_plugin(path, check_context):
+ """Loads the given check or check include plugin into the given
+ check context.
+
+ To improve loading speed the files are not read directly. The files are
+ python byte-code compiled before in case it has not been done before. In
+ case there is already a compiled file that is newer than the current one,
+ then the precompiled file is loaded."""
+
+ precompiled_path = _precompiled_plugin_path(path)
+
+ if not _is_plugin_precompiled(path, precompiled_path):
+ console.vverbose("Precompile %s to %s\n" % (path, precompiled_path))
+ _precompile_plugin(path, precompiled_path)
+
+ exec(marshal.load(open(precompiled_path)), check_context)
+
+
+def _is_plugin_precompiled(path, precompiled_path):
+ return os.path.exists(precompiled_path) \
+ and os.stat(path).st_mtime < os.stat(precompiled_path).st_mtime
+
+
+def _precompile_plugin(path, precompiled_path):
+ code = compile(open(path).read(), path, "exec")
+
+ if not os.path.exists(os.path.dirname(precompiled_path)):
+ os.makedirs(os.path.dirname(precompiled_path))
+
+ with open(precompiled_path, "w") as compiled_file:
+ marshal.dump(code, compiled_file)
+
+
+def _precompiled_plugin_path(path):
+ is_local = path.startswith(cmk.paths.local_checks_dir)
+ return os.path.join(cmk.paths.precompiled_checks_dir,
+ "local" if is_local else "",
+ os.path.basename(path))
+
+
def check_variable_names():
return _check_variables.keys()
Module: check_mk
Branch: master
Commit: 9e2aef2f5a5fde00ce123a99a647e3ebbdf2341c
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=9e2aef2f5a5fde…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Mon Apr 30 10:29:43 2018 +0200
6053 FIX Fixed sync after user notification rule changes
When a user changes his personal notification rules in a distributed
environment, the user profile synchronization was only performed for the
configured "login sites". These are the sites where user logins are allowed (by
the site configuration). In case of notification setting changes the notification
rules need to be synchronized to all sites instead.
Change-Id: I6d9f4b9ee0cecba116d16fc917973c04ca610ff5
---
.werks/6053 | 15 +++++++++++++++
web/htdocs/wato.py | 11 ++++++-----
web/htdocs/watolib.py | 5 +++++
3 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/.werks/6053 b/.werks/6053
new file mode 100644
index 0000000..d9dfaba
--- /dev/null
+++ b/.werks/6053
@@ -0,0 +1,15 @@
+Title: Fixed sync after user notification rule changes
+Level: 1
+Component: notifications
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1525074833
+
+When a user changes his personal notification rules in a distributed
+environment, the user profile synchronization was only performed for the
+configured "login sites". These are the sites where user logins are allowed (by
+the site configuration). In case of notification setting changes the notification
+rules need to be synchronized to all sites instead.
diff --git a/web/htdocs/wato.py b/web/htdocs/wato.py
index f15da8b..639c28d 100644
--- a/web/htdocs/wato.py
+++ b/web/htdocs/wato.py
@@ -9105,7 +9105,7 @@ def mode_user_notifications(phase, profilemode):
return
if notification_rule_start_async_repl:
- user_profile_async_replication_dialog()
+ user_profile_async_replication_dialog(sites=get_notification_sync_sites())
notification_rule_start_async_repl = False
html.h3(_('Notification Rules'))
@@ -9224,7 +9224,7 @@ def mode_notification_rule(phase, profilemode):
return "notifications"
if notification_rule_start_async_repl:
- user_profile_async_replication_dialog()
+ user_profile_async_replication_dialog(sites=get_notification_sync_sites())
notification_rule_start_async_repl = False
return
@@ -14799,12 +14799,12 @@ def user_profile_async_replication_page():
html.context_button(_('User Profile'), 'user_profile.py', 'back')
html.end_context_buttons()
- user_profile_async_replication_dialog()
+ user_profile_async_replication_dialog(sites=config.user.authorized_login_sites())
html.footer()
-def user_profile_async_replication_dialog():
+def user_profile_async_replication_dialog(sites):
repstatus = watolib.load_replication_status()
html.message(_('In order to activate your changes available on all remote sites, your user profile needs '
@@ -14815,7 +14815,8 @@ def user_profile_async_replication_dialog():
html.h3(_('Replication States'))
html.open_div(id_="profile_repl")
num_replsites = 0
- for site_id, site in config.user.authorized_login_sites():
+ for site_id in sites:
+ site = config.sites[site_id]
srs = repstatus.get(site_id, {})
if not "secret" in site:
diff --git a/web/htdocs/watolib.py b/web/htdocs/watolib.py
index d5f145c..918aa16 100644
--- a/web/htdocs/watolib.py
+++ b/web/htdocs/watolib.py
@@ -4321,6 +4321,11 @@ def get_event_console_sync_sites():
return [ s[0] for s in config.get_event_console_site_choices() ]
+def get_notification_sync_sites():
+ return sorted([ site_id for site_id, site in wato_slave_sites()
+ if not config.site_is_local(site_id) ])
+
+
# TODO: cleanup all call sites to this name
site_choices = config.site_choices
Module: check_mk
Branch: master
Commit: 3b71c921a9145bb3259b78758060a984a4ca8722
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=3b71c921a9145b…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Wed Apr 25 09:41:10 2018 +0200
6031 FIX Improved initial activation step performance in some cases
When activating the changed configuration, the first step "Initializing activation" may
take a lot of time when there are a lot of files located below the "local" hierarchy of
the site.
This is caused by a backup mechanism that is executed in this step. It saves snapshots
of your WATO configuration (not the whole site nor the whole config of the site) to
archives which are located below "var/check_mk/wato/snapshots". These can be used for
reverting the changes in the GUI and for recovery of configuration files. This is not
primarily a backup mechanism, better use the Check_MK site backup for this task.
These archives were also containing the whole local/ hierarchy. In case there are a lot
of small files or some larger files in this hierarchy, this slowed down the activation
significantly. We've decided to remove those files from the config snapshot now.
Change-Id: Ib215fa78ef16f03f2e17d294478d62b98e2d6f9e
---
.werks/6031 | 23 +++++++++++++++++++++++
web/plugins/wato/backup_domains.py | 1 -
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/.werks/6031 b/.werks/6031
new file mode 100644
index 0000000..5357ee8
--- /dev/null
+++ b/.werks/6031
@@ -0,0 +1,23 @@
+Title: Improved initial activation step performance in some cases
+Level: 1
+Component: wato
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1524641789
+
+When activating the changed configuration, the first step "Initializing activation" may
+take a lot of time when there are a lot of files located below the "local" hierarchy of
+the site.
+
+This is caused by a backup mechanism that is executed in this step. It saves snapshots
+of your WATO configuration (not the whole site nor the whole config of the site) to
+archives which are located below "var/check_mk/wato/snapshots". These can be used for
+reverting the changes in the GUI and for recovery of configuration files. This is not
+primarily a backup mechanism, better use the Check_MK site backup for this task.
+
+These archives were also containing the whole local/ hierarchy. In case there are a lot
+of small files or some larger files in this hierarchy, this slowed down the activation
+significantly. We've decided to remove those files from the config snapshot now.
diff --git a/web/plugins/wato/backup_domains.py b/web/plugins/wato/backup_domains.py
index e877c14..83399e8 100644
--- a/web/plugins/wato/backup_domains.py
+++ b/web/plugins/wato/backup_domains.py
@@ -76,6 +76,5 @@ watolib.backup_domains.update({
("dir", "var/check_mk/packages" ),
("dir", "local" ),
],
- "default" : True,
},
})
Module: check_mk
Branch: master
Commit: 8505241cfdf84c041cde2cd32cf26e2de2c2ee5d
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=8505241cfdf84c…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Mon Apr 30 07:01:22 2018 +0200
6045 FIX df: Disable for docker containers
Change-Id: I985f80c869b97e87b99e7de710c1025a39b9d12c
---
.werks/6045 | 11 +++++++++++
agents/check_mk_agent.linux | 44 ++++++++++++++++++++++++++------------------
2 files changed, 37 insertions(+), 18 deletions(-)
diff --git a/.werks/6045 b/.werks/6045
new file mode 100644
index 0000000..7cbbe30
--- /dev/null
+++ b/.werks/6045
@@ -0,0 +1,11 @@
+Title: df: Disable for docker containers
+Level: 1
+Component: checks
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1525064482
+
+
diff --git a/agents/check_mk_agent.linux b/agents/check_mk_agent.linux
index 9df613a..85c7d4c 100755
--- a/agents/check_mk_agent.linux
+++ b/agents/check_mk_agent.linux
@@ -153,6 +153,31 @@ function section_uptime()
fi
}
+# Print out Partitions / Filesystems. (-P gives non-wrapped POSIXed output)
+# Heads up: NFS-mounts are generally supressed to avoid agent hangs.
+# If hard NFS mounts are configured or you have too large nfs retry/timeout
+# settings, accessing those mounts from the agent would leave you with
+# thousands of agent processes and, ultimately, a dead monitored system.
+# These should generally be monitored on the NFS server, not on the clients.
+function section_df()
+{
+ if [ -n "$MK_IN_CONTAINER" ]; then
+ return
+ fi
+
+ echo '<<<df>>>'
+ # The exclusion list is getting a bit of a problem. -l should hide any remote FS but seems
+ # to be all but working.
+ local excludefs="-x smbfs -x cifs -x iso9660 -x udf -x nfsv4 -x nfs -x mvfs -x zfs -x prl_fs"
+ df -PTlk "$excludefs" | sed 1d
+
+ # df inodes information
+ echo '<<<df>>>'
+ echo '[df_inodes_start]'
+ df -PTli "$excludefs" | sed 1d
+ echo '[df_inodes_end]'
+}
+
function run_mrpe() {
local descr=$1
shift
@@ -333,24 +358,7 @@ then
sed -n '/^service[[:space:]]*check_mk/,/}/s/^[[:space:]]*only_from[[:space:]]*=[[:space:]]*\(.*\)/\1/p' /etc/xinetd.d/* | head -n1; echo
fi
-# Print out Partitions / Filesystems. (-P gives non-wrapped POSIXed output)
-# Heads up: NFS-mounts are generally supressed to avoid agent hangs.
-# If hard NFS mounts are configured or you have too large nfs retry/timeout
-# settings, accessing those mounts from the agent would leave you with
-# thousands of agent processes and, ultimately, a dead monitored system.
-# These should generally be monitored on the NFS server, not on the clients.
-
-echo '<<<df>>>'
-# The exclusion list is getting a bit of a problem. -l should hide any remote FS but seems
-# to be all but working.
-excludefs="-x smbfs -x cifs -x iso9660 -x udf -x nfsv4 -x nfs -x mvfs -x zfs -x prl_fs"
-df -PTlk "$excludefs" | sed 1d
-
-# df inodes information
-echo '<<<df>>>'
-echo '[df_inodes_start]'
-df -PTli "$excludefs" | sed 1d
-echo '[df_inodes_end]'
+section_df
# Filesystem usage for ZFS
if type zfs > /dev/null 2>&1 ; then
Module: check_mk
Branch: master
Commit: 65dc62f5715b9d9b1e91d2c3de06672c0ec6b461
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=65dc62f5715b9d…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Fri Apr 27 21:09:30 2018 +0200
6044 FIX Nagios: Fixed crash when discovered checks were disabled
When all services discovered by a single check were disabled a
"KeyError: 'logwatch'" exception occured while checking the host.
Change-Id: I74b2a5e41e25247889ddebef4a71bc4da7d85149
---
.werks/6044 | 12 ++++++++++++
cmk_base/check_table.py | 12 ++++++------
cmk_base/core_nagios.py | 15 ++++++++++-----
3 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/.werks/6044 b/.werks/6044
new file mode 100644
index 0000000..5bd84da
--- /dev/null
+++ b/.werks/6044
@@ -0,0 +1,12 @@
+Title: Nagios: Fixed crash when discovered checks were disabled
+Level: 1
+Component: core
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.6.0i1
+Date: 1524855837
+
+When all services discovered by a single check were disabled a
+"KeyError: 'logwatch'" exception occured while checking the host.
diff --git a/cmk_base/check_table.py b/cmk_base/check_table.py
index 58e3aa0..fedcfc3 100644
--- a/cmk_base/check_table.py
+++ b/cmk_base/check_table.py
@@ -45,7 +45,7 @@ import cmk_base.item_state as item_state
# filter_mode: "include_clustered" -> returns checks of own host, including clustered checks
# filter_mode: "only_clustered" -> returns only checks belonging to clusters
def get_check_table(hostname, remove_duplicates=False, use_cache=True,
- world='config', skip_autochecks=False, filter_mode=None):
+ world='config', skip_autochecks=False, filter_mode=None, skip_ignored=True):
import cmk_base.discovery as discovery
if config.is_ping_host(hostname):
@@ -131,7 +131,7 @@ def get_check_table(hostname, remove_duplicates=False, use_cache=True,
if rulesets.hosttags_match_taglist(hosttags, tags) and \
rulesets.in_extraconf_hostlist(hostlist, hostname):
descr = config.service_description(hostname, checkname, item)
- if config.service_ignored(hostname, checkname, descr):
+ if skip_ignored and config.service_ignored(hostname, checkname, descr):
return
svc_is_mine = hostname == config.host_of_clustered_service(hostname, descr)
@@ -192,8 +192,8 @@ def get_check_table(hostname, remove_duplicates=False, use_cache=True,
return check_table
-def get_precompiled_check_table(hostname, remove_duplicates=True, world="config", filter_mode=None):
- host_checks = get_sorted_check_table(hostname, remove_duplicates, world, filter_mode=filter_mode)
+def get_precompiled_check_table(hostname, remove_duplicates=True, world="config", filter_mode=None, skip_ignored=True):
+ host_checks = get_sorted_check_table(hostname, remove_duplicates, world, filter_mode=filter_mode, skip_ignored=skip_ignored)
precomp_table = []
for check_plugin_name, item, params, description, _unused_deps in host_checks:
# make these globals available to the precompile function
@@ -270,13 +270,13 @@ def _remove_duplicate_checks(check_table):
# if there already is a TCP based one with the same
# description. E.g: df vs hr_fs.
# TODO: Clean this up!
-def get_sorted_check_table(hostname, remove_duplicates=False, world="config", filter_mode=None):
+def get_sorted_check_table(hostname, remove_duplicates=False, world="config", filter_mode=None, skip_ignored=True):
# Convert from dictionary into simple tuple list. Then sort
# it according to the service dependencies.
unsorted = [ (checkname, item, params, descr, deps)
for ((checkname, item), (params, descr, deps))
in get_check_table(hostname, remove_duplicates=remove_duplicates, world=world,
- filter_mode=filter_mode).items() ]
+ filter_mode=filter_mode, skip_ignored=skip_ignored).items() ]
def cmp(a, b):
if a[3] < b[3]:
diff --git a/cmk_base/core_nagios.py b/cmk_base/core_nagios.py
index 62ebc25..9cfd5fe 100644
--- a/cmk_base/core_nagios.py
+++ b/cmk_base/core_nagios.py
@@ -951,7 +951,8 @@ def _precompile_hostcheck(hostname):
pass
# check table, enriched with addition precompiled information.
- host_check_table = check_table.get_precompiled_check_table(hostname, filter_mode="include_clustered")
+ host_check_table = check_table.get_precompiled_check_table(hostname,
+ filter_mode="include_clustered", skip_ignored=False)
if not host_check_table:
console.verbose("(no Check_MK checks)\n")
return
@@ -1009,10 +1010,9 @@ if '-d' in sys.argv:
""")
- # Do we need to load the SNMP module? This is the case, if the host
- # has at least one SNMP based check. Also collect the needed check
- # types and sections.
needed_check_plugin_names = set([])
+
+ # Collect the needed check plugin names using the host check table
for check_plugin_name, _unused_item, _unused_param, descr in host_check_table:
if check_plugin_name not in checks.check_info:
sys.stderr.write('Warning: Ignoring missing check %s.\n' % check_plugin_name)
@@ -1029,7 +1029,7 @@ if '-d' in sys.argv:
# the autochecks of the nodes
if config.is_cluster(hostname):
for node in config.nodes_of(hostname):
- needed_check_plugin_names.update([ e[0] for e in check_table.get_precompiled_check_table(node) ])
+ needed_check_plugin_names.update([ e[0] for e in check_table.get_precompiled_check_table(node, skip_ignored=False) ])
# check info table
# We need to include all those plugins that are referenced in the host's
@@ -1061,6 +1061,11 @@ if '-d' in sys.argv:
for check_plugin_name in sorted(needed_check_plugin_names):
console.verbose(" %s%s%s", tty.green, check_plugin_name, tty.normal, stream=sys.stderr)
+ # Disable this check. We don't have all check plugins loaded and have not all possible
+ # check config variables known. These issues don't have to be validated by the single
+ # precompiled checks.
+ output.write("config.verify_non_invalid_variables = lambda x: None\n")
+
output.write("config.load(validate_hosts=False)\n")
# handling of clusters