Module: check_mk
Branch: master
Commit: 970cbe7a825cbc4724b262c36af2bef5cb3f5065
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=970cbe7a825cbc…
Author: Andreas Boesl <ab(a)mathias-kettner.de>
Date: Fri Jan 25 13:29:34 2019 +0100
6702 Introduced various performance improvements for cmc config generation (e.g. multiprocessing)
Previous versions only used one CPU core for the config generation.
The time to generate the config highly depends on the number of hosts, services and especially rulesets.
Lots of the underlying computation code has been changed, caches and functions were optimized.
As a result Check_MK is now able to distribute the work load of the config generation over several CPUs.
Per default, this feature is activated. Unless configured otherwise, it uses up to 75% of the available
CPUs during the configuration generation, leaving some CPUs for running monitoring core.
A new configuration option <tt>Generate monitoring configuration via multiprocessing</tt> has been introduced.
You can either switch of multiprocessing or configure the number of used CPUs manually.
Change-Id: I0d3da33f45fa41cc963aca013f815b23d87bd8f5
---
.werks/6702 | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/.werks/6702 b/.werks/6702
new file mode 100644
index 0000000..55527dc
--- /dev/null
+++ b/.werks/6702
@@ -0,0 +1,21 @@
+Title: Introduced various performance improvements for cmc config generation (e.g. multiprocessing)
+Level: 2
+Component: cmc
+Compatible: compat
+Edition: cee
+Version: 1.6.0i1
+Date: 1548418345
+Class: feature
+
+Previous versions only used one CPU core for the config generation.
+The time to generate the config highly depends on the number of hosts, services and especially rulesets.
+
+Lots of the underlying computation code has been changed, caches and functions were optimized.
+As a result Check_MK is now able to distribute the work load of the config generation over several CPUs.
+
+Per default, this feature is activated. Unless configured otherwise, it uses up to 75% of the available
+CPUs during the config generation, leaving some CPUs for running monitoring core.
+A new configuration option <tt>Generate monitoring configuration via multiprocessing</tt> has been introduced.
+You can either switch of multiprocessing or configure the number of used CPUs manually.
+
+Tests have shown that the performance can increased by a factor of 5-10 on a 8 CPU core setup.
Module: check_mk
Branch: master
Commit: d1fa72f658154c3c70319fda225cbb003fa5d78b
URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=d1fa72f658154c…
Author: Andreas Boesl <ab(a)mathias-kettner.de>
Date: Thu Jan 24 16:50:13 2019 +0100
Further speedups during config generation
Change-Id: Ifbdb9f120cd0bfeb7dcaea17235a2645c160b664
---
cmk_base/config.py | 30 +++++++++++++++++++++++-------
1 file changed, 23 insertions(+), 7 deletions(-)
diff --git a/cmk_base/config.py b/cmk_base/config.py
index 5d834b5..fafb574 100644
--- a/cmk_base/config.py
+++ b/cmk_base/config.py
@@ -2869,7 +2869,20 @@ class ConfigCache(object):
return self._hosttags.get(hostname, [])
def set_all_processed_hosts(self, all_processed_hosts):
- self._all_processed_hosts = all_processed_hosts
+ self._all_processed_hosts = set(all_processed_hosts)
+
+ nodes_and_clusters = set()
+ for hostname in self._all_processed_hosts:
+ nodes_and_clusters.update(self._nodes_of_cache.get(hostname, []))
+ nodes_and_clusters.update(self._clusters_of_cache.get(hostname, []))
+ self._all_processed_hosts.update(nodes_and_clusters)
+
+ # The folder host lookup includes a list of all -processed- hosts within a given
+ # folder. Any update with set_all_processed hosts invalidates this cache, because
+ # the scope of relevant hosts has changed. This is -good-, since the values in this
+ # lookup are iterated one by one later on in all_matching_hosts
+ self._folder_host_lookup = {}
+
self._adjust_processed_hosts_similarity()
def _adjust_processed_hosts_similarity(self):
@@ -2912,17 +2925,19 @@ class ConfigCache(object):
self._hosts_grouped_by_tags.setdefault(group_ref, set()).add(hostname)
self._host_grouped_ref[hostname] = group_ref
- def get_hosts_within_folder(self, folder_path):
- if folder_path not in self._folder_host_lookup:
+ def get_hosts_within_folder(self, folder_path, with_foreign_hosts):
+ cache_id = with_foreign_hosts, folder_path
+ if cache_id not in self._folder_host_lookup:
hosts_in_folder = set()
# Strip off "+"
folder_path_tmp = folder_path[:-1]
- for hostname in self._all_configured_hosts:
+ relevant_hosts = self._all_configured_hosts if with_foreign_hosts else self._all_processed_hosts
+ for hostname in relevant_hosts:
if self._host_paths[hostname].startswith(folder_path_tmp):
hosts_in_folder.add(hostname)
- self._folder_host_lookup[folder_path] = hosts_in_folder
+ self._folder_host_lookup[cache_id] = hosts_in_folder
return hosts_in_folder
- return self._folder_host_lookup[folder_path]
+ return self._folder_host_lookup[cache_id]
def get_autochecks_of(self, hostname, world):
try:
@@ -2990,7 +3005,8 @@ class ConfigCache(object):
# Thin out the valid hosts further. If the rule is located in a folder
# we only need the intersection of the folders hosts and the previously determined valid_hosts
- valid_hosts = self.get_hosts_within_folder(rule_path).intersection(valid_hosts)
+ valid_hosts = self.get_hosts_within_folder(rule_path,
+ with_foreign_hosts).intersection(valid_hosts)
# Contains matched hosts