[checkmk-commits] 5886 FIX Improved performane of "ineffective ruleset" searching

Module: check_mk Branch: master Commit: d7afc50b9c5f30470a3bd02e3fe1f05df245b774 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=d7afc50b9c5f30… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Mon Mar 5 14:47:55 2018 +0100 5886 FIX Improved performane of "ineffective ruleset" searching Pages that do analyze the matching of rules on folders / hosts were not performing good when having a lot of rules and match objects configured. This has been improved by introducing some internal caches. Change-Id: Ia18e46df823b718ddcab85d2a37ab32df1ddad93 --- .werks/5886 | 13 +++++++++++ web/htdocs/watolib.py | 61 +++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/.werks/5886 b/.werks/5886 new file mode 100644 index 0000000..63153c2 --- /dev/null +++ b/.werks/5886 @@ -0,0 +1,13 @@ +Title: Improved performane of "ineffective ruleset" searching +Level: 1 +Component: wato +Class: fix +Compatible: compat +Edition: cre +State: unknown +Version: 1.5.0i4 +Date: 1520257450 + +Pages that do analyze the matching of rules on folders / hosts were +not performing good when having a lot of rules and match objects +configured. This has been improved by introducing some internal caches. diff --git a/web/htdocs/watolib.py b/web/htdocs/watolib.py index 7bfa980..64ae001 100644 --- a/web/htdocs/watolib.py +++ b/web/htdocs/watolib.py @@ -813,6 +813,7 @@ class WithPermissionsAndAttributes(WithPermissions): def __init__(self): super(WithPermissionsAndAttributes, self).__init__() self._attributes = {} + self._effective_attributes = None # .--------------------------------------------------------------------. # | ATTRIBUTES | @@ -846,6 +847,22 @@ class WithPermissionsAndAttributes(WithPermissions): del self.attributes()[attrname] + def drop_caches(self): + self._effective_attributes = None + + + def _cache_effective_attributes(self, effective): + self._effective_attributes = effective.copy() + + + def _get_cached_effective_attributes(self): + if self._effective_attributes is None: + raise KeyError("Not cached") + else: + return self._effective_attributes.copy() + + + #. # .--BaseFolder----------------------------------------------------------. # | ____ _____ _ _ | @@ -1195,7 +1212,6 @@ class CREFolder(BaseFolder): self._subfolders = {} self._choices_for_moving_host = None - self._effective_attributes = None self._root_dir = root_dir if self._root_dir: @@ -1418,6 +1434,13 @@ class CREFolder(BaseFolder): self.need_permission("write") if self._hosts != None: self._save_hosts_file() + + # Clean up caches of all hosts in this folder, just to be sure. We could also + # check out all call sites of save_hosts() and partially drop the caches of + # individual hosts to optimize this. + for host in self._hosts.values(): + host.drop_caches() + call_hook_hosts_changed(self) @@ -1672,12 +1695,15 @@ class CREFolder(BaseFolder): def drop_caches(self): + super(CREFolder, self).drop_caches() self._choices_for_moving_host = None - self._effective_attributes = None for subfolder in self._subfolders.values(): subfolder.drop_caches() + for host in self._hosts.values(): + host.drop_caches() + # .-----------------------------------------------------------------------. # | ELEMENT ACCESS | @@ -1880,8 +1906,10 @@ class CREFolder(BaseFolder): def effective_attributes(self): - if self._effective_attributes != None: - return self._effective_attributes.copy() # cached :-) + try: + return self._get_cached_effective_attributes() # cached :-) + except KeyError: + pass effective = {} for folder in self.parent_folder_chain(): @@ -1894,8 +1922,7 @@ class CREFolder(BaseFolder): if attrname not in effective: effective.setdefault(attrname, host_attribute.default_value()) - self._effective_attributes = effective.copy() - + self._cache_effective_attributes(effective) return effective @@ -2702,12 +2729,18 @@ class CREHost(WithPermissionsAndAttributes): self._name = host_name self._attributes = attributes self._cluster_nodes = cluster_nodes + self._cached_host_tags = None def __repr__(self): return "Host(%r)" % (self._name) + def drop_caches(self): + super(CREHost, self).drop_caches() + self._cached_host_tags = None + + # .--------------------------------------------------------------------. # | ELEMENT ACCESS | # '--------------------------------------------------------------------' @@ -2760,6 +2793,9 @@ class CREHost(WithPermissionsAndAttributes): # Compute tags from settings of each individual tag. We've got # the current value for each individual tag. Also other attributes # can set tags (e.g. the SiteAttribute) + if self._cached_host_tags is not None: + return self._cached_host_tags # Cached :-) + tags = set([]) effective = self.effective_attributes() for attr, topic in all_host_attributes(): @@ -2773,6 +2809,7 @@ class CREHost(WithPermissionsAndAttributes): if "no-snmp" in tags and "no-agent" in tags: tags.add("ping") + self._cached_host_tags = tags return tags @@ -2800,8 +2837,14 @@ class CREHost(WithPermissionsAndAttributes): def effective_attributes(self): + try: + return self._get_cached_effective_attributes() # cached :-) + except KeyError: + pass + effective = self.folder().effective_attributes() effective.update(self.attributes()) + self._cache_effective_attributes(effective) return effective @@ -8302,9 +8345,6 @@ class Rule(object): if not self._matches_hostname(hostname): yield _("The host name does not match.") - if not self.folder.is_transitive_parent_of(host_folder): - yield _("The rule does not apply to the folder of the host.") - host_tags = host.tags() for tag in self.tag_specs: if tag[0] != '/' and tag[0] != '!' and tag not in host_tags: @@ -8312,6 +8352,9 @@ class Rule(object): elif tag[0] == '!' and tag[1:] in host_tags: yield _("The host has the tag %s") % tag + if not self.folder.is_transitive_parent_of(host_folder): + yield _("The rule does not apply to the folder of the host.") + if item != NO_ITEM and self.ruleset.item_type(): if not self.matches_item(item): yield _("The %s \"%s\" does not match this rule.") % \