Checkmk git commits March 2018

checkmk-commits@lists.checkmk.com

10 participants
326 discussions

Fixed pylint checks after removal of PYTHONPATH from sites environment

by Lars Michelsen

Module: check_mk Branch: master Commit: e721bafdf797fa2d9fe73e5c29d2b3985f5e0b45 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e721bafdf797fa… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Mar 1 09:26:05 2018 +0100 Fixed pylint checks after removal of PYTHONPATH from sites environment Change-Id: Ic05c84ec15a5072128b3aa3fb5e249177f85a546 --- tests/testlib/pylint_cmk.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/testlib/pylint_cmk.py b/tests/testlib/pylint_cmk.py index c73a329..3e278cf 100644 --- a/tests/testlib/pylint_cmk.py +++ b/tests/testlib/pylint_cmk.py @@ -38,9 +38,11 @@ def add_file(f, path): def run_pylint(base_path, check_files=None): #, cleanup_test_dir=False): - pylint_args = os.environ.get("PYLINT_ARGS", "") - if pylint_args: - pylint_args += " " + args = os.environ.get("PYLINT_ARGS", "") + if args: + pylint_args = args.split(" ") + else: + pylint_args = [] pylint_cfg = repo_path() + "/pylintrc" @@ -49,10 +51,14 @@ def run_pylint(base_path, check_files=None): #, cleanup_test_dir=False): print "Nothing to do..." return 0 # nothing to do + cmd = [ + "python", "-m", "pylint", + "--rcfile", pylint_cfg, + ] + pylint_args + check_files + os.putenv("TEST_PATH", repo_path() + "/tests") - cmd = "pylint --rcfile=\"%s\" %s%s" % (pylint_cfg, pylint_args, " ".join(check_files)) - print("Running pylint with: %s" % cmd) - p = subprocess.Popen(cmd, shell=True, cwd=base_path) + print("Running pylint in '%s' with: %s" % (base_path, subprocess.list2cmdline(cmd))) + p = subprocess.Popen(cmd, shell=False, cwd=base_path) exit_code = p.wait() print("Finished with exit code: %d" % exit_code)

6 years, 3 months

5871 FIX Improved service view rendering performance when showing perfometers

by Lars Michelsen

Module: check_mk Branch: master Commit: c5ac34f94e01a9e6d771d0017b927890664ae17c URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c5ac34f94e01a9… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Tue Feb 27 18:05:40 2018 +0100 5871 FIX Improved service view rendering performance when showing perfometers When displaying service views which conain the "Perf-O-Meter" painter, the rendering of the page, especially with a larger number of rows (>1000), could take a lot of time (multiple seconds) for rendering. This was caused by an unoptimized perfometer rendering/selection algorithm. A workaround with older versions is to remove the Perf-O-Meter painter from the views. Change-Id: Ia3273b52f79417ae0d67a1ac1974c4ca2dbe77f0 --- .werks/5871 | 16 +++++++ web/htdocs/metrics.py | 122 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 104 insertions(+), 34 deletions(-) diff --git a/.werks/5871 b/.werks/5871 new file mode 100644 index 0000000..47f8c21 --- /dev/null +++ b/.werks/5871 @@ -0,0 +1,16 @@ +Title: Improved service view rendering performance when showing perfometers +Level: 1 +Component: multisite +Class: fix +Compatible: compat +Edition: cre +State: unknown +Version: 1.5.0i4 +Date: 1519744948 + +When displaying service views which conain the "Perf-O-Meter" painter, the rendering +of the page, especially with a larger number of rows (>1000), could take a lot of time +(multiple seconds) for rendering. + +This was caused by an unoptimized perfometer rendering/selection algorithm. A workaround +with older versions is to remove the Perf-O-Meter painter from the views. diff --git a/web/htdocs/metrics.py b/web/htdocs/metrics.py index a9287d3..48294bc 100644 --- a/web/htdocs/metrics.py +++ b/web/htdocs/metrics.py @@ -34,6 +34,7 @@ # unit: The definition-dict of a unit like in unit_info # graph_template: Template for a graph. Essentially a dict with the key "metrics" import math, time, colorsys, shlex, operator, random +import string import config, pagetypes, table import sites import traceback @@ -93,7 +94,6 @@ def load_plugins(force): global graph_info ; graph_info = AutomaticDict("manual_graph_template") load_web_plugins("metrics", globals()) - loaded_with_language = current_language fixup_graph_info() fixup_unit_info() @@ -113,15 +113,16 @@ def fixup_unit_info(): unit.setdefault("description", unit["title"]) +def fixup_perfometer_info(): + _convert_legacy_tuple_perfometers(perfometer_info) + _precalculate_some_perfometer_caches(perfometer_info) + + # During implementation of the metric system the perfometers were first defined using # tuples. This has been replaced with a dict based syntax. This function converts the # old known formats from tuple to dict. # All shipped perfometers have been converted to the dict format with 1.5.0i3. # TODO: Remove this one day. -def fixup_perfometer_info(): - _convert_legacy_tuple_perfometers(perfometer_info) - - def _convert_legacy_tuple_perfometers(perfometers): for index, perfometer in reversed(list(enumerate(perfometers))): if type(perfometer) == dict: @@ -156,6 +157,72 @@ def _convert_legacy_tuple_perfometers(perfometers): logger.warning(_("Could not convert perfometer to dict format: %r. Ignoring this one.") % perfometer) perfometers.pop(index) + +def _precalculate_some_perfometer_caches(perfometers): + for perfometer in perfometers: + # Precalculate the list of metric expressions of the perfometers + required_expressions = _perfometer_expressions(perfometer) + + # And also precalculate the trivial metric names that can later be used to filter + # perfometers without the need to evaluate the expressions. + required_trivial_metric_names = _required_trivial_metric_names(required_expressions) + + perfometer["_required"] = required_expressions + perfometer["_required_names"] = required_trivial_metric_names + + +def _perfometer_expressions(perfometer): + """Returns all metric expressions of a perfometer + This is used for checking which perfometer can be displayed for a given service later. + """ + required = [] + + if perfometer["type"] == "linear": + required += perfometer["segments"][:] + + elif perfometer["type"] == "logarithmic": + required.append(perfometer["metric"]) + + elif perfometer["type"] in ("stacked", "dual"): + if "perfometers" not in perfometer: + raise MKGeneralException(_("Perfometers of type 'stacked' and 'dual' need " + "the element 'perfometers' (%r)") % perfometer) + + for sub_perfometer in perfometer["perfometers"]: + required += _perfometer_expressions(sub_perfometer) + + else: + raise NotImplementedError(_("Invalid perfometer type: %s") % perfometer["type"]) + + + if "label" in perfometer and perfometer["label"] != None: + required.append(perfometer["label"][0]) + if "total" in perfometer: + required.append(perfometer["total"]) + + return required + + +def _required_trivial_metric_names(required_expressions): + """Extract the trivial metric names from a list of expressions. + Ignores numeric parts. Returns None in case there is a non trivial + metric found. This means the trivial filtering can not be used. + """ + required_metric_names = set() + + allowed_chars = string.ascii_letters + string.digits + "_" + + for entry in required_expressions: + if type(entry) in [ str, unicode ]: + if any(char not in allowed_chars for char in entry): + # Found a non trivial metric expression. Totally skip this mechanism + return None + + required_metric_names.add(entry) + + return required_metric_names + + #. # .--Constants-----------------------------------------------------------. # | ____ _ _ | @@ -824,10 +891,11 @@ class Perfometers(object): return perfometers - # TODO: We will run into a performance problem here when we - # have more and more Perf-O-Meter definitions. def _perfometer_possible(self, perfometer, translated_metrics): - for req in self._get_perfometer_expressions(perfometer, translated_metrics): + if self._skip_perfometer_by_trivial_metrics(perfometer["_required_names"], translated_metrics): + return False + + for req in perfometer["_required"]: try: evaluate(req, translated_metrics) except: @@ -844,34 +912,20 @@ class Perfometers(object): return True - def _get_perfometer_expressions(self, perfometer, translated_metrics): - required = [] + def _skip_perfometer_by_trivial_metrics(self, required_metric_names, translated_metrics): + """Whether or not a perfometer can be skipped by simple metric name matching instead of expression evaluation - if perfometer["type"] == "linear": - required = perfometer["segments"][:] - - elif perfometer["type"] == "logarithmic": - required = [ perfometer["metric"] ] - - elif perfometer["type"] in ("stacked", "dual"): - if "perfometers" not in perfometer: - raise MKGeneralException(_("Perfometers of type 'stacked' and 'dual' need " - "the element 'perfometers' (%r)") % perfometer) - - for sub_perfometer in perfometer["perfometers"]: - required += self._get_perfometer_expressions(sub_perfometer, translated_metrics) - - else: - raise NotImplementedError(_("Invalid perfometer type: %s") % perfometer["type"]) - - - if "label" in perfometer and perfometer["label"] != None: - required.append(perfometer["label"][0]) - - if "total" in perfometer: - required.append(perfometer["total"]) + Performance optimization: Try to reduce the amount of perfometers to evaluate by + comparing the strings in the "required" metrics with the translated metrics. + We only look at the simple "requried expressions" that don't make use of formulas. + In case there is a formula, we can not skip the perfometer and have to evaluate + it. + """ + if required_metric_names is None: + return False - return required + available_metric_names = set(translated_metrics.keys()) + return not required_metric_names.issubset(available_metric_names)

6 years, 3 months

Fixed another bug in #5853

by Lars Michelsen

Module: check_mk Branch: master Commit: efd302b3f868371cf7c70a952afa6b05da7536fb URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=efd302b3f86837… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Tue Feb 27 18:22:18 2018 +0100 Fixed another bug in #5853 Change-Id: Id99357db6358b3e8c0e680f59b55fa16a593ff3f --- web/htdocs/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/htdocs/views.py b/web/htdocs/views.py index 4cd9f23..29c9b4f 100644 --- a/web/htdocs/views.py +++ b/web/htdocs/views.py @@ -805,6 +805,9 @@ class Cell(object): else: txt = img_filename + if isinstance(txt, HTML): + txt = "%s" % txt + txt = html.escaper.unescape_attributes(txt) txt = html.strip_tags(txt)

6 years, 3 months

5872 FIX Do not bake VANILLA/GENERIC implicit when bake for list of hosts

by Lars Michelsen

Module: check_mk Branch: master Commit: 02a87258a8717b18b7440c2293a56d0ac45311d3 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=02a87258a8717b… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Mar 1 07:04:22 2018 +0100 5872 FIX Do not bake VANILLA/GENERIC implicit when bake for list of hosts When the agent bakery is invoked with "cmk -A HOST1" to bake the agents for the host with the name HOST1, previous Check_Mk versions were baking the VANILLA and GENERIC agents in addition to the given host. This has been changed. Now the bakery only bakes the agents for the hosts the user requested with the call. Change-Id: I69c888a06bbd65a2b2f9711e8c95dd7cafb79afd --- .werks/5872 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.werks/5872 b/.werks/5872 new file mode 100644 index 0000000..24de1f7 --- /dev/null +++ b/.werks/5872 @@ -0,0 +1,14 @@ +Title: Do not bake VANILLA/GENERIC implicit when bake for list of hosts +Level: 1 +Component: agents +Compatible: compat +Edition: cee +Version: 1.5.0i4 +Date: 1519854141 +Class: fix + +When the agent bakery is invoked with "cmk -A HOST1" to bake the agents +for the host with the name HOST1, previous Check_Mk versions were baking +the VANILLA and GENERIC agents in addition to the given host. This has +been changed. Now the bakery only bakes the agents for the hosts the +user requested with the call.

6 years, 3 months

Refactored WATO folder page to WatoMode

by Lars Michelsen

Module: check_mk Branch: master Commit: 10746024c0e3e6e8b93648175ff3bb14fc7efd92 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=10746024c0e3e6… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Mar 1 08:25:55 2018 +0100 Refactored WATO folder page to WatoMode Change-Id: I3e21db97e1c4fa5a0a673f9705bb0faeba6cbc83 --- web/htdocs/wato.py | 958 +++++++++++++++++++++++++++-------------------------- 1 file changed, 484 insertions(+), 474 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=10746024c0…

6 years, 3 months

Fixed pylint imports from site specific paths

by Lars Michelsen

Module: check_mk Branch: master Commit: b4af2add3bae8f4317087e9e3a494358beb031ed URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=b4af2add3bae8f… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Mar 1 07:01:51 2018 +0100 Fixed pylint imports from site specific paths Change-Id: Iaf690658b2a72911115877a216dad5f4c544760d --- tests/testlib/pylint_cmk.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/testlib/pylint_cmk.py b/tests/testlib/pylint_cmk.py index c73a329..c989066 100644 --- a/tests/testlib/pylint_cmk.py +++ b/tests/testlib/pylint_cmk.py @@ -10,6 +10,7 @@ import tempfile from pylint.reporters.text import ColorizedTextReporter, ParseableTextReporter from pylint.utils import Message +from pylint.lint import Run from testlib import repo_path, cmk_path, cmc_path, cme_path @@ -39,8 +40,6 @@ def add_file(f, path): def run_pylint(base_path, check_files=None): #, cleanup_test_dir=False): pylint_args = os.environ.get("PYLINT_ARGS", "") - if pylint_args: - pylint_args += " " pylint_cfg = repo_path() + "/pylintrc" @@ -50,10 +49,25 @@ def run_pylint(base_path, check_files=None): #, cleanup_test_dir=False): return 0 # nothing to do os.putenv("TEST_PATH", repo_path() + "/tests") - cmd = "pylint --rcfile=\"%s\" %s%s" % (pylint_cfg, pylint_args, " ".join(check_files)) - print("Running pylint with: %s" % cmd) - p = subprocess.Popen(cmd, shell=True, cwd=base_path) - exit_code = p.wait() + + args = [] + if pylint_args: + args += pylint_args.split(" ") + + args += [ "--rcfile=%s" % pylint_cfg ] + args += check_files + + print("Running pylint in %r with args: %r" % (base_path, args)) + + try: + orig_wd = os.getcwd() + os.chdir(base_path) + + run = Run(args, exit=False) + exit_code = run.linter.msg_status + finally: + os.chdir(orig_wd) + print("Finished with exit code: %d" % exit_code) #if exit_code == 0 and cleanup_test_dir:

6 years, 3 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

Checkmk git commits March 2018