Checkmk git commits September 2018

checkmk-commits@lists.checkmk.com

12 participants
371 discussions

Reworked output handling of background job processes

by Lars Michelsen

Module: check_mk Branch: master Commit: 83aa0342ebc1675148d0d9da0ffb9599a027cc24 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=83aa0342ebc167… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Mon Aug 27 20:32:55 2018 +0200 Reworked output handling of background job processes * The output of background jobs was previously handled like this: stdout: Was set to a IO buffer object which was written from the executing function (in the executing thread) and continuously read from the main thread for writing it to the process status file. The underlying stdout file descriptor was not set which means that the output of subprocesses or libraries (e.g. libldap debug output) was not captured. stderr: Was also captured using an IO buffer and transported to var/log/web.log of the site instead of logging it to the job output. The underlying stdout file descriptor was not set which means that the output of subprocesses or libraries (e.g. libldap debug output) was not captured. * Now both, stdout and stderr, are set to a tempfile.TemporaryFile() sys.stdout and sys.stderr are set and also the FDs 1 and 2 are set. This means that all output of the background job is written to the temporary file. It is continuously read from the main thread of the background job and written to the status file. * The logging produced by the background job, which normally goes to var/log/web.log is now also written to stdout which is finally shown in the progress info. Change-Id: I6a674e2469479fa838f3933bf9f90c40a793eb88 --- cmk/gui/background_job.py | 53 +++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/cmk/gui/background_job.py b/cmk/gui/background_job.py index 1b7a433..5de69b2 100644 --- a/cmk/gui/background_job.py +++ b/cmk/gui/background_job.py @@ -28,14 +28,16 @@ import os import sys import time +import tempfile import pprint import threading import multiprocessing -from cStringIO import StringIO +import io import psutil import shutil import signal import traceback +import logging from cmk.gui.i18n import _ from cmk.gui.globals import html @@ -213,8 +215,8 @@ class BackgroundProcess(multiprocessing.Process): if not self._logger: self._logger = cmk.log.logger - sys.stderr = StreamLogger(self._logger, cmk.log.ERROR) - sys.stdout = StringIO() + self._open_stdout_and_stderr() + self._enable_logging_to_stdout() self._job_parameters["logger"] = self._logger @@ -223,15 +225,16 @@ class BackgroundProcess(multiprocessing.Process): def _execute_function(self): - # The specific function is called in a separate thread - # The main thread collects the stdout from the function-thread and updates the status file accordingly + # The specific function is called in a separate thread The main thread + # collects the output produced by the function-thread and updates the + # status file accordingly t = threading.Thread(target=self._call_function_with_exception_handling, args=[self._job_parameters]) t.start() last_progress_info = "" while t.isAlive(): time.sleep(0.2) - progress_info = sys.stdout.getvalue() # pylint: disable=no-member + progress_info = self._read_output() if progress_info != last_progress_info: self._jobstatus.update_status({"progress_info": BackgroundProcessInterface.parse_progress_info(progress_info)}) last_progress_info = progress_info @@ -239,8 +242,7 @@ class BackgroundProcess(multiprocessing.Process): # Final progress info update job_status_update = {} - progress_info = sys.stdout.getvalue() # pylint: disable=no-member - + progress_info = self._read_output() if progress_info != last_progress_info: job_status_update.update({"progress_info": BackgroundProcessInterface.parse_progress_info(progress_info)}) @@ -272,25 +274,27 @@ class BackgroundProcess(multiprocessing.Process): job_interface.send_exception(_("Exception: %s") % (e)) - -# TODO(ab): Wouldn't it be better to also write the stderr to the job results instead of just writing -# it to the web.log? -class StreamLogger(object): - """File like stream object to redirects writes to the given logger""" - def __init__(self, logger, level): - super(StreamLogger, self).__init__() - self._logger = logger - self._level = level - self._linebuf = '' + def _open_stdout_and_stderr(self): + """Create a temporary file and use it as stdout / stderr buffer""" + # We can not use io.BytesIO() or similar because we need real file descriptors + # to be able to catch the (debug) output of libraries like libldap or subproccesses + sys.stdout = sys.stderr = tempfile.TemporaryFile(mode='w+b') + os.dup2(sys.stdout.fileno(), 1) + os.dup2(sys.stderr.fileno(), 2) - def write(self, buf): - for line in buf.rstrip().splitlines(): - self._logger.log(self._level, line.rstrip()) + def _read_output(self): + sys.stderr.flush() + sys.stderr.seek(0, io.SEEK_SET) + return sys.stderr.read() - def flush(self): - pass + def _enable_logging_to_stdout(self): + """In addition to the web.log we also want to see the job specific logs + in stdout (which results in job progress info)""" + handler = logging.StreamHandler(stream=sys.stdout) + handler.setFormatter(cmk.log.get_formatter()) + cmk.log.logger.addHandler(handler) @@ -521,6 +525,9 @@ class BackgroundJob(object): p.start() p.join() + job_status = self.get_status() + self._logger.debug("Started job \"%s\" (PID: %d)" % (self._job_id, job_status["pid"])) + def _start_background_subprocess(self, job_parameters): try:

6 years

LDAP sync: Cleaned up debug logging

by Lars Michelsen

Module: check_mk Branch: master Commit: db47d1efb370bedb407715fa6e3246742cef6609 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=db47d1efb370be… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Tue Aug 28 08:50:51 2018 +0200 LDAP sync: Cleaned up debug logging * Remove special stderr buffering for LDAP connect that is used when logging is set to Debug to record libldap low level debug info. * Our detailed diagnose entries are now logged with "informational" log level for LDAP. * The low level libldap and TLS entries are now logged in addition when LDAP log level is set to debug. Change-Id: I71354096b80a390e16f13df621ab0160056a72bd --- cmk/gui/plugins/userdb/ldap_connector.py | 99 ++++++++------------------------ 1 file changed, 23 insertions(+), 76 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=db47d1efb3…

6 years

6476 FIX apc_symmetra: Fixed transposed default levels for battery capacity

by Marcel Arentz

Module: check_mk Branch: master Commit: c7d85066fdc06b341d87ff2bd7cb6b87181dc887 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c7d85066fdc06b… Author: Marcel Arentz <ma(a)mathias-kettner.de> Date: Thu Sep 6 12:57:23 2018 +0200 6476 FIX apc_symmetra: Fixed transposed default levels for battery capacity You'll need to restart the core for this change to take effect. Change-Id: Id2dc3d609a81843ca5f82cc117a4e23698d9c253 --- .werks/6476 | 10 ++++++++++ checks/apc_symmetra | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.werks/6476 b/.werks/6476 new file mode 100644 index 0000000..2725f3f --- /dev/null +++ b/.werks/6476 @@ -0,0 +1,10 @@ +Title: apc_symmetra: Fixed transposed default levels for battery capacity +Level: 1 +Component: checks +Compatible: incomp +Edition: cre +Version: 1.6.0i1 +Date: 1536231288 +Class: fix + +You'll need to restart the core for this change to take effect. diff --git a/checks/apc_symmetra b/checks/apc_symmetra index 50fdfa7..388287a 100644 --- a/checks/apc_symmetra +++ b/checks/apc_symmetra @@ -114,7 +114,7 @@ def parse_apc_symmetra(info): # crit_capacity, crit_sys_temp, crit_batt_curr, crit_voltage = levels # Temperature default now 60C: regadring to a apc technician a temperature up tp 70C is possible factory_settings["apc_default_levels"] = { - "capacity" : ( 80, 95 ), + "capacity" : ( 95, 80 ), "calibration_state" : 0 }

6 years

Fixed 'omd umount' exit code.

by Sven Panne

Module: check_mk Branch: master Commit: a18058fd38ada2aac17f8d64f7c6e4f0951e1642 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=a18058fd38ada2… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Sep 6 09:28:24 2018 +0200 Fixed 'omd umount' exit code. Change-Id: Iec3de33bea3c7a538c54591e9eb4c692063f359f --- omd/packages/omd/omd | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/omd/packages/omd/omd b/omd/packages/omd/omd index c30b77e..678afc4 100644 --- a/omd/packages/omd/omd +++ b/omd/packages/omd/omd @@ -3312,8 +3312,8 @@ def main_umount(args, options=None): only_version = options.get("version") # if no site is selected, all sites are affected + exit_status = 0 if not g_sitename: - exit_status = 0 for site in all_sites(): # Set global vars for the current site g_sitename = site @@ -3330,17 +3330,14 @@ def main_umount(args, options=None): sys.stdout.write("%sUnmounting tmpfs of site %s%s..." % (tty_bold, site, tty_normal)) sys.stdout.flush() - if not unmount_tmpfs(site, False, kill="kill" in options): - sys.stdout.write(tty_error + "\n") + if not show_success(unmount_tmpfs(site, False, kill="kill" in options)): exit_status = 1 - else: - sys.stdout.write(tty_ok + "\n") else: # Skip the site even when it is partly running if not site_is_stopped(g_sitename): bail_out("Cannot unmount tmpfs of site '%s' while it is running." % g_sitename) unmount_tmpfs(g_sitename, kill="kill" in options) - sys.exit(0) + sys.exit(exit_status) def main_init_action(command, args, options=None):

6 years

6579 FIX Fixed host renaming in notification rules

by Sven Panne

Module: check_mk Branch: master Commit: c4ae341176eb294fb47b27c8589c7ffaa7818589 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c4ae341176eb29… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Sep 6 09:39:07 2018 +0200 6579 FIX Fixed host renaming in notification rules Under certain circumstances, it was possible that hosts in notification rules of a user were not rewritten during host renaming. This has been fixed. Change-Id: I58e9c38f193c18574a60f2b155461d5c43c19d5b --- .werks/6579 | 12 ++++++++++++ cmk/gui/wato/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.werks/6579 b/.werks/6579 new file mode 100644 index 0000000..a05d084 --- /dev/null +++ b/.werks/6579 @@ -0,0 +1,12 @@ +Title: Fixed host renaming in notification rules +Level: 1 +Component: wato +Compatible: compat +Edition: cre +Version: 1.6.0i1 +Date: 1536219419 +Class: fix + +Under certain circumstances, it was possible that hosts in notification +rules of a user were not rewritten during host renaming. This has been +fixed. diff --git a/cmk/gui/wato/__init__.py b/cmk/gui/wato/__init__.py index 20e3541..6a222f1 100644 --- a/cmk/gui/wato/__init__.py +++ b/cmk/gui/wato/__init__.py @@ -2327,7 +2327,7 @@ def rename_host_in_event_rules(oldname, newname): num_changed = rename_in_event_rules(rules) if num_changed: actions += [ "notify_user" ] * num_changed - some_changed = True + some_user_changed = True rules = watolib.load_notification_rules() num_changed = rename_in_event_rules(rules)

6 years

Activate unused-variable warnings for everything but the checks.

by Sven Panne

Module: check_mk Branch: master Commit: 93f38f2b9b9914fb9145789337fe080467f237d2 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=93f38f2b9b9914… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Sep 6 08:16:21 2018 +0200 Activate unused-variable warnings for everything but the checks. Of course the suppression in the checks is a temporary measure, but doing it this way, at least things don't get worse in the rest of the code... :-} Change-Id: I78add1b778734ec6f3a836eed6b81ec36ad4ea0e --- .pylintrc | 4 ---- tests/pylint/test_pylint.py | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.pylintrc b/.pylintrc index 19bd203..74d62b0 100644 --- a/.pylintrc +++ b/.pylintrc @@ -27,10 +27,6 @@ disable= # through our code and fix a few obscure places. abstract-method, #--------------------------------------------------------------------------- - # We should really enable this, there could be some typos hiding which are - # real bugs. Just some work to do... - unused-variable, - #--------------------------------------------------------------------------- # Enabling this would be very desirable, it vastly improves readability and # it might even be necessary for tools like mypy. Fixing this involves some # amount of relatively easy work, especially if we want to avoid code diff --git a/tests/pylint/test_pylint.py b/tests/pylint/test_pylint.py index 425e41a..8380537 100755 --- a/tests/pylint/test_pylint.py +++ b/tests/pylint/test_pylint.py @@ -127,6 +127,9 @@ def inv_tree(path, default_value=None): pylint_cmk.add_file(f, repo_path() + "/cmk_base/check_api.py") pylint_cmk.add_file(f, repo_path() + "/cmk_base/inventory_plugins.py") + # TODO: Remove this when all our checks are fixed. + f.write("# pylint: disable=unused-variable\n") + # Now add the checks for path in pylint_cmk.check_files(repo_path() + "/checks"): pylint_cmk.add_file(f, path)

6 years

Reserved 10 Werk IDS

by Sven Panne

Module: check_mk Branch: master Commit: eb33211e0a6582b128c09d8c96940cd63b45ef6d URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=eb33211e0a6582… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Sep 6 09:36:22 2018 +0200 Reserved 10 Werk IDS Change-Id: I29b4b6d1c14a532f875fd7d01001b19bf1927ac2 --- .werks/first_free | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.werks/first_free b/.werks/first_free index f1f8de3..9af4a61 100644 --- a/.werks/first_free +++ b/.werks/first_free @@ -1 +1 @@ -6579 +6589

6 years

Fixed even more unused-variables warnings.

by Sven Panne

Module: check_mk Branch: master Commit: c0769db06c334a4ac8f34e38d78e16f3f839a624 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=c0769db06c334a… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Wed Sep 5 16:33:52 2018 +0200 Fixed even more unused-variables warnings. Change-Id: I5bcd4727f2884c1dd08c59d7c52dde983956e419 --- agents/plugins/mk_logwatch | 5 +- agents/plugins/mtr | 2 +- agents/special/agent_allnet_ip_sensoric | 2 +- agents/special/agent_jolokia | 2 +- agents/special/agent_netapp | 2 - agents/special/agent_prism | 2 +- agents/special/agent_storeonce | 4 - agents/special/agent_tinkerforge | 3 +- agents/special/agent_ucs_bladecenter | 2 +- agents/special/agent_vsphere | 9 +-- agents/special/agent_vsphere.pysphere | 4 +- bin/mkbench | 14 ++-- cmk/daemon.py | 3 +- cmk/gui/sidebar.py | 3 + cmk/gui/wato/__init__.py | 134 ++++++++++++-------------------- omd/packages/omd/omd | 39 ++++------ 16 files changed, 93 insertions(+), 137 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=c0769db06c…

6 years

Fixed more unused-variables warnings.

by Sven Panne

Module: check_mk Branch: master Commit: 5a5fd478c10c85751f1539f27243b54cd1a0a0e1 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=5a5fd478c10c85… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Wed Sep 5 12:17:07 2018 +0200 Fixed more unused-variables warnings. Change-Id: Ib5530ed2580a46167336434a0d19fdb363f24b2f --- cmk/ec/history.py | 2 -- cmk/gui/plugins/sidebar/shipped.py | 6 ++---- cmk/gui/plugins/sidebar/site_status.py | 2 +- cmk/gui/wato/mkeventd.py | 9 ++------- cmk/gui/wato/pages/analyze_configuration.py | 2 +- cmk/gui/wato/pages/audit_log.py | 5 ++--- cmk/gui/wato/pages/global_settings.py | 6 +++--- cmk/gui/wato/pages/timeperiods.py | 15 +++++++-------- 8 files changed, 18 insertions(+), 29 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=5a5fd478c1…

6 years

6569 FIX mk_logwatch: options are no longer applied globally

by Moritz Kiemer

Module: check_mk Branch: master Commit: 87155056526ad3b8a8273327bff424bee5bb4068 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=87155056526ad3… Author: Moritz Kiemer <mo(a)mathias-kettner.de> Date: Wed Sep 5 09:52:02 2018 +0200 6569 FIX mk_logwatch: options are no longer applied globally Until this fix, a specified option had overwritten any previously defined values for said option, and was applied globally. This has now been fixed: The scope of an option is limited to the files on the same line. In case the same file is referenced in different lines, the options for this specific file are updated with the new ones, keeping the allready defined ones. Change-Id: Ib0bf6045283862797f44e8fc0a2c422cf3abbe6e --- .werks/6569 | 14 +++ agents/plugins/mk_logwatch | 225 ++++++++++++++++++++++++++------------------- 2 files changed, 144 insertions(+), 95 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=8715505652…

6 years

← Newer
1
...
29
30
31
32
33
34
35
...
38
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

Checkmk git commits September 2018