[checkmk-commits] 6780 FIX Fixed random alert / notify helper crashes on some platforms

Module: check_mk Branch: master Commit: 9bb9a12ade18e92333aafd4afed4b6b51ad75124 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=9bb9a12ade18e9… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Oct 11 20:46:24 2018 +0200 6780 FIX Fixed random alert / notify helper crashes on some platforms The alert and notify helpers of the Microcore could crash "randomly" after hours of operation. The crash was visible in form of "ImportError: cannot import name" messages in the var/log/notify.log or var/log/alert.log. Due to another problem, which will also be fixed soon, the crash of the helper process made the Microcore process crash. As far as we know this happened only on Debian Jessie, CentOS/RedHat 7 and SLES12 SP1/SP2. CMK-1085 Change-Id: I50abefc35f89ba688752e1b83a16c6acdc9122ef --- .werks/6780 | 23 +++++++++++++++++++++++ cmk/daemon.py | 18 ++++++++++++++++++ cmk/gui/background_job.py | 16 ++-------------- cmk_base/events.py | 10 ++++++++++ 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/.werks/6780 b/.werks/6780 new file mode 100644 index 0000000..b11376e --- /dev/null +++ b/.werks/6780 @@ -0,0 +1,23 @@ +Title: Fixed random alert / notify helper crashes on some platforms +Level: 2 +Component: core +Class: fix +Compatible: compat +Edition: cee +State: unknown +Version: 1.6.0i1 +Date: 1539283109 + +The alert and notify helpers of the Microcore could crash "randomly" +after hours of operation. + +The crash was visible in form of "ImportError: cannot import name" messages +in the var/log/notify.log or var/log/alert.log. + +Due to another problem, which will also be fixed soon, the crash of the +helper process made the Microcore process crash. + +As far as we know this happened only on Debian Jessie, CentOS/RedHat 7 and +SLES12 SP1/SP2. + +CMK-1085 diff --git a/cmk/daemon.py b/cmk/daemon.py index 7578b8f..24bbf3f 100644 --- a/cmk/daemon.py +++ b/cmk/daemon.py @@ -81,6 +81,24 @@ def daemonize(user=0, group=0): os.close(so) +def closefrom(lowfd): + """Closes all file descriptors starting with "lowfd", ignoring errors + + Deletes all open file descriptors greater than or equal to lowfd from the + per-process object reference table. Any errors encountered while closing + file descriptors are ignored. + + Difference to os.closerange() is that this automatically determines the + highest fd number to close. + """ + try: + highfd = os.sysconf("SC_OPEN_MAX") + except ValueError: + highfd = 1024 + + os.closerange(lowfd, highfd) + + def lock_with_pid_file(path): """ Use this after daemonizing or in foreground mode to ensure there is only diff --git a/cmk/gui/background_job.py b/cmk/gui/background_job.py index 76f9c64..0766d67 100644 --- a/cmk/gui/background_job.py +++ b/cmk/gui/background_job.py @@ -44,6 +44,7 @@ import cmk import cmk.log import cmk.daemon as daemon import cmk.store as store +import cmk.daemon as daemon from cmk.exceptions import MKGeneralException import cmk.gui.log @@ -183,9 +184,7 @@ class BackgroundProcess(multiprocessing.Process): os.setsid() daemon.set_procname(BackgroundJobDefines.process_name) - - # Close file descpriptors. This also closes logfile handles! - self._close_fds() + daemon.closefrom(0) ##################### ALL HANDLES HAVE BEEN CLOSED BOUNDARY ########################### @@ -205,17 +204,6 @@ class BackgroundProcess(multiprocessing.Process): "state": JobStatus.state_exception}) - - def _close_fds(self): - try: - MAXFD = os.sysconf("SC_OPEN_MAX") - except: - MAXFD = 256 - - # Close all file descriptors - os.closerange(0, MAXFD) - - def initialize_environment(self): if not self._logger: self._logger = cmk.gui.log.logger diff --git a/cmk_base/events.py b/cmk_base/events.py index 5f0494d..6da2963 100755 --- a/cmk_base/events.py +++ b/cmk_base/events.py @@ -38,6 +38,7 @@ import urllib import livestatus import cmk from cmk.regex import regex +import cmk.daemon import cmk_base.config as config import cmk_base.core @@ -72,7 +73,16 @@ def event_keepalive(event_function, log_function, call_every_loop=None, loop_int log_function("Configuration has changed. Restarting myself.") if shutdown_function: shutdown_function() + os.putenv("CMK_EVENT_RESTART", "1") + + # Close all unexpected file descriptors before invoking + # execvp() to prevent inheritance of them. In CMK-1085 we + # had an issue related to os.urandom() which kept FDs open. + # This specific issue of Python 2.7.9 should've been fixed + # since Python 2.7.10. Just to be sure we keep cleaning up. + cmk.daemon.closefrom(3) + os.execvp("cmk", sys.argv) data = ""