[checkmk-commits] 3856 Improved handling of Check_MK GUI request timeouts

Module: check_mk Branch: master Commit: 0bc1f344ef2911fa22661c33888f372fcd0f2047 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=0bc1f344ef2911… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Wed Sep 14 10:34:21 2016 +0200 3856 Improved handling of Check_MK GUI request timeouts In previous Check_MK versions, there was no real user friendly and correct handling of too long web request processing. So if a user issued a request to the Multisite GUI which took longer than the system apache request timeout, it resulted in a default webserver error page showing a "proxy timeout" error message. Another issue: The timeout was dependent on your system apache configuration and, if you did not change the settings on your own, on the Linux distribution defaults. We have now changed the timeout mechanism as follows: The system apache process will end the communication with the client after the timeout configured for the proxy connection from system apache to site apache. This is done in /omd/sites/[site]/etc/apache/proxy-port.conf file in the "timeout=x" parameter of the ProxyPass statement. The client timeout has been configured to 120 seconds. The application (Check_MK GUI) request timeout should always be lower than the client timeout to make it possible to abort the page processing and send a helpful response page to the client. The default timeout is set to 110 seconds. Developer note: It is possible to disable the applications request timeout (temoporarily) or totally for specific calls, but the timeout to the client will always be applied by the system webserver. So the client will always get a error page while the site apache continues processing the request (until the first try to write anything to the client) which will result in an exception. --- .werks/3856 | 40 ++++++++++++++++++++++++++++++ ChangeLog | 1 + web/htdocs/html_mod_python.py | 7 +++--- web/htdocs/htmllib.py | 57 +++++++++++++++++++++++++++++++++++++++++-- web/htdocs/wato.py | 2 -- 5 files changed, 100 insertions(+), 7 deletions(-) diff --git a/.werks/3856 b/.werks/3856 new file mode 100644 index 0000000..dfb36b9 --- /dev/null +++ b/.werks/3856 @@ -0,0 +1,40 @@ +Title: Improved handling of Check_MK GUI request timeouts +Level: 1 +Component: multisite +Compatible: compat +Version: 1.4.0i1 +Date: 1473841576 +Class: feature + +In previous Check_MK versions, there was no real user friendly and correct +handling of too long web request processing. So if a user issued a request +to the Multisite GUI which took longer than the system apache request timeout, +it resulted in a default webserver error page showing a "proxy timeout" error +message. + +Another issue: The timeout was dependent on your system apache configuration +and, if you did not change the settings on your own, on the Linux distribution +defaults. + +We have now changed the timeout mechanism as follows: + +The system apache process will end the communication with the client after +the timeout configured for the proxy connection from system apache to site +apache. This is done in /omd/sites/[site]/etc/apache/proxy-port.conf file +in the "timeout=x" parameter of the ProxyPass statement. The client timeout +has been configured to 120 seconds. + +The application (Check_MK GUI) request timeout should always be lower than +the client timeout to make it possible to abort the page processing and send +a helpful response page to the client. The default timeout is set to 110 +seconds. + +Developer note: + +It is possible to disable the applications request timeout (temoporarily) +or totally for specific calls, but the timeout to the client will always +be applied by the system webserver. So the client will always get a error +page while the site apache continues processing the request (until the +first try to write anything to the client) which will result in an +exception. + diff --git a/ChangeLog b/ChangeLog index c370b64..d1a7724 100644 --- a/ChangeLog +++ b/ChangeLog @@ -412,6 +412,7 @@ * 3780 availability: now limit of fetched logfile rows is configurable. * 3864 tactical overview now shows the number of stale hosts and services * 3847 Painter options can now be reset to defaults... + * 3856 Improved handling of Check_MK GUI request timeouts... * 3059 FIX: Fixed highlighting of availability timeline time slices * 3175 FIX: Fix timeranges of graphs that are embedded in a dashboard... * 3076 FIX: fixed broken views in check_mk raw edition diff --git a/web/htdocs/html_mod_python.py b/web/htdocs/html_mod_python.py index 6dad47c..c14d40b 100644 --- a/web/htdocs/html_mod_python.py +++ b/web/htdocs/html_mod_python.py @@ -25,7 +25,7 @@ # Boston, MA 02110-1301 USA. from mod_python import Cookie, util, apache -from lib import make_utf8, MKGeneralException +from lib import make_utf8, MKGeneralException, MKException import htmllib import os, time, config, weblib, re import defaults @@ -45,6 +45,8 @@ class html_mod_python(htmllib.html): def __init__(self, req, fields): htmllib.html.__init__(self) + self.enable_request_timeout() + req.content_type = "text/html; charset=UTF-8" req.header_sent = False @@ -69,8 +71,7 @@ class html_mod_python(htmllib.html): self.set_output_format(self.var("output_format", "html")) - # The web servers configured request timeout (Timeout in case of apache) - def request_timeout(self): + def client_request_timeout(self): return int(self.req.server.timeout) diff --git a/web/htdocs/htmllib.py b/web/htdocs/htmllib.py index 5a7e597..1887994 100644 --- a/web/htdocs/htmllib.py +++ b/web/htdocs/htmllib.py @@ -60,13 +60,14 @@ import urllib import random import re import __builtin__ +import signal try: import simplejson as json except ImportError: import json -from cmk.exceptions import MKGeneralException +from cmk.exceptions import MKGeneralException, MKException from lib import MKUserError # Information about uri @@ -76,6 +77,10 @@ class InvalidUserInput(Exception): self.text = text +class RequestTimeout(MKException): + pass + + # This is a simple class which wraps a string provided by the caller # to make html.attrencode() know that this string should not be # encoded, html.attrencode() will then return the unmodified value. @@ -130,6 +135,7 @@ class html(object): self.body_classes = ['main'] self._default_javascripts = [ "checkmk", "graphs" ] self._default_stylesheets = [ "check_mk", "graphs" ] + self._request_timeout = 110 # seconds # Time measurement self.times = {} @@ -159,7 +165,9 @@ class html(object): raise NotImplementedError() - def request_timeout(self): + # The system web servers configured request timeout. This is the time + # before the request is terminated from the view of the client. + def client_request_timeout(self): raise NotImplementedError() @@ -1959,3 +1967,48 @@ class html(object): elapsed = now - self.last_measurement self.times[name] += elapsed self.last_measurement = now + + + # + # Request timeout handling + # + # The system apache process will end the communication with the client after + # the timeout configured for the proxy connection from system apache to site + # apache. This is done in /omd/sites/[site]/etc/apache/proxy-port.conf file + # in the "timeout=x" parameter of the ProxyPass statement. + # + # The regular request timeout configured here should always be lower to make + # it possible to abort the page processing and send a helpful answer to the + # client. + # + # It is possible to disable the applications request timeout (temoporarily) + # or totally for specific calls, but the timeout to the client will always + # be applied by the system webserver. So the client will always get a error + # page while the site apache continues processing the request (until the + # first try to write anything to the client) which will result in an + # exception. + # + + # The timeout of the Check_MK GUI request processing. When the timeout handling + # has been enabled with enable_request_timeout(), after this time an alarm signal + # will be raised to give the application the option to end the processing in a + # gentle way. + def request_timeout(self): + return self._request_timeout + + + def enable_request_timeout(self): + signal.signal(signal.SIGALRM, self.handle_request_timeout) + signal.alarm(self.request_timeout()) + + + def disable_request_timeout(self): + signal.alarm(0) + + + def handle_request_timeout(self, signum, frame): + raise RequestTimeout(_("Your request timed out after %d seconds. This issue may be " + "related to a local configuration problem or a request which works " + "with a too large number of objects. But if you think this " + "issue is a bug, please send a crash report.") % + self.request_timeout()) diff --git a/web/htdocs/wato.py b/web/htdocs/wato.py index eb150a7..d90a640 100644 --- a/web/htdocs/wato.py +++ b/web/htdocs/wato.py @@ -3436,8 +3436,6 @@ def mode_bulk_discovery(phase): arguments = [ "@raiseerrors" ] + arguments timeout = html.request_timeout() - # To have a chance to print an error reduce the timeout a bit - timeout = int(timeout * 0.8) unlock_exclusive() # Avoid freezing WATO when hosts do not respond timely counts, failed_hosts = check_mk_automation(site_id, "inventory",