Module: check_mk
Branch: master
Commit: 77b3bfc33f30b980db5e61351ae4a86b315ba357
URL:
http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=77b3bfc33f30b9…
Author: Lars Michelsen <lm(a)mathias-kettner.de>
Date: Mon Jun 12 14:55:55 2017 +0200
4755 FIX Fixed possible error when processing piggyback monitoring data
An error "OSError: [Errno 2] No such file or directory:
'/omd/sites/*/tmp/check_mk/piggyback/'"
could occur when using piggyback agent data (like it's done when monitoring ESX hosts
/ vcenters).
The error message was visible in the cmc.log and looked like this:
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 371, in
get_host_info
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> ignore_check_interval=True)
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 506, in
get_realhost_info
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> piggy_output =
get_piggyback_info(hostname) + get_piggyback_info(ipaddress)
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 635, in
get_piggyback_info
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> for sourcehost, file_path in
get_piggyback_files(hostname):
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 612, in
get_piggyback_files
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> for sourcehost in
os.listdir(dir):
2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> OSError: [Errno 2] No such file
or directory: '/omd/sites/mysite/tmp/check_mk/piggyback/backedhost'
This was some kind of race condition when one Check_MK helper cleaned up outdated
piggyback files
while another helper was reading the piggyback data.
Change-Id: I09e1d61f473bf6d110421d7cf6330c6d37cbe74b
---
.werks/4755 | 27 ++++++++++++++++++++++++
modules/check_mk_base.py | 53 ++++++++++++++++++++++++++++++++++++------------
2 files changed, 67 insertions(+), 13 deletions(-)
diff --git a/.werks/4755 b/.werks/4755
new file mode 100644
index 0000000..bbc8180
--- /dev/null
+++ b/.werks/4755
@@ -0,0 +1,27 @@
+Title: Fixed possible error when processing piggyback monitoring data
+Level: 1
+Component: core
+Class: fix
+Compatible: compat
+Edition: cre
+State: unknown
+Version: 1.5.0i1
+Date: 1497271655
+
+An error "OSError: [Errno 2] No such file or directory:
'/omd/sites/*/tmp/check_mk/piggyback/'"
+could occur when using piggyback agent data (like it's done when monitoring ESX hosts
/ vcenters).
+
+The error message was visible in the cmc.log and looked like this:
+
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 371, in
get_host_info
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> ignore_check_interval=True)
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 506, in
get_realhost_info
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> piggy_output =
get_piggyback_info(hostname) + get_piggyback_info(ipaddress)
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 635, in
get_piggyback_info
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> for sourcehost, file_path in
get_piggyback_files(hostname):
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> File
"/omd/sites/mysite/share/check_mk/modules/check_mk_base.py", line 612, in
get_piggyback_files
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> for sourcehost in
os.listdir(dir):
+2017-06-06 06:39:36 [4] Check_MK helper [7231]: : >> OSError: [Errno 2] No such
file or directory: '/omd/sites/mysite/tmp/check_mk/piggyback/backedhost'
+
+This was some kind of race condition when one Check_MK helper cleaned up outdated
piggyback files
+while another helper was reading the piggyback data.
diff --git a/modules/check_mk_base.py b/modules/check_mk_base.py
index 0e1705b..90c0427 100644
--- a/modules/check_mk_base.py
+++ b/modules/check_mk_base.py
@@ -528,19 +528,46 @@ def add_persisted_info(hostname, info):
def get_piggyback_files(hostname):
files = []
dir = cmk.paths.tmp_dir + "/piggyback/" + hostname
- if os.path.exists(dir):
- for sourcehost in os.listdir(dir):
- if sourcehost not in ['.', '..'] \
- and not sourcehost.startswith(".new."):
- file_path = dir + "/" + sourcehost
-
- if cachefile_age(file_path) > piggyback_max_cachefile_age:
- console.verbose("Piggyback file %s is outdated by %d seconds.
Deleting it.\n" %
- (file_path, cachefile_age(file_path) -
piggyback_max_cachefile_age))
- os.remove(file_path)
- continue
-
- files.append((sourcehost, file_path))
+
+ # remove_piggyback_info_from() may remove stale piggyback files of one source
+ # host and also the directory "hostname" when the last piggyback file for
the
+ # current host was removed. This may cause the os.listdir() to fail. We treat
+ # this as regular case: No piggyback files for the current host.
+ try:
+ source_hosts = os.listdir(dir)
+ except OSError, e:
+ if e.errno == 2: # No such file or directory
+ return files
+ else:
+ raise
+
+ for sourcehost in source_hosts:
+ if sourcehost in ['.', '..'] or
sourcehost.startswith(".new."):
+ continue
+
+ file_path = dir + "/" + sourcehost
+
+ try:
+ file_age = cachefile_age(file_path)
+ except MKGeneralException, e:
+ continue # File might've been deleted. That's ok.
+
+ # Cleanup outdated files
+ if file_age > piggyback_max_cachefile_age:
+ console.verbose("Piggyback file %s is outdated by %d seconds. Deleting
it.\n" %
+ (file_path, file_age - piggyback_max_cachefile_age))
+
+ try:
+ os.remove(file_path)
+ except OSError, e:
+ if e.errno == 2: # No such file or directory
+ pass # Deleted in the meantime. That's ok.
+ else:
+ raise
+
+ continue
+
+ files.append((sourcehost, file_path))
return files