Checkmk git commits January 2019

checkmk-commits@lists.checkmk.com

15 participants
490 discussions

Remove reference to special_agent_api.tar.gz in setup.sh

by Tom Baerwinkel

Module: check_mk Branch: master Commit: f01e0ae398f06556c3ac48e5f96638b4dd4b90fe URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=f01e0ae398f065… Author: Tom Baerwinkel <tb(a)mathias-kettner.de> Date: Thu Jan 10 15:36:13 2019 +0100 Remove reference to special_agent_api.tar.gz in setup.sh Change-Id: I995310d4d8a04ab44099774697fd37d6c32456f7 --- scripts/setup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/setup.sh b/scripts/setup.sh index 65ad4aa..5a73685 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -742,7 +742,6 @@ do mkdir -p $DESTDIR$python_lib_dir && tar xzf $SRCDIR/lib.tar.gz -C $DESTDIR$python_lib_dir && tar xzf $SRCDIR/base.tar.gz -C $DESTDIR$python_lib_dir && - tar xzf $SRCDIR/special_agent_api.tar.gz -C $DESTDIR$python_lib_dir && mkdir -p $DESTDIR$sharedir/werks && tar xzf $SRCDIR/werks.tar.gz -C $DESTDIR$sharedir/werks && mkdir -p $DESTDIR$localedir &&

5 years, 8 months

7018 Livestatus can now be configured to connect via IPv6

by Lars Michelsen

Module: check_mk Branch: master Commit: 712b6a52d80bec4ee2f73913b1fe014ab56f47fa URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=712b6a52d80bec… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Fri Jan 4 20:40:46 2019 +0100 7018 Livestatus can now be configured to connect via IPv6 In previous versions it was not possible to connect the GUI to a remote site via Livestatus using IPv6. This is now possible and can be configured from the "Distributed Monitoring" configuration. Technically this was prevented by several smaller things. The internal Livestatus xinetd configuration now allows ::/0 besides 0.0.0.0 by default. In case you have modified this setting and want to use IPv6, you may have to add the IPv6 addresses of your choice to this option. The site configuration GUI is now able to handle IPv6 addresses properly. The internally used livestatus.py Livestatus client implementation supports IPv6 now. Livestatus proxy can now connect to Livestatus via IPv6. Added tests for livestatus proxy connection initiation using the different kind of sockets. The cascading livestatus proxy feature, which is used to make the local unix socket of a site available via the network, can now be used with IPv6 in addition to the already existing IPv4 support. Change-Id: I6c9cada5e21cd7d43a9133256a172d36d7db4497 --- .werks/7018 | 26 +++++++++++++++ cmk/gui/watolib.py | 15 ++++++--- livestatus/api/python/livestatus.py | 12 ++++--- .../mk-livestatus/LIVESTATUS_TCP_ONLY_FROM | 6 ++-- tests/unit/livestatus/test_livestatus_unit.py | 38 ++++++++++++++++++++++ 5 files changed, 84 insertions(+), 13 deletions(-) diff --git a/.werks/7018 b/.werks/7018 new file mode 100644 index 0000000..47b5b15 --- /dev/null +++ b/.werks/7018 @@ -0,0 +1,26 @@ +Title: Livestatus can now be configured to connect via IPv6 +Level: 2 +Component: multisite +Compatible: compat +Edition: cre +Version: 1.6.0i1 +Date: 1546113685 +Class: feature + +In previous versions it was not possible to connect the GUI to a remote site +via Livestatus using IPv6. This is now possible and can be configured from the +"Distributed Monitoring" configuration. + +Technically this was prevented by several smaller things. + +The internal Livestatus xinetd configuration now allows ::/0 besides 0.0.0.0 by +default. In case you have modified this setting and want to use IPv6, you may +have to add the IPv6 addresses of your choice to this option. + +The site configuration GUI is now able to handle IPv6 addresses properly. + +The internally used livestatus.py Livestatus client implementation supports +IPv6 now. Livestatus proxy can now connect to Livestatus via IPv6 and also the +cascading proxy feature, which is used to make the local unix socket of a site +available via the network, can now be used with IPv6 in addition to the already +existing IPv4 support. diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py index 745738b..4d02163 100644 --- a/cmk/gui/watolib.py +++ b/cmk/gui/watolib.py @@ -102,8 +102,9 @@ from cmk.gui.exceptions import MKGeneralException, MKAuthException, MKUserError, from cmk.gui.valuespec import ( Dictionary, Integer, + HostAddress, ListOfStrings, - IPv4Network, + IPNetwork, Checkbox, Transform, DropdownChoice, @@ -4028,7 +4029,8 @@ class SiteManagement(object): None, totext="", )), - ("tcp", _("Connect via TCP (IPv4)"), cls._tcp_socket_valuespec()), + ("tcp", _("Connect via TCP (IPv4)"), cls._tcp_socket_valuespec(ipv6=False)), + ("tcp6", _("Connect via TCP (IPv6)"), cls._tcp_socket_valuespec(ipv6=True)), ("unix", _("Connect via UNIX socket"), Dictionary( elements=[ @@ -4044,7 +4046,7 @@ class SiteManagement(object): return conn_choices @classmethod - def _tcp_socket_valuespec(cls): + def _tcp_socket_valuespec(cls, ipv6): return Dictionary( elements=[ ("address", @@ -4052,10 +4054,12 @@ class SiteManagement(object): title=_("TCP address to connect to"), orientation="float", elements=[ - TextAscii( + HostAddress( label=_("Host:"), allow_empty=False, size=15, + allow_ipv4_address=not ipv6, + allow_ipv6_address=ipv6, ), Integer( label=_("Port:"), @@ -10789,9 +10793,10 @@ class LivestatusViaTCP(Dictionary): help=_("The access to Livestatus via TCP will only be allowed from the " "configured source IP addresses. You can either configure specific " "IP addresses or networks in the syntax <tt>10.3.3.0/24</tt>."), - valuespec=IPv4Network(), + valuespec=IPNetwork(), orientation="horizontal", allow_empty=False, + default_value=["0.0.0.0", "::/0"], )), ] kwargs["optional_keys"] = ["only_from"] diff --git a/livestatus/api/python/livestatus.py b/livestatus/api/python/livestatus.py index f458bbc..1b0a70a 100644 --- a/livestatus/api/python/livestatus.py +++ b/livestatus/api/python/livestatus.py @@ -317,17 +317,19 @@ class SingleSiteConnection(Helpers): if family_txt == "unix": return socket.AF_UNIX, url - elif family_txt == "tcp": + elif family_txt in ["tcp", "tcp6"]: try: host, port_txt = url.rsplit(":", 1) port = int(port_txt) except ValueError: - raise MKLivestatusConfigError("Invalid livestatus tcp URL '%s'. " - "Correct example is 'tcp:somehost:6557'" % url) - return socket.AF_INET, (host, port) + raise MKLivestatusConfigError( + "Invalid livestatus tcp URL '%s'. " + "Correct example is 'tcp:somehost:6557' or 'tcp6:somehost:6557'" % url) + address_family = socket.AF_INET if family_txt == "tcp" else socket.AF_INET6 + return address_family, (host, port) raise MKLivestatusConfigError("Invalid livestatus URL '%s'. " - "Must begin with 'tcp:' or 'unix:'" % url) + "Must begin with 'tcp:', 'tcp6:' or 'unix:'" % url) def disconnect(self): self.socket = None diff --git a/omd/packages/mk-livestatus/LIVESTATUS_TCP_ONLY_FROM b/omd/packages/mk-livestatus/LIVESTATUS_TCP_ONLY_FROM index 7efa9ba..ed77503 100755 --- a/omd/packages/mk-livestatus/LIVESTATUS_TCP_ONLY_FROM +++ b/omd/packages/mk-livestatus/LIVESTATUS_TCP_ONLY_FROM @@ -5,12 +5,12 @@ # Description: # If Livestatus is configured to listen on a TCP port, you # can configure the IP addresses that are allowed to -# connect to livestatus here. The setting 0.0.0.0 makes the -# port available to all clients. +# connect to livestatus here. The setting "0.0.0.0 ::/0" makes the +# port available to all IPv4 and IPv6 clients, case "$1" in default) - echo "0.0.0.0" + echo "0.0.0.0 ::/0" ;; choices) echo "(?:(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})(/[0-9]{1,2})?\s?)+" diff --git a/tests/unit/livestatus/test_livestatus_unit.py b/tests/unit/livestatus/test_livestatus_unit.py index 49fafec..08347c4 100644 --- a/tests/unit/livestatus/test_livestatus_unit.py +++ b/tests/unit/livestatus/test_livestatus_unit.py @@ -1,4 +1,5 @@ import socket +from contextlib import closing from pathlib2 import Path import pytest # type: ignore @@ -54,11 +55,48 @@ def test_livestatus_local_connection(sock_path): assert isinstance(live, livestatus.SingleSiteConnection) +def test_livestatus_ipv4_connection(): + with closing(socket.socket(socket.AF_INET)) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # pylint: disable=no-member + + # Pick a random port + sock.bind(("127.0.0.1", 0)) # pylint: disable=no-member + port = sock.getsockname()[1] # pylint: disable=no-member + + sock.listen(1) # pylint: disable=no-member + + live = livestatus.SingleSiteConnection("tcp:127.0.0.1:%d" % port) + live.connect() + + +def test_livestatus_ipv6_connection(): + with closing(socket.socket(socket.AF_INET6)) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # pylint: disable=no-member + + # Pick a random port + try: + sock.bind(("::1", 0)) # pylint: disable=no-member + except socket.error as e: + # Skip this test in case ::1 can not be bound to + # (happened in docker container with IPv6 disabled) + if e.errno == 99: # Cannot assign requested address + pytest.skip("Unable to bind to ::1 (%s)" % e) + + port = sock.getsockname()[1] # pylint: disable=no-member + + sock.listen(1) # pylint: disable=no-member + + live = livestatus.SingleSiteConnection("tcp6:::1:%d" % port) + live.connect() + + @pytest.mark.parametrize("socket_url,result", [ ("unix:/omd/sites/heute/tmp/run/live", (socket.AF_UNIX, "/omd/sites/heute/tmp/run/live")), ("unix:/omd/sites/heute/tmp/run/li:ve", (socket.AF_UNIX, "/omd/sites/heute/tmp/run/li:ve")), ("tcp:127.0.0.1:1234", (socket.AF_INET, ("127.0.0.1", 1234))), ("tcp:126.0.0.1:abc", None), + ("tcp6:::1:1234", (socket.AF_INET6, ("::1", 1234))), + ("tcp6:::1:abc", None), ("xyz:bla", None), ]) def test_single_site_connection_socketurl(socket_url, result, monkeypatch):

5 years, 8 months

Speed up test-* targets a bit.

by Sven Panne

Module: check_mk Branch: master Commit: 860f13040cb637967a565e355da33f9228bc802d URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=860f13040cb637… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Jan 10 10:55:08 2019 +0100 Speed up test-* targets a bit. Change-Id: I7d104cd96fb22e6ec704261c9eddbc2eaa08927a --- tests/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index d1e2f66..ce75da0 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -60,16 +60,16 @@ test-format-python: $(VENV) ../.style.yapf sed -e 's/^+++ //' -e 's/[[:blank:]]*(reformatted)$$/:1:1: error: file needs formatting/' test-gui-crawl: $(VENV) - $(PYTEST) -T gui_crawl + $(PYTEST) -T gui_crawl $(realpath integration/cmk/gui/test_crawl.py) test-integration: $(VENV) - $(PYTEST) -T integration + $(PYTEST) -T integration $(realpath integration) test-packaging: $(VENV) - $(PYTEST) -T packaging + $(PYTEST) -T packaging packaging test-pylint: $(VENV) - $(PYTEST) -T pylint + $(PYTEST) -T pylint pylint test-shellcheck: @CMK_DIR="$(realpath ..)" ; \

5 years, 8 months

Deobfuscate is_read_only_mode_enabled().

by Sven Panne

Module: check_mk Branch: master Commit: 88a4fb3a1870fbad8a85a456551ffb101d277ebe URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=88a4fb3a1870fb… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Jan 10 11:34:38 2019 +0100 Deobfuscate is_read_only_mode_enabled(). Change-Id: I2ef2f5a1944bc0a0f667c5d7abeed87fceffa920 --- cmk/gui/watolib.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py index 455f03c..745738b 100644 --- a/cmk/gui/watolib.py +++ b/cmk/gui/watolib.py @@ -8893,19 +8893,12 @@ def read_only_message(): def is_read_only_mode_enabled(): if not config.wato_read_only: return False - - enabled = False if config.wato_read_only["enabled"] is True: - enabled = True - elif isinstance(config.wato_read_only['enabled'], tuple): + return True + if isinstance(config.wato_read_only['enabled'], tuple): start_time, end_time = config.wato_read_only['enabled'] - now = time.time() - enabled = now >= start_time and now <= end_time - - if not enabled: - return False - - return True + return start_time <= time.time() <= end_time + return False def may_override_read_only_mode():

5 years, 8 months

Simplify store's lock handling a bit.

by Sven Panne

Module: check_mk Branch: master Commit: cd2e9f297a639d95fb2e3880b5d78ae90e1746b3 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=cd2e9f297a639d… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Jan 10 09:43:43 2019 +0100 Simplify store's lock handling a bit. Change-Id: I987c628a70b50fe0f5e7a2030c25405679618de3 --- cmk/utils/store.py | 51 ++++++++++++++------------------------ tests/unit/cmk/utils/test_store.py | 4 +-- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/cmk/utils/store.py b/cmk/utils/store.py index 437af43..62d96ba 100644 --- a/cmk/utils/store.py +++ b/cmk/utils/store.py @@ -34,7 +34,7 @@ import os import pprint import tempfile import time -from typing import List # pylint: disable=unused-import +from typing import Dict, List # pylint: disable=unused-import import pathlib2 as pathlib @@ -266,12 +266,11 @@ def save_to_mk_file(path, key, value, pprint_value=False): # | wait forever. | # '----------------------------------------------------------------------' -g_aquired_locks = [] -g_locked_paths = [] # type: List[str] +_acquired_locks = {} # type: Dict[str, int] def aquire_lock(path, blocking=True): - if path in g_locked_paths: + if have_lock(path): return True # No recursive locking logger.debug("Try aquire lock on %s", path) @@ -297,11 +296,9 @@ def aquire_lock(path, blocking=True): os.close(fd) fd = fd_new + _acquired_locks[path] = fd logger.debug("Got lock on %s", path) - g_aquired_locks.append((path, fd)) - g_locked_paths.append(path) - def try_aquire_lock(path): try: @@ -314,43 +311,31 @@ def try_aquire_lock(path): def release_lock(path): - if path not in g_locked_paths: + if not have_lock(path): return # no unlocking needed - logger.debug("Releasing lock on %s", path) - - for lock_path, fd in g_aquired_locks: - if lock_path != path: - continue - - try: - os.close(fd) - except OSError as e: - if e.errno != errno.EBADF: # Bad file number - raise - - g_aquired_locks.remove((lock_path, fd)) - break - - g_locked_paths.remove(path) + fd = _acquired_locks.get(path) + if fd is None: + return + try: + os.close(fd) + except OSError as e: + if e.errno != errno.EBADF: # Bad file number + raise + _acquired_locks.pop(path, None) logger.debug("Released lock on %s", path) def have_lock(path): - return path in g_locked_paths + return path in _acquired_locks def release_all_locks(): - global g_aquired_locks, g_locked_paths logger.debug("Releasing all locks") - logger.debug("g_aquired_locks: %r", g_aquired_locks) - logger.debug("g_locked_paths: %r", g_locked_paths) - - for path, _unused_fd in g_aquired_locks[:]: + logger.debug("_acquired_locks: %r", _acquired_locks) + for path in list(_acquired_locks.iterkeys()): release_lock(path) - - g_aquired_locks = [] - g_locked_paths = [] + _acquired_locks.clear() # Experimental but not used yet. diff --git a/tests/unit/cmk/utils/test_store.py b/tests/unit/cmk/utils/test_store.py index 90393ea..361aed4 100644 --- a/tests/unit/cmk/utils/test_store.py +++ b/tests/unit/cmk/utils/test_store.py @@ -160,7 +160,7 @@ def test_release_lock_already_closed(tmpdir): store.aquire_lock(path) assert store.have_lock(path) == True - os.close([ fd for p, fd in store.g_aquired_locks if p == path ][0]) + os.close(store._acquired_locks[path]) store.release_lock(path) assert store.have_lock(path) == False @@ -198,7 +198,7 @@ def test_release_all_locks_already_closed(tmpdir): store.aquire_lock(path) assert store.have_lock(path) == True - os.close([ fd for p, fd in store.g_aquired_locks if p == path ][0]) + os.close(store._acquired_locks[path]) store.release_all_locks() assert store.have_lock(path) == False

5 years, 8 months

Make pylint a bit more happy.

by Sven Panne

Module: check_mk Branch: master Commit: 739deb0d230c595dd97e8a6030214b8e81979733 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=739deb0d230c59… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Jan 10 09:56:48 2019 +0100 Make pylint a bit more happy. Change-Id: I2b27e2aea415567257b7f016e2372069c08693df --- tests/unit/cmk/utils/test_store.py | 78 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/tests/unit/cmk/utils/test_store.py b/tests/unit/cmk/utils/test_store.py index 361aed4..7c2effd 100644 --- a/tests/unit/cmk/utils/test_store.py +++ b/tests/unit/cmk/utils/test_store.py @@ -1,14 +1,15 @@ # encoding: utf-8 -import cmk.utils.store as store -from cmk.utils.exceptions import MKGeneralException -import imp import sys import threading import time import os + import pytest +import cmk.utils.store as store +from cmk.utils.exceptions import MKGeneralException + def test_load_data_from_file_not_existing(tmpdir): data = store.load_data_from_file("%s/x" % tmpdir) assert data is None @@ -28,8 +29,8 @@ def test_load_data_not_locked(tmpdir): locked_file = tmpdir.join("locked_file") locked_file.write("[1, 2]") - data = store.load_data_from_file("%s" % locked_file) - assert store.have_lock("%s" % locked_file) == False + store.load_data_from_file("%s" % locked_file) + assert store.have_lock("%s" % locked_file) is False def test_load_data_from_file_locking(tmpdir): @@ -38,7 +39,7 @@ def test_load_data_from_file_locking(tmpdir): data = store.load_data_from_file("%s" % locked_file, lock=True) assert data == [1, 2] - assert store.have_lock("%s" % locked_file) == True + assert store.have_lock("%s" % locked_file) is True def test_load_data_from_not_permitted_file(tmpdir): @@ -57,9 +58,9 @@ def test_load_data_from_file_dict(tmpdir): locked_file.write(repr({"1": 2, "ä": u"ß"})) data = store.load_data_from_file("%s" % locked_file) - assert type(data) == dict + assert isinstance(data, dict) assert data["1"] == 2 - assert type(data["ä"]) == unicode + assert isinstance(data["ä"], unicode) assert data["ä"] == u"ß" @@ -115,9 +116,9 @@ def test_acquire_lock(tmpdir): path = "%s" % locked_file - assert store.have_lock(path) == False + assert store.have_lock(path) is False store.aquire_lock(path) - assert store.have_lock(path) == True + assert store.have_lock(path) is True def test_acquire_lock_twice(tmpdir): @@ -126,11 +127,11 @@ def test_acquire_lock_twice(tmpdir): path = "%s" % locked_file - assert store.have_lock(path) == False + assert store.have_lock(path) is False store.aquire_lock(path) - assert store.have_lock(path) == True + assert store.have_lock(path) is True store.aquire_lock(path) - assert store.have_lock(path) == True + assert store.have_lock(path) is True def test_release_lock_not_locked(): @@ -143,11 +144,11 @@ def test_release_lock(tmpdir): path = "%s" % locked_file - assert store.have_lock(path) == False + assert store.have_lock(path) is False store.aquire_lock(path) - assert store.have_lock(path) == True + assert store.have_lock(path) is True store.release_lock(path) - assert store.have_lock(path) == False + assert store.have_lock(path) is False def test_release_lock_already_closed(tmpdir): @@ -156,14 +157,14 @@ def test_release_lock_already_closed(tmpdir): path = "%s" % locked_file - assert store.have_lock(path) == False + assert store.have_lock(path) is False store.aquire_lock(path) - assert store.have_lock(path) == True + assert store.have_lock(path) is True os.close(store._acquired_locks[path]) store.release_lock(path) - assert store.have_lock(path) == False + assert store.have_lock(path) is False def test_release_all_locks(tmpdir): @@ -175,17 +176,17 @@ def test_release_all_locks(tmpdir): path1 = "%s" % locked_file1 path2 = "%s" % locked_file2 - assert store.have_lock(path1) == False + assert store.have_lock(path1) is False store.aquire_lock(path1) - assert store.have_lock(path1) == True + assert store.have_lock(path1) is True - assert store.have_lock(path2) == False + assert store.have_lock(path2) is False store.aquire_lock(path2) - assert store.have_lock(path2) == True + assert store.have_lock(path2) is True store.release_all_locks() - assert store.have_lock(path1) == False - assert store.have_lock(path2) == False + assert store.have_lock(path1) is False + assert store.have_lock(path2) is False def test_release_all_locks_already_closed(tmpdir): @@ -194,14 +195,14 @@ def test_release_all_locks_already_closed(tmpdir): path = "%s" % locked_file - assert store.have_lock(path) == False + assert store.have_lock(path) is False store.aquire_lock(path) - assert store.have_lock(path) == True + assert store.have_lock(path) is True os.close(store._acquired_locks[path]) store.release_all_locks() - assert store.have_lock(path) == False + assert store.have_lock(path) is False class LockTestThread(threading.Thread): @@ -217,15 +218,15 @@ class LockTestThread(threading.Thread): def run(self): while True: if self.do == "lock": - assert self.store.have_lock(self.path) == False + assert self.store.have_lock(self.path) is False self.store.aquire_lock(self.path) - assert self.store.have_lock(self.path) == True + assert self.store.have_lock(self.path) is True self.do = None elif self.do == "unlock": - assert self.store.have_lock(self.path) == True + assert self.store.have_lock(self.path) is True self.store.release_lock(self.path) - assert self.store.have_lock(self.path) == False + assert self.store.have_lock(self.path) is False self.do = None else: @@ -234,9 +235,10 @@ class LockTestThread(threading.Thread): def test_locking(tmpdir): + # HACK: We abuse modules as data containers, so we have to do this Kung Fu... store1 = sys.modules["cmk.utils.store"] del sys.modules["cmk.utils.store"] - import cmk.utils.store as store + import cmk.utils.store as _store # pylint: disable=reimported store2 = sys.modules["cmk.utils.store"] assert store1 != store2 @@ -256,13 +258,13 @@ def test_locking(tmpdir): if store1.have_lock(path): break time.sleep(0.01) - assert store1.have_lock(path) == True + assert store1.have_lock(path) is True # Now try to get lock with store2 t2.do = "lock" time.sleep(0.2) - assert store1.have_lock(path) == True - assert store2.have_lock(path) == False + assert store1.have_lock(path) is True + assert store2.have_lock(path) is False # And now unlock store1 and check whether store2 has the lock now t1.do = "unlock" @@ -270,6 +272,6 @@ def test_locking(tmpdir): if not store1.have_lock(path): break time.sleep(0.01) - assert store1.have_lock(path) == False + assert store1.have_lock(path) is False time.sleep(0.2) - assert store2.have_lock(path) == True + assert store2.have_lock(path) is True

5 years, 8 months

graphs: rename hacky winperf util metric

by Moritz Kiemer

Module: check_mk Branch: master Commit: 75cc9872cd47352a880be2fa5fd33a8101454e70 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=75cc9872cd4735… Author: Moritz Kiemer <mo(a)mathias-kettner.de> Date: Thu Jan 10 11:52:12 2019 +0100 graphs: rename hacky winperf util metric Change-Id: Ie2b8f000137a39efb39ddcddbcaa5ae085443e0f --- cmk/gui/plugins/metrics/check_mk.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmk/gui/plugins/metrics/check_mk.py b/cmk/gui/plugins/metrics/check_mk.py index 1dcc593..d6910e6 100644 --- a/cmk/gui/plugins/metrics/check_mk.py +++ b/cmk/gui/plugins/metrics/check_mk.py @@ -5939,7 +5939,7 @@ check_metrics["check_mk-vms_sys.util"] = { check_metrics["check_mk-winperf.cpuusage"] = { "cpuusage": { - "name": "util" + "name": "util_numcpu_as_max" }, } @@ -8355,16 +8355,16 @@ graph_info["cpu_average_2"] = { "range": (0, 100), } -graph_info["cpu_utilization_1"] = { - "title": _("CPU utilization (%(util:max@count) CPU Threads)"), +graph_info["cpu_utilization_numcpus"] = { + "title": _("CPU utilization (%(util_numcpu_as_max:max@count) CPU Threads)"), "metrics": [ - ("util,user,-#ff6000", "stack", _("Privileged")), + ("util_numcpu_as_max,user,-#ff6000", "stack", _("Privileged")), ("user", "area"), - ("util#008000", "line", _("Total")), + ("util_numcpu_as_max#008000", "line", _("Total")), ], "scalars": [ - "util:warn", - "util:crit", + "util_numcpu_as_max:warn", + "util_numcpu_as_max:crit", ], "range": (0, 100), }

5 years, 8 months

Use errno module. Tiny simplifications.

by Sven Panne

Module: check_mk Branch: master Commit: 2da805bf65b9545eae4ddb188e237033cf81d4e4 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=2da805bf65b954… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Jan 10 09:37:34 2019 +0100 Use errno module. Tiny simplifications. Change-Id: I70b612ea8ba264a57964f1ad57eb8eeae3d9c9a9 --- cmk/utils/store.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/cmk/utils/store.py b/cmk/utils/store.py index 7d2ced0..437af43 100644 --- a/cmk/utils/store.py +++ b/cmk/utils/store.py @@ -28,6 +28,7 @@ functionality is the locked file opening realized with the File() context manager.""" import ast +import errno import fcntl import os import pprint @@ -102,12 +103,10 @@ def load_mk_file(path, default=None, lock=False): try: try: execfile(path, globals(), default) - return default except IOError as e: - if e.errno == 2: # IOError: [Errno 2] No such file or directory - return default - else: + if e.errno != errno.ENOENT: # No such file or directory raise + return default except MKTimeout: raise @@ -145,10 +144,9 @@ def load_data_from_file(path, default=None, lock=False): return ast.literal_eval(content) except IOError as e: - if e.errno == 2: # IOError: [Errno 2] No such file or directory - return default - - raise + if e.errno != errno.ENOENT: # No such file or directory + raise + return default except MKTimeout: raise @@ -232,9 +230,7 @@ def save_file(path, content, mode=0660): if tmp_path: os.unlink(tmp_path) except IOError as e: - if e.errno == 2: # No such file or directory - pass - else: + if e.errno != errno.ENOENT: # No such file or directory raise # TODO: How to handle debug mode or logging? @@ -312,10 +308,9 @@ def try_aquire_lock(path): aquire_lock(path, blocking=False) return True except IOError as e: - if e.errno == 11: # Resource temporarily unavailable - return False - else: + if e.errno != errno.EAGAIN: # Try again raise + return False def release_lock(path): @@ -331,9 +326,7 @@ def release_lock(path): try: os.close(fd) except OSError as e: - if e.errno == 9: # OSError: [Errno 9] Bad file descriptor - pass - else: + if e.errno != errno.EBADF: # Bad file number raise g_aquired_locks.remove((lock_path, fd))

5 years, 8 months

Make WATO's exclusive lock a context manager.

by Sven Panne

Module: check_mk Branch: master Commit: 2cc032a4cf04f97ab65b3e2df692a8103f249b9c URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=2cc032a4cf04f9… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Jan 10 08:48:25 2019 +0100 Make WATO's exclusive lock a context manager. There are a few very obscure unbalanced lock/unlock call sites left, we should *really* fix these: Locking must be done in a balanced way locally, otherwise hard-to-find bugs are basically guaranteed.... Change-Id: Ia6a90eee5ba329e0b49512d2fd5ab2d1c710357b --- cmk/gui/wato/__init__.py | 22 ++++----- cmk/gui/wato/pages/bulk_discovery.py | 12 ++--- cmk/gui/wato/pages/parentscan.py | 9 ++-- cmk/gui/watolib.py | 86 ++++++++++++++++++++---------------- 4 files changed, 62 insertions(+), 67 deletions(-) diff --git a/cmk/gui/wato/__init__.py b/cmk/gui/wato/__init__.py index 7bd898a..77ef646 100644 --- a/cmk/gui/wato/__init__.py +++ b/cmk/gui/wato/__init__.py @@ -868,23 +868,19 @@ def add_scanned_hosts_to_folder(folder, found): if not watolib.Host.host_exists(host_name): entries.append((host_name, attrs, None)) - watolib.lock_exclusive() - folder.create_hosts(entries) - folder.save() - watolib.unlock_exclusive() + with watolib.exclusive_lock(): + folder.create_hosts(entries) + folder.save() def save_network_scan_result(folder, result): # Reload the folder, lock WATO before to protect against concurrency problems. - watolib.lock_exclusive() - - # A user might have changed the folder somehow since starting the scan. Load the - # folder again to get the current state. - write_folder = watolib.Folder.folder(folder.path()) - write_folder.set_attribute("network_scan_result", result) - write_folder.save() - - watolib.unlock_exclusive() + with watolib.exclusive_lock(): + # A user might have changed the folder somehow since starting the scan. Load the + # folder again to get the current state. + write_folder = watolib.Folder.folder(folder.path()) + write_folder.set_attribute("network_scan_result", result) + write_folder.save() #. diff --git a/cmk/gui/wato/pages/bulk_discovery.py b/cmk/gui/wato/pages/bulk_discovery.py index edf971f..8064f01 100644 --- a/cmk/gui/wato/pages/bulk_discovery.py +++ b/cmk/gui/wato/pages/bulk_discovery.py @@ -132,23 +132,17 @@ class BulkDiscoveryBackgroundJob(WatoBackgroundJob): return counts, failed_hosts def _process_discovery_results(self, task, job_interface, counts, failed_hosts): - try: - # The following code updates the host config. The progress from loading the WATO folder - # until it has been saved needs to be locked. - watolib.lock_exclusive() - + # The following code updates the host config. The progress from loading the WATO folder + # until it has been saved needs to be locked. + with watolib.exclusive_lock(): watolib.Folder.invalidate_caches() folder = watolib.Folder.folder(task.folder_path) - for hostname in task.host_names: self._process_service_counts_for_host(counts[hostname]) msg = self._process_discovery_result_for_host( folder.host(hostname), failed_hosts.get(hostname, False), counts[hostname]) job_interface.send_progress_update("%s: %s" % (hostname, msg)) - finally: - watolib.unlock_exclusive() - def _process_service_counts_for_host(self, host_counts): self._num_services_added += host_counts[0] self._num_services_removed += host_counts[1] diff --git a/cmk/gui/wato/pages/parentscan.py b/cmk/gui/wato/pages/parentscan.py index 93940bf..e7d5466 100644 --- a/cmk/gui/wato/pages/parentscan.py +++ b/cmk/gui/wato/pages/parentscan.py @@ -127,13 +127,10 @@ class ParentScanBackgroundJob(WatoBackgroundJob): state, skipped_gateways, error = gateways[0][1:] if state in ["direct", "root", "gateway"]: - try: - # The following code updates the host config. The progress from loading the WATO folder - # until it has been saved needs to be locked. - watolib.lock_exclusive() + # The following code updates the host config. The progress from loading the WATO folder + # until it has been saved needs to be locked. + with watolib.exclusive_lock(): self._configure_host_and_gateway(task, settings, state, gateway) - finally: - watolib.unlock_exclusive() else: self._logger.error(error) diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py index fa67d43..455f03c 100644 --- a/cmk/gui/watolib.py +++ b/cmk/gui/watolib.py @@ -46,6 +46,7 @@ import abc import ast import base64 import cStringIO +from contextlib import contextmanager import copy import glob from hashlib import sha256 @@ -236,16 +237,16 @@ def init_wato_datastructures(with_wato_lock=False): not _need_to_create_sample_config(): return - if with_wato_lock: - lock_exclusive() - - if not os.path.exists(ConfigDomainCACertificates.trusted_cas_file): - ConfigDomainCACertificates().activate() - - _create_sample_config() + def init(): + if not os.path.exists(ConfigDomainCACertificates.trusted_cas_file): + ConfigDomainCACertificates().activate() + _create_sample_config() if with_wato_lock: - unlock_exclusive() + with exclusive_lock(): + init() + else: + init() # TODO: Create a hook here and move CEE and other specific things away @@ -5459,35 +5460,33 @@ class ActivateChangesManager(ActivateChanges): # Lock WATO modifications during snapshot creation def _create_snapshots(self): - lock_exclusive() - - if not self._changes: - raise MKUserError(None, _("Currently there are no changes to activate.")) + with exclusive_lock(): + if not self._changes: + raise MKUserError(None, _("Currently there are no changes to activate.")) - if self._get_last_change_id() != self._activate_until: - raise MKUserError( - None, - _("Another change has been made in the meantime. Please review it " - "to ensure you also want to activate it now and start the " - "activation again.")) - - # Create (legacy) WATO config snapshot - start = time.time() - logger.debug("Snapshot creation started") - # TODO: Remove/Refactor once new changes mechanism has been implemented - # This single function is responsible for the slow activate changes (python tar packaging..) - create_snapshot(self._comment) - - work_dir = os.path.join(self.activation_tmp_base_dir, self._activation_id) - if cmk.is_managed_edition(): - import cmk.gui.cme.managed_snapshots as managed_snapshots - managed_snapshots.CMESnapshotManager( - work_dir, self._get_site_configurations()).generate_snapshots() - else: - self._generate_snapshots(work_dir) + if self._get_last_change_id() != self._activate_until: + raise MKUserError( + None, + _("Another change has been made in the meantime. Please review it " + "to ensure you also want to activate it now and start the " + "activation again.")) + + # Create (legacy) WATO config snapshot + start = time.time() + logger.debug("Snapshot creation started") + # TODO: Remove/Refactor once new changes mechanism has been implemented + # This single function is responsible for the slow activate changes (python tar packaging..) + create_snapshot(self._comment) + + work_dir = os.path.join(self.activation_tmp_base_dir, self._activation_id) + if cmk.is_managed_edition(): + import cmk.gui.cme.managed_snapshots as managed_snapshots + managed_snapshots.CMESnapshotManager( + work_dir, self._get_site_configurations()).generate_snapshots() + else: + self._generate_snapshots(work_dir) - logger.debug("Snapshot creation took %.4f" % (time.time() - start)) - unlock_exclusive() + logger.debug("Snapshot creation took %.4f" % (time.time() - start)) def _get_site_configurations(self): site_configurations = {} @@ -5664,8 +5663,7 @@ class ActivateChangesManager(ActivateChanges): # Cleanup stale activations? def _do_housekeeping(self): - lock_exclusive() - try: + with exclusive_lock(): for activation_id in self._existing_activation_ids(): # skip the current activation_id if self._activation_id == activation_id: @@ -5690,8 +5688,6 @@ class ActivateChangesManager(ActivateChanges): if delete: shutil.rmtree("%s/%s" % (ActivateChangesManager.activation_tmp_base_dir, activation_id)) - finally: - unlock_exclusive() def _existing_activation_ids(self): ids = [] @@ -10536,10 +10532,22 @@ def make_action_link(vars_): return folder_preserving_link(vars_ + [("_transid", html.transaction_manager.get())]) +@contextmanager +def exclusive_lock(): + path = cmk.utils.paths.default_config_dir + "/multisite.mk" + store.aquire_lock(path) + try: + yield + finally: + store.release_lock(path) + + +# TODO: Use exclusive_lock() and nuke this! def lock_exclusive(): store.aquire_lock(cmk.utils.paths.default_config_dir + "/multisite.mk") +# TODO: Use exclusive_lock() and nuke this! def unlock_exclusive(): store.release_lock(cmk.utils.paths.default_config_dir + "/multisite.mk")

5 years, 8 months

graphs: move and rename averaged graph definition

by Moritz Kiemer

Module: check_mk Branch: master Commit: a57d026f0e96f8da0823bca22c19e8d8ba86aee9 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=a57d026f0e96f8… Author: Moritz Kiemer <mo(a)mathias-kettner.de> Date: Thu Jan 10 11:38:16 2019 +0100 graphs: move and rename averaged graph definition Change-Id: I012d32952370d9ae3afa717a7c49102b3636e4bf --- cmk/gui/plugins/metrics/check_mk.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmk/gui/plugins/metrics/check_mk.py b/cmk/gui/plugins/metrics/check_mk.py index 233aab0..1dcc593 100644 --- a/cmk/gui/plugins/metrics/check_mk.py +++ b/cmk/gui/plugins/metrics/check_mk.py @@ -8333,7 +8333,7 @@ graph_info["fgpa_utilization"] = { # CPU UTILIZATION # -graph_info["util_average"] = { +graph_info["util_average_1"] = { "metrics": [ ("util", "area"), ("util_average", "line"), @@ -8345,6 +8345,16 @@ graph_info["util_average"] = { "range": ("util:min", "util:max"), } +graph_info["cpu_average_2"] = { + "title": _("CPU utilization"), + "metrics": [("util1", "area"), ("util15", "line")], + "scalars": [ + "util1:warn", + "util1:crit", + ], + "range": (0, 100), +} + graph_info["cpu_utilization_1"] = { "title": _("CPU utilization (%(util:max@count) CPU Threads)"), "metrics": [ @@ -8359,16 +8369,6 @@ graph_info["cpu_utilization_1"] = { "range": (0, 100), } -graph_info["cpu_utilization_2"] = { - "title": _("CPU utilization"), - "metrics": [("util1", "area"), ("util15", "line")], - "scalars": [ - "util1:warn", - "util1:crit", - ], - "range": (0, 100), -} - graph_info["cpu_utilization_3"] = { "title": _("CPU utilization"), "metrics": [

5 years, 8 months

← Newer
1
...
32
33
34
35
36
37
38
...
49
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

Checkmk git commits January 2019