Checkmk git commits December 2018

checkmk-commits@lists.checkmk.com

15 participants
342 discussions

by Simon Betz

Module: check_mk Branch: master Commit: e11cde0bf0c6f088d5e9e9aae0d942a8bff0cc5a URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e11cde0bf0c6f0… Author: Simon Betz <si(a)mathias-kettner.de> Date: Thu Dec 13 11:22:02 2018 +0100 Minor cleanups - Renamed AWS/S3 buckets summary section for uniformity - Added a few headers - Reformatted AWSSections' init method Change-Id: I1269678c79e407fcf9306f914b91bc08bb8e541f --- agents/special/agent_aws | 100 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 24 deletions(-) diff --git a/agents/special/agent_aws b/agents/special/agent_aws index d29ac4e..89b7590 100755 --- a/agents/special/agent_aws +++ b/agents/special/agent_aws @@ -58,12 +58,12 @@ def _datetime_converter(o): #. -# .--sections------------------------------------------------------------. -# | _ _ | -# | ___ ___ ___| |_(_) ___ _ __ ___ | -# | / __|/ _ \/ __| __| |/ _ \| '_ \/ __| | -# | \__ \ __/ (__| |_| | (_) | | | \__ \ | -# | |___/\___|\___|\__|_|\___/|_| |_|___/ | +# .--section API---------------------------------------------------------. +# | _ _ _ ____ ___ | +# | ___ ___ ___| |_(_) ___ _ __ / \ | _ \_ _| | +# | / __|/ _ \/ __| __| |/ _ \| '_ \ / _ \ | |_) | | | +# | \__ \ __/ (__| |_| | (_) | | | | / ___ \| __/| | | +# | |___/\___|\___|\__|_|\___/|_| |_| /_/ \_\_| |___| | # | | # '----------------------------------------------------------------------' @@ -184,6 +184,17 @@ class AWSSection(object): self._received_results.setdefault(sender.name, content) +#. +# .--EC2-----------------------------------------------------------------. +# | _____ ____ ____ | +# | | ____/ ___|___ \ | +# | | _|| | __) | | +# | | |__| |___ / __/ | +# | |_____\____|_____| | +# | | +# '----------------------------------------------------------------------' + + class EC2Summary(AWSSection): @property def name(self): @@ -286,10 +297,21 @@ class EC2(AWSSection): ] -class S3Buckets(AWSSection): +#. +# .--S3------------------------------------------------------------------. +# | ____ _____ | +# | / ___|___ / | +# | \___ \ |_ \ | +# | ___) |__) | | +# | |____/____/ | +# | | +# '----------------------------------------------------------------------' + + +class S3Summary(AWSSection): @property def name(self): - return "s3_buckets" + return "s3_summary" @property def interval(self): @@ -337,7 +359,7 @@ class S3(AWSSection): def _get_metrics(self): bucket_ids = [] - for entry in self._received_results.get('s3_buckets', []): + for entry in self._received_results.get('s3_summary', []): bucket_ids.append(entry['Name']) metrics = [] idx = 0 @@ -405,7 +427,7 @@ class S3Objects(AWSSection): def _get_bucket_ids(self): bucket_ids = [] - for entry in self._received_results.get('s3_buckets', []): + for entry in self._received_results.get('s3_summary', []): bucket_ids.append(entry['Name']) return bucket_ids @@ -426,6 +448,17 @@ class S3Objects(AWSSection): ] +#. +# .--ELB-----------------------------------------------------------------. +# | _____ _ ____ | +# | | ____| | | __ ) | +# | | _| | | | _ \ | +# | | |___| |___| |_) | | +# | |_____|_____|____/ | +# | | +# '----------------------------------------------------------------------' + + class ELBSummary(AWSSection): @property def name(self): @@ -564,6 +597,17 @@ class ELB(AWSSection): ] +#. +# .--EBS-----------------------------------------------------------------. +# | _____ ____ ____ | +# | | ____| __ ) ___| | +# | | _| | _ \___ \ | +# | | |___| |_) |__) | | +# | |_____|____/____/ | +# | | +# '----------------------------------------------------------------------' + + class EBSSummary(AWSSection): @property def name(self): @@ -705,7 +749,15 @@ class EBS(AWSSection): ] -# ---sections handler----------------------------------------------------- +#. +# .--sections------------------------------------------------------------. +# | _ _ | +# | ___ ___ ___| |_(_) ___ _ __ ___ | +# | / __|/ _ \/ __| __| |/ _ \| '_ \/ __| | +# | \__ \ __/ (__| |_| | (_) | | | \__ \ | +# | |___/\___|\___|\__|_|\___/|_| |_|___/ | +# | | +# '----------------------------------------------------------------------' class AWSSections(object): @@ -724,41 +776,41 @@ class AWSSections(object): #---distributors---------------------------------------------------- ec2_summary_distributor = ResultDistributor() - s3_buckets_distributor = ResultDistributor() + s3_summary_distributor = ResultDistributor() elb_summary_distributor = ResultDistributor() ebs_summary_distributor = ResultDistributor() - #---sections-------------------------------------------------------- + #---sections with distributors-------------------------------------- ec2_summary = EC2Summary(ec2_client, ec2_summary_distributor) - ec2 = EC2(cloudwatch_client) + s3_summary = S3Summary(s3_client, s3_summary_distributor) + elb_summary = ELBSummary(elb_client, elb_summary_distributor) + ebs_summary = EBSSummary(ec2_client, ebs_summary_distributor) - s3_buckets = S3Buckets(s3_client, s3_buckets_distributor) + #---sections-------------------------------------------------------- + ec2 = EC2(cloudwatch_client) s3_objects = S3Objects(s3_client) s3 = S3(cloudwatch_client) - - elb_summary = ELBSummary(elb_client, elb_summary_distributor) elb_health = ELBHealth(elb_client) elb = ELB(cloudwatch_client) - - ebs_summary = EBSSummary(ec2_client, ebs_summary_distributor) ebs_health = EBSHealth(ec2_client) ebs = EBS(cloudwatch_client) - #---distributors registry------------------------------------------- + #---register sections to distributors------------------------------- ec2_summary_distributor.add(ec2) - s3_buckets_distributor.add(s3) - s3_buckets_distributor.add(s3_objects) + s3_summary_distributor.add(s3_objects) + s3_summary_distributor.add(s3) elb_summary_distributor.add(elb_health) elb_summary_distributor.add(elb) ebs_summary_distributor.add(ebs_health) ebs_summary_distributor.add(ebs) - #---section registry------------------------------------------------ + #---register sections with distributors for execution--------------- self._sections.append(ec2_summary) - self._sections.append(s3_buckets) + self._sections.append(s3_summary) self._sections.append(elb_summary) self._sections.append(ebs_summary) + #---register sections for execution--------------------------------- self._sections.append(ec2) self._sections.append(s3_objects) self._sections.append(s3)

5 years, 9 months

Started AWS/EBS checks

by Simon Betz

Module: check_mk Branch: master Commit: adef19bfbdfd5ea67155282c5e3919c7dd293b3a URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=adef19bfbdfd5e… Author: Simon Betz <si(a)mathias-kettner.de> Date: Thu Dec 13 10:02:10 2018 +0100 Started AWS/EBS checks Change-Id: I1cc949759370a4e01e426022688af22c4233c8e5 --- agents/special/agent_aws | 151 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/agents/special/agent_aws b/agents/special/agent_aws index 0b0d733..d29ac4e 100755 --- a/agents/special/agent_aws +++ b/agents/special/agent_aws @@ -564,6 +564,147 @@ class ELB(AWSSection): ] +class EBSSummary(AWSSection): + @property + def name(self): + return "ebs_summary" + + @property + def interval(self): + return 300 + + def _get_raw_result(self): + return self._client.describe_volumes() + + def _extract_content_from(self, raw_result): + try: + return raw_result['Volumes'] + except KeyError, e: + logging.info("%s: KeyError %s ", self.name, e) + return [] + + def _format_content(self, content): + return [AWSSectionResult("", content)] + + +class EBSHealth(AWSSection): + @property + def name(self): + return "ebs_health" + + @property + def interval(self): + return 300 + + def _get_raw_result(self): + volumes = [] + for entry in self._received_results.get('ebs_summary', []): + volumes.append(entry['VolumeId']) + return self._client.describe_volume_status(VolumeIds=volumes) + + def _extract_content_from(self, raw_result): + content = {} + for row in raw_result.get('VolumeStatuses', []): + try: + content.setdefault(row['VolumeId'], row['VolumeStatus']) + except KeyError, e: + logging.info("%s: KeyError %s ", self.name, e) + return content + + def _format_content(self, content): + # parse function of aws.include extends a list, thus + # we put details as an element in this list + return [ + AWSSectionResult(piggyback_hostname, [details]) + for piggyback_hostname, details in content.iteritems() + ] + + +class EBS(AWSSection): + @property + def name(self): + return "ebs" + + @property + def interval(self): + return 300 + + def _get_raw_result(self): + end_time = time.time() + start_time = end_time - self.period + metrics = self._get_metrics() + if not metrics: + return {} + return self._client.get_metric_data( + MetricDataQueries=metrics, + StartTime=start_time, + EndTime=end_time, + ) + + def _get_metrics(self): + volumes = [] + for entry in self._received_results.get('ebs_summary', []): + volumes.append((entry['VolumeId'], entry['VolumeType'])) + metrics = [] + idx = 0 + for volume_name, volume_type in volumes: + for metric_name, volume_types in [ + # Disk IO + ("VolumeReadOps", []), + ("VolumeWriteOps", []), + ("VolumeReadBytes", []), + ("VolumeWriteBytes", []), + ("VolumeThroughputPercentage", ["io1"]), + ("VolumeConsumedReadWriteOps", ["io1"]), + # Latency + ("VolumeTotalReadTime", []), + ("VolumeTotalWriteTime", []), + # Disk activity + ("VolumeQueueLength", []), + ("VolumeIdleTime", []), + ("BurstBalance", ["gp2", "st1", "sc1"]), + # Status checks + ("VolumeStatus", []), + ("IOPerformance", ["io1"]), + ]: + if volume_types and volume_type not in volume_types: + continue + metrics.append({ + 'Id': "%s_%s" % (metric_name.lower(), idx), + 'Label': volume_name, + 'MetricStat': { + 'Metric': { + 'Namespace': 'AWS/EBS', + 'MetricName': metric_name, + 'Dimensions': [{ + 'Name': "VolumeID", + 'Value': volume_name, + }] + }, + 'Period': self.period, + 'Stat': 'Average', + }, + }) + idx += 1 + return metrics + + def _extract_content_from(self, raw_result): + try: + return raw_result['MetricDataResults'] + except KeyError, e: + logging.info("%s: KeyError %s ", self.name, e) + return [] + + def _format_content(self, content): + content_by_piggyback_hosts = {} + for row in content: + content_by_piggyback_hosts.setdefault(row['Label'], []).append(row) + return [ + AWSSectionResult(piggyback_hostname, rows) + for piggyback_hostname, rows in content_by_piggyback_hosts.iteritems() + ] + + # ---sections handler----------------------------------------------------- @@ -585,6 +726,7 @@ class AWSSections(object): ec2_summary_distributor = ResultDistributor() s3_buckets_distributor = ResultDistributor() elb_summary_distributor = ResultDistributor() + ebs_summary_distributor = ResultDistributor() #---sections-------------------------------------------------------- ec2_summary = EC2Summary(ec2_client, ec2_summary_distributor) @@ -598,23 +740,32 @@ class AWSSections(object): elb_health = ELBHealth(elb_client) elb = ELB(cloudwatch_client) + ebs_summary = EBSSummary(ec2_client, ebs_summary_distributor) + ebs_health = EBSHealth(ec2_client) + ebs = EBS(cloudwatch_client) + #---distributors registry------------------------------------------- ec2_summary_distributor.add(ec2) s3_buckets_distributor.add(s3) s3_buckets_distributor.add(s3_objects) elb_summary_distributor.add(elb_health) elb_summary_distributor.add(elb) + ebs_summary_distributor.add(ebs_health) + ebs_summary_distributor.add(ebs) #---section registry------------------------------------------------ self._sections.append(ec2_summary) self._sections.append(s3_buckets) self._sections.append(elb_summary) + self._sections.append(ebs_summary) self._sections.append(ec2) self._sections.append(s3_objects) self._sections.append(s3) self._sections.append(elb_health) self._sections.append(elb) + self._sections.append(ebs_health) + self._sections.append(ebs) def _init_client(self, client_key): try:

5 years, 9 months

fixed pylint error

by Andreas Boesl

Module: check_mk Branch: master Commit: f46ddf45f846d1dc5ea18f61d3ec1a161e4e0187 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=f46ddf45f846d1… Author: Andreas Boesl <ab(a)mathias-kettner.de> Date: Thu Dec 13 14:54:55 2018 +0100 fixed pylint error Change-Id: Ib9c30cb2e451680e67fe02f342b7c9a029ddf828 --- cmk/gui/plugins/userdb/ldap_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmk/gui/plugins/userdb/ldap_connector.py b/cmk/gui/plugins/userdb/ldap_connector.py index f425d79..c19b06c 100644 --- a/cmk/gui/plugins/userdb/ldap_connector.py +++ b/cmk/gui/plugins/userdb/ldap_connector.py @@ -2594,7 +2594,7 @@ def synchronize_profiles_to_sites(logger, profiles_to_synchronize): for site_id, site in remote_sites: jobs.append( pool.apply_async(_sychronize_profile_worker, - ((states, site_id, site, user_id, profile),))) + ((states, site_id, site, profiles_to_synchronize)))) results = [] start_time = time.time()

5 years, 9 months

mk_filestats: rework

by Moritz Kiemer

Module: check_mk Branch: master Commit: 8599e9fda238fc2d743eecddc1e534195eb286ea URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=8599e9fda238fc… Author: Moritz Kiemer <mo(a)mathias-kettner.de> Date: Thu Dec 13 09:18:06 2018 +0100 mk_filestats: rework * only create one single check * plugin always reports count * use existing metric name * change service description to 'File group %s' Change-Id: Ibaf0ef3e672c0218344c1b4bc36ad689768b4301 --- agents/plugins/mk_filestats.py | 21 ++-- checkman/filestats | 5 +- checkman/filestats.count_only | 31 ------ checkman/filestats.extremes_only | 32 ------ checks/filestats | 119 +++------------------ .../checks/generictests/datasets/filestats_1.py | 57 +++++----- .../datasets/filestats_broken_sections.py | 38 +++++++ .../generictests/datasets/filestats_count_only.py | 31 ------ 8 files changed, 101 insertions(+), 233 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=8599e9fda2…

5 years, 9 months

Fixed order of method string-related calls: First decode, then strip.

by Sven Panne

Module: check_mk Branch: master Commit: 3cd19edb1e8501d9ecbedd5aec27c1ec20156ef9 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=3cd19edb1e8501… Author: Sven Panne <sp(a)mathias-kettner.de> Date: Thu Dec 13 12:19:54 2018 +0100 Fixed order of method string-related calls: First decode, then strip. Doing it the other way round is wrong in general and only works by accident for UTF-8. Fold the decoding step directly into the file handling in a few places, see Python's "Unicode HOWTO". Change-Id: Id6dba07fdd70ac579a7dee6442da9bdcdd47d334 --- cmk/gui/plugins/views/perfometer.py | 2 +- cmk/gui/wato/pages/audit_log.py | 19 +++--- cmk/gui/watolib.py | 2 +- cmk/man_pages.py | 129 +++++++++++++++++++----------------- cmk_base/notify.py | 2 +- 5 files changed, 80 insertions(+), 74 deletions(-) diff --git a/cmk/gui/plugins/views/perfometer.py b/cmk/gui/plugins/views/perfometer.py index 29d1f82..dbdc1ac 100644 --- a/cmk/gui/plugins/views/perfometer.py +++ b/cmk/gui/plugins/views/perfometer.py @@ -58,7 +58,7 @@ class Perfometer(object): self._parse_perf_data() def _parse_perf_data(self): - perf_data_string = self._row["service_perf_data"].strip().decode("utf-8") + perf_data_string = self._row["service_perf_data"].decode("utf-8").strip() if not perf_data_string: return diff --git a/cmk/gui/wato/pages/audit_log.py b/cmk/gui/wato/pages/audit_log.py index 46778f6..b6e1df8 100644 --- a/cmk/gui/wato/pages/audit_log.py +++ b/cmk/gui/wato/pages/audit_log.py @@ -25,6 +25,7 @@ # Boston, MA 02110-1301 USA. """Handling of the audit logfiles""" +import codecs import os import re import time @@ -393,18 +394,18 @@ class ModeAuditLog(WatoMode): return [] entries = [] - for line in file(self.log_path): - line = line.rstrip().decode("utf-8") - splitted = line.split(None, 4) + with codecs.open(self.log_path, encoding="utf-8") as fp: + for line in fp: + splitted = line.rstrip().split(None, 4) - if len(splitted) == 5 and splitted[0].isdigit(): - splitted[0] = int(splitted[0]) + if len(splitted) == 5 and splitted[0].isdigit(): + splitted[0] = int(splitted[0]) - user, action, text = splitted[2:] - if self._filter_entry(user, action, text): - continue + user, action, text = splitted[2:] + if self._filter_entry(user, action, text): + continue - entries.append(splitted) + entries.append(splitted) entries.reverse() diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py index 659895b..7ea6af7 100644 --- a/cmk/gui/watolib.py +++ b/cmk/gui/watolib.py @@ -7512,7 +7512,7 @@ def load_user_scripts_from(adir): try: lines = file(path) lines.next() - line = lines.next().strip().decode("utf-8") + line = lines.next().decode("utf-8").strip() if line.startswith("#") and "encoding:" in line: line = lines.next().strip() if line.startswith("#"): diff --git a/cmk/man_pages.py b/cmk/man_pages.py index a6cd908..931d9c5 100644 --- a/cmk/man_pages.py +++ b/cmk/man_pages.py @@ -30,6 +30,7 @@ used as base for the list of supported checks and catalogs of checks. These man pages are in a Check_MK specific format an not real Linux/Unix man pages""" +import codecs import os import re import sys @@ -267,9 +268,10 @@ def print_man_page_table(): def _get_title_from_man_page(path): - for line in file(path): - if line.startswith("title:"): - return line.split(":", 1)[1].strip().decode("utf-8") + with codecs.open(path, encoding="utf-8") as fp: + for line in fp: + if line.startswith("title:"): + return line.split(":", 1)[1].strip() raise MKGeneralException(_("Invalid man page: Failed to get the title")) @@ -407,16 +409,17 @@ def _run_dialog(args): def _create_fallback_man_page(name, path, error_message): - return { - "name": name, - "path": path, - "description": file(path).read().strip().decode("utf-8"), - "title": _("%s: Cannot parse man page: %s") % (name, error_message), - "agents": "", - "license": "unknown", - "distribution": "unknown", - "catalog": ["generic"], - } + with codecs.open(path, encoding="utf-8") as fp: + return { + "name": name, + "path": path, + "description": fp.read().strip(), + "title": _("%s: Cannot parse man page: %s") % (name, error_message), + "agents": "", + "license": "unknown", + "distribution": "unknown", + "catalog": ["generic"], + } def _parse_man_page_header(name, path): @@ -426,24 +429,25 @@ def _parse_man_page_header(name, path): } key = None lineno = 0 - for line in file(path): - line = line.rstrip().decode("utf-8") - lineno += 1 - try: - if not line: - parsed[key] += "\n\n" - elif line[0] == ' ': - parsed[key] += "\n" + line.lstrip() - elif line[0] == '[': - break # End of header - else: - key, rest = line.split(":", 1) - parsed[key] = rest.lstrip() - except Exception: - if cmk.debug.enabled(): - raise - sys.stderr.write("ERROR: Invalid line %d in man page %s\n%s" % (lineno, path, line)) - break + with codecs.open(path, encoding="utf-8") as fp: + for line in fp: + line = line.rstrip() + lineno += 1 + try: + if not line: + parsed[key] += "\n\n" + elif line[0] == ' ': + parsed[key] += "\n" + line.lstrip() + elif line[0] == '[': + break # End of header + else: + key, rest = line.split(":", 1) + parsed[key] = rest.lstrip() + except Exception: + if cmk.debug.enabled(): + raise + sys.stderr.write("ERROR: Invalid line %d in man page %s\n%s" % (lineno, path, line)) + break # verify mandatory keys. FIXME: This list may be incomplete for key in [ @@ -475,40 +479,41 @@ def load_man_page(name): man_page['header'] = current_section empty_line_count = 0 - for lineno, line in enumerate(file(path)): - try: - line = line.decode("utf-8") - if line.startswith(' ') and line.strip() != "": # continuation line - empty_line_count = 0 - if current_variable: - name, curval = current_section[-1] - if curval.strip() == "": - current_section[-1] = (name, line.rstrip()[1:]) + with codecs.open(path, encoding="utf-8") as fp: + for lineno, line in enumerate(fp): + try: + if line.startswith(' ') and line.strip() != "": # continuation line + empty_line_count = 0 + if current_variable: + name, curval = current_section[-1] + if curval.strip() == "": + current_section[-1] = (name, line.rstrip()[1:]) + else: + current_section[-1] = (name, curval + "\n" + line.rstrip()[1:]) else: - current_section[-1] = (name, curval + "\n" + line.rstrip()[1:]) - else: - raise Exception - continue - - line = line.strip() - if line == "": - empty_line_count += 1 - if empty_line_count == 1 and current_variable: - name, curval = current_section[-1] - current_section[-1] = (name, curval + "\n<br>\n") - continue - empty_line_count = 0 + raise Exception + continue + + line = line.strip() + if line == "": + empty_line_count += 1 + if empty_line_count == 1 and current_variable: + name, curval = current_section[-1] + current_section[-1] = (name, curval + "\n<br>\n") + continue + empty_line_count = 0 - if line[0] == '[' and line[-1] == ']': - section_header = line[1:-1] - current_section, current_variable = [], None - man_page[section_header] = current_section - else: - current_variable, restofline = line.split(':', 1) - current_section.append((current_variable, restofline.lstrip())) + if line[0] == '[' and line[-1] == ']': + section_header = line[1:-1] + current_section, current_variable = [], None + man_page[section_header] = current_section + else: + current_variable, restofline = line.split(':', 1) + current_section.append((current_variable, restofline.lstrip())) - except Exception as e: - raise MKGeneralException("Syntax error in %s line %d (%s).\n" % (path, lineno + 1, e)) + except Exception as e: + raise MKGeneralException( + "Syntax error in %s line %d (%s).\n" % (path, lineno + 1, e)) header = {} for key, value in man_page['header']: diff --git a/cmk_base/notify.py b/cmk_base/notify.py index 4247fcb..0293e9d 100755 --- a/cmk_base/notify.py +++ b/cmk_base/notify.py @@ -1346,7 +1346,7 @@ def call_notification_script(plugin, plugin_context): # one - potentially huge - memory buffer line = p.stdout.readline() if line != '': - plugin_log("Output: %s" % line.rstrip().decode('utf-8')) + plugin_log("Output: %s" % line.decode('utf-8').rstrip()) if _log_to_stdout: console.output(line) else:

5 years, 9 months

added WK6695: User synchronization: Fixed scenario where an update thread could get stuck indefinitely

by Andreas Boesl

Module: check_mk Branch: master Commit: 7d48e639d05fa0f6c560df88efda394060f68e9a URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=7d48e639d05fa0… Author: Andreas Boesl <ab(a)mathias-kettner.de> Date: Thu Dec 13 11:55:43 2018 +0100 added WK6695: User synchronization: Fixed scenario where an update thread could get stuck indefinitely Change-Id: I692e9d62f4fc3fea82942119ce726a14c4572a19 --- .werks/6695 | 11 ++++++++++ cmk/gui/plugins/userdb/ldap_connector.py | 35 +++++++++++++++++++++++++------- cmk/gui/watolib.py | 17 ++++++++++------ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/.werks/6695 b/.werks/6695 new file mode 100644 index 0000000..57a65e9 --- /dev/null +++ b/.werks/6695 @@ -0,0 +1,11 @@ +Title: User synchronization: Fixed scenario where an update thread could get stuck indefinitely +Level: 1 +Component: wato +Class: fix +Compatible: compat +Edition: cre +State: unknown +Version: 1.6.0i1 +Date: 1543835110 + +In this scenario the background job never finished, preventing any new updates. diff --git a/cmk/gui/plugins/userdb/ldap_connector.py b/cmk/gui/plugins/userdb/ldap_connector.py index c9773c9..8773ddd 100644 --- a/cmk/gui/plugins/userdb/ldap_connector.py +++ b/cmk/gui/plugins/userdb/ldap_connector.py @@ -49,6 +49,7 @@ import copy from multiprocessing.pool import ThreadPool +from multiprocessing.pool import TimeoutError import os import re import shutil @@ -2582,16 +2583,33 @@ def synchronize_profiles_to_sites(logger, profiles_to_synchronize): logger.info('Credentials changed for %s. Trying to sync to %d sites' % (", ".join( profiles_to_synchronize.keys()), len(remote_sites))) - synchronization_jobs = [] states = sites.states() - for site_id, site in remote_sites: - synchronization_jobs.append((states, site_id, site, profiles_to_synchronize)) - pool = ThreadPool() - results = pool.map(_sychronize_profile_worker, synchronization_jobs) - pool.close() - pool.join() + jobs = [] + for site_id, site in remote_sites: + jobs.append( + pool.apply_async(_sychronize_profile_worker, + ((states, site_id, site, user_id, profile),))) + + results = [] + start_time = time.time() + while time.time() - start_time < 10: + for job in jobs[:]: + try: + results.append(job.get(timeout=0.5)) + jobs.remove(job) + except TimeoutError: + pass + if not jobs: + break + + contacted_sites = set([x[0] for x in remote_sites]) + working_sites = set([result.site_id for result in results]) + for site_id in contacted_sites - working_sites: + results.append( + SynchronizationResult( + site_id, error_text=_("No response from update thread"), failed=True)) for result in results: if result.error_text: @@ -2604,6 +2622,9 @@ def synchronize_profiles_to_sites(logger, profiles_to_synchronize): sites=[result.site_id], need_restart=False) + pool.terminate() + pool.join() + num_failed = sum([1 for result in results if result.failed]) num_disabled = sum([1 for result in results if result.disabled]) num_succeeded = sum([1 for result in results if result.succeeded]) diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py index 7889ff0..659895b 100644 --- a/cmk/gui/watolib.py +++ b/cmk/gui/watolib.py @@ -4570,13 +4570,14 @@ def do_site_login(site_id, name, password): raise MKAutomationException(response) -def get_url(url, insecure, auth=None, data=None, files=None): +def get_url(url, insecure, auth=None, data=None, files=None, timeout=None): response = requests.post( url, data=data, verify=not insecure, auth=auth, files=files, + timeout=timeout, ) response.encoding = "utf-8" # Always decode with utf-8 @@ -4653,7 +4654,7 @@ def sync_changes_before_remote_automation(site_id): state.get("_status_details")) -def do_remote_automation(site, command, vars_): +def do_remote_automation(site, command, vars_, timeout=None): base_url = site["multisiteurl"] secret = site.get("secret") if not secret: @@ -4666,7 +4667,7 @@ def do_remote_automation(site, command, vars_): ("debug", config.debug and '1' or '') ]) - response = get_url(url, site.get('insecure', False), data=dict(vars_)) + response = get_url(url, site.get('insecure', False), data=dict(vars_), timeout=timeout) if not response: raise MKAutomationException(_("Empty output from remote site.")) @@ -4856,10 +4857,13 @@ def legacy_push_user_profile_to_site(site, user_id, profile): ]) response = get_url( - url, site.get('insecure', False), data={ + url, + site.get('insecure', False), + data={ 'user_id': user_id, 'profile': mk_repr(profile), - }) + }, + timeout=60) if not response: raise MKAutomationException(_("Empty output from remote site.")) @@ -4873,7 +4877,8 @@ def legacy_push_user_profile_to_site(site, user_id, profile): def push_user_profiles_to_site(site, user_profiles): - return do_remote_automation(site, "push-profiles", [("profiles", repr(user_profiles))]) + return do_remote_automation( + site, "push-profiles", [("profiles", repr(user_profiles))], timeout=60) PushUserProfilesRequest = NamedTuple("PushUserProfilesRequest", [("user_profiles", dict)])

5 years, 9 months

WK6700: LDAP synchronization no longer syncs user profiles repeatedly to remote sites on password changes

by Andreas Boesl

Module: check_mk Branch: master Commit: 95356aa17f635c50082bdd54f8729bd19e24c5c3 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=95356aa17f635c… Author: Andreas Boesl <ab(a)mathias-kettner.de> Date: Thu Dec 13 12:04:37 2018 +0100 WK6700: LDAP synchronization no longer syncs user profiles repeatedly to remote sites on password changes Change-Id: I7c84b2f8a535ceb206dc96d2a8a01f679b893c7d --- .werks/6700 | 12 ++++++++++++ cmk/gui/plugins/userdb/ldap_connector.py | 6 +++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.werks/6700 b/.werks/6700 new file mode 100644 index 0000000..1aab705 --- /dev/null +++ b/.werks/6700 @@ -0,0 +1,12 @@ +Title: LDAP synchronization no longer syncs user profiles repeatedly to remote sites on password changes +Level: 1 +Component: multisite +Class: fix +Compatible: compat +Edition: cre +State: unknown +Version: 1.6.0i1 +Date: 1544696760 + +LDAP password changes were not correctly applied to the master site. +This could also trigger needless updates to the slave sites. diff --git a/cmk/gui/plugins/userdb/ldap_connector.py b/cmk/gui/plugins/userdb/ldap_connector.py index 8773ddd..f425d79 100644 --- a/cmk/gui/plugins/userdb/ldap_connector.py +++ b/cmk/gui/plugins/userdb/ldap_connector.py @@ -1162,6 +1162,7 @@ class LDAPUserConnector(UserConnector): del users[user_id] # remove the user changes.append(_("LDAP [%s]: Removed user %s") % (connection_id, user_id)) + has_changed_passwords = False profiles_to_synchronize = {} for user_id, ldap_user in ldap_users.items(): mode_create, user = load_user(user_id) @@ -1231,6 +1232,9 @@ class LDAPUserConnector(UserConnector): del changed['serial'] pw_changed = True + if pw_changed: + has_changed_passwords = True + # Synchronize new user profile to remote sites if needed if pw_changed and not changed and config.has_wato_slave_sites(): profiles_to_synchronize[user_id] = user @@ -1254,7 +1258,7 @@ class LDAPUserConnector(UserConnector): if changes and config.wato_enabled and not config.is_wato_slave_site(): watolib.add_change("edit-users", "<br>\n".join(changes), add_user=False) - if changes: + if changes or has_changed_passwords: userdb.save_users(users) else: userdb.release_users_lock()

5 years, 9 months

Started AWS/ELB checks

by Simon Betz

Module: check_mk Branch: master Commit: af89cea492835cc5c097c4db63e0ca03264507f7 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=af89cea492835c… Author: Simon Betz <si(a)mathias-kettner.de> Date: Tue Dec 11 11:14:27 2018 +0100 Started AWS/ELB checks Change-Id: Ib684a01a78344fee7f33d15fb254983368ac5090 --- agents/special/agent_aws | 153 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/agents/special/agent_aws b/agents/special/agent_aws index c814bd8..0b0d733 100755 --- a/agents/special/agent_aws +++ b/agents/special/agent_aws @@ -426,6 +426,144 @@ class S3Objects(AWSSection): ] +class ELBSummary(AWSSection): + @property + def name(self): + return "elb_summary" + + @property + def interval(self): + return 300 + + def _get_raw_result(self): + return self._client.describe_load_balancers() + + def _extract_content_from(self, raw_result): + try: + return raw_result['LoadBalancerDescriptions'] + except KeyError, e: + logging.info("%s: KeyError %s ", self.name, e) + return [] + + def _format_content(self, content): + return [AWSSectionResult("", content)] + + +class ELBHealth(AWSSection): + @property + def name(self): + return "elb_health" + + @property + def interval(self): + return 300 + + def _get_raw_result(self): + load_balancers = {} + for entry in self._received_results.get('elb_summary', []): + name = entry['LoadBalancerName'] + load_balancers.setdefault(name, + self._client.describe_instance_health(LoadBalancerName=name)) + return load_balancers + + def _extract_content_from(self, raw_result): + content = {} + for name, attrs in raw_result.iteritems(): + try: + content.setdefault(name, attrs['InstanceStates']) + except KeyError, e: + logging.info("%s: KeyError %s ", self.name, e) + return content + + def _format_content(self, content): + return [ + AWSSectionResult(piggyback_hostname, rows) + for piggyback_hostname, rows in content.iteritems() + ] + + +class ELB(AWSSection): + @property + def name(self): + return "elb" + + @property + def interval(self): + return 300 + + def _get_raw_result(self): + end_time = time.time() + start_time = end_time - self.period + metrics = self._get_metrics() + if not metrics: + return {} + return self._client.get_metric_data( + MetricDataQueries=metrics, + StartTime=start_time, + EndTime=end_time, + ) + + def _get_metrics(self): + load_balancers = [] + for entry in self._received_results.get('elb_summary', []): + load_balancers.append(entry['LoadBalancerName']) + metrics = [] + idx = 0 + for load_balancer in load_balancers: + for metric_name, unit in [ + # Load balancer metrics + ("RequestCount", ""), + ("SurgeQueueLength", ""), + ("SpilloverCount", ""), + ("HTTPCode_ELB_4XX", ""), + ("HTTPCode_ELB_5XX", ""), + # Backend metrics + ("HealthyHostCount", ""), + ("UnHealthyHostCount", ""), + ("Latency", ""), + ("HTTPCode_Backend_2XX", ""), + ("HTTPCode_Backend_3XX", ""), + ("HTTPCode_Backend_4XX", ""), + ("HTTPCode_Backend_5XX", ""), + ("BackendConnectionErrors", ""), + ]: + metrics.append({ + 'Id': "%s_%s" % (metric_name.lower(), idx), + 'Label': load_balancer, + 'MetricStat': { + 'Metric': { + 'Namespace': 'AWS/ELB', + 'MetricName': metric_name, + 'Dimensions': [{ + 'Name': "LoadBalancerName", + 'Value': load_balancer, + }] + }, + 'Period': self.period, + 'Stat': 'Average', + 'Unit': unit, + }, + }) + idx += 1 + return metrics + + def _extract_content_from(self, raw_result): + try: + return raw_result['MetricDataResults'] + except KeyError, e: + logging.info("%s: KeyError %s ", self.name, e) + return [] + + def _format_content(self, content): + content_by_piggyback_hosts = {} + for row in content: + content_by_piggyback_hosts.setdefault(row['Label'], []).append(row) + return [ + AWSSectionResult(piggyback_hostname, rows) + for piggyback_hostname, rows in content_by_piggyback_hosts.iteritems() + ] + + # ---sections handler----------------------------------------------------- @@ -440,30 +578,43 @@ class AWSSections(object): #---clients--------------------------------------------------------- ec2_client = self._init_client('ec2') s3_client = self._init_client('s3') + elb_client = self._init_client('elb') cloudwatch_client = self._init_client('cloudwatch') #---distributors---------------------------------------------------- ec2_summary_distributor = ResultDistributor() s3_buckets_distributor = ResultDistributor() + elb_summary_distributor = ResultDistributor() #---sections-------------------------------------------------------- ec2_summary = EC2Summary(ec2_client, ec2_summary_distributor) ec2 = EC2(cloudwatch_client) + s3_buckets = S3Buckets(s3_client, s3_buckets_distributor) s3_objects = S3Objects(s3_client) s3 = S3(cloudwatch_client) + elb_summary = ELBSummary(elb_client, elb_summary_distributor) + elb_health = ELBHealth(elb_client) + elb = ELB(cloudwatch_client) + #---distributors registry------------------------------------------- ec2_summary_distributor.add(ec2) s3_buckets_distributor.add(s3) s3_buckets_distributor.add(s3_objects) + elb_summary_distributor.add(elb_health) + elb_summary_distributor.add(elb) #---section registry------------------------------------------------ self._sections.append(ec2_summary) self._sections.append(s3_buckets) + self._sections.append(elb_summary) + self._sections.append(ec2) - self._sections.append(s3) self._sections.append(s3_objects) + self._sections.append(s3) + self._sections.append(elb_health) + self._sections.append(elb) def _init_client(self, client_key): try:

5 years, 9 months

added WK6699: Rule and pack choices now always show the id in addition to the title

by Andreas Boesl

Module: check_mk Branch: master Commit: 49deef34fa6302042a5e0e1353539f7fd276fa41 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=49deef34fa6302… Author: Andreas Boesl <ab(a)mathias-kettner.de> Date: Thu Dec 13 11:35:46 2018 +0100 added WK6699: Rule and pack choices now always show the id in addition to the title Change-Id: Ie837d52c824d960f2c894872b263673a051de0ea --- .werks/6699 | 11 +++++++++++ cmk/gui/plugins/wato/bi.py | 7 +++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.werks/6699 b/.werks/6699 new file mode 100644 index 0000000..8db52cd --- /dev/null +++ b/.werks/6699 @@ -0,0 +1,11 @@ +Title: BI configuration: Rule and pack choices now always show the id in addition to the title +Level: 1 +Component: bi +Class: feature +Compatible: compat +Edition: cre +State: unknown +Version: 1.6.0i1 +Date: 1544603575 + + diff --git a/cmk/gui/plugins/wato/bi.py b/cmk/gui/plugins/wato/bi.py index 9aa682e..d73625c 100644 --- a/cmk/gui/plugins/wato/bi.py +++ b/cmk/gui/plugins/wato/bi.py @@ -522,10 +522,9 @@ class ModeBI(WatoMode, BIManagement): choices = [] for pack_id, pack in sorted(self._packs.iteritems()): if self.may_use_rules_in_pack(pack): - pack_choices = [ - (rule_id, rule["title"]) for rule_id, rule in pack["rules"].iteritems() - ] - choices.append((pack_id, pack["title"], + pack_choices = [(rule_id, "%s (%s)" % (rule["title"], rule["id"])) + for rule_id, rule in pack["rules"].iteritems()] + choices.append((pack_id, "%s (%s)" % (pack["title"], pack["id"]), DropdownChoice(choices=sorted(pack_choices)))) return choices

5 years, 9 months

pylint: Enable global-variable-undefined checker

by Lars Michelsen

Module: check_mk Branch: master Commit: 11655b2068c8d9d41ff3fd2e4908fb807c181f9e URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=11655b2068c8d9… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Dec 13 07:22:54 2018 +0100 pylint: Enable global-variable-undefined checker Change-Id: I073069299365f4240c090ddfe245595827cec01b --- .pylintrc | 4 - active_checks/check_form_submit | 15 ++-- agents/plugins/mk_sap | 1 + bin/livedump | 3 + bin/mkbackup | 3 + cmk/gui/bi.py | 25 +++--- cmk/gui/dashboard.py | 8 +- cmk/gui/forms.py | 18 ++--- cmk/gui/notify.py | 15 ++-- cmk/gui/sidebar.py | 4 +- cmk/gui/views.py | 5 ++ cmk/gui/visuals.py | 6 +- cmk/gui/watolib.py | 13 +-- cmk/man_pages.py | 12 +-- cmk_base/config.py | 3 + cmk_base/core_nagios.py | 170 +++++++++++++++++++++------------------- cmk_base/notify.py | 5 +- omd/packages/omd/omdlib/main.py | 25 +++--- 18 files changed, 182 insertions(+), 153 deletions(-) Diff: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commitdiff;h=11655b2068…

5 years, 9 months

← Newer
1
...
17
18
19
20
21
22
23
...
35
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

Checkmk git commits December 2018