[checkmk-commits] agent_aws: Split metric queries info 100-sized chunks (

pagination not supported for cloudwatch.get_metric_data) Message-ID: <5c1b4c1e.F5sfVtSmrQD3DFy9%si(a)mathias-kettner.de> User-Agent: Heirloom mailx 12.5 6/20/10 MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Module: check_mk Branch: master Commit: ed0f6d0d9e4685ddb8b00a78eb81d2a2fe786b60 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=ed0f6d0d9e4685… Author: Simon Betz <si(a)mathias-kettner.de> Date: Wed Dec 19 15:24:35 2018 +0100 agent_aws: Split metric queries info 100-sized chunks (pagination not supported for cloudwatch.get_metric_data) Change-Id: I15940f23b260eba088199746411c07ad05916493 --- agents/special/agent_aws | 102 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 25 deletions(-) diff --git a/agents/special/agent_aws b/agents/special/agent_aws index d3fdb18..934a5bf 100755 --- a/agents/special/agent_aws +++ b/agents/special/agent_aws @@ -57,6 +57,10 @@ def _datetime_converter(o): return o.__str__() +def _chunks(list_, length=100): + return [list_[i:i + length] for i in xrange(0, len(list_), length)] + + #. # .--section API---------------------------------------------------------. # | _ _ _ ____ ___ | @@ -233,11 +237,24 @@ class EC2(AWSSection): metrics = self._get_metrics() if not metrics: return {} - return self._client.get_metric_data( - MetricDataQueries=metrics, - StartTime=start_time, - EndTime=end_time, - ) + + # A single GetMetricData call can include up to 100 MetricDataQuery structures + # There's no pagination for this operation: + # self._client.can_paginate('get_metric_data') = False + raw_result = [] + for chunk in _chunks(metrics): + response = self._client.get_metric_data( + MetricDataQueries=chunk, + StartTime=start_time, + EndTime=end_time, + ) + try: + metrics = response['MetricDataResults'] + except KeyError, e: + logging.info("%s: KeyError %s; Available are %s", self.name, e, response.keys()) + continue + raw_result.extend(metrics) + return raw_result def _get_metrics(self): instance_ids = [] @@ -357,11 +374,24 @@ class S3(AWSSection): metrics = self._get_metrics() if not metrics: return {} - return self._client.get_metric_data( - MetricDataQueries=metrics, - StartTime=start_time, - EndTime=end_time, - ) + + # A single GetMetricData call can include up to 100 MetricDataQuery structures + # There's no pagination for this operation: + # self._client.can_paginate('get_metric_data') = False + raw_result = [] + for chunk in _chunks(metrics): + response = self._client.get_metric_data( + MetricDataQueries=chunk, + StartTime=start_time, + EndTime=end_time, + ) + try: + metrics = response['MetricDataResults'] + except KeyError, e: + logging.info("%s: KeyError %s; Available are %s", self.name, e, response.keys()) + continue + raw_result.extend(metrics) + return raw_result def _get_metrics(self): bucket_ids = [] @@ -403,11 +433,7 @@ class S3(AWSSection): return metrics def _extract_content_from(self, raw_result): - try: - return raw_result['MetricDataResults'] - except KeyError, e: - logging.info("%s: KeyError %s; Available are %s", self.name, e, raw_result.keys()) - return [] + return raw_result def _format_content(self, content): content_by_piggyback_hosts = {} @@ -536,11 +562,24 @@ class ELB(AWSSection): metrics = self._get_metrics() if not metrics: return {} - return self._client.get_metric_data( - MetricDataQueries=metrics, - StartTime=start_time, - EndTime=end_time, - ) + + # A single GetMetricData call can include up to 100 MetricDataQuery structures + # There's no pagination for this operation: + # self._client.can_paginate('get_metric_data') = False + raw_result = [] + for chunk in _chunks(metrics): + response = self._client.get_metric_data( + MetricDataQueries=chunk, + StartTime=start_time, + EndTime=end_time, + ) + try: + metrics = response['MetricDataResults'] + except KeyError, e: + logging.info("%s: KeyError %s; Available are %s", self.name, e, response.keys()) + continue + raw_result.extend(metrics) + return raw_result def _get_metrics(self): load_balancers = [] @@ -699,11 +738,24 @@ class EBS(AWSSection): metrics = self._get_metrics() if not metrics: return {} - return self._client.get_metric_data( - MetricDataQueries=metrics, - StartTime=start_time, - EndTime=end_time, - ) + + # A single GetMetricData call can include up to 100 MetricDataQuery structures + # There's no pagination for this operation: + # self._client.can_paginate('get_metric_data') = False + raw_result = [] + for chunk in _chunks(metrics): + response = self._client.get_metric_data( + MetricDataQueries=chunk, + StartTime=start_time, + EndTime=end_time, + ) + try: + metrics = response['MetricDataResults'] + except KeyError, e: + logging.info("%s: KeyError %s; Available are %s", self.name, e, response.keys()) + continue + raw_result.extend(metrics) + return raw_result def _get_metrics(self): volumes = []