pagination not supported for cloudwatch.get_metric_data)
Message-ID: <5c1b4c1e.F5sfVtSmrQD3DFy9%si(a)mathias-kettner.de>
User-Agent: Heirloom mailx 12.5 6/20/10
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Module: check_mk
Branch: master
Commit: ed0f6d0d9e4685ddb8b00a78eb81d2a2fe786b60
URL:
http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=ed0f6d0d9e4685…
Author: Simon Betz <si(a)mathias-kettner.de>
Date: Wed Dec 19 15:24:35 2018 +0100
agent_aws: Split metric queries info 100-sized chunks (pagination not supported for
cloudwatch.get_metric_data)
Change-Id: I15940f23b260eba088199746411c07ad05916493
---
agents/special/agent_aws | 102 +++++++++++++++++++++++++++++++++++------------
1 file changed, 77 insertions(+), 25 deletions(-)
diff --git a/agents/special/agent_aws b/agents/special/agent_aws
index d3fdb18..934a5bf 100755
--- a/agents/special/agent_aws
+++ b/agents/special/agent_aws
@@ -57,6 +57,10 @@ def _datetime_converter(o):
return o.__str__()
+def _chunks(list_, length=100):
+ return [list_[i:i + length] for i in xrange(0, len(list_), length)]
+
+
#.
# .--section API---------------------------------------------------------.
# | _ _ _ ____ ___ |
@@ -233,11 +237,24 @@ class EC2(AWSSection):
metrics = self._get_metrics()
if not metrics:
return {}
- return self._client.get_metric_data(
- MetricDataQueries=metrics,
- StartTime=start_time,
- EndTime=end_time,
- )
+
+ # A single GetMetricData call can include up to 100 MetricDataQuery structures
+ # There's no pagination for this operation:
+ # self._client.can_paginate('get_metric_data') = False
+ raw_result = []
+ for chunk in _chunks(metrics):
+ response = self._client.get_metric_data(
+ MetricDataQueries=chunk,
+ StartTime=start_time,
+ EndTime=end_time,
+ )
+ try:
+ metrics = response['MetricDataResults']
+ except KeyError, e:
+ logging.info("%s: KeyError %s; Available are %s", self.name, e,
response.keys())
+ continue
+ raw_result.extend(metrics)
+ return raw_result
def _get_metrics(self):
instance_ids = []
@@ -357,11 +374,24 @@ class S3(AWSSection):
metrics = self._get_metrics()
if not metrics:
return {}
- return self._client.get_metric_data(
- MetricDataQueries=metrics,
- StartTime=start_time,
- EndTime=end_time,
- )
+
+ # A single GetMetricData call can include up to 100 MetricDataQuery structures
+ # There's no pagination for this operation:
+ # self._client.can_paginate('get_metric_data') = False
+ raw_result = []
+ for chunk in _chunks(metrics):
+ response = self._client.get_metric_data(
+ MetricDataQueries=chunk,
+ StartTime=start_time,
+ EndTime=end_time,
+ )
+ try:
+ metrics = response['MetricDataResults']
+ except KeyError, e:
+ logging.info("%s: KeyError %s; Available are %s", self.name, e,
response.keys())
+ continue
+ raw_result.extend(metrics)
+ return raw_result
def _get_metrics(self):
bucket_ids = []
@@ -403,11 +433,7 @@ class S3(AWSSection):
return metrics
def _extract_content_from(self, raw_result):
- try:
- return raw_result['MetricDataResults']
- except KeyError, e:
- logging.info("%s: KeyError %s; Available are %s", self.name, e,
raw_result.keys())
- return []
+ return raw_result
def _format_content(self, content):
content_by_piggyback_hosts = {}
@@ -536,11 +562,24 @@ class ELB(AWSSection):
metrics = self._get_metrics()
if not metrics:
return {}
- return self._client.get_metric_data(
- MetricDataQueries=metrics,
- StartTime=start_time,
- EndTime=end_time,
- )
+
+ # A single GetMetricData call can include up to 100 MetricDataQuery structures
+ # There's no pagination for this operation:
+ # self._client.can_paginate('get_metric_data') = False
+ raw_result = []
+ for chunk in _chunks(metrics):
+ response = self._client.get_metric_data(
+ MetricDataQueries=chunk,
+ StartTime=start_time,
+ EndTime=end_time,
+ )
+ try:
+ metrics = response['MetricDataResults']
+ except KeyError, e:
+ logging.info("%s: KeyError %s; Available are %s", self.name, e,
response.keys())
+ continue
+ raw_result.extend(metrics)
+ return raw_result
def _get_metrics(self):
load_balancers = []
@@ -699,11 +738,24 @@ class EBS(AWSSection):
metrics = self._get_metrics()
if not metrics:
return {}
- return self._client.get_metric_data(
- MetricDataQueries=metrics,
- StartTime=start_time,
- EndTime=end_time,
- )
+
+ # A single GetMetricData call can include up to 100 MetricDataQuery structures
+ # There's no pagination for this operation:
+ # self._client.can_paginate('get_metric_data') = False
+ raw_result = []
+ for chunk in _chunks(metrics):
+ response = self._client.get_metric_data(
+ MetricDataQueries=chunk,
+ StartTime=start_time,
+ EndTime=end_time,
+ )
+ try:
+ metrics = response['MetricDataResults']
+ except KeyError, e:
+ logging.info("%s: KeyError %s; Available are %s", self.name, e,
response.keys())
+ continue
+ raw_result.extend(metrics)
+ return raw_result
def _get_metrics(self):
volumes = []