Module: check_mk
Branch: master
Commit: e828341d1b706b88a0d5774976be44f58ecef680
URL:
http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e828341d1b706b…
Author: Simon Betz <si(a)mathias-kettner.de>
Date: Mon Jan 28 08:26:19 2019 +0100
agent_aws: Allow choosing specific buckets and fetch tagging information for S3 services
Change-Id: Ie4be4185bf4216310c31751d185602e7d9f560ae
---
agents/special/agent_aws | 51 ++++++++++++++++++++---------
checks/agent_aws | 10 ++++++
checks/aws_s3 | 10 +++---
cmk/gui/plugins/wato/datasource_programs.py | 15 +++++++--
tests/unit/checks/test_agent_aws.py | 11 ++++++-
5 files changed, 73 insertions(+), 24 deletions(-)
diff --git a/agents/special/agent_aws b/agents/special/agent_aws
index 6f0913d..08eb0e3 100755
--- a/agents/special/agent_aws
+++ b/agents/special/agent_aws
@@ -441,6 +441,20 @@ class EC2(AWSSectionCloudwatch):
class S3(AWSSectionGeneric):
+ def __init__(self, client, distributor=None, buckets=None):
+ super(S3, self).__init__(client, distributor=distributor)
+ self._buckets = self._determine_buckets(buckets)
+
+ def _determine_buckets(self, buckets):
+ if buckets:
+ return [{"Name": b} for b in buckets]
+ response = self._client.list_buckets()
+ try:
+ return response['Buckets']
+ except KeyError as e:
+ logging.info("%s: KeyError %s; Available are %s", self.name, e,
response.keys())
+ return []
+
@property
def name(self):
return "s3"
@@ -450,15 +464,8 @@ class S3(AWSSectionGeneric):
return 86400
def _extract_content_from_raw_result(self):
- response = self._client.list_buckets()
- try:
- buckets = response['Buckets']
- except KeyError as e:
- logging.info("%s: KeyError %s; Available are %s", self.name, e,
response.keys())
- return []
-
paginator = self._client.get_paginator('list_objects')
- for bucket in buckets:
+ for bucket in self._buckets:
bucket_name = bucket['Name']
response = self._client.get_bucket_location(Bucket=bucket_name)
try:
@@ -469,6 +476,18 @@ class S3(AWSSectionGeneric):
else:
bucket['LocationConstraint'] = location
+ try:
+ response = self._client.get_bucket_tagging(Bucket=bucket_name)
+ tagging = response['TagSet']
+ except botocore.exceptions.ClientError as e:
+ # If there are no tags attached to a bucket we receive a
'ClientError'
+ logging.info("%s/%s: No tags set, %s", self.name, bucket_name,
e)
+ except KeyError as e:
+ logging.info("%s/%s: KeyError %s; Available are %s", self.name,
bucket_name, e,
+ response.keys())
+ else:
+ bucket['Tagging'] = tagging
+
contents = []
pages = paginator.paginate(Bucket=bucket_name)
for page in pages:
@@ -480,7 +499,7 @@ class S3(AWSSectionGeneric):
else:
contents += objects
bucket['Contents'] = contents
- return buckets
+ return self._buckets
def _format_content(self, content):
return [AWSSectionResult("", content)]
@@ -958,20 +977,24 @@ class AWSSectionsUSEast(AWSSections):
def init_sections(self, **kwargs):
#---clients---------------------------------------------------------
ce_client = self._init_client('ce')
+ s3_client = self._init_client('s3')
#---sections--------------------------------------------------------
ce = CostsAndUsage(ce_client)
+ s3 = S3(s3_client, buckets=kwargs.get('buckets'))
#---register sections for execution---------------------------------
if 'ce' in self._services:
self._sections.append(ce)
+ if 's3' in self._services:
+ self._sections.append(s3)
+
class AWSSectionsGeneric(AWSSections):
def init_sections(self, **kwargs):
#---clients---------------------------------------------------------
ec2_client = self._init_client('ec2')
- s3_client = self._init_client('s3')
elb_client = self._init_client('elb')
cloudtrail_client = self._init_client('cloudtrail')
glacier_client = self._init_client('glacier')
@@ -990,7 +1013,6 @@ class AWSSectionsGeneric(AWSSections):
elb_summary = ELBSummary(elb_client, elb_summary_distributor)
#---sections--------------------------------------------------------
- s3 = S3(s3_client)
elb_health = ELBHealth(elb_client)
cloudtrail = CloudTrail(cloudtrail_client)
glacier = Glacier(glacier_client)
@@ -1040,10 +1062,6 @@ class AWSSectionsGeneric(AWSSections):
self._sections.append(elb_health)
self._sections.append(elb)
- # No dependencies
- if 's3' in self._services:
- self._sections.append(s3)
-
if 'cloudtrail' in self._services:
self._sections.append(cloudtrail)
@@ -1112,6 +1130,7 @@ def parse_arguments(argv):
nargs='+',
required=True,
help="Services to monitor:\n%s" % "\n".join(["%-15s
%s" % e for e in AWSServices]))
+ parser.add_argument('--buckets', nargs='+', help="Buckets to
monitor")
parser.add_argument(
"--no-cache", action="store_true", help="Execute all
sections, do not rely on cached data.")
parser.add_argument("--hostname", required=True)
@@ -1153,7 +1172,7 @@ def main(args=None):
try:
session = create_session(access_key_id, secret_access_key,
"us-east-1")
aws_sections_us_east = AWSSectionsUSEast(hostname, session, services)
- aws_sections_us_east.init_sections()
+ aws_sections_us_east.init_sections(buckets=args.buckets)
aws_sections_us_east.run(use_cache=use_cache)
except Exception as e:
if args.debug:
diff --git a/checks/agent_aws b/checks/agent_aws
index 6904ada..01b7171 100644
--- a/checks/agent_aws
+++ b/checks/agent_aws
@@ -42,6 +42,16 @@ def agent_aws_arguments(params, hostname, ipaddress):
# in order to create reliable tests
args += sorted(services.keys())
+ # {'s3': {'buckets': 'all'} already handled by
services.keys()
+ # {'s3': {'buckets', ('buckets', ['cmk-dev',
'foo'])}}
+ try:
+ _, buckets = services.get('s3', {}).get('buckets')
+ except (TypeError, ValueError):
+ pass
+ else:
+ args.append("--buckets")
+ args += buckets
+
args += [
"--hostname",
hostname,
diff --git a/checks/aws_s3 b/checks/aws_s3
index 9381d6c..8a8e287 100644
--- a/checks/aws_s3
+++ b/checks/aws_s3
@@ -51,8 +51,7 @@ def check_aws_s3_summary(item, params, parsed):
if not region:
region = "unknown"
buckets_by_region.setdefault(region, []).append(row)
- long_output.append("Bucket: %s, Region: %s, Creation date: %s"\
- % (bucket_name, region, row['CreationDate']))
+ long_output.append("Bucket: %s, Region: %s" % (bucket_name, region))
for region, buckets in buckets_by_region.iteritems():
yield 0, "%s: %s" % (region, len(buckets))
@@ -93,9 +92,10 @@ def check_aws_s3_objects(item, params, metrics):
if size >= largest_size:
largest = row['Key']
largest_size = size
- yield 0, 'Number of object: %s' % num_objects
- yield 0, 'Size: %s' % get_bytes_human_readable(sum_size)
- yield 0, 'Largest: %s, %s' % (largest,
get_bytes_human_readable(largest_size))
+ yield 0, 'Bucket size: %s' % get_bytes_human_readable(sum_size)
+ yield 0, 'Number of objects: %s' % num_objects
+ if largest:
+ yield 0, 'Largest object: %s, %s' % (largest,
get_bytes_human_readable(largest_size))
check_info['aws_s3.objects'] = {
diff --git a/cmk/gui/plugins/wato/datasource_programs.py
b/cmk/gui/plugins/wato/datasource_programs.py
index 7c5eeaf..1ab244f 100644
--- a/cmk/gui/plugins/wato/datasource_programs.py
+++ b/cmk/gui/plugins/wato/datasource_programs.py
@@ -1852,8 +1852,19 @@ class RulespecSpecialAgentsAws(HostRulespec):
None, totext="", title=_("Elastic Compute
Cloud (EC2)"))),
("ebs", FixedValue(
None, totext="", title=_("Elastic Block
Storage (EBS)"))),
- ("s3", FixedValue(None, totext="",
- title=_("Simple Storage Service
(S3)"))),
+ ("s3",
+ Dictionary(
+ title=_("Simple Storage Service (S3)"),
+ elements=[('buckets',
+ CascadingDropdown(
+ title=_("Buckets"),
+ choices=[
+ ('all', _("All
buckets")),
+ ('buckets', _("Specific
buckets"),
+ ListOfStrings()),
+ ]))],
+ optional_keys=[],
+ )),
],
)),
],
diff --git a/tests/unit/checks/test_agent_aws.py b/tests/unit/checks/test_agent_aws.py
index 030b7f2..a5f907d 100644
--- a/tests/unit/checks/test_agent_aws.py
+++ b/tests/unit/checks/test_agent_aws.py
@@ -12,10 +12,19 @@ import pytest
"--aws-access-key-id": "user",
"--aws-secret-access-key": "d1ng",
"--regions": ['region-eu'],
- "--services": {'ec2': None, 's3': None},
+ "--services": {'ec2': None, 's3': {'buckets':
'all'}},
}, [
'--aws-access-key-id', 'user', '--aws-secret-access-key',
'd1ng',
'--regions', 'region-eu', '--services', 'ec2',
's3', '--hostname', 'host'
+]),({
+ "--aws-access-key-id": "user",
+ "--aws-secret-access-key": "d1ng",
+ "--regions": ['region-eu'],
+ "--services": {'ec2': None, 's3': {'buckets':
('buckets', ['A', 'B'])}},
+}, [
+ '--aws-access-key-id', 'user', '--aws-secret-access-key',
'd1ng',
+ '--regions', 'region-eu', '--services', 'ec2',
's3', '--buckets', 'A', 'B',
+ '--hostname', 'host'
])])
def test_aws(check_manager, params, result):
agent = check_manager.get_special_agent("agent_aws")