[checkmk-commits] agent_aws: Allow choosing specific buckets and fetch tagging information for S3 services

Module: check_mk Branch: master Commit: e828341d1b706b88a0d5774976be44f58ecef680 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=e828341d1b706b… Author: Simon Betz <si(a)mathias-kettner.de> Date: Mon Jan 28 08:26:19 2019 +0100 agent_aws: Allow choosing specific buckets and fetch tagging information for S3 services Change-Id: Ie4be4185bf4216310c31751d185602e7d9f560ae --- agents/special/agent_aws | 51 ++++++++++++++++++++--------- checks/agent_aws | 10 ++++++ checks/aws_s3 | 10 +++--- cmk/gui/plugins/wato/datasource_programs.py | 15 +++++++-- tests/unit/checks/test_agent_aws.py | 11 ++++++- 5 files changed, 73 insertions(+), 24 deletions(-) diff --git a/agents/special/agent_aws b/agents/special/agent_aws index 6f0913d..08eb0e3 100755 --- a/agents/special/agent_aws +++ b/agents/special/agent_aws @@ -441,6 +441,20 @@ class EC2(AWSSectionCloudwatch): class S3(AWSSectionGeneric): + def __init__(self, client, distributor=None, buckets=None): + super(S3, self).__init__(client, distributor=distributor) + self._buckets = self._determine_buckets(buckets) + + def _determine_buckets(self, buckets): + if buckets: + return [{"Name": b} for b in buckets] + response = self._client.list_buckets() + try: + return response['Buckets'] + except KeyError as e: + logging.info("%s: KeyError %s; Available are %s", self.name, e, response.keys()) + return [] + @property def name(self): return "s3" @@ -450,15 +464,8 @@ class S3(AWSSectionGeneric): return 86400 def _extract_content_from_raw_result(self): - response = self._client.list_buckets() - try: - buckets = response['Buckets'] - except KeyError as e: - logging.info("%s: KeyError %s; Available are %s", self.name, e, response.keys()) - return [] - paginator = self._client.get_paginator('list_objects') - for bucket in buckets: + for bucket in self._buckets: bucket_name = bucket['Name'] response = self._client.get_bucket_location(Bucket=bucket_name) try: @@ -469,6 +476,18 @@ class S3(AWSSectionGeneric): else: bucket['LocationConstraint'] = location + try: + response = self._client.get_bucket_tagging(Bucket=bucket_name) + tagging = response['TagSet'] + except botocore.exceptions.ClientError as e: + # If there are no tags attached to a bucket we receive a 'ClientError' + logging.info("%s/%s: No tags set, %s", self.name, bucket_name, e) + except KeyError as e: + logging.info("%s/%s: KeyError %s; Available are %s", self.name, bucket_name, e, + response.keys()) + else: + bucket['Tagging'] = tagging + contents = [] pages = paginator.paginate(Bucket=bucket_name) for page in pages: @@ -480,7 +499,7 @@ class S3(AWSSectionGeneric): else: contents += objects bucket['Contents'] = contents - return buckets + return self._buckets def _format_content(self, content): return [AWSSectionResult("", content)] @@ -958,20 +977,24 @@ class AWSSectionsUSEast(AWSSections): def init_sections(self, **kwargs): #---clients--------------------------------------------------------- ce_client = self._init_client('ce') + s3_client = self._init_client('s3') #---sections-------------------------------------------------------- ce = CostsAndUsage(ce_client) + s3 = S3(s3_client, buckets=kwargs.get('buckets')) #---register sections for execution--------------------------------- if 'ce' in self._services: self._sections.append(ce) + if 's3' in self._services: + self._sections.append(s3) + class AWSSectionsGeneric(AWSSections): def init_sections(self, **kwargs): #---clients--------------------------------------------------------- ec2_client = self._init_client('ec2') - s3_client = self._init_client('s3') elb_client = self._init_client('elb') cloudtrail_client = self._init_client('cloudtrail') glacier_client = self._init_client('glacier') @@ -990,7 +1013,6 @@ class AWSSectionsGeneric(AWSSections): elb_summary = ELBSummary(elb_client, elb_summary_distributor) #---sections-------------------------------------------------------- - s3 = S3(s3_client) elb_health = ELBHealth(elb_client) cloudtrail = CloudTrail(cloudtrail_client) glacier = Glacier(glacier_client) @@ -1040,10 +1062,6 @@ class AWSSectionsGeneric(AWSSections): self._sections.append(elb_health) self._sections.append(elb) - # No dependencies - if 's3' in self._services: - self._sections.append(s3) - if 'cloudtrail' in self._services: self._sections.append(cloudtrail) @@ -1112,6 +1130,7 @@ def parse_arguments(argv): nargs='+', required=True, help="Services to monitor:\n%s" % "\n".join(["%-15s %s" % e for e in AWSServices])) + parser.add_argument('--buckets', nargs='+', help="Buckets to monitor") parser.add_argument( "--no-cache", action="store_true", help="Execute all sections, do not rely on cached data.") parser.add_argument("--hostname", required=True) @@ -1153,7 +1172,7 @@ def main(args=None): try: session = create_session(access_key_id, secret_access_key, "us-east-1") aws_sections_us_east = AWSSectionsUSEast(hostname, session, services) - aws_sections_us_east.init_sections() + aws_sections_us_east.init_sections(buckets=args.buckets) aws_sections_us_east.run(use_cache=use_cache) except Exception as e: if args.debug: diff --git a/checks/agent_aws b/checks/agent_aws index 6904ada..01b7171 100644 --- a/checks/agent_aws +++ b/checks/agent_aws @@ -42,6 +42,16 @@ def agent_aws_arguments(params, hostname, ipaddress): # in order to create reliable tests args += sorted(services.keys()) + # {'s3': {'buckets': 'all'} already handled by services.keys() + # {'s3': {'buckets', ('buckets', ['cmk-dev', 'foo'])}} + try: + _, buckets = services.get('s3', {}).get('buckets') + except (TypeError, ValueError): + pass + else: + args.append("--buckets") + args += buckets + args += [ "--hostname", hostname, diff --git a/checks/aws_s3 b/checks/aws_s3 index 9381d6c..8a8e287 100644 --- a/checks/aws_s3 +++ b/checks/aws_s3 @@ -51,8 +51,7 @@ def check_aws_s3_summary(item, params, parsed): if not region: region = "unknown" buckets_by_region.setdefault(region, []).append(row) - long_output.append("Bucket: %s, Region: %s, Creation date: %s"\ - % (bucket_name, region, row['CreationDate'])) + long_output.append("Bucket: %s, Region: %s" % (bucket_name, region)) for region, buckets in buckets_by_region.iteritems(): yield 0, "%s: %s" % (region, len(buckets)) @@ -93,9 +92,10 @@ def check_aws_s3_objects(item, params, metrics): if size >= largest_size: largest = row['Key'] largest_size = size - yield 0, 'Number of object: %s' % num_objects - yield 0, 'Size: %s' % get_bytes_human_readable(sum_size) - yield 0, 'Largest: %s, %s' % (largest, get_bytes_human_readable(largest_size)) + yield 0, 'Bucket size: %s' % get_bytes_human_readable(sum_size) + yield 0, 'Number of objects: %s' % num_objects + if largest: + yield 0, 'Largest object: %s, %s' % (largest, get_bytes_human_readable(largest_size)) check_info['aws_s3.objects'] = { diff --git a/cmk/gui/plugins/wato/datasource_programs.py b/cmk/gui/plugins/wato/datasource_programs.py index 7c5eeaf..1ab244f 100644 --- a/cmk/gui/plugins/wato/datasource_programs.py +++ b/cmk/gui/plugins/wato/datasource_programs.py @@ -1852,8 +1852,19 @@ class RulespecSpecialAgentsAws(HostRulespec): None, totext="", title=_("Elastic Compute Cloud (EC2)"))), ("ebs", FixedValue( None, totext="", title=_("Elastic Block Storage (EBS)"))), - ("s3", FixedValue(None, totext="", - title=_("Simple Storage Service (S3)"))), + ("s3", + Dictionary( + title=_("Simple Storage Service (S3)"), + elements=[('buckets', + CascadingDropdown( + title=_("Buckets"), + choices=[ + ('all', _("All buckets")), + ('buckets', _("Specific buckets"), + ListOfStrings()), + ]))], + optional_keys=[], + )), ], )), ], diff --git a/tests/unit/checks/test_agent_aws.py b/tests/unit/checks/test_agent_aws.py index 030b7f2..a5f907d 100644 --- a/tests/unit/checks/test_agent_aws.py +++ b/tests/unit/checks/test_agent_aws.py @@ -12,10 +12,19 @@ import pytest "--aws-access-key-id": "user", "--aws-secret-access-key": "d1ng", "--regions": ['region-eu'], - "--services": {'ec2': None, 's3': None}, + "--services": {'ec2': None, 's3': {'buckets': 'all'}}, }, [ '--aws-access-key-id', 'user', '--aws-secret-access-key', 'd1ng', '--regions', 'region-eu', '--services', 'ec2', 's3', '--hostname', 'host' +]),({ + "--aws-access-key-id": "user", + "--aws-secret-access-key": "d1ng", + "--regions": ['region-eu'], + "--services": {'ec2': None, 's3': {'buckets': ('buckets', ['A', 'B'])}}, +}, [ + '--aws-access-key-id', 'user', '--aws-secret-access-key', 'd1ng', + '--regions', 'region-eu', '--services', 'ec2', 's3', '--buckets', 'A', 'B', + '--hostname', 'host' ])]) def test_aws(check_manager, params, result): agent = check_manager.get_special_agent("agent_aws")