[checkmk-commits] Check_MK Git: check_mk: New option --compress-history for optimizing monitoring history files

Module: check_mk Branch: master Commit: 336c84a2f451e6c0d7e6cc5f8017e22d70d9a3fe URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=336c84a2f451e6… Author: Mathias Kettner &lt;mk(a)mathias-kettner.de&gt; Date: Thu Jul 30 09:13:24 2015 +0200 New option --compress-history for optimizing monitoring history files Note: this is still experimental. Also only use this when using CMC, please. --- modules/check_mk.py | 9 ++- modules/compresslog.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 2 deletions(-) diff --git a/modules/check_mk.py b/modules/check_mk.py index ee7dd36..7394b81 100755 --- a/modules/check_mk.py +++ b/modules/check_mk.py @@ -4505,6 +4505,7 @@ def usage(): cmk --notify used to send notifications from core cmk --create-rrd [--keepalive|SPEC] create round robin database (only CEE) cmk --convert-rrds [--split] [H...] convert exiting RRD to new format (only CEE) + cmk --compress-history FILES... optimize monitoring history files for CMC cmk --handle-alerts alert handling, always in keepalive mode (only CEE) cmk -i, --inventory [HOST1 HOST2...] Do a HW/SW-Inventory of some ar all hosts cmk --inventory-as-check HOST Do HW/SW-Inventory, behave like check plugin @@ -5756,7 +5757,7 @@ long_options = [ "help", "version", "verbose", "compile", "debug", "interactive" "snmptranslate", "bake-agents", "force", "show-snmp-stats", "usewalk", "scan-parents", "procs=", "automation=", "handle-alerts", "notify", "snmpget=", "profile", "keepalive", "keepalive-fd=", "create-rrd", - "convert-rrds", "split-rrds", + "convert-rrds", "compress-history", "split-rrds", "no-cache", "update", "restart", "reload", "dump", "fake-dns=", "man", "nowiki", "config-check", "backup=", "restore=", "check-inventory=", "check-discovery=", "paths", @@ -5766,7 +5767,7 @@ long_options = [ "help", "version", "verbose", "compile", "debug", "interactive" non_config_options = ['-L', '--list-checks', '-P', '--package', '-M', '--handle-alerts', '--notify', '--man', '-V', '--version' ,'-h', '--help', '--automation', - '--create-rrd', '--convert-rrds', '--keepalive', '--cap' ] + '--create-rrd', '--convert-rrds', '--compress-history', '--keepalive', '--cap' ] try: opts, args = getopt.getopt(sys.argv[1:], short_options, long_options) @@ -5984,6 +5985,10 @@ try: execfile(modules_dir + "/rrd.py") do_convert_rrds(args) done = True + elif o == '--compress-history': + execfile(modules_dir + "/compresslog.py") + do_compress_history(args) + done = True elif o in [ '-A', '--bake-agents' ]: if 'do_bake_agents' not in globals(): bail_out("Agent baking is not implemented in your version of Check_MK. Sorry.") diff --git a/modules/compresslog.py b/modules/compresslog.py new file mode 100755 index 0000000..0a0e195 --- /dev/null +++ b/modules/compresslog.py @@ -0,0 +1,151 @@ +#!/usr/bin/python +# -*- encoding: utf-8; py-indent-offset: 4 -*- +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2014 mk(a)mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# Helper file for more effectively organizing monitoring log files. +# Rewrites existing logfiles for CMC. You can concatenate several +# logfiles and then compress them. Do *not* compress compressed +# files again. + + +def do_compress_history(args): + if not args: + bail_out("Please specify files to compress.") + + for filename in args: + try: + verbose("%s..." % filename) + compress_history_file(filename, filename + ".compressed") + verbose("OK\n") + except Exception, e: + if opt_debug: + raise + bail_out(str(e)) + + +def compress_history_file(input_path, output_path): + known_services = {} + machine_state = "START" + + output = file(output_path, "w") + for line in file(input_path): + skip_this_line = False + timestamp = int(line[1:11]) + line_type, host, service = parse_history_line(line) + + vverbose("%s (%s) %s / %s / %s\n" % (line, machine_state, line_type, host, service)) + + if line_type == "RESTART" or line_type == "LOGGING_INITIAL": + if machine_state != "START": + machine_state = "AFTER_RESTART" + services_after_reload = {} + if line_type == "LOGGING_INITIAL": + skip_this_line = True + + elif line_type == "CURRENT": + if machine_state not in ("START", "CURRENT", "AFTER_RESTART"): + raise Exception("Unexpected line %s (while in state %s)" % (line, machine_state)) + machine_state = "CURRENT" + known_services.setdefault(host, set([])).add(service) + + elif line_type == "INITIAL": + if machine_state == "OPERATION": + pass # happens at CMC. That does not create a log entry on reload + elif machine_state == "START": + machine_state = "INITIAL" + known_services.setdefault(host, set([])).add(service) + services_after_reload = {} + elif machine_state not in ("AFTER_RESTART", "INITIAL"): + raise Exception("Unexpected line %s (while in state %s)" % (line, machine_state)) + else: + machine_state = "INITIAL" + services_after_reload.setdefault(host, set([])).add(service) + if host in known_services and service in known_services[host]: + skip_this_line = True + + elif line_type == "OPERATION": + if machine_state != "START": + if machine_state == "INITIAL": + for host in known_services.keys(): + if host not in services_after_reload: + for service in known_services[host]: + log_vanished_object(output, timestamp, host, service) + del known_services[host] + else: + known = known_services[host] + after_reload = services_after_reload[host] + for service in list(known): + if service not in after_reload: + log_vanished_object(output, timestamp, host, service) + known.remove(service) + machine_state = "OPERATION" + else: + pass + + + + if not skip_this_line: + output.write(line) + + +def parse_history_line(line): + command = get_line_command(line) + if "INITIAL" in command: + host, service = get_host_service_from_history_line(command, line) + return "INITIAL", host, service + elif "CURRENT" in command: + host, service = get_host_service_from_history_line(command, line) + return "CURRENT", host, service + elif "logging intitial" in command \ + or "logging initial" in command: + return "LOGGING_INITIAL", None, None + elif "LOG ROTATION" in command \ + or "LOG VERSION" in command: + return "RESTART", None, None + else: + return "OPERATION", None, None + + +def get_host_service_from_history_line(command, line): + arguments = line.split(":")[1].strip().split(";") + if "HOST" in command: + return arguments[0], None + else: + return arguments[0], arguments[1] + + +def get_line_command(line): + if ":" in line: + return line.split(":")[0].split("]")[1].strip() + else: + return line.split("]")[1].strip() + + +def log_vanished_object(output, timestamp, host, service): + if service: + output.write("[%s] VANISHED SERVICE: %s;%s\n" % (timestamp, host, service)) + else: + output.write("[%s] VANISHED HOST: %s\n" % (timestamp, host)) +