[checkmk-commits] Check_MK Git: check_mk: ldap: supporting paginated searches now

Module: check_mk Branch: master Commit: a557f4f7061bfad6b04086e254a578158c43a263 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=a557f4f7061bfa… Author: Lars Michelsen <lm(a)mathias-kettner.de> Date: Thu Jun 27 10:25:11 2013 +0200 ldap: supporting paginated searches now Using asynchronous searches / added optional support for paginated searches (Can be enabled in connection settings) --- ChangeLog | 3 ++ web/plugins/userdb/ldap.py | 78 +++++++++++++++++++++++++++- web/plugins/wato/check_mk_configuration.py | 15 +++++- 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index ed9938f..c97fe8f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -63,6 +63,9 @@ Can be activated in global settings * New global multisite options: Default downtime duration and comment Display current date in dashboard + * LDAP: Using asynchronous searches / added optional support for paginated + searches (Can be enabled in connection settings) + WATO: * Allow to configure check-/retry_interval in second precision * FIX: do validation of check items in rule editor diff --git a/web/plugins/userdb/ldap.py b/web/plugins/userdb/ldap.py index ae51e41..6129538 100644 --- a/web/plugins/userdb/ldap.py +++ b/web/plugins/userdb/ldap.py @@ -31,6 +31,15 @@ try: # docs: http://www.python-ldap.org/doc/html/index.html import ldap import ldap.filter + from ldap.controls import SimplePagedResultsControl + + # be compatible to both python-ldap below 2.4 and above + try: + LDAP_CONTROL_PAGED_RESULTS = ldap.LDAP_CONTROL_PAGE_OID + ldap_compat = False + except: + LDAP_CONTROL_PAGED_RESULTS = ldap.CONTROL_PAGEDRESULTS + ldap_compat = True except: pass from lib import * @@ -180,6 +189,68 @@ def ldap_bind(username, password, catch = True): else: raise +def ldap_async_search(base, scope, filt, columns): + ldap_log(' ASYNC SEARCH') + # issue the ldap search command (async) + msgid = ldap_connection.search_ext(base, scope, filt, columns) + + results = [] + while True: + restype, resdata = ldap_connection.result(msgid = msgid, + timeout = config.ldap_connection.get('response_timeout', 5)) + + results.extend(resdata) + if restype == ldap.RES_SEARCH_RESULT or not resdata: + break + + # no limit at the moment + #if sizelimit and len(users) >= sizelimit: + # ldap_connection.abandon_ext(msgid) + # break + time.sleep(0.1) + + return results + +def ldap_paged_async_search(base, scope, filt, columns): + ldap_log(' PAGED ASYNC SEARCH') + page_size = config.ldap_connection.get('page_size', 100) + + if ldap_compat: + lc = SimplePagedResultsControl(size = page_size, cookie = '') + else: + lc = SimplePagedResultsControl( + LDAP_CONTROL_PAGED_RESULTS, True, (page_size, '') + ) + + results = [] + while True: + # issue the ldap search command (async) + msgid = ldap_connection.search_ext(base, scope, filt, columns, serverctrls = [lc]) + + unused_code, response, unused_msgid, serverctrls = ldap_connection.result3( + msgid = msgid, timeout = config.ldap_connection.get('response_timeout', 5) + ) + + for result in response: + results.append(result) + + # Mark current position in pagination control for next loop + cookie = None + for serverctrl in serverctrls: + if serverctrl.controlType == LDAP_CONTROL_PAGED_RESULTS: + if ldap_compat: + cookie = serverctrl.cookie + if cookie: + lc.cookie = cookie + else: + cookie = serverctrl.controlValue[1] + if cookie: + lc.controlValue = (page_size, cookie) + break + if not cookie: + break + return results + def ldap_search(base, filt = '(objectclass=*)', columns = [], scope = None): if scope: config_scope = scope @@ -196,14 +267,17 @@ def ldap_search(base, filt = '(objectclass=*)', columns = [], scope = None): ldap_log('LDAP_SEARCH "%s" "%s" "%s" "%r"' % (base, scope, filt, columns)) start_time = time.time() - # Convert all keys to lower case! result = [] try: - for dn, obj in ldap_connection.search_s(base, scope, filt, columns): + + search_func = config.ldap_connection.get('page_size') \ + and ldap_paged_async_search or ldap_async_search + for dn, obj in search_func(base, scope, filt, columns): if dn is None: continue # skip unwanted answers new_obj = {} for key, val in obj.iteritems(): + # Convert all keys to lower case! new_obj[key.lower().decode('utf-8')] = [ i.decode('utf-8') for i in val ] result.append((dn, new_obj)) except ldap.NO_SUCH_OBJECT, e: diff --git a/web/plugins/wato/check_mk_configuration.py b/web/plugins/wato/check_mk_configuration.py index 991e110..ba15b70 100644 --- a/web/plugins/wato/check_mk_configuration.py +++ b/web/plugins/wato/check_mk_configuration.py @@ -478,8 +478,21 @@ register_configvar(group, ), ], )), + ("page_size", Integer( + title = _("LDAP Page Size"), + help = _("LDAP searches can be performed in paginated mode, for example to improve " + "the performance. This enables pagination and configures the size of the pages."), + minvalue = 1, + default_value = 100, + )), + ("response_timeout", Integer( + title = _("LDAP Response Timeout (sec)"), + help = _("Timeout for LDAP query responses."), + minvalue = 0, + default_value = 5, + )), ], - optional_keys = ['no_persistent', 'use_ssl', 'bind', ], + optional_keys = ['no_persistent', 'use_ssl', 'bind', 'page_size', 'response_timeout'], ), domain = "multisite", )