[checkmk-commits] added WK 6691: LDAP sync: Sped up user profile synchronization to slave sites

Module: check_mk Branch: master Commit: 68b68730033ab28b7bf1835f399614cb1c758331 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=68b68730033ab2… Author: Andreas Boesl &lt;ab(a)mathias-kettner.de&gt; Date: Thu Nov 22 17:24:20 2018 +0100 added WK 6691: LDAP sync: Sped up user profile synchronization to slave sites Change-Id: I95abf6734227b44d83daa3f5568650c760c4267e --- .werks/6691 | 14 +++++ cmk/gui/plugins/userdb/ldap_connector.py | 94 +++++++++++++++++++++----------- 2 files changed, 75 insertions(+), 33 deletions(-) diff --git a/.werks/6691 b/.werks/6691 new file mode 100644 index 0000000..8dece46 --- /dev/null +++ b/.werks/6691 @@ -0,0 +1,14 @@ +Title: LDAP sync: Sped up user profile synchronization to slave sites +Level: 1 +Component: wato +Class: feature +Compatible: compat +Edition: cre +State: unknown +Version: 1.6.0i1 +Date: 1542890636 + +The background job doing the user synchronization contacted one site after +another when updating the credentials for one user. +This werk changes the update mechanism. It now pushes the updated profile +to all sites at once through multithreading. diff --git a/cmk/gui/plugins/userdb/ldap_connector.py b/cmk/gui/plugins/userdb/ldap_connector.py index eac85c7..b3fc8b9 100644 --- a/cmk/gui/plugins/userdb/ldap_connector.py +++ b/cmk/gui/plugins/userdb/ldap_connector.py @@ -66,6 +66,8 @@ import ldap import ldap.filter from ldap.controls import SimplePagedResultsControl +from multiprocessing.pool import ThreadPool + import cmk import cmk.paths import cmk.log @@ -92,9 +94,11 @@ from cmk.gui.valuespec import ( ) from cmk.gui.i18n import _ from cmk.gui.globals import html -from cmk.gui.exceptions import MKGeneralException, MKUserError +from cmk.gui.exceptions import MKGeneralException, MKUserError, RequestTimeout from cmk.gui.plugins.userdb.utils import UserConnector, user_connector_registry +import cmk.gui.sites as sites + if cmk.is_managed_edition(): import cmk.gui.cme.managed as managed else: @@ -1249,7 +1253,7 @@ class LDAPUserConnector(UserConnector): self._logger.info( 'SYNC FINISHED - Duration: %0.3f sec, Queries: %d' % (duration, self._num_queries)) - import cmk.gui.watolib as watolib # TODO: Cleanup + import cmk.gui.watolib as watolib if changes and config.wato_enabled and not config.is_wato_slave_site(): watolib.add_change("edit-users", "<br>\n".join(changes), add_user=False) @@ -2100,7 +2104,7 @@ def ldap_needed_attributes_alias(connection, params): ldap_attribute_plugins['alias'] = { 'title': _('Alias'), - 'help': _('Populates the alias attribute of the WATO user by syncrhonizing an attribute ' + 'help': _('Populates the alias attribute of the WATO user by synchronizing an attribute ' 'from the LDAP user account. By default the LDAP attribute <tt>cn</tt> is used.'), 'needed_attributes': ldap_needed_attributes_alias, 'sync_func': ldap_sync_alias, @@ -2548,7 +2552,6 @@ ldap_attribute_plugins['groups_to_roles'] = { # | | # '----------------------------------------------------------------------' - # In case the sync is done on the master of a distributed setup the auth serial # is increased on the master, but not on the slaves. The user can not access the # slave sites anymore with the master sites cookie since the serials differ. In @@ -2562,49 +2565,74 @@ ldap_attribute_plugins['groups_to_roles'] = { # time. In this case the implementation does not scale well. We would need to # change this to some kind of profile bulk sync per site. # TODO: Should we move this to watolib? -def synchronize_profile_to_sites(logger, user_id, profile): - import cmk.gui.sites as sites - import cmk.gui.watolib as watolib # TODO: Cleanup + +class SynchronizationResult(object): + def __init__(self, site_id, error_text=None, disabled=False, succeeded=False, failed=False): + self.site_id = site_id + self.error_text = error_text + self.failed = failed + self.disabled = disabled + self.succeeded = succeeded + + +def synchronize_profile_to_sites(logger, user_id, profile): + import cmk.gui.watolib as watolib remote_sites = [(site_id, config.site(site_id)) for site_id in config.get_login_sites()] logger.info( 'Credentials changed: %s. Trying to sync to %d sites' % (user_id, len(remote_sites))) - num_disabled = 0 - num_succeeded = 0 - num_failed = 0 + synchronization_jobs = [] + states = sites.states() + for site_id, site in remote_sites: - if not site.get("replication"): - num_disabled += 1 - continue + synchronization_jobs.append((states, site_id, site, user_id, profile)) - if site.get("disabled"): - num_disabled += 1 - continue + pool = ThreadPool() + results = pool.map(_sychronize_profile_worker, synchronization_jobs) + pool.close() + pool.join() - status = sites.state(site_id, {}).get("state", "unknown") - if status == "dead": - result = "Site is dead" - else: - try: - result = watolib.push_user_profile_to_site(site, user_id, profile) - except Exception as e: - result = "%s" % e - - if result == True: - num_succeeded += 1 - else: - num_failed += 1 - logger.info(' FAILED [%s]: %s' % (site_id, result)) - # Add pending entry to make sync possible later for admins + for result in results: + if result.error_text: + logger.info(' FAILED [%s]: %s' % (result.site_id, result.error_text)) if config.wato_enabled: watolib.add_change( "edit-users", - _('Password changed (sync failed: %s)') % result, + _('Password changed (sync failed: %s)') % result.error_text, add_user=False, - sites=[site_id], + sites=[result.site_id], need_restart=False) + num_failed = sum([1 for result in results if result.failed]) + num_disabled = sum([1 for result in results if result.disabled]) + num_succeeded = sum([1 for result in results if result.succeeded]) logger.info( ' Disabled: %d, Succeeded: %d, Failed: %d' % (num_disabled, num_succeeded, num_failed)) + + +def _sychronize_profile_worker(site_configuration): + import cmk.gui.watolib as watolib + states, site_id, site, user_id, profile = site_configuration + + if not site.get("replication"): + return SynchronizationResult(site_id, disabled=True) + + if site.get("disabled"): + return SynchronizationResult(site_id, disabled=True) + + status = states.get(site_id, {}).get("state", "unknown") + if status == "dead": + return SynchronizationResult( + site_id, error_text=_("Site %s is dead") % site_id, failed=True) + + try: + result = watolib.push_user_profile_to_site(site, user_id, profile) + return SynchronizationResult(site_id, succeeded=True) + except RequestTimeout: + # This function is currently only used by the background job + # which does not have any request timeout set, just in case... + raise + except Exception, e: + return SynchronizationResult(site_id, error_text="%s" % e, failed=True)