[checkmk-commits] 6694 Increased performance of user profile synchronization between sites

Module: check_mk Branch: master Commit: 9a48b9ddd3c97017d709b6f9889b6d39195c6721 URL: http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=9a48b9ddd3c970… Author: Andreas Boesl <ab(a)mathias-kettner.de> Date: Fri Nov 30 09:27:19 2018 +0100 6694 Increased performance of user profile synchronization between sites User profiles may now be synchronized in bulks. CMK-1358 Change-Id: Ia57bacde37eaf65ac9ce638c18f5ba1515419225 --- .werks/6694 | 10 +++++++ cmk/gui/plugins/userdb/ldap_connector.py | 25 ++++++++++------ cmk/gui/wato/user_profile.py | 4 ++- cmk/gui/watolib.py | 49 ++++++++++++++++++++++++++++++-- tests/unit/cmk/gui/test_watolib.py | 1 + 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/.werks/6694 b/.werks/6694 new file mode 100644 index 0000000..4c6be39 --- /dev/null +++ b/.werks/6694 @@ -0,0 +1,10 @@ +Title: Increased performance of user profile synchronization between sites +Level: 1 +Component: multisite +Compatible: compat +Edition: cre +Version: 1.6.0i1 +Date: 1543567554 +Class: feature + +User profiles may now be synchronized in bulks. diff --git a/cmk/gui/plugins/userdb/ldap_connector.py b/cmk/gui/plugins/userdb/ldap_connector.py index 951f820..adafa28 100644 --- a/cmk/gui/plugins/userdb/ldap_connector.py +++ b/cmk/gui/plugins/userdb/ldap_connector.py @@ -1168,6 +1168,7 @@ class LDAPUserConnector(UserConnector): del users[user_id] # remove the user changes.append(_("LDAP [%s]: Removed user %s") % (connection_id, user_id)) + profiles_to_synchronize = {} for user_id, ldap_user in ldap_users.items(): mode_create, user = load_user(user_id) user_connection_id = userdb.cleanup_connection_id(user.get('connector')) @@ -1238,7 +1239,7 @@ class LDAPUserConnector(UserConnector): # Synchronize new user profile to remote sites if needed if pw_changed and not changed and config.has_wato_slave_sites(): - synchronize_profile_to_sites(self._logger, user_id, user) + profiles_to_synchronize[user_id] = user if changed: for key, (old_value, new_value) in sorted(changed.items()): @@ -1249,6 +1250,8 @@ class LDAPUserConnector(UserConnector): _("LDAP [%s]: Modified user %s (%s)") % (connection_id, user_id, ', '.join(details))) + synchronize_profiles_to_sites(self._logger, profiles_to_synchronize) + duration = time.time() - start_time self._logger.info( 'SYNC FINISHED - Duration: %0.3f sec, Queries: %d' % (duration, self._num_queries)) @@ -2576,18 +2579,21 @@ class SynchronizationResult(object): self.succeeded = succeeded -def synchronize_profile_to_sites(logger, user_id, profile): +def synchronize_profiles_to_sites(logger, profiles_to_synchronize): + if not profiles_to_synchronize: + return + import cmk.gui.watolib as watolib - remote_sites = [(site_id, config.site(site_id)) for site_id in config.get_login_sites()] + remote_sites = [(site_id, config.site(site_id)) for site_id in config.get_login_slave_sites()] - logger.info( - 'Credentials changed: %s. Trying to sync to %d sites' % (user_id, len(remote_sites))) + logger.info('Credentials changed for %s. Trying to sync to %d sites' % (", ".join( + profiles_to_synchronize.keys()), len(remote_sites))) synchronization_jobs = [] states = sites.states() for site_id, site in remote_sites: - synchronization_jobs.append((states, site_id, site, user_id, profile)) + synchronization_jobs.append((states, site_id, site, profiles_to_synchronize)) pool = ThreadPool() results = pool.map(_sychronize_profile_worker, synchronization_jobs) @@ -2614,7 +2620,7 @@ def synchronize_profile_to_sites(logger, user_id, profile): def _sychronize_profile_worker(site_configuration): import cmk.gui.watolib as watolib - states, site_id, site, user_id, profile = site_configuration + states, site_id, site, profiles_to_synchronize = site_configuration if not site.get("replication"): return SynchronizationResult(site_id, disabled=True) @@ -2628,7 +2634,10 @@ def _sychronize_profile_worker(site_configuration): site_id, error_text=_("Site %s is dead") % site_id, failed=True) try: - watolib.push_user_profile_to_site(site, user_id, profile) + result = watolib.push_user_profiles_to_site_transitional_wrapper( + site, profiles_to_synchronize) + if result != True: + return SynchronizationResult(site_id, error_text=result, failed=True) return SynchronizationResult(site_id, succeeded=True) except RequestTimeout: # This function is currently only used by the background job diff --git a/cmk/gui/wato/user_profile.py b/cmk/gui/wato/user_profile.py index b2a0c53..8ce0e37 100644 --- a/cmk/gui/wato/user_profile.py +++ b/cmk/gui/wato/user_profile.py @@ -372,7 +372,9 @@ class ModeAjaxProfileReplication(WatoWebApiMode): raise MKUserError(None, _('The requested user does not exist')) start = time.time() - result = watolib.push_user_profile_to_site(site, user_id, users[user_id]) + result = watolib.push_user_profiles_to_site_transitional_wrapper( + site, {user_id: users[user_id]}) + duration = time.time() - start watolib.ActivateChanges().update_activation_time( site_id, watolib.ACTIVATION_TIME_PROFILE_SYNC, duration) diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py index 7ff47fb..562f31f 100644 --- a/cmk/gui/watolib.py +++ b/cmk/gui/watolib.py @@ -4830,8 +4830,25 @@ class AutomationPushSnapshot(AutomationCommand): shutil.rmtree(tmp_dir) -# TODO: Recode to new sync? -def push_user_profile_to_site(site, user_id, profile): +def push_user_profiles_to_site_transitional_wrapper(site, user_profiles): + try: + return push_user_profiles_to_site(site, user_profiles) + except MKAutomationException as e: + if "Invalid automation command: push-profiles" in "%s" % e: + failed_info = [] + for user_id, user in user_profiles.iteritems(): + result = legacy_push_user_profile_to_site(site, user_id, user) + if result != True: + failed_info.append(result) + + if failed_info: + return "\n".join(failed_info) + return True + else: + raise + + +def legacy_push_user_profile_to_site(site, user_id, profile): url = site["multisiteurl"] + "automation.py?" + html.urlencode_vars([ ("command", "push-profile"), ("secret", site["secret"]), @@ -4856,6 +4873,34 @@ def push_user_profile_to_site(site, user_id, profile): return response +def push_user_profiles_to_site(site, user_profiles): + return do_remote_automation(site, "push-profiles", [("profiles", repr(user_profiles))]) + + +PushUserProfilesRequest = typing.NamedTuple("PushUserProfilesRequest", [("user_profiles", dict)]) + + +(a)automation_command_registry.register +class PushUserProfilesToSite(AutomationCommand): + def command_name(self): + return "push-profiles" + + def get_request(self): + return PushUserProfilesRequest(ast.literal_eval(html.var("profiles"))) + + def execute(self, request): + user_profiles = request.user_profiles + + if not user_profiles: + raise MKGeneralException(_('Invalid call: No profiles set.')) + + users = userdb.load_users(lock=True) + for user_id, profile in user_profiles.iteritems(): + users[user_id] = profile + userdb.save_users(users) + return True + + # Add pending entry to make sync possible later for admins def add_profile_replication_change(site_id, result): add_change( diff --git a/tests/unit/cmk/gui/test_watolib.py b/tests/unit/cmk/gui/test_watolib.py index 731ce48..4d545d4 100644 --- a/tests/unit/cmk/gui/test_watolib.py +++ b/tests/unit/cmk/gui/test_watolib.py @@ -65,6 +65,7 @@ def test_registered_automation_commands(): registered = sorted(watolib.automation_command_registry.keys()) assert registered == sorted([ 'activate-changes', + 'push-profiles', 'check-analyze-config', 'execute-dcd-command', 'network-scan',