Module: check_mk
Branch: master
Commit: 9a48b9ddd3c97017d709b6f9889b6d39195c6721
URL:
http://git.mathias-kettner.de/git/?p=check_mk.git;a=commit;h=9a48b9ddd3c970…
Author: Andreas Boesl <ab(a)mathias-kettner.de>
Date: Fri Nov 30 09:27:19 2018 +0100
6694 Increased performance of user profile synchronization between sites
User profiles may now be synchronized in bulks.
CMK-1358
Change-Id: Ia57bacde37eaf65ac9ce638c18f5ba1515419225
---
.werks/6694 | 10 +++++++
cmk/gui/plugins/userdb/ldap_connector.py | 25 ++++++++++------
cmk/gui/wato/user_profile.py | 4 ++-
cmk/gui/watolib.py | 49 ++++++++++++++++++++++++++++++--
tests/unit/cmk/gui/test_watolib.py | 1 +
5 files changed, 78 insertions(+), 11 deletions(-)
diff --git a/.werks/6694 b/.werks/6694
new file mode 100644
index 0000000..4c6be39
--- /dev/null
+++ b/.werks/6694
@@ -0,0 +1,10 @@
+Title: Increased performance of user profile synchronization between sites
+Level: 1
+Component: multisite
+Compatible: compat
+Edition: cre
+Version: 1.6.0i1
+Date: 1543567554
+Class: feature
+
+User profiles may now be synchronized in bulks.
diff --git a/cmk/gui/plugins/userdb/ldap_connector.py
b/cmk/gui/plugins/userdb/ldap_connector.py
index 951f820..adafa28 100644
--- a/cmk/gui/plugins/userdb/ldap_connector.py
+++ b/cmk/gui/plugins/userdb/ldap_connector.py
@@ -1168,6 +1168,7 @@ class LDAPUserConnector(UserConnector):
del users[user_id] # remove the user
changes.append(_("LDAP [%s]: Removed user %s") %
(connection_id, user_id))
+ profiles_to_synchronize = {}
for user_id, ldap_user in ldap_users.items():
mode_create, user = load_user(user_id)
user_connection_id =
userdb.cleanup_connection_id(user.get('connector'))
@@ -1238,7 +1239,7 @@ class LDAPUserConnector(UserConnector):
# Synchronize new user profile to remote sites if needed
if pw_changed and not changed and config.has_wato_slave_sites():
- synchronize_profile_to_sites(self._logger, user_id, user)
+ profiles_to_synchronize[user_id] = user
if changed:
for key, (old_value, new_value) in sorted(changed.items()):
@@ -1249,6 +1250,8 @@ class LDAPUserConnector(UserConnector):
_("LDAP [%s]: Modified user %s (%s)") % (connection_id,
user_id,
',
'.join(details)))
+ synchronize_profiles_to_sites(self._logger, profiles_to_synchronize)
+
duration = time.time() - start_time
self._logger.info(
'SYNC FINISHED - Duration: %0.3f sec, Queries: %d' % (duration,
self._num_queries))
@@ -2576,18 +2579,21 @@ class SynchronizationResult(object):
self.succeeded = succeeded
-def synchronize_profile_to_sites(logger, user_id, profile):
+def synchronize_profiles_to_sites(logger, profiles_to_synchronize):
+ if not profiles_to_synchronize:
+ return
+
import cmk.gui.watolib as watolib
- remote_sites = [(site_id, config.site(site_id)) for site_id in
config.get_login_sites()]
+ remote_sites = [(site_id, config.site(site_id)) for site_id in
config.get_login_slave_sites()]
- logger.info(
- 'Credentials changed: %s. Trying to sync to %d sites' % (user_id,
len(remote_sites)))
+ logger.info('Credentials changed for %s. Trying to sync to %d sites' %
(", ".join(
+ profiles_to_synchronize.keys()), len(remote_sites)))
synchronization_jobs = []
states = sites.states()
for site_id, site in remote_sites:
- synchronization_jobs.append((states, site_id, site, user_id, profile))
+ synchronization_jobs.append((states, site_id, site, profiles_to_synchronize))
pool = ThreadPool()
results = pool.map(_sychronize_profile_worker, synchronization_jobs)
@@ -2614,7 +2620,7 @@ def synchronize_profile_to_sites(logger, user_id, profile):
def _sychronize_profile_worker(site_configuration):
import cmk.gui.watolib as watolib
- states, site_id, site, user_id, profile = site_configuration
+ states, site_id, site, profiles_to_synchronize = site_configuration
if not site.get("replication"):
return SynchronizationResult(site_id, disabled=True)
@@ -2628,7 +2634,10 @@ def _sychronize_profile_worker(site_configuration):
site_id, error_text=_("Site %s is dead") % site_id, failed=True)
try:
- watolib.push_user_profile_to_site(site, user_id, profile)
+ result = watolib.push_user_profiles_to_site_transitional_wrapper(
+ site, profiles_to_synchronize)
+ if result != True:
+ return SynchronizationResult(site_id, error_text=result, failed=True)
return SynchronizationResult(site_id, succeeded=True)
except RequestTimeout:
# This function is currently only used by the background job
diff --git a/cmk/gui/wato/user_profile.py b/cmk/gui/wato/user_profile.py
index b2a0c53..8ce0e37 100644
--- a/cmk/gui/wato/user_profile.py
+++ b/cmk/gui/wato/user_profile.py
@@ -372,7 +372,9 @@ class ModeAjaxProfileReplication(WatoWebApiMode):
raise MKUserError(None, _('The requested user does not exist'))
start = time.time()
- result = watolib.push_user_profile_to_site(site, user_id, users[user_id])
+ result = watolib.push_user_profiles_to_site_transitional_wrapper(
+ site, {user_id: users[user_id]})
+
duration = time.time() - start
watolib.ActivateChanges().update_activation_time(
site_id, watolib.ACTIVATION_TIME_PROFILE_SYNC, duration)
diff --git a/cmk/gui/watolib.py b/cmk/gui/watolib.py
index 7ff47fb..562f31f 100644
--- a/cmk/gui/watolib.py
+++ b/cmk/gui/watolib.py
@@ -4830,8 +4830,25 @@ class AutomationPushSnapshot(AutomationCommand):
shutil.rmtree(tmp_dir)
-# TODO: Recode to new sync?
-def push_user_profile_to_site(site, user_id, profile):
+def push_user_profiles_to_site_transitional_wrapper(site, user_profiles):
+ try:
+ return push_user_profiles_to_site(site, user_profiles)
+ except MKAutomationException as e:
+ if "Invalid automation command: push-profiles" in "%s" % e:
+ failed_info = []
+ for user_id, user in user_profiles.iteritems():
+ result = legacy_push_user_profile_to_site(site, user_id, user)
+ if result != True:
+ failed_info.append(result)
+
+ if failed_info:
+ return "\n".join(failed_info)
+ return True
+ else:
+ raise
+
+
+def legacy_push_user_profile_to_site(site, user_id, profile):
url = site["multisiteurl"] + "automation.py?" +
html.urlencode_vars([
("command", "push-profile"),
("secret", site["secret"]),
@@ -4856,6 +4873,34 @@ def push_user_profile_to_site(site, user_id, profile):
return response
+def push_user_profiles_to_site(site, user_profiles):
+ return do_remote_automation(site, "push-profiles", [("profiles",
repr(user_profiles))])
+
+
+PushUserProfilesRequest = typing.NamedTuple("PushUserProfilesRequest",
[("user_profiles", dict)])
+
+
+(a)automation_command_registry.register
+class PushUserProfilesToSite(AutomationCommand):
+ def command_name(self):
+ return "push-profiles"
+
+ def get_request(self):
+ return PushUserProfilesRequest(ast.literal_eval(html.var("profiles")))
+
+ def execute(self, request):
+ user_profiles = request.user_profiles
+
+ if not user_profiles:
+ raise MKGeneralException(_('Invalid call: No profiles set.'))
+
+ users = userdb.load_users(lock=True)
+ for user_id, profile in user_profiles.iteritems():
+ users[user_id] = profile
+ userdb.save_users(users)
+ return True
+
+
# Add pending entry to make sync possible later for admins
def add_profile_replication_change(site_id, result):
add_change(
diff --git a/tests/unit/cmk/gui/test_watolib.py b/tests/unit/cmk/gui/test_watolib.py
index 731ce48..4d545d4 100644
--- a/tests/unit/cmk/gui/test_watolib.py
+++ b/tests/unit/cmk/gui/test_watolib.py
@@ -65,6 +65,7 @@ def test_registered_automation_commands():
registered = sorted(watolib.automation_command_registry.keys())
assert registered == sorted([
'activate-changes',
+ 'push-profiles',
'check-analyze-config',
'execute-dcd-command',
'network-scan',