config: Allow reconfiguration of crypto options

Needs new knet crypto API.

If it's not available, then fall back to the old
API and forbid changing crypto while running.

To avoid us being dependant on the leader node, each
node sends its own crypto_reconfig_phase messages so
we can guarantee that the reconfiguration always completes
on each node.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

configure.ac

@@ -187,6 +187,8 @@ OLDLIBS="$LIBS"
 LIBS="$knet_LIBS $LIBS"
 AC_CHECK_LIB([knet],[knet_handle_enable_access_lists],
 	     [AC_DEFINE_UNQUOTED([HAVE_KNET_ACCESS_LIST], 1, [have knet access list])])
+AC_CHECK_LIB([knet],[knet_handle_crypto_set_config],
+	     [AC_DEFINE_UNQUOTED([HAVE_KNET_CRYPTO_RECONF], 1, [have knet crypto reconfig support])])
 LIBS="$OLDLIBS"
 
 # Checks for library functions.

exec/cfg.c

@@ -74,7 +74,8 @@ enum cfg_message_req_types {
         MESSAGE_REQ_EXEC_CFG_RINGREENABLE = 0,
 	MESSAGE_REQ_EXEC_CFG_KILLNODE = 1,
 	MESSAGE_REQ_EXEC_CFG_SHUTDOWN = 2,
-	MESSAGE_REQ_EXEC_CFG_RELOAD_CONFIG = 3
+	MESSAGE_REQ_EXEC_CFG_RELOAD_CONFIG = 3,
+	MESSAGE_REQ_EXEC_CFG_CRYPTO_RECONFIG = 4
 };
 
 #define DEFAULT_SHUTDOWN_TIMEOUT 5
@@ -130,6 +131,10 @@ static void message_handler_req_exec_cfg_reload_config (
         const void *message,
         unsigned int nodeid);
 
+static void message_handler_req_exec_cfg_reconfig_crypto (
+        const void *message,
+        unsigned int nodeid);
+
 static void exec_cfg_killnode_endian_convert (void *msg);
 
 static void message_handler_req_lib_cfg_ringstatusget (
@@ -225,6 +230,9 @@ static struct corosync_exec_handler cfg_exec_engine[] =
 	},
 	{ /* 3 */
 		.exec_handler_fn = message_handler_req_exec_cfg_reload_config,
+	},
+	{ /* 4 */
+		.exec_handler_fn = message_handler_req_exec_cfg_reconfig_crypto,
 	}
 };
 
@@ -263,6 +271,11 @@ struct req_exec_cfg_reload_config {
 	mar_message_source_t source __attribute__((aligned(8)));
 };
 
+struct req_exec_cfg_crypto_reconfig {
+	struct qb_ipc_request_header header __attribute__((aligned(8)));
+	mar_uint32_t phase __attribute__((aligned(8)));
+};
+
 struct req_exec_cfg_killnode {
 	struct qb_ipc_request_header header __attribute__((aligned(8)));
         mar_uint32_t nodeid __attribute__((aligned(8)));
@@ -577,11 +590,13 @@ static void delete_and_notify_if_changed(icmap_map_t temp_map, const char *key_n
  */
 static void remove_ro_entries(icmap_map_t temp_map)
 {
+#ifndef HAVE_KNET_CRYPTO_RECONF
 	delete_and_notify_if_changed(temp_map, "totem.secauth");
 	delete_and_notify_if_changed(temp_map, "totem.crypto_hash");
 	delete_and_notify_if_changed(temp_map, "totem.crypto_cipher");
 	delete_and_notify_if_changed(temp_map, "totem.keyfile");
 	delete_and_notify_if_changed(temp_map, "totem.key");
+#endif
 	delete_and_notify_if_changed(temp_map, "totem.version");
 	delete_and_notify_if_changed(temp_map, "totem.threads");
 	delete_and_notify_if_changed(temp_map, "totem.ip_version");
@@ -679,6 +694,9 @@ static void message_handler_req_exec_cfg_reload_config (
 	log_printf(LOGSYS_LEVEL_NOTICE, "Config reload requested by node " CS_PRI_NODE_ID, nodeid);
 
 	icmap_set_uint8("config.totemconfig_reload_in_progress", 1);
+
+	/* Make sure there is no rubbish in this that might be checked, even on error */
+	memset(&new_config, 0, sizeof(new_config));
 	/*
 	 * Set up a new hashtable as a staging area.
 	 */
@@ -717,6 +735,7 @@ static void message_handler_req_exec_cfg_reload_config (
 	assert(new_config.orig_interfaces != NULL);
 
 	totempg_get_config(&new_config);
+	new_config.crypto_changed = 0;
 
 	new_config.interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX);
 	assert(new_config.interfaces != NULL);
@@ -740,6 +759,13 @@ static void message_handler_req_exec_cfg_reload_config (
 	/* Read from temp_map into new_config */
 	totem_volatile_config_read(&new_config, temp_map, NULL);
 
+	/* Get updated crypto parameters. Will set a flag in new_config if things have changed */
+	if (totem_reread_crypto_config(&new_config, temp_map, &error_string) == -1) {
+		log_printf (LOGSYS_LEVEL_ERROR, "Crypto configuration is not valid: %s\n", error_string);
+		res = CS_ERR_INVALID_PARAM;
+		goto reload_fini;
+	}
+
 	/* Validate dynamic parameters */
 	if (totem_volatile_config_validate(&new_config, temp_map, &error_string) == -1) {
 		log_printf (LOGSYS_LEVEL_ERROR, "Configuration is not valid: %s\n", error_string);
@@ -747,6 +773,16 @@ static void message_handler_req_exec_cfg_reload_config (
 		goto reload_fini;
 	}
 
+	/* Save this here so we can get at it for the later phases of crypto change */
+	if (new_config.crypto_changed) {
+#ifndef HAVE_KNET_CRYPTO_RECONF
+		new_config.crypto_changed = 0;
+		log_printf (LOGSYS_LEVEL_ERROR, "Crypto reconfiguration is not supported by the linked version of knet\n");
+		res = CS_ERR_INVALID_PARAM;
+		goto reload_fini;
+#endif
+	}
+
 	/*
 	 * Copy new keys into live config.
 	 */
@@ -774,6 +810,27 @@ reload_fini_nofree:
 	icmap_fini_r(temp_map);
 
 reload_fini_nomap:
+
+	/* If crypto was changed, now it's loaded on all nodes we can enable it.
+	 * Each node sends its own PHASE message so we're not relying on the leader
+	 * node to survive the transition
+	 */
+	if (new_config.crypto_changed) {
+		struct req_exec_cfg_crypto_reconfig req_exec_cfg_crypto_reconfig;
+		struct iovec iovec;
+
+		req_exec_cfg_crypto_reconfig.header.size =
+			sizeof (struct req_exec_cfg_crypto_reconfig);
+		req_exec_cfg_crypto_reconfig.header.id = SERVICE_ID_MAKE (CFG_SERVICE,
+									  MESSAGE_REQ_EXEC_CFG_CRYPTO_RECONFIG);
+		req_exec_cfg_crypto_reconfig.phase = CRYPTO_RECONFIG_PHASE_ACTIVATE;
+
+		iovec.iov_base = (char *)&req_exec_cfg_crypto_reconfig;
+		iovec.iov_len = sizeof (struct req_exec_cfg_crypto_reconfig);
+
+		assert (api->totem_mcast (&iovec, 1, TOTEM_SAFE) == 0);
+	}
+
 	/* All done, return result to the caller if it was on this system */
 	if (nodeid == api->totem_nodeid_get()) {
 		res_lib_cfg_reload_config.header.size = sizeof(res_lib_cfg_reload_config);
@@ -788,6 +845,46 @@ reload_fini_nomap:
 	LEAVE();
 }
 
+/* Handle the phases of crypto reload
+ * The first time we are called is after the new crypto config has been loaded
+ * but not activated.
+ *
+ * 1 - activate the new crypto configuration
+ * 2 - clear out the old configuration
+ */
+static void message_handler_req_exec_cfg_reconfig_crypto (
+        const void *message,
+        unsigned int nodeid)
+{
+	const struct req_exec_cfg_crypto_reconfig *req_exec_cfg_crypto_reconfig = message;
+
+	/* Got our own reconfig message */
+	if (nodeid == api->totem_nodeid_get()) {
+		log_printf (LOGSYS_LEVEL_DEBUG, "Crypto reconfiguration phase %d", req_exec_cfg_crypto_reconfig->phase);
+
+		/* Do the deed */
+		totempg_crypto_reconfigure_phase(req_exec_cfg_crypto_reconfig->phase);
+
+		/* Move to the next phase if not finished */
+		if (req_exec_cfg_crypto_reconfig->phase < CRYPTO_RECONFIG_PHASE_CLEANUP) {
+			struct req_exec_cfg_crypto_reconfig req_exec_cfg_crypto_reconfig2;
+			struct iovec iovec;
+
+			req_exec_cfg_crypto_reconfig2.header.size =
+				sizeof (struct req_exec_cfg_crypto_reconfig);
+			req_exec_cfg_crypto_reconfig2.header.id = SERVICE_ID_MAKE (CFG_SERVICE,
+										   MESSAGE_REQ_EXEC_CFG_CRYPTO_RECONFIG);
+			req_exec_cfg_crypto_reconfig2.phase = CRYPTO_RECONFIG_PHASE_CLEANUP;
+
+			iovec.iov_base = (char *)&req_exec_cfg_crypto_reconfig2;
+			iovec.iov_len = sizeof (struct req_exec_cfg_crypto_reconfig);
+
+			assert (api->totem_mcast (&iovec, 1, TOTEM_SAFE) == 0);
+		}
+	}
+}
+
+
 /*
  * Library Interface Implementation
  */

exec/main.c

@@ -1380,12 +1380,6 @@ int main (int argc, char **argv, char **envp)
 		log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist.");
 	}
 
-	res = totem_config_keyread (&totem_config, &error_string);
-	if (res == -1) {
-		log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
-		corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD);
-	}
-
 	res = totem_config_validate (&totem_config, &error_string);
 	if (res == -1) {
 		log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);

exec/totemconfig.c

@@ -527,7 +527,7 @@ parse_error:
 
 }
 
-static int totem_get_crypto(struct totem_config *totem_config, const char **error_string)
+static int totem_get_crypto(struct totem_config *totem_config, icmap_map_t map, const char **error_string)
 {
 	char *str;
 	const char *tmp_cipher;
@@ -538,7 +538,7 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro
 	tmp_cipher = "none";
 	tmp_model = "none";
 
-	if (icmap_get_string("totem.crypto_model", &str) == CS_OK) {
+	if (icmap_get_string_r(map, "totem.crypto_model", &str) == CS_OK) {
 		if (strcmp(str, "nss") == 0) {
 			tmp_model = "nss";
 		}
@@ -550,7 +550,7 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro
 		tmp_model = "nss";
 	}
 
-	if (icmap_get_string("totem.secauth", &str) == CS_OK) {
+	if (icmap_get_string_r(map, "totem.secauth", &str) == CS_OK) {
 		if (strcmp(str, "on") == 0) {
 			tmp_cipher = "aes256";
 			tmp_hash = "sha256";
@@ -558,7 +558,7 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro
 		free(str);
 	}
 
-	if (icmap_get_string("totem.crypto_cipher", &str) == CS_OK) {
+	if (icmap_get_string_r(map, "totem.crypto_cipher", &str) == CS_OK) {
 		if (strcmp(str, "none") == 0) {
 			tmp_cipher = "none";
 		}
@@ -574,7 +574,7 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro
 		free(str);
 	}
 
-	if (icmap_get_string("totem.crypto_hash", &str) == CS_OK) {
+	if (icmap_get_string_r(map, "totem.crypto_hash", &str) == CS_OK) {
 		if (strcmp(str, "none") == 0) {
 			tmp_hash = "none";
 		}
@@ -607,6 +607,12 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro
 		return -1;
 	}
 
+	if (strcmp(tmp_cipher, totem_config->crypto_cipher_type) ||
+	    strcmp(tmp_hash, totem_config->crypto_hash_type) ||
+	    strcmp(tmp_model, totem_config->crypto_model)) {
+		totem_config->crypto_changed = 1;
+	    }
+
 	strncpy(totem_config->crypto_cipher_type, tmp_cipher, CONFIG_STRING_LEN_MAX);
 	strncpy(totem_config->crypto_hash_type, tmp_hash, CONFIG_STRING_LEN_MAX);
 	strncpy(totem_config->crypto_model, tmp_model, CONFIG_STRING_LEN_MAX);
@@ -1761,9 +1767,15 @@ extern int totem_config_read (
 
 	icmap_get_uint32("totem.version", (uint32_t *)&totem_config->version);
 
-	if (totem_get_crypto(totem_config, error_string) != 0) {
+	/* initial crypto load */
+	if (totem_get_crypto(totem_config, icmap_get_global_map(), error_string) != 0) {
+		return -1;
+	}
+	if (totem_config_keyread(totem_config, icmap_get_global_map(), error_string) != 0) {
 		return -1;
 	}
+	totem_config->crypto_index = 1;
+	totem_config->crypto_changed = 0;
 
 	if (icmap_get_string("totem.link_mode", &str) == CS_OK) {
 		if (strlen(str) >= TOTEM_LINK_MODE_BYTES) {
@@ -2134,12 +2146,19 @@ parse_error:
 
 int totem_config_keyread (
 	struct totem_config *totem_config,
+	icmap_map_t map,
 	const char **error_string)
 {
 	int got_key = 0;
 	char *key_location = NULL;
 	int res;
 	size_t key_len;
+	char old_key[TOTEM_PRIVATE_KEY_LEN_MAX];
+	size_t old_key_len;
+
+	/* Take a copy so we can see if it has changed */
+	memcpy(old_key, totem_config->private_key, sizeof(totem_config->private_key));
+	old_key_len = totem_config->private_key_len;
 
 	memset (totem_config->private_key, 0, sizeof(totem_config->private_key));
 	totem_config->private_key_len = 0;
@@ -2150,7 +2169,7 @@ int totem_config_keyread (
 	}
 
 	/* cmap may store the location of the key file */
-	if (icmap_get_string("totem.keyfile", &key_location) == CS_OK) {
+	if (icmap_get_string_r(map, "totem.keyfile", &key_location) == CS_OK) {
 		res = read_keyfile(key_location, totem_config, error_string);
 		free(key_location);
 		if (res)  {
@@ -2158,7 +2177,7 @@ int totem_config_keyread (
 		}
 		got_key = 1;
 	} else { /* Or the key itself may be in the cmap */
-		if (icmap_get("totem.key", NULL, &key_len, NULL) == CS_OK) {
+		if (icmap_get_r(map, "totem.key", NULL, &key_len, NULL) == CS_OK) {
 			if (key_len > sizeof(totem_config->private_key)) {
 				sprintf(error_string_response, "key is too long");
 				goto key_error;
@@ -2167,7 +2186,7 @@ int totem_config_keyread (
 				sprintf(error_string_response, "key is too short");
 				goto key_error;
 			}
-			if (icmap_get("totem.key", totem_config->private_key, &key_len, NULL) == CS_OK) {
+			if (icmap_get_r(map, "totem.key", totem_config->private_key, &key_len, NULL) == CS_OK) {
 				totem_config->private_key_len = key_len;
 				got_key = 1;
 			} else {
@@ -2184,6 +2203,11 @@ int totem_config_keyread (
 			goto key_error;
 	}
 
+	if (old_key_len != totem_config->private_key_len ||
+	    memcmp(old_key, totem_config->private_key, sizeof(totem_config->private_key))) {
+		totem_config->crypto_changed = 1;
+	}
+
 	return (0);
 
 key_error:
@@ -2192,6 +2216,17 @@ key_error:
 
 }
 
+int totem_reread_crypto_config(struct totem_config *totem_config, icmap_map_t map, const char **error_string)
+{
+	if (totem_get_crypto(totem_config, map, error_string) != 0) {
+		return -1;
+	}
+	if (totem_config_keyread(totem_config, map, error_string) != 0) {
+		return -1;
+	}
+	return 0;
+}
+
 static void debug_dump_totem_config(const struct totem_config *totem_config)
 {
 

exec/totemconfig.h

@@ -58,6 +58,7 @@ extern int totem_config_validate (
 
 extern int totem_config_keyread (
 	struct totem_config *totem_config,
+	icmap_map_t map,
 	const char **error_string);
 
 extern int totem_config_find_local_addr_in_nodelist(
@@ -65,12 +66,16 @@ extern int totem_config_find_local_addr_in_nodelist(
 	const char *ipaddr_key_prefix,
 	unsigned int *node_pos);
 
-
 extern void totem_volatile_config_read(
 	struct totem_config *totem_config,
 	icmap_map_t temp_map,
 	const char *deleted_key);
 
+extern int totem_reread_crypto_config(
+	struct totem_config *totem_config,
+	icmap_map_t map,
+	const char **error_string);
+
 extern int totem_volatile_config_validate(
 	struct totem_config *totem_config,
 	icmap_map_t temp_map,

exec/totemknet.c

@@ -195,6 +195,11 @@ struct work_item {
 int totemknet_member_list_rebind_ip (
 	void *knet_context);
 
+
+static int totemknet_configure_compression (
+	void *knet_context,
+	struct totem_config *totem_config);
+
 static void totemknet_start_merge_detect_timeout(
 	void *knet_context);
 
@@ -900,6 +905,68 @@ static void totemknet_add_config_notifications(struct totemknet_instance *instan
 	LEAVE();
 }
 
+static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
+{
+	struct knet_handle_crypto_cfg crypto_cfg;
+	int res;
+
+	/* These have already been validated */
+	memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
+	memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
+	memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
+	memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
+	crypto_cfg.private_key_len = instance->totem_config->private_key_len;
+
+#ifdef HAVE_KNET_CRYPTO_RECONF
+
+	knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
+			crypto_cfg.crypto_model,
+			crypto_cfg.crypto_cipher_type,
+			crypto_cfg.crypto_hash_type,
+			instance->totem_config->crypto_index
+		);
+
+	/* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
+	if (strcmp(instance->totem_config->crypto_cipher_type, "none") == 0) {
+		res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
+		if (res) {
+			knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
+		}
+	}
+
+	/* use_config will be called later when all nodes are synced */
+	res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
+	if (res == -1) {
+		knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
+		goto exit_error;
+	}
+	if (res == -2) {
+		knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
+		goto exit_error;
+	}
+#else
+	knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
+			crypto_cfg.crypto_model,
+			crypto_cfg.crypto_cipher_type,
+			crypto_cfg.crypto_hash_type
+		);
+
+	res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
+	if (res == -1) {
+	knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
+		goto exit_error;
+	}
+	if (res == -2) {
+		knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
+		goto exit_error;
+	}
+#endif
+
+
+exit_error:
+	return res;
+}
+
 /*
  * Create an instance
  */
@@ -1040,36 +1107,47 @@ int totemknet_initialize (
 	}
 
 	/* Enable crypto if requested */
+#ifdef HAVE_KNET_CRYPTO_RECONF
 	if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
-		struct knet_handle_crypto_cfg crypto_cfg;
-
-		assert(strlen(instance->totem_config->crypto_model) < sizeof(crypto_cfg.crypto_model));
-		strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
-
-		assert(strlen(instance->totem_config->crypto_cipher_type) < sizeof(crypto_cfg.crypto_cipher_type));
-		strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
-
-		assert(strlen(instance->totem_config->crypto_hash_type) < sizeof(crypto_cfg.crypto_hash_type));
-		strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
-
-		assert(instance->totem_config->private_key_len <= sizeof(crypto_cfg.private_key));
-		memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
-		crypto_cfg.private_key_len = instance->totem_config->private_key_len;
+	        res = totemknet_set_knet_crypto(instance);
+		if (res == 0) {
+			res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
+			if (res) {
+				knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
+				goto exit_error;
+			}
+		} else {
+			knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
+			goto exit_error;
+		}
+		res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
+		if (res) {
+			knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
+			goto exit_error;
+		}
 
-		res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
-		if (res == -1) {
-			knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
+	} else {
+		res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
+		if (res) {
+			knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
 			goto exit_error;
 		}
-		if (res == -2) {
-			knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
+	}
+#else
+	if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
+		res = totemknet_set_knet_crypto(instance);
+		if (res) {
+			knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
 			goto exit_error;
 		}
-		knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
 	}
+#endif
 
 	/* Set up compression */
-	totemknet_reconfigure(instance, instance->totem_config);
+	if (strcmp(totem_config->knet_compression_model, "none") != 0) {
+		/* Not fatal, but will log */
+		(void)totemknet_configure_compression(knet_context, totem_config);
+	}
 
 	knet_handle_setfwd(instance->knet_handle, 1);
 
@@ -1469,7 +1547,8 @@ int totemknet_member_list_rebind_ip (
 	return (0);
 }
 
-int totemknet_reconfigure (
+
+static int totemknet_configure_compression (
 	void *knet_context,
 	struct totem_config *totem_config)
 {
@@ -1487,17 +1566,98 @@ int totemknet_reconfigure (
 	if (res) {
 		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
 	}
+	return res;
+}
+
+int totemknet_reconfigure (
+	void *knet_context,
+	struct totem_config *totem_config)
+{
+	struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
+	int res = 0;
+
+	(void)totemknet_configure_compression(knet_context, totem_config);
 
 #ifdef HAVE_LIBNOZZLE
-	/* Set up nozzle device(s). Return code is ignored, because unability
+	/* Set up nozzle device(s). Return code is ignored, because inability
 	 * configure nozzle is not fatal problem, errors are logged and
 	 * there is not much else we can do */
 	(void)setup_nozzle(instance);
 #endif
+
+	if (totem_config->crypto_changed) {
+		/* Flip crypto_index */
+		totem_config->crypto_index = 3-totem_config->crypto_index;
+		res = totemknet_set_knet_crypto(instance);
+
+		knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
+				totem_config->crypto_model,
+				totem_config->crypto_cipher_type,
+				totem_config->crypto_hash_type);
+	}
 	return (res);
 }
 
 
+int totemknet_crypto_reconfigure_phase (
+	void *knet_context,
+	struct totem_config *totem_config,
+	cfg_message_crypto_reconfig_phase_t phase)
+{
+#ifdef HAVE_KNET_CRYPTO_RECONF
+	int res;
+	int config_to_use;
+	int config_to_clear;
+	struct knet_handle_crypto_cfg crypto_cfg;
+	struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
+
+	knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
+
+	switch (phase) {
+		case CRYPTO_RECONFIG_PHASE_ACTIVATE:
+			config_to_use = totem_config->crypto_index;
+			if (strcmp(instance->totem_config->crypto_cipher_type, "none") == 0) {
+				config_to_use = 0; /* we are clearing it */
+			}
+
+			/* Enable the new config on this node */
+			res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
+			if (res == -1) {
+				knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
+			}
+			break;
+
+		case CRYPTO_RECONFIG_PHASE_CLEANUP:
+			/*
+			 * All nodes should now have the new config. clear the old one out
+			 * OR disable crypto entirely if that's what the new config insists on.
+			 */
+			config_to_clear = 3-totem_config->crypto_index;
+			knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
+
+			strcpy(crypto_cfg.crypto_model, "none");
+			strcpy(crypto_cfg.crypto_cipher_type, "none");
+			strcpy(crypto_cfg.crypto_hash_type, "none");
+			res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
+			if (res == -1) {
+				knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
+			}
+			if (res == -2) {
+				knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
+			}
+
+			/* If crypto is enabled then disable all cleartext reception */
+			if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
+				res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
+				if (res) {
+					knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
+				}
+			}
+	}
+#endif
+	return 0;
+}
+
 void totemknet_stats_clear (
 	void *knet_context)
 {

exec/totemknet.h

@@ -143,6 +143,11 @@ extern int totemknet_reconfigure (
 	void *knet_context,
 	struct totem_config *totem_config);
 
+extern int totemknet_crypto_reconfigure_phase (
+	void *knet_context,
+	struct totem_config *totem_config,
+	cfg_message_crypto_reconfig_phase_t phase);
+
 extern void totemknet_stats_clear (
 	void *knet_context);
 

exec/totemnet.c

@@ -153,6 +153,11 @@ struct transport {
 		void *net_context,
 		struct totem_config *totem_config);
 
+	int (*crypto_reconfigure_phase) (
+		void *net_context,
+		struct totem_config *totem_config,
+		cfg_message_crypto_reconfig_phase_t phase);
+
 	void (*stats_clear) (
 		void *net_context);
 };
@@ -179,7 +184,8 @@ struct transport transport_entries[] = {
 		.recv_mcast_empty = totemudp_recv_mcast_empty,
 		.member_add = totemudp_member_add,
 		.member_remove = totemudp_member_remove,
-		.reconfigure = totemudp_reconfigure
+		.reconfigure = totemudp_reconfigure,
+		.crypto_reconfigure_phase = NULL
 	},
 	{
 		.name = "UDP/IP Unicast",
@@ -202,7 +208,8 @@ struct transport transport_entries[] = {
 		.recv_mcast_empty = totemudpu_recv_mcast_empty,
 		.member_add = totemudpu_member_add,
 		.member_remove = totemudpu_member_remove,
-		.reconfigure = totemudpu_reconfigure
+		.reconfigure = totemudpu_reconfigure,
+		.crypto_reconfigure_phase = NULL
 	},
 	{
 		.name = "Kronosnet",
@@ -226,6 +233,7 @@ struct transport transport_entries[] = {
 		.member_add = totemknet_member_add,
 		.member_remove = totemknet_member_remove,
 		.reconfigure = totemknet_reconfigure,
+		.crypto_reconfigure_phase = totemknet_crypto_reconfigure_phase,
 		.stats_clear = totemknet_stats_clear
 	}
 };
@@ -571,6 +579,22 @@ int totemnet_reconfigure (
 	return (res);
 }
 
+int totemnet_crypto_reconfigure_phase (
+	void *net_context,
+	struct totem_config *totem_config,
+	cfg_message_crypto_reconfig_phase_t phase)
+{
+	struct totemnet_instance *instance = (struct totemnet_instance *)net_context;
+	unsigned int res = 0;
+
+	if (instance->transport->crypto_reconfigure_phase) {
+		res = instance->transport->crypto_reconfigure_phase (
+			instance->transport_context,
+			totem_config, phase);
+	}
+	return (res);
+}
+
 void totemnet_stats_clear (
 	void *net_context)
 {

exec/totemnet.h

@@ -119,6 +119,8 @@ extern int totemnet_net_mtu_adjust (void *net_context, struct totem_config *tote
 
 extern int totemnet_reconfigure (void *net_context, struct totem_config *totem_config);
 
+extern int totemnet_crypto_reconfigure_phase (void *net_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase);
+
 extern void totemnet_stats_clear (void *net_context);
 
 extern const char *totemnet_iface_print (void *net_context);

exec/totempg.c

@@ -1561,6 +1561,11 @@ extern int totempg_reconfigure (void)
 	return totemsrp_reconfigure (totemsrp_context, totempg_totem_config);
 }
 
+extern int totempg_crypto_reconfigure_phase (cfg_message_crypto_reconfig_phase_t phase)
+{
+	return totemsrp_crypto_reconfigure_phase (totemsrp_context, totempg_totem_config, phase);
+}
+
 extern void totempg_stats_clear (int flags)
 {
 	if (flags & TOTEMPG_STATS_CLEAR_TOTEM) {

exec/totemsrp.c

@@ -5181,6 +5181,15 @@ int totemsrp_reconfigure (void *context, struct totem_config *totem_config)
 	return (res);
 }
 
+int totemsrp_crypto_reconfigure_phase (void *context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
+{
+	struct totemsrp_instance *instance = (struct totemsrp_instance *)context;
+	int res;
+
+	res = totemnet_crypto_reconfigure_phase (instance->totemnet_context, totem_config, phase);
+	return (res);
+}
+
 void totemsrp_stats_clear (void *context, int flags)
 {
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)context;

exec/totemsrp.h

@@ -151,6 +151,11 @@ int totemsrp_reconfigure (
 	void *context,
 	struct totem_config *totem_config);
 
+int totemsrp_crypto_reconfigure_phase (
+	void *context,
+	struct totem_config *totem_config,
+	cfg_message_crypto_reconfig_phase_t phase);
+
 void totemsrp_stats_clear (
 	void *srp_context, int flags);
 

include/corosync/totem/totem.h

@@ -151,6 +151,11 @@ struct memb_ring_id {
 	unsigned long long seq;
 } __attribute__((packed));
 
+typedef enum {
+	CRYPTO_RECONFIG_PHASE_ACTIVATE = 1,
+	CRYPTO_RECONFIG_PHASE_CLEANUP = 2,
+} cfg_message_crypto_reconfig_phase_t;
+
 struct totem_config {
 	int version;
 
@@ -221,6 +226,10 @@ struct totem_config {
 
 	char crypto_hash_type[CONFIG_STRING_LEN_MAX];
 
+	int crypto_index; /* Num of crypto config currently loaded into knet ( 1 or 2 ) */
+
+	int crypto_changed; /* Has crypto changed since last time? (it's expensive to reload) */
+
 	char knet_compression_model[CONFIG_STRING_LEN_MAX];
 
 	uint32_t knet_compression_threshold;

include/corosync/totem/totempg.h

@@ -192,6 +192,8 @@ extern void totempg_trans_ack (void);
 
 extern int totempg_reconfigure (void);
 
+extern int totempg_crypto_reconfigure_phase (cfg_message_crypto_reconfig_phase_t phase);
+
 extern void totempg_force_gather (void);
 
 extern void totempg_get_config(struct totem_config *config);