Add option to force cluster into GATHER state

Signed-off-by: Chris Walker <cwalker@cray.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

exec/main.c

@@ -969,6 +969,41 @@ static void corosync_fplay_control_init (void)
 			NULL, &track);
 }
 
+static void force_gather_notify_fn(
+	int32_t event,
+	const char *key_name,
+	struct icmap_notify_value new_val,
+	struct icmap_notify_value old_val,
+	void *user_data)
+{
+	char *key_val;
+
+	if (icmap_get_string(key_name, &key_val) == CS_OK && strcmp(key_val, "no") == 0)
+		goto out;
+
+	icmap_set_string("runtime.force_gather", "no");
+
+	if (strcmp(key_name, "runtime.force_gather") == 0) {
+		log_printf(LOGSYS_LEVEL_ERROR, "Forcing into GATHER state\n");
+		totempg_force_gather();
+	}
+
+out:
+	free(key_val);
+}
+
+static void corosync_force_gather_init (void)
+{
+	icmap_track_t track = NULL;
+
+	icmap_set_string("runtime.force_gather", "no");
+
+	icmap_track_add("runtime.force_gather",
+			ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY,
+			force_gather_notify_fn,
+			NULL, &track);
+}
+
 /*
  * Set RO flag for keys, which ether doesn't make sense to change by user (statistic)
  * or which when changed are not reflected by runtime (totem.crypto_cipher, ...).
@@ -1023,6 +1058,7 @@ static void main_service_ready (void)
 	cs_ipcs_init();
 	corosync_totem_stats_init ();
 	corosync_fplay_control_init ();
+	corosync_force_gather_init ();
 	sync_init (
 		corosync_sync_callbacks_retrieve,
 		corosync_sync_completed);

exec/totempg.c

@@ -1552,3 +1552,7 @@ void totempg_trans_ack (void)
 	totemsrp_trans_ack (totemsrp_context);
 }
 
+void totempg_force_gather (void)
+{
+	totemsrp_force_gather(totemsrp_context);
+}

exec/totemsrp.c

@@ -5167,3 +5167,8 @@ void totemsrp_stats_clear (void *context, int flags)
 		totemnet_stats_clear (instance->totemnet_context);
 	}
 }
+
+void totemsrp_force_gather (void *context)
+{
+	timer_function_orf_token_timeout(context);
+}

exec/totemsrp.h

@@ -154,4 +154,7 @@ int totemsrp_reconfigure (
 void totemsrp_stats_clear (
 	void *srp_context, int flags);
 
+void totemsrp_force_gather (
+	void *context);
+
 #endif /* TOTEMSRP_H_DEFINED */

include/corosync/totem/totempg.h

@@ -192,6 +192,8 @@ extern void totempg_trans_ack (void);
 
 extern int totempg_reconfigure (void);
 
+extern void totempg_force_gather (void);
+
 #ifdef __cplusplus
 }
 #endif

man/cmap_keys.8

@@ -66,7 +66,7 @@ key is same as in the configuration file. All values are of string type.
 .TP
 nodelist.*
 Values read from the configuration file. Each node element in the configuration file gets
-assigned it's position starting from zero. So the first node from the config file has
+assigned its position starting from zero. So the first node from the config file has
 nodelist.node.0. prefix. To be a valid entry, each node must have
 .B ring0_addr
 key.
@@ -81,9 +81,14 @@ out nodeid/ring addresses of the local node directly from cmap.
 
 .TP
 runtime.blackbox.*
-Trigger keys for storing fplay data. It's recommended that you the corosync-blackbox command
+Trigger keys for storing fplay data. It's recommended that you use the corosync-blackbox command
 to change keys in this prefix.
 
+.TP
+runtime.force_gather
+Set to 'yes' to force the processor to move into the GATHER state.  This operation
+is dangerous and is not recommended.
+
 .TP
 runtime.config.*
 Contains the values actually in use by the totem membership protocol.
@@ -94,7 +99,7 @@ on individual keys please refer to the man page
 
 .TP
 runtime.services.*
-Prefix with statistics for service engines. Each service has it's own
+Prefix with statistics for service engines. Each service has its own
 .B service_id
 key in the prefix with the name runtime.services.SERVICE., where SERVICE is the lower case
 name of the service. Inside the service prefix is the number of messages received and sent