votequorum: add leave_remove option

this also cleanup NODESTATE for good. JOINING was never used

Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>

exec/coroparse.c

@@ -371,6 +371,7 @@ static int main_config_parser_cb(const char *path,
 			}
 
 			if ((strcmp(path, "quorum.two_node") == 0) ||
+			    (strcmp(path, "quorum.leave_remove") == 0) ||
 			    (strcmp(path, "quorum.wait_for_all") == 0) ||
 			    (strcmp(path, "quorum.auto_tie_breaker") == 0) ||
 			    (strcmp(path, "quorum.last_man_standing") == 0)) {

exec/votequorum.c

@@ -83,6 +83,8 @@ static int lowest_node_id = -1;
 static uint8_t last_man_standing = 0;
 static uint32_t last_man_standing_window = DEFAULT_LMS_WIN;
 
+static uint8_t leave_remove = 0;
+static uint32_t ev_barrier = 0;
 /*
  * votequorum_exec defines/structs/forward definitions
  */
@@ -135,13 +137,15 @@ static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid,
  * votequorum internal node status/view
  */
 
+#define NODE_FLAGS_LEAVING 1
+
 #define NODEID_US 0
 #define NODEID_QDEVICE UINT32_MAX
 
 typedef enum {
-	NODESTATE_JOINING=1,
-	NODESTATE_MEMBER,
-	NODESTATE_DEAD
+	NODESTATE_MEMBER=1,
+	NODESTATE_DEAD,
+	NODESTATE_LEAVING
 } nodestate_t;
 
 struct cluster_node {
@@ -214,6 +218,7 @@ static quorum_set_quorate_fn_t quorum_callback;
  */
 
 static char *votequorum_exec_init_fn (struct corosync_api_v1 *api);
+static int votequorum_exec_exit_fn (void);
 
 static void message_handler_req_exec_votequorum_nodeinfo (
 	const void *message,
@@ -328,6 +333,7 @@ static struct corosync_service_engine votequorum_service_engine = {
 	.lib_engine			= quorum_lib_service,
 	.lib_engine_count		= sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler),
 	.exec_init_fn			= votequorum_exec_init_fn,
+	.exec_exit_fn			= votequorum_exec_exit_fn,
 	.exec_engine			= votequorum_exec_engine,
 	.exec_engine_count		= sizeof (votequorum_exec_engine) / sizeof (struct corosync_exec_handler),
 	.confchg_fn			= votequorum_confchg_fn,
@@ -488,6 +494,10 @@ static int calculate_quorum(int allow_decrease, unsigned int max_expected, unsig
 
 	ENTER();
 
+	if ((leave_remove) && (allow_decrease) && (max_expected)) {
+		max_expected = max(ev_barrier, max_expected);
+	}
+
 	list_iterate(nodelist, &cluster_members_list) {
 		node = list_entry(nodelist, struct cluster_node, list);
 
@@ -724,6 +734,7 @@ static char *votequorum_readconfig_static(void)
 		wait_for_all = 1;
 	}
 
+	icmap_get_uint8("quorum.leave_remove", &leave_remove);
 	icmap_get_uint8("quorum.wait_for_all", &wait_for_all);
 	icmap_get_uint8("quorum.auto_tie_breaker", &auto_tie_breaker);
 	icmap_get_uint8("quorum.last_man_standing", &last_man_standing);
@@ -759,6 +770,8 @@ static void votequorum_readconfig_dynamic(void)
 
 	icmap_get_uint32("quorum.expected_votes", &us->expected_votes);
 
+	ev_barrier = us->expected_votes;
+
 #ifdef EXPERIMENTAL_QUORUM_DEVICE_API
 	if (icmap_get_uint32("quorum.quorumdev_poll", &quorumdev_poll) != CS_OK) {
 		quorumdev_poll = DEFAULT_QDEV_POLL;
@@ -1012,10 +1025,18 @@ static void message_handler_req_exec_votequorum_nodeinfo (
 	nodestate_t old_state;
 	int new_node = 0;
 	int allow_downgrade = 0;
+	int by_node = 0;
 
 	ENTER();
 
 	log_printf(LOGSYS_LEVEL_DEBUG, "got nodeinfo message from cluster node %u", nodeid);
+	log_printf(LOGSYS_LEVEL_DEBUG, "nodeinfo message[%u]: votes: %d, expected: %d wfa: %d quorate: %d flags: %d",
+					nodeid,
+					req_exec_quorum_nodeinfo->votes,
+					req_exec_quorum_nodeinfo->expected_votes,
+					req_exec_quorum_nodeinfo->wait_for_all_status,
+					req_exec_quorum_nodeinfo->quorate,
+					req_exec_quorum_nodeinfo->flags);
 
 	node = find_node_by_nodeid(nodeid);
 	if (!node) {
@@ -1036,7 +1057,14 @@ static void message_handler_req_exec_votequorum_nodeinfo (
 	/* Update node state */
 	node->votes = req_exec_quorum_nodeinfo->votes;
 	node->flags = req_exec_quorum_nodeinfo->flags;
-	node->state = NODESTATE_MEMBER;
+
+	if (node->flags & NODE_FLAGS_LEAVING) {
+		node->state = NODESTATE_LEAVING;
+		allow_downgrade = 1;
+		by_node = 1;
+	} else {
+		node->state = NODESTATE_MEMBER;
+	}
 
 	if ((!cluster_is_quorate) &&
 	    (req_exec_quorum_nodeinfo->quorate)) {
@@ -1050,12 +1078,6 @@ static void message_handler_req_exec_votequorum_nodeinfo (
 		node->expected_votes = us->expected_votes;
 	}
 
-	log_printf(LOGSYS_LEVEL_DEBUG, "nodeinfo message: votes: %d, expected: %d wfa: %d quorate: %d",
-					req_exec_quorum_nodeinfo->votes,
-					req_exec_quorum_nodeinfo->expected_votes,
-					req_exec_quorum_nodeinfo->wait_for_all_status,
-					req_exec_quorum_nodeinfo->quorate);
-
 	if ((last_man_standing) && (req_exec_quorum_nodeinfo->votes > 1)) {
 		log_printf(LOGSYS_LEVEL_WARNING, "Last Man Standing feature is supported only when all"
 						 "cluster nodes votes are set to 1. Disabling LMS.");
@@ -1072,7 +1094,7 @@ static void message_handler_req_exec_votequorum_nodeinfo (
 	    old_expected != node->expected_votes ||
 	    old_flags != node->flags ||
 	    old_state != node->state) {
-		recalculate_quorum(allow_downgrade, 0);
+		recalculate_quorum(allow_downgrade, by_node);
 	}
 
 	if (!nodeid) {
@@ -1129,6 +1151,7 @@ static void message_handler_req_exec_votequorum_reconfigure (
 			}
 		}
 		votequorum_exec_send_expectedvotes_notification();
+		ev_barrier = req_exec_quorum_reconfigure->value;
 		recalculate_quorum(1, 0);  /* Allow decrease */
 		break;
 
@@ -1142,6 +1165,21 @@ static void message_handler_req_exec_votequorum_reconfigure (
 	LEAVE();
 }
 
+static int votequorum_exec_exit_fn (void)
+{
+	int ret = 0;
+
+	ENTER();
+
+	if (leave_remove) {
+		us->flags |= NODE_FLAGS_LEAVING;
+		ret = votequorum_exec_send_nodeinfo();
+	}
+
+	LEAVE();
+	return ret;
+}
+
 static char *votequorum_exec_init_fn (struct corosync_api_v1 *api)
 {
 #ifdef COROSYNC_SOLARIS
@@ -1413,6 +1451,9 @@ static void message_handler_req_lib_votequorum_getinfo (void *conn, const void *
 		if (auto_tie_breaker) {
 			res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_AUTO_TIE_BREAKER;
 		}
+		if (leave_remove) {
+			res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_LEAVE_REMOVE;
+		}
 	} else {
 		error = CS_ERR_NOT_EXIST;
 	}

include/corosync/ipc_votequorum.h

@@ -108,6 +108,7 @@ struct res_lib_votequorum_status {
 #define VOTEQUORUM_INFO_WAIT_FOR_ALL	        4
 #define VOTEQUORUM_INFO_LAST_MAN_STANDING       8
 #define VOTEQUORUM_INFO_AUTO_TIE_BREAKER       16
+#define VOTEQUORUM_INFO_LEAVE_REMOVE           32
 
 struct res_lib_votequorum_getinfo {
 	struct qb_ipc_response_header header __attribute__((aligned(8)));

include/corosync/votequorum.h

@@ -51,13 +51,14 @@ typedef uint64_t votequorum_handle_t;
 #define VOTEQUORUM_INFO_WAIT_FOR_ALL            4
 #define VOTEQUORUM_INFO_LAST_MAN_STANDING       8
 #define VOTEQUORUM_INFO_AUTO_TIE_BREAKER       16
+#define VOTEQUORUM_INFO_LEAVE_REMOVE           32
 
 #define VOTEQUORUM_NODEID_US 0
 #define VOTEQUORUM_NODEID_QDEVICE -1
 
-#define NODESTATE_JOINING    1
-#define NODESTATE_MEMBER     2
-#define NODESTATE_DEAD       3
+#define NODESTATE_MEMBER     1
+#define NODESTATE_DEAD       2
+#define NODESTATE_LEAVING    3
 
 /** @} */
 

man/votequorum.5

@@ -270,9 +270,55 @@ quorum {
 }
 
 .fi
+.PP
+.B leave_remove: 1
+.PP
+Enables leave remove (LR) feature (default: 0).
+.PP
+The general behaviour of votequorum is to never decrease expected votes or quorum.
+.PP
+When LR is enabled, both expected votes and quorum are recalculated when
+a node leaves the cluster in a clean state (normal corosync shutdown process) down
+to configured expected_votes.
+.PP
+Example use case:
+.PP
+.nf
+1) N node cluster (where N is any value higher than 3)
+
+2) expected_votes set to 3 in corosync.conf
+
+3) only 3 nodes are running
+
+4) admin requires to increase processing power and adds 10 nodes
+
+5) internal expected_votes is automatically set to 13
+
+6) minimum expected_votes is 3 (from configuration)
+
+- up to this point this is standard votequorum behavior -
+
+7) once the work is done, admin wants to remove nodes from the cluster
+
+8) using an ordered shutdown the admin can reduce the cluster size
+   automatically back to 3, but not below 3, where normal quorum
+   operation will work as usual.
+
+.fi
+.PP
+Example configuration:
+.nf
+
+quorum {
+    provider: corosync_votequorum
+    expected_votes: 3
+    leave_remove: 1
+}
+.fi
+.PP
 .SH VARIOUS NOTES
 .PP
-* WFA / LMS / ATB can be used combined together.
+* WFA / LMS / ATB / LR can be used combined together.
 .PP
 * In order to change the default votes for a node there are two options:
 .nf

man/votequorum_getinfo.3

@@ -67,6 +67,7 @@ struct votequorum_info {
 #define VOTEQUORUM_INFO_WAIT_FOR_ALL            4
 #define VOTEQUORUM_INFO_LAST_MAN_STANDING       8
 #define VOTEQUORUM_INFO_AUTO_TIE_BREAKER       16
+#define VOTEQUORUM_INFO_LEAVE_REMOVE           32
 
 .ta
 .fi

test/testvotequorum1.c

@@ -48,9 +48,6 @@ static votequorum_handle_t g_handle;
 static const char *node_state(int state)
 {
 	switch (state) {
-	case NODESTATE_JOINING:
-		return "Joining";
-		break;
 	case NODESTATE_MEMBER:
 		return "Member";
 		break;
@@ -130,6 +127,7 @@ int main(int argc, char *argv[])
 		if (info.flags & VOTEQUORUM_INFO_WAIT_FOR_ALL) printf("WaitForAll ");
 		if (info.flags & VOTEQUORUM_INFO_LAST_MAN_STANDING) printf("LastManStanding ");
 		if (info.flags & VOTEQUORUM_INFO_AUTO_TIE_BREAKER) printf("AutoTieBreaker ");
+		if (info.flags & VOTEQUORUM_INFO_LEAVE_REMOVE) printf("LeaveRemove ");
 
 		printf("\n");
 	}
@@ -159,6 +157,7 @@ int main(int argc, char *argv[])
 			if (info.flags & VOTEQUORUM_INFO_WAIT_FOR_ALL) printf("WaitForAll ");
 			if (info.flags & VOTEQUORUM_INFO_LAST_MAN_STANDING) printf("LastManStanding ");
 			if (info.flags & VOTEQUORUM_INFO_AUTO_TIE_BREAKER) printf("AutoTieBreaker ");
+			if (info.flags & VOTEQUORUM_INFO_LEAVE_REMOVE) printf("LeaveRemove ");
 			printf("\n");
 		}
 	}

tools/corosync-quorumtool.c

@@ -341,6 +341,7 @@ static int display_quorum_data(int is_quorate, int loop)
 		if (info.flags & VOTEQUORUM_INFO_WAIT_FOR_ALL) printf("WaitForAll ");
 		if (info.flags & VOTEQUORUM_INFO_LAST_MAN_STANDING) printf("LastManStanding ");
 		if (info.flags & VOTEQUORUM_INFO_AUTO_TIE_BREAKER) printf("AutoTieBreaker ");
+		if (info.flags & VOTEQUORUM_INFO_LEAVE_REMOVE) printf("LeaveRemove ");
 		printf("\n");
 	} else {
 		fprintf(stderr, "votequorum_getinfo FAILED: %d\n", err);