votequorum: delegate qdevice_master_wins setting to qdevice

votequorum has no business to device if master_wins setting is correct or not.
only the qdevice can decide and should set the value for votequorum.

Logic is:

- user requests master_wins from config
- corosync starts
- qdevice starts
- qdevice reads cmap values / register with votequorum
- qdevice decides if the node can support master_wins or not and tells votequorum
- at this point votequorum can check if an unquorate node is part of the master_wins
  partition

it is the qdevice responsibility to keep that value up to date in votequorum and the
value can be changed at runtime.

this commit also exchange per node master_wins information to lay down the infrastructure
to verify discrepancies in node config for master_wins (coming next on this channel).

Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

exec/votequorum.c

@@ -153,13 +153,14 @@ static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid,
  * votequorum internal node status/view
  */
 
-#define NODE_FLAGS_QUORATE              1
-#define NODE_FLAGS_LEAVING              2
-#define NODE_FLAGS_WFASTATUS            4
-#define NODE_FLAGS_FIRST                8
-#define NODE_FLAGS_QDEVICE_REGISTERED  16
-#define NODE_FLAGS_QDEVICE_ALIVE       32
-#define NODE_FLAGS_QDEVICE_CAST_VOTE   64
+#define NODE_FLAGS_QUORATE               1
+#define NODE_FLAGS_LEAVING               2
+#define NODE_FLAGS_WFASTATUS             4
+#define NODE_FLAGS_FIRST                 8
+#define NODE_FLAGS_QDEVICE_REGISTERED   16
+#define NODE_FLAGS_QDEVICE_ALIVE        32
+#define NODE_FLAGS_QDEVICE_CAST_VOTE    64
+#define NODE_FLAGS_QDEVICE_MASTER_WINS 128
 
 #define NODEID_QDEVICE 0
 
@@ -319,6 +320,9 @@ static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
 static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
 							     const void *message);
 
+static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn,
+							     const void *message);
+
 static struct corosync_lib_handler quorum_lib_service[] =
 {
 	{ /* 0 */
@@ -356,6 +360,10 @@ static struct corosync_lib_handler quorum_lib_service[] =
 	{ /* 8 */
 		.lib_handler_fn		= message_handler_req_lib_votequorum_qdevice_poll,
 		.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED
+	},
+	{ /* 9 */
+		.lib_handler_fn		= message_handler_req_lib_votequorum_qdevice_master_wins,
+		.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED
 	}
 };
 
@@ -559,14 +567,15 @@ static int check_qdevice_master(void)
 static void decode_flags(uint32_t flags)
 {
 	log_printf(LOGSYS_LEVEL_DEBUG,
-		   "flags: quorate: %s Leaving: %s WFA Status: %s First: %s Qdevice: %s QdeviceAlive: %s QdeviceCastVote: %s",
+		   "flags: quorate: %s Leaving: %s WFA Status: %s First: %s Qdevice: %s QdeviceAlive: %s QdeviceCastVote: %s QdeviceMasterWins: %s",
 		   (flags & NODE_FLAGS_QUORATE)?"Yes":"No",
 		   (flags & NODE_FLAGS_LEAVING)?"Yes":"No",
 		   (flags & NODE_FLAGS_WFASTATUS)?"Yes":"No",
 		   (flags & NODE_FLAGS_FIRST)?"Yes":"No",
 		   (flags & NODE_FLAGS_QDEVICE_REGISTERED)?"Yes":"No",
 		   (flags & NODE_FLAGS_QDEVICE_ALIVE)?"Yes":"No",
-		   (flags & NODE_FLAGS_QDEVICE_CAST_VOTE)?"Yes":"No");
+		   (flags & NODE_FLAGS_QDEVICE_CAST_VOTE)?"Yes":"No",
+		   (flags & NODE_FLAGS_QDEVICE_MASTER_WINS)?"Yes":"No");
 }
 
 static void update_wait_for_all_status(uint8_t wfa_status)
@@ -598,6 +607,12 @@ static void update_qdevice_can_operate(uint8_t status)
 	icmap_set_uint8("runtime.votequorum.qdevice_can_operate", qdevice_can_operate);
 }
 
+static void update_qdevice_master_wins(uint8_t allow)
+{
+	qdevice_master_wins = allow;
+	icmap_set_uint8("runtime.votequorum.qdevice_master_wins", qdevice_master_wins);
+}
+
 /*
  * quorum calculation core bits
  */
@@ -892,9 +907,6 @@ static int votequorum_qdevice_is_configured(uint32_t *qdevice_votes)
 		if (icmap_get_uint32("quorum.device.timeout", &qdevice_timeout) != CS_OK) {
 			qdevice_timeout = DEFAULT_QDEVICE_TIMEOUT;
 		}
-		if (icmap_get_uint8("quorum.device.master_wins", &qdevice_master_wins) != CS_OK) {
-			qdevice_master_wins = 0;
-		}
 		update_qdevice_can_operate(1);
 		return 1;
 	}
@@ -2052,6 +2064,9 @@ static void message_handler_req_lib_votequorum_getinfo (void *conn, const void *
 		if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) {
 			res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QDEVICE_CAST_VOTE;
 		}
+		if (node->flags & NODE_FLAGS_QDEVICE_MASTER_WINS) {
+			res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QDEVICE_MASTER_WINS;
+		}
 	} else {
 		error = CS_ERR_NOT_EXIST;
 	}
@@ -2291,6 +2306,7 @@ static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
 		us->flags &= ~NODE_FLAGS_QDEVICE_REGISTERED;
 		us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
 		us->flags &= ~NODE_FLAGS_QDEVICE_CAST_VOTE;
+		us->flags &= ~NODE_FLAGS_QDEVICE_MASTER_WINS;
 		votequorum_exec_send_nodeinfo(us->node_id);
 		votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER,
 						 req_lib_votequorum_qdevice_unregister->name);
@@ -2391,3 +2407,48 @@ out:
 
 	LEAVE();
 }
+
+static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn,
+							     const void *message)
+{
+	const struct req_lib_votequorum_qdevice_master_wins *req_lib_votequorum_qdevice_master_wins = message;
+	struct res_lib_votequorum_status res_lib_votequorum_status;
+	cs_error_t error = CS_OK;
+	uint32_t oldflags = us->flags;
+
+	ENTER();
+
+	if (!qdevice_can_operate) {
+		error = CS_ERR_ACCESS;
+		goto out;
+	}
+
+	if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
+		if (strncmp(req_lib_votequorum_qdevice_master_wins->name, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
+			error = CS_ERR_INVALID_PARAM;
+			goto out;
+		}
+
+		if (req_lib_votequorum_qdevice_master_wins->allow) {
+			us->flags |= NODE_FLAGS_QDEVICE_MASTER_WINS;
+		} else {
+			us->flags &= ~NODE_FLAGS_QDEVICE_MASTER_WINS;
+		}
+
+		if (us->flags != oldflags) {
+			votequorum_exec_send_nodeinfo(us->node_id);
+		}
+
+		update_qdevice_master_wins(req_lib_votequorum_qdevice_master_wins->allow);
+	} else {
+		error = CS_ERR_NOT_EXIST;
+	}
+
+out:
+	res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status);
+	res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS;
+	res_lib_votequorum_status.header.error = error;
+	corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status));
+
+	LEAVE();
+}

include/corosync/ipc_votequorum.h

@@ -47,7 +47,8 @@ enum req_votequorum_types {
 	MESSAGE_REQ_VOTEQUORUM_QDEVICE_REGISTER,
 	MESSAGE_REQ_VOTEQUORUM_QDEVICE_UNREGISTER,
 	MESSAGE_REQ_VOTEQUORUM_QDEVICE_UPDATE,
-	MESSAGE_REQ_VOTEQUORUM_QDEVICE_POLL
+	MESSAGE_REQ_VOTEQUORUM_QDEVICE_POLL,
+	MESSAGE_REQ_VOTEQUORUM_QDEVICE_MASTER_WINS
 };
 
 enum res_votequorum_types {
@@ -80,6 +81,12 @@ struct req_lib_votequorum_qdevice_poll {
 	int cast_vote;
 };
 
+struct req_lib_votequorum_qdevice_master_wins {
+	struct qb_ipc_request_header header __attribute__((aligned(8)));
+	char name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
+	unsigned int allow;
+};
+
 struct req_lib_votequorum_setvotes {
 	struct qb_ipc_request_header header __attribute__((aligned(8)));
 	unsigned int votes;
@@ -119,6 +126,7 @@ struct res_lib_votequorum_status {
 #define VOTEQUORUM_INFO_QDEVICE_REGISTERED     64
 #define VOTEQUORUM_INFO_QDEVICE_ALIVE         128
 #define VOTEQUORUM_INFO_QDEVICE_CAST_VOTE     256
+#define VOTEQUORUM_INFO_QDEVICE_MASTER_WINS   512
 
 #define VOTEQUORUM_NODESTATE_MEMBER     1
 #define VOTEQUORUM_NODESTATE_DEAD       2

include/corosync/votequorum.h

@@ -51,6 +51,7 @@ typedef uint64_t votequorum_handle_t;
 #define VOTEQUORUM_INFO_QDEVICE_REGISTERED     64
 #define VOTEQUORUM_INFO_QDEVICE_ALIVE         128
 #define VOTEQUORUM_INFO_QDEVICE_CAST_VOTE     256
+#define VOTEQUORUM_INFO_QDEVICE_MASTER_WINS   512
 
 #define VOTEQUORUM_NODEID_QDEVICE 0
 #define VOTEQUORUM_MAX_QDEVICE_NAME_LEN 255
@@ -205,6 +206,14 @@ cs_error_t votequorum_qdevice_poll (
 	const char *name,
 	unsigned int cast_vote);
 
+/**
+ * Allow qdevice to tell votequorum if master_wins can be enabled or not
+ */
+cs_error_t votequorum_qdevice_master_wins (
+	votequorum_handle_t handle,
+	const char *name,
+	unsigned int allow);
+
 #ifdef __cplusplus
 }
 #endif

lib/votequorum.c

@@ -646,6 +646,53 @@ error_exit:
 	return (error);
 }
 
+cs_error_t votequorum_qdevice_master_wins (
+	votequorum_handle_t handle,
+	const char *name,
+	unsigned int allow)
+{
+	cs_error_t error;
+	struct votequorum_inst *votequorum_inst;
+	struct iovec iov;
+	struct req_lib_votequorum_qdevice_master_wins req_lib_votequorum_qdevice_master_wins;
+	struct res_lib_votequorum_status res_lib_votequorum_status;
+
+	if (strlen(name) >= VOTEQUORUM_MAX_QDEVICE_NAME_LEN) {
+		return CS_ERR_INVALID_PARAM;
+	}
+
+	error = hdb_error_to_cs(hdb_handle_get (&votequorum_handle_t_db, handle, (void *)&votequorum_inst));
+	if (error != CS_OK) {
+		return (error);
+	}
+
+	req_lib_votequorum_qdevice_master_wins.header.size = sizeof (struct req_lib_votequorum_qdevice_master_wins);
+	req_lib_votequorum_qdevice_master_wins.header.id = MESSAGE_REQ_VOTEQUORUM_QDEVICE_MASTER_WINS;
+	strcpy(req_lib_votequorum_qdevice_master_wins.name, name);
+	req_lib_votequorum_qdevice_master_wins.allow = allow;
+
+	iov.iov_base = (char *)&req_lib_votequorum_qdevice_master_wins;
+	iov.iov_len = sizeof (struct req_lib_votequorum_qdevice_master_wins);
+
+	error = qb_to_cs_error(qb_ipcc_sendv_recv (
+		votequorum_inst->c,
+		&iov,
+		1,
+		&res_lib_votequorum_status,
+		sizeof (struct res_lib_votequorum_status), CS_IPC_TIMEOUT_MS));
+
+	if (error != CS_OK) {
+		goto error_exit;
+	}
+
+	error = res_lib_votequorum_status.header.error;
+
+error_exit:
+	hdb_handle_put (&votequorum_handle_t_db, handle);
+
+	return (error);
+}
+
 cs_error_t votequorum_qdevice_unregister (
 	votequorum_handle_t handle,
 	const char *name)

man/votequorum_getinfo.3.in

@@ -85,6 +85,7 @@ The flags are defined as:
 #define VOTEQUORUM_INFO_QDEVICE_REGISTERED     64
 #define VOTEQUORUM_INFO_QDEVICE_ALIVE         128
 #define VOTEQUORUM_INFO_QDEVICE_CAST_VOTE     256
+#define VOTEQUORUM_INFO_QDEVICE_MASTER_WINS   512
 
 .fi
 .PP

test/testvotequorum2.c

@@ -60,6 +60,9 @@ static void print_info(int ok_to_fail)
 		if (info.flags & VOTEQUORUM_INFO_QDEVICE_CAST_VOTE) {
 			printf("cast vote\n");
 		}
+		if (info.flags & VOTEQUORUM_INFO_QDEVICE_MASTER_WINS) {
+			printf("master wins\n");
+		}
 		printf("\n");
 	}
 }
@@ -67,7 +70,7 @@ static void print_info(int ok_to_fail)
 int main(int argc, char *argv[])
 {
 	int ret = 0;
-	int cast_vote = 1;
+	int cast_vote = 1, master_wins = 1;
 	int pollcount=0, polltime=1;
 	int err;
 
@@ -87,6 +90,9 @@ int main(int argc, char *argv[])
 	if (argc >= 4) {
 		cast_vote = atoi(argv[3]);
 	}
+	if (argc >= 5) {
+		master_wins = atoi(argv[4]);
+	}
 
 	if (argc >= 2) {
 		if ( (err=votequorum_qdevice_register(handle, "QDEVICE")) != CS_OK) {
@@ -95,6 +101,12 @@ int main(int argc, char *argv[])
 			goto out;
 		}
 
+		if ( (err=votequorum_qdevice_master_wins(handle, "QDEVICE", master_wins)) != CS_OK) {
+			fprintf(stderr, "qdevice_master_wins FAILED: %d\n", err);
+			ret = -1;
+			goto out;
+		}
+
 		while (pollcount--) {
 			print_info(0);
 			if ((err=votequorum_qdevice_poll(handle, "QDEVICE", cast_vote)) != CS_OK) {

tools/corosync-quorumtool.c

@@ -354,9 +354,10 @@ static void display_nodes_data(uint32_t nodeid, nodeid_format_t nodeid_format, n
 				printf("0x%08x ", VOTEQUORUM_NODEID_QDEVICE);
 			}
 			print_uint32_padded(info.qdevice_votes);
-			printf("%s (%s/%s)\n", info.qdevice_name,
+			printf("%s (%s/%s/%s)\n", info.qdevice_name,
 			       info.flags & VOTEQUORUM_INFO_QDEVICE_ALIVE?"Alive":"Not alive",
-			       info.flags & VOTEQUORUM_INFO_QDEVICE_CAST_VOTE?"Voting":"Not voting");
+			       info.flags & VOTEQUORUM_INFO_QDEVICE_CAST_VOTE?"Voting":"Not voting",
+			       info.flags & VOTEQUORUM_INFO_QDEVICE_MASTER_WINS?"MasterWins":"NoMasterWins");
 		}
 	}
 }