votequorum: Fix up quorum/nodelist callbacks

This patch tidies the two state change callbacks and explains them
in the man page:

The difference between votequorum_nodelist_notification_t and
votequorum_quorum_notification_t is subtle but important.
The 'nodelist' callback is sent at the start of a cluster state
transition and contains the new ring_id and only the list of
nodes that are included in the sync state - ie only active nodes. No
quorum information is included this callback because it is not
available at that time.

The 'quorum' callback is sent after the cluster state transition has
completed and does contain quorum information.
In addition, the nodelist contains a list of all nodes known to
votequorum (whether up or down) and their state as well
as information about the quorum device attached (if any). quorum
callbacks will not be sent for qdevice up and down
events unless they affect quorum.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>

cts/agents/votequorum_test_agent.c

@@ -57,10 +57,12 @@
 static quorum_handle_t q_handle = 0;
 static votequorum_handle_t vq_handle = 0;
 
-static void votequorum_quorum_notification_fn(
+static void votequorum_notification_fn(
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate)
+	uint32_t quorate,
+	uint32_t node_list_entries,
+	votequorum_node_t node_list[])
 {
 	qb_log (LOG_INFO, "VQ notification quorate: %d", quorate);
 }
@@ -118,7 +120,7 @@ static int q_lib_init(void)
 
 	if (vq_handle == 0) {
 		qb_log (LOG_INFO, "votequorum_initialize");
-		vq_callbacks.votequorum_quorum_notify_fn = votequorum_quorum_notification_fn;
+		vq_callbacks.votequorum_quorum_notify_fn = votequorum_notification_fn;
 		vq_callbacks.votequorum_expectedvotes_notify_fn = NULL;
 		ret = CS_ERR_NOT_EXIST;
 		while (ret == CS_ERR_NOT_EXIST && retry > 0) {

exec/votequorum.c

@@ -820,7 +820,7 @@ static int load_ev_tracking_barrier(void)
 
 static void update_wait_for_all_status(uint8_t wfa_status)
 {
-	ENTER(); 
+	ENTER();
 
 	wait_for_all_status = wfa_status;
 	if (wait_for_all_status) {
@@ -1732,24 +1732,45 @@ static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context
 {
 	struct res_lib_votequorum_quorum_notification *res_lib_votequorum_notification;
 	struct list_head *tmp;
+	struct cluster_node *node;
+	int i = 0;
+	int cluster_members = 0;
 	int size;
 	char buf[sizeof(struct res_lib_votequorum_quorum_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
 
 	ENTER();
 
-
 	log_printf(LOGSYS_LEVEL_DEBUG, "Sending quorum callback, quorate = %d", cluster_is_quorate);
 
-	size = sizeof(struct res_lib_votequorum_quorum_notification);
+	list_iterate(tmp, &cluster_members_list) {
+		node = list_entry(tmp, struct cluster_node, list);
+		cluster_members++;
+        }
+	if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
+		cluster_members++;
+	}
+
+	size = sizeof(struct res_lib_votequorum_quorum_notification) + sizeof(struct votequorum_node) * cluster_members;
 
 	res_lib_votequorum_notification = (struct res_lib_votequorum_quorum_notification *)&buf;
 	res_lib_votequorum_notification->quorate = cluster_is_quorate;
 	res_lib_votequorum_notification->context = context;
-
+	res_lib_votequorum_notification->node_list_entries = cluster_members;
 	res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_QUORUM_NOTIFICATION;
 	res_lib_votequorum_notification->header.size = size;
 	res_lib_votequorum_notification->header.error = CS_OK;
 
+	/* Send all known nodes and their states */
+	list_iterate(tmp, &cluster_members_list) {
+		node = list_entry(tmp, struct cluster_node, list);
+		res_lib_votequorum_notification->node_list[i].nodeid = node->node_id;
+		res_lib_votequorum_notification->node_list[i++].state = node->state;
+        }
+	if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
+		res_lib_votequorum_notification->node_list[i].nodeid = VOTEQUORUM_QDEVICE_NODEID;
+		res_lib_votequorum_notification->node_list[i++].state = qdevice->state;
+	}
+
 	/* Send it to all interested parties */
 	if (conn) {
 		int ret = corosync_api->ipc_dispatch_send(conn, &buf, size);
@@ -1773,41 +1794,27 @@ static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context
 static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t context)
 {
 	struct res_lib_votequorum_nodelist_notification *res_lib_votequorum_notification;
-	struct list_head *tmp;
-	struct cluster_node *node;
-	int cluster_members = 0;
 	int i = 0;
 	int size;
-	char buf[sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
+	struct list_head *tmp;
+	char buf[sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(uint32_t) * quorum_members_entries];
 
 	ENTER();
 
 	log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = %d/%lld", quorum_ringid.rep.nodeid, quorum_ringid.seq);
 
-	list_iterate(tmp, &cluster_members_list) {
-		node = list_entry(tmp, struct cluster_node, list);
-		cluster_members++;
-        }
-	if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
-		cluster_members++;
-	}
-
-	size = sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(struct votequorum_node) * cluster_members;
+	size = sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(uint32_t) * quorum_members_entries;
 
 	res_lib_votequorum_notification = (struct res_lib_votequorum_nodelist_notification *)&buf;
-	res_lib_votequorum_notification->node_list_entries = cluster_members;
+	res_lib_votequorum_notification->node_list_entries = quorum_members_entries;
 	res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.rep.nodeid;
 	res_lib_votequorum_notification->ring_id.seq = quorum_ringid.seq;
 	res_lib_votequorum_notification->context = context;
-	list_iterate(tmp, &cluster_members_list) {
-		node = list_entry(tmp, struct cluster_node, list);
-		res_lib_votequorum_notification->node_list[i].nodeid = node->node_id;
-		res_lib_votequorum_notification->node_list[i++].state = node->state;
-        }
-	if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
-		res_lib_votequorum_notification->node_list[i].nodeid = VOTEQUORUM_QDEVICE_NODEID;
-		res_lib_votequorum_notification->node_list[i++].state = qdevice->state;
+
+	for (i=0; i<quorum_members_entries; i++) {
+		res_lib_votequorum_notification->node_list[i] = quorum_members[i];
 	}
+
 	res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_NODELIST_NOTIFICATION;
 	res_lib_votequorum_notification->header.size = size;
 	res_lib_votequorum_notification->header.error = CS_OK;
@@ -1897,7 +1904,7 @@ static void exec_votequorum_qdevice_reg_endian_convert (void *message)
 
 	ENTER();
 
-	req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation);	
+	req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation);
 
 	LEAVE();
 }
@@ -2111,7 +2118,7 @@ static void message_handler_req_exec_votequorum_nodeinfo (
 recalculate:
 	if ((new_node) ||
 	    (nodeid == us->node_id) ||
-	    (node->flags & NODE_FLAGS_FIRST) || 
+	    (node->flags & NODE_FLAGS_FIRST) ||
 	    (old_votes != node->votes) ||
 	    (old_expected != node->expected_votes) ||
 	    (old_flags != node->flags) ||
@@ -2421,6 +2428,7 @@ static int votequorum_sync_process (void)
 static void votequorum_sync_activate (void)
 {
 	recalculate_quorum(0, 0);
+	log_printf(LOGSYS_LEVEL_DEBUG, "CC: quorum_callback sent from sync_activate");
 	quorum_callback(quorum_members, quorum_members_entries,
 			cluster_is_quorate, &quorum_ringid);
 	votequorum_exec_send_quorum_notification(NULL, 0L);

include/corosync/ipc_votequorum.h

@@ -215,6 +215,8 @@ struct res_lib_votequorum_quorum_notification {
 	struct qb_ipc_response_header header __attribute__((aligned(8)));
 	mar_uint32_t quorate __attribute__((aligned(8)));
 	mar_uint64_t context __attribute__((aligned(8)));
+	mar_uint32_t node_list_entries __attribute__((aligned(8)));
+	struct votequorum_node node_list[] __attribute__((aligned(8)));
 };
 
 struct res_lib_votequorum_nodelist_notification {
@@ -222,7 +224,7 @@ struct res_lib_votequorum_nodelist_notification {
 	mar_uint64_t context __attribute__((aligned(8)));
 	struct mar_votequorum_ring_id ring_id __attribute__((aligned(8)));
 	mar_uint32_t node_list_entries __attribute__((aligned(8)));
-	struct votequorum_node node_list[] __attribute__((aligned(8)));
+	mar_uint32_t node_list[] __attribute__((aligned(8)));
 };
 
 /**

include/corosync/votequorum.h

@@ -107,14 +107,16 @@ typedef struct {
 typedef void (*votequorum_quorum_notification_fn_t) (
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate);
+	uint32_t quorate,
+	uint32_t node_list_entries,
+	votequorum_node_t node_list[]);
 
 typedef void (*votequorum_nodelist_notification_fn_t) (
 	votequorum_handle_t handle,
 	uint64_t context,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
-	votequorum_node_t node_list[]);
+	uint32_t node_list[]);
 
 /**
  * @brief The votequorum_expectedvotes_notification_fn_t callback

lib/votequorum.c

@@ -512,7 +512,9 @@ cs_error_t votequorum_dispatch (
 
 			callbacks.votequorum_quorum_notify_fn ( handle,
 								res_lib_votequorum_quorum_notification->context,
-								res_lib_votequorum_quorum_notification->quorate );
+								res_lib_votequorum_quorum_notification->quorate,
+								res_lib_votequorum_quorum_notification->node_list_entries,
+								(votequorum_node_t *)res_lib_votequorum_quorum_notification->node_list );
 			break;
 
 		case MESSAGE_RES_VOTEQUORUM_NODELIST_NOTIFICATION:
@@ -526,7 +528,7 @@ cs_error_t votequorum_dispatch (
 								  res_lib_votequorum_nodelist_notification->context,
 								  ring_id,
 								  res_lib_votequorum_nodelist_notification->node_list_entries,
-								  (votequorum_node_t *)res_lib_votequorum_nodelist_notification->node_list );
+								  res_lib_votequorum_nodelist_notification->node_list );
 			break;
 
 		case MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION:

man/votequorum_initialize.3.in

@@ -51,24 +51,47 @@ argument to uniquely identify the connection.  The
 argument is then used in other function calls to identify the connection to be used
 for communication with the votequorum service.
 .PP
-Every time the voting configuraton changes (eg a node joins or leave the cluster), the callback is called.
+Every time the voting configuration is about to change (eg a node joins or leave the cluster), the callback is called.
 The callback function is described by the following type definitions:
 
 .nf
-typedef void (*votequorum_notification_fn_t) (
+typedef void (*votequorum_nodelist_notification_fn_t) (
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate,
 	uint32_t node_list_entries,
-	votequorum_node_t node_list[]
+	uint32_t node_list[]
 	);
 
 .fi
 
-Current ring_id (one get in votequorum_notification_fn) must be passed to
+Current ring_id (one get in votequorum_quorum_notification_fn) must be passed to
 .B votequorum_qdevice_poll
 to make qdevice voting valid.
 
+.PP
+Every time the quorum state changes (eg a node joins or leave the cluster), the callback is called.
+The callback function is described by the following type definitions:
+
+.nf
+typedef void (*votequorum_quorum_notification_fn_t) (
+	votequorum_handle_t handle,
+	uint64_t context,
+	uint32_t quorate,
+	uint32_t node_list_entries,
+	votequorum_node_t node_list[]
+	);
+
+.fi
+
+The difference between votequorum_nodelist_notification_t and votequorum_quorum_notification_t is subtle but important.
+The 'nodelist' callback is sent at the start of a cluster state transition and contains the new ring_id and only the list of
+nodes that are included in the sync state - ie only active nodes. No quorum information is included this callback 
+because it is not available at that time.
+The 'quorum' callback is sent after the cluster state transition has completed and does contain quorum information. 
+In addition, the nodelist contains a list of all nodes known to votequorum (whether up or down) and their state as well 
+as information about the quorum device attached (if any). quorum callbacks will not be sent for qdevice up and down
+events unless they affect quorum.
+
 .PP
 Every time the expected votes are changed, the callback is called.
 The expected votes callback function is described by the following type definitions:
@@ -86,8 +109,9 @@ argument is of the type:
 
 .nf
 typedef struct {
-	votequorum_notification_fn_t votequorum_notify_fn;
+	votequorum_quorum_notification_fn_t votequorum_quorum_notify_fn;
 	votequorum_expectedvotes_notification_fn_t votequorum_expectedvotes_notify_fn;
+	votequorum_nodelist_notification_fn_t votequorum_nodelist_notify_fn;
 } votequorum_callbacks_t;
 .fi
 .PP

test/testvotequorum1.c

@@ -79,11 +79,21 @@ static void votequorum_expectedvotes_notification_fn(
 static void votequorum_quorum_notification_fn(
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate
+	uint32_t quorate,
+	uint32_t node_list_entries,
+	votequorum_node_t node_list[]
 	)
 {
+	int i;
+
 	printf("votequorum quorum notification called \n");
+	printf("  number of nodes = %d\n", node_list_entries);
 	printf("  quorate         = %d\n", quorate);
+
+	for (i = 0; i< node_list_entries; i++) {
+		printf("      %d: %s\n", node_list[i].nodeid, node_state(node_list[i].state));
+	}
+
 }
 
 static void votequorum_nodelist_notification_fn(
@@ -91,7 +101,7 @@ static void votequorum_nodelist_notification_fn(
 	uint64_t context,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
-	votequorum_node_t node_list[]
+	uint32_t node_list[]
 	)
 {
 	int i;
@@ -99,11 +109,12 @@ static void votequorum_nodelist_notification_fn(
 	printf("votequorum nodelist notification called \n");
 	printf("  number of nodes = %d\n", node_list_entries);
 	printf("  current ringid  = (%u.%"PRIu64")\n", ring_id.nodeid, ring_id.seq);
+	printf("  nodes: ");
 
 	for (i = 0; i< node_list_entries; i++) {
-		printf("      %d: %s\n", node_list[i].nodeid, node_state(node_list[i].state));
+		printf("%d ", node_list[i]);
 	}
-	printf("\n");
+	printf("\n\n");
 }
 
 

test/testvotequorum2.c

@@ -81,7 +81,7 @@ static void votequorum_nodelist_notification_fn(
 	uint64_t context,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
-	votequorum_node_t node_list[])
+	uint32_t node_list[])
 {
 
 	printf("votequorum nodelist notification called \n");

tools/corosync-quorumtool.c

@@ -127,7 +127,7 @@ static void votequorum_notification_fn(
 	uint64_t context,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
-	votequorum_node_t node_list[]);
+	uint32_t node_list[]);
 static votequorum_handle_t v_handle;
 static votequorum_callbacks_t v_callbacks = {
 	.votequorum_quorum_notify_fn = NULL,
@@ -358,7 +358,7 @@ static void votequorum_notification_fn(
 	uint64_t context,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
-	votequorum_node_t node_list[])
+	uint32_t node_list[])
 {
 	g_ring_id_rep_node = ring_id.nodeid;
 	g_vq_called = 1;