votequorum: split callbacks into nodelist and quorum

This split is needed for qdevice, so that it gets the ring_id and
nodelist as part of the sync process and not afterwards - when quorum
has been calculated.

As this is and unsupported API I'm not too worried about breaking
existing code - all the clients I know of are using the quorum API
anyway as they should be.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>

cts/agents/votequorum_test_agent.c

@@ -57,13 +57,10 @@
 static quorum_handle_t q_handle = 0;
 static votequorum_handle_t vq_handle = 0;
 
-static void votequorum_notification_fn(
+static void votequorum_quorum_notification_fn(
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate,
-	votequorum_ring_id_t ring_id,
-	uint32_t node_list_entries,
-	votequorum_node_t node_list[])
+	uint32_t quorate)
 {
 	qb_log (LOG_INFO, "VQ notification quorate: %d", quorate);
 }
@@ -121,7 +118,7 @@ static int q_lib_init(void)
 
 	if (vq_handle == 0) {
 		qb_log (LOG_INFO, "votequorum_initialize");
-		vq_callbacks.votequorum_notify_fn = votequorum_notification_fn;
+		vq_callbacks.votequorum_quorum_notify_fn = votequorum_quorum_notification_fn;
 		vq_callbacks.votequorum_expectedvotes_notify_fn = NULL;
 		ret = CS_ERR_NOT_EXIST;
 		while (ret == CS_ERR_NOT_EXIST && retry > 0) {

exec/votequorum.c

@@ -147,6 +147,7 @@ struct req_exec_quorum_qdevice_reconfigure {
 
 static void votequorum_exec_send_expectedvotes_notification(void);
 static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context);
+static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t context);
 
 #define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES 1
 #define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES     2
@@ -1729,16 +1730,60 @@ static int votequorum_exec_send_qdevice_reg(uint32_t operation, const char *qdev
 
 static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context)
 {
-	struct res_lib_votequorum_notification *res_lib_votequorum_notification;
+	struct res_lib_votequorum_quorum_notification *res_lib_votequorum_notification;
+	struct list_head *tmp;
+	int size;
+	char buf[sizeof(struct res_lib_votequorum_quorum_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
+
+	ENTER();
+
+
+	log_printf(LOGSYS_LEVEL_DEBUG, "Sending quorum callback, quorate = %d", cluster_is_quorate);
+
+	size = sizeof(struct res_lib_votequorum_quorum_notification);
+
+	res_lib_votequorum_notification = (struct res_lib_votequorum_quorum_notification *)&buf;
+	res_lib_votequorum_notification->quorate = cluster_is_quorate;
+	res_lib_votequorum_notification->context = context;
+
+	res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_QUORUM_NOTIFICATION;
+	res_lib_votequorum_notification->header.size = size;
+	res_lib_votequorum_notification->header.error = CS_OK;
+
+	/* Send it to all interested parties */
+	if (conn) {
+		int ret = corosync_api->ipc_dispatch_send(conn, &buf, size);
+		LEAVE();
+		return ret;
+	} else {
+		struct quorum_pd *qpd;
+
+		list_iterate(tmp, &trackers_list) {
+			qpd = list_entry(tmp, struct quorum_pd, list);
+			res_lib_votequorum_notification->context = qpd->tracking_context;
+			corosync_api->ipc_dispatch_send(qpd->conn, &buf, size);
+		}
+	}
+
+	LEAVE();
+
+	return 0;
+}
+
+static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t context)
+{
+	struct res_lib_votequorum_nodelist_notification *res_lib_votequorum_notification;
 	struct list_head *tmp;
 	struct cluster_node *node;
 	int cluster_members = 0;
 	int i = 0;
 	int size;
-	char buf[sizeof(struct res_lib_votequorum_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
+	char buf[sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
 
 	ENTER();
 
+	log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = %d/%lld", quorum_ringid.rep.nodeid, quorum_ringid.seq);
+
 	list_iterate(tmp, &cluster_members_list) {
 		node = list_entry(tmp, struct cluster_node, list);
 		cluster_members++;
@@ -1747,10 +1792,9 @@ static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context
 		cluster_members++;
 	}
 
-	size = sizeof(struct res_lib_votequorum_notification) + sizeof(struct votequorum_node) * cluster_members;
+	size = sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(struct votequorum_node) * cluster_members;
 
-	res_lib_votequorum_notification = (struct res_lib_votequorum_notification *)&buf;
-	res_lib_votequorum_notification->quorate = cluster_is_quorate;
+	res_lib_votequorum_notification = (struct res_lib_votequorum_nodelist_notification *)&buf;
 	res_lib_votequorum_notification->node_list_entries = cluster_members;
 	res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.rep.nodeid;
 	res_lib_votequorum_notification->ring_id.seq = quorum_ringid.seq;
@@ -1764,7 +1808,7 @@ static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context
 		res_lib_votequorum_notification->node_list[i].nodeid = VOTEQUORUM_QDEVICE_NODEID;
 		res_lib_votequorum_notification->node_list[i++].state = qdevice->state;
 	}
-	res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_NOTIFICATION;
+	res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_NODELIST_NOTIFICATION;
 	res_lib_votequorum_notification->header.size = size;
 	res_lib_votequorum_notification->header.error = CS_OK;
 
@@ -2352,7 +2396,6 @@ static void votequorum_sync_init (
 
 static int votequorum_sync_process (void)
 {
-
 	if (!sync_nodeinfo_sent) {
 		votequorum_exec_send_nodeinfo(us->node_id);
 		votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
@@ -2360,6 +2403,7 @@ static int votequorum_sync_process (void)
 			votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
 							 qdevice_name);
 		}
+		votequorum_exec_send_nodelist_notification(NULL, 0LL);
 		sync_nodeinfo_sent = 1;
 	}
 
@@ -2686,6 +2730,7 @@ static void message_handler_req_lib_votequorum_trackstart (void *conn,
 	if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CURRENT ||
 	    req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES) {
 		log_printf(LOGSYS_LEVEL_DEBUG, "sending initial status to %p", conn);
+		votequorum_exec_send_nodelist_notification(conn, req_lib_votequorum_trackstart->context);
 		votequorum_exec_send_quorum_notification(conn, req_lib_votequorum_trackstart->context);
 	}
 

include/corosync/ipc_votequorum.h

@@ -63,8 +63,9 @@ enum res_votequorum_types {
 	MESSAGE_RES_VOTEQUORUM_STATUS = 0,
 	MESSAGE_RES_VOTEQUORUM_GETINFO,
 	MESSAGE_RES_VOTEQUORUM_TRACKSTART,
-	MESSAGE_RES_VOTEQUORUM_NOTIFICATION,
-	MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION
+	MESSAGE_RES_VOTEQUORUM_QUORUM_NOTIFICATION,
+	MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION,
+	MESSAGE_RES_VOTEQUORUM_NODELIST_NOTIFICATION,
 };
 
 /**
@@ -208,12 +209,17 @@ struct votequorum_node {
 };
 
 /**
- * @brief The res_lib_votequorum_notification struct
+ * @brief The res_lib_votequorum_quorum_notification struct
  */
-struct res_lib_votequorum_notification {
+struct res_lib_votequorum_quorum_notification {
 	struct qb_ipc_response_header header __attribute__((aligned(8)));
 	mar_uint32_t quorate __attribute__((aligned(8)));
 	mar_uint64_t context __attribute__((aligned(8)));
+};
+
+struct res_lib_votequorum_nodelist_notification {
+	struct qb_ipc_response_header header __attribute__((aligned(8)));
+	mar_uint64_t context __attribute__((aligned(8)));
 	struct mar_votequorum_ring_id ring_id __attribute__((aligned(8)));
 	mar_uint32_t node_list_entries __attribute__((aligned(8)));
 	struct votequorum_node node_list[] __attribute__((aligned(8)));

include/corosync/votequorum.h

@@ -102,12 +102,16 @@ typedef struct {
 } votequorum_ring_id_t;
 
 /**
- * @brief The votequorum_notification_fn_t callback
+ * @brief The votequorum_quorum_notification_fn_t callback
  */
-typedef void (*votequorum_notification_fn_t) (
+typedef void (*votequorum_quorum_notification_fn_t) (
+	votequorum_handle_t handle,
+	uint64_t context,
+	uint32_t quorate);
+
+typedef void (*votequorum_nodelist_notification_fn_t) (
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
 	votequorum_node_t node_list[]);
@@ -124,8 +128,9 @@ typedef void (*votequorum_expectedvotes_notification_fn_t) (
  * @brief The votequorum_callbacks_t struct
  */
 typedef struct {
-	votequorum_notification_fn_t votequorum_notify_fn;
+	votequorum_quorum_notification_fn_t votequorum_quorum_notify_fn;
 	votequorum_expectedvotes_notification_fn_t votequorum_expectedvotes_notify_fn;
+	votequorum_nodelist_notification_fn_t votequorum_nodelist_notify_fn;
 } votequorum_callbacks_t;
 
 /**

lib/votequorum.c

@@ -435,7 +435,8 @@ cs_error_t votequorum_dispatch (
 	struct votequorum_inst *votequorum_inst;
 	votequorum_callbacks_t callbacks;
 	struct qb_ipc_response_header *dispatch_data;
-	struct res_lib_votequorum_notification *res_lib_votequorum_notification;
+	struct res_lib_votequorum_quorum_notification *res_lib_votequorum_quorum_notification;
+	struct res_lib_votequorum_nodelist_notification *res_lib_votequorum_nodelist_notification;
 	struct res_lib_votequorum_expectedvotes_notification *res_lib_votequorum_expectedvotes_notification;
 	char dispatch_buf[IPC_DISPATCH_SIZE];
 	votequorum_ring_id_t ring_id;
@@ -503,20 +504,29 @@ cs_error_t votequorum_dispatch (
 		 */
 		switch (dispatch_data->id) {
 
-		case MESSAGE_RES_VOTEQUORUM_NOTIFICATION:
-			if (callbacks.votequorum_notify_fn == NULL) {
+		case MESSAGE_RES_VOTEQUORUM_QUORUM_NOTIFICATION:
+			if (callbacks.votequorum_quorum_notify_fn == NULL) {
 				break;
 			}
-			res_lib_votequorum_notification = (struct res_lib_votequorum_notification *)dispatch_data;
-			marshall_from_mar_votequorum_ring_id (&ring_id, &res_lib_votequorum_notification->ring_id);
-
-			callbacks.votequorum_notify_fn ( handle,
-							 res_lib_votequorum_notification->context,
-							 res_lib_votequorum_notification->quorate,
-							 ring_id,
-							 res_lib_votequorum_notification->node_list_entries,
-							 (votequorum_node_t *)res_lib_votequorum_notification->node_list );
-				;
+			res_lib_votequorum_quorum_notification = (struct res_lib_votequorum_quorum_notification *)dispatch_data;
+
+			callbacks.votequorum_quorum_notify_fn ( handle,
+								res_lib_votequorum_quorum_notification->context,
+								res_lib_votequorum_quorum_notification->quorate );
+			break;
+
+		case MESSAGE_RES_VOTEQUORUM_NODELIST_NOTIFICATION:
+			if (callbacks.votequorum_nodelist_notify_fn == NULL) {
+				break;
+			}
+			res_lib_votequorum_nodelist_notification = (struct res_lib_votequorum_nodelist_notification *)dispatch_data;
+			marshall_from_mar_votequorum_ring_id (&ring_id, &res_lib_votequorum_nodelist_notification->ring_id);
+
+			callbacks.votequorum_nodelist_notify_fn ( handle,
+								  res_lib_votequorum_nodelist_notification->context,
+								  ring_id,
+								  res_lib_votequorum_nodelist_notification->node_list_entries,
+								  (votequorum_node_t *)res_lib_votequorum_nodelist_notification->node_list );
 			break;
 
 		case MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION:

test/testvotequorum1.c

@@ -76,10 +76,19 @@ static void votequorum_expectedvotes_notification_fn(
 }
 
 
-static void votequorum_notification_fn(
+static void votequorum_quorum_notification_fn(
+	votequorum_handle_t handle,
+	uint64_t context,
+	uint32_t quorate
+	)
+{
+	printf("votequorum quorum notification called \n");
+	printf("  quorate         = %d\n", quorate);
+}
+
+static void votequorum_nodelist_notification_fn(
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
 	votequorum_node_t node_list[]
@@ -87,8 +96,7 @@ static void votequorum_notification_fn(
 {
 	int i;
 
-	printf("votequorum notification called \n");
-	printf("  quorate         = %d\n", quorate);
+	printf("votequorum nodelist notification called \n");
 	printf("  number of nodes = %d\n", node_list_entries);
 	printf("  current ringid  = (%u.%"PRIu64")\n", ring_id.nodeid, ring_id.seq);
 
@@ -110,7 +118,8 @@ int main(int argc, char *argv[])
 		return 0;
 	}
 
-	callbacks.votequorum_notify_fn = votequorum_notification_fn;
+	callbacks.votequorum_quorum_notify_fn = votequorum_quorum_notification_fn;
+	callbacks.votequorum_nodelist_notify_fn = votequorum_nodelist_notification_fn;
 	callbacks.votequorum_expectedvotes_notify_fn = votequorum_expectedvotes_notification_fn;
 
 	if ( (err=votequorum_initialize(&g_handle, &callbacks)) != CS_OK)

test/testvotequorum2.c

@@ -76,16 +76,15 @@ static int print_info(int ok_to_fail)
 	return 0;
 }
 
-static void votequorum_notification_fn(
+static void votequorum_nodelist_notification_fn(
 	votequorum_handle_t vq_handle,
 	uint64_t context,
-	uint32_t quorate,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
 	votequorum_node_t node_list[])
 {
 
-	printf("votequorum notification called \n");
+	printf("votequorum nodelist notification called \n");
 	printf("  current ringid  = (%u.%"PRIu64")\n", ring_id.nodeid, ring_id.seq);
 	printf("\n");
 
@@ -119,7 +118,7 @@ int main(int argc, char *argv[])
 	const char *options = "F:n:p:t:cmq1h";
 
 	memset(&callbacks, 0, sizeof(callbacks));
-	callbacks.votequorum_notify_fn = votequorum_notification_fn;
+	callbacks.votequorum_nodelist_notify_fn = votequorum_nodelist_notification_fn;
 
 	while ((opt = getopt(argc, argv, options)) != -1) {
 		switch (opt) {

tools/corosync-quorumtool.c

@@ -125,14 +125,14 @@ static uint32_t g_vq_called;
 static void votequorum_notification_fn(
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
 	votequorum_node_t node_list[]);
 static votequorum_handle_t v_handle;
 static votequorum_callbacks_t v_callbacks = {
-	.votequorum_notify_fn = votequorum_notification_fn,
-	.votequorum_expectedvotes_notify_fn = NULL
+	.votequorum_quorum_notify_fn = NULL,
+	.votequorum_expectedvotes_notify_fn = NULL,
+	.votequorum_nodelist_notify_fn = votequorum_notification_fn,
 };
 static uint32_t our_nodeid = 0;
 
@@ -356,12 +356,10 @@ static const char *node_name(uint32_t nodeid, name_format_t name_format)
 static void votequorum_notification_fn(
 	votequorum_handle_t handle,
 	uint64_t context,
-	uint32_t quorate,
 	votequorum_ring_id_t ring_id,
 	uint32_t node_list_entries,
 	votequorum_node_t node_list[])
 {
-	fprintf(stderr, "CC: VQ notify, ring nodeid is %d\n", ring_id.nodeid);
 	g_ring_id_rep_node = ring_id.nodeid;
 	g_vq_called = 1;
 }
@@ -708,9 +706,13 @@ static int monitor_status(nodeid_format_t nodeid_format, name_format_t name_form
 			goto quorum_err;
 		}
 		if (using_votequorum()) {
-			err = votequorum_dispatch(v_handle, CS_DISPATCH_ONE);
-			if (err != CS_OK) {
-				fprintf(stderr, "Unable to dispatch votequorum status: %s\n", cs_strerror(err));
+			g_vq_called = 0;
+			while (!g_vq_called) {
+				err = votequorum_dispatch(v_handle, CS_DISPATCH_ONE);
+				if (err != CS_OK) {
+					fprintf(stderr, "Unable to dispatch votequorum status: %s\n", cs_strerror(err));
+					goto quorum_err;
+				}
 			}
 		}