quorum: change API to return quorum type at initialization time

corosync internal theory of operation is that without a quorum provider
the cluster is always quorate. This is fine for membership free clusters
but it does pose a problem for applications that need membership and
"real" quorum.

this change add quorum_type to quorum_initialize call to return QUORUM_FREE
or QUORUM_SET. Applications can then make their own decisions to error out
or continue operating.

The only other way to know if a quorum provider is enabled/configured is
to poke at confdb/objdb, but adds an unnecessary burden to applications
that really don't need to use an entire library for a boolean value.

Reviewed-by: Steven Dake <sdake@redhat.com>
Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>

cts/agents/votequorum_test_agent.c

@@ -151,9 +151,10 @@ static int q_lib_init(void)
 		}
 	}
 	if (q_handle == 0) {
+		uint32_t q_type;
 		syslog (LOG_INFO, "quorum_initialize");
 		q_callbacks.quorum_notify_fn = quorum_notification_fn;
-		ret = quorum_initialize (&q_handle, &q_callbacks);
+		ret = quorum_initialize (&q_handle, &q_callbacks, &q_type);
 		if (ret != CS_OK) {
 			syslog (LOG_ERR, "quorum_initialize FAILED: %d\n", ret);
 			q_handle = 0;

exec/vsf_quorum.c

@@ -90,6 +90,8 @@ static void message_handler_req_lib_quorum_trackstart (void *conn,
 						       const void *msg);
 static void message_handler_req_lib_quorum_trackstop (void *conn,
 						      const void *msg);
+static void message_handler_req_lib_quorum_gettype (void *conn,
+						       const void *msg);
 static void send_library_notification(void *conn);
 static void send_internal_notification(void);
 static int quorum_exec_init_fn (struct corosync_api_v1 *api);
@@ -97,6 +99,7 @@ static int quorum_lib_init_fn (void *conn);
 static int quorum_lib_exit_fn (void *conn);
 
 static int primary_designated = 0;
+static int quorum_type = 0;
 static struct corosync_api_v1 *corosync_api;
 static struct list_head lib_trackers_list;
 static struct list_head internal_trackers_list;
@@ -171,6 +174,10 @@ static struct corosync_lib_handler quorum_lib_service[] =
 	{ /* 2 */
 		.lib_handler_fn				= message_handler_req_lib_quorum_trackstop,
 		.flow_control				= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
+	},
+	{ /* 3 */
+		.lib_handler_fn				= message_handler_req_lib_quorum_gettype,
+		.flow_control				= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
 	}
 };
 
@@ -329,6 +336,7 @@ static int quorum_exec_init_fn (struct corosync_api_v1 *api)
 
 		quorum_iface = (struct quorum_services_api_ver1 *)quorum_iface_p;
 		quorum_iface->init (api, quorum_api_set_quorum);
+		quorum_type = 1;
 		free(quorum_module);
 	}
 	if (!quorum_iface) {
@@ -336,6 +344,7 @@ static int quorum_exec_init_fn (struct corosync_api_v1 *api)
                  * With no quorum provider, we are always quorate
                  */
 		primary_designated = 1;
+		quorum_type = 0;
 	}
 
 	return (0);
@@ -434,7 +443,6 @@ static void message_handler_req_lib_quorum_getquorate (void *conn,
 	corosync_api->ipc_response_send(conn, &res_lib_quorum_getquorate, sizeof(res_lib_quorum_getquorate));
 }
 
-
 static void message_handler_req_lib_quorum_trackstart (void *conn,
 						       const void *msg)
 {
@@ -495,3 +503,19 @@ static void message_handler_req_lib_quorum_trackstop (void *conn, const void *ms
 	res.error = CS_OK;
 	corosync_api->ipc_response_send(conn, &res, sizeof(struct qb_ipc_response_header));
 }
+
+static void message_handler_req_lib_quorum_gettype (void *conn,
+						       const void *msg)
+{
+	struct res_lib_quorum_gettype res_lib_quorum_gettype;
+
+	log_printf(LOGSYS_LEVEL_DEBUG, "got quorum_type request on %p\n", conn);
+
+	/* send status */
+	res_lib_quorum_gettype.quorum_type = quorum_type;
+	res_lib_quorum_gettype.header.size = sizeof(res_lib_quorum_gettype);
+	res_lib_quorum_gettype.header.id = MESSAGE_RES_QUORUM_GETTYPE;
+	res_lib_quorum_gettype.header.error = CS_OK;
+	corosync_api->ipc_response_send(conn, &res_lib_quorum_gettype, sizeof(res_lib_quorum_gettype));
+}
+

include/corosync/ipc_quorum.h

@@ -40,14 +40,16 @@
 enum req_quorum_types {
 	MESSAGE_REQ_QUORUM_GETQUORATE = 0,
 	MESSAGE_REQ_QUORUM_TRACKSTART,
-	MESSAGE_REQ_QUORUM_TRACKSTOP
+	MESSAGE_REQ_QUORUM_TRACKSTOP,
+	MESSAGE_REQ_QUORUM_GETTYPE
 };
 
 enum res_quorum_types {
 	MESSAGE_RES_QUORUM_GETQUORATE = 0,
 	MESSAGE_RES_QUORUM_TRACKSTART,
 	MESSAGE_RES_QUORUM_TRACKSTOP,
-	MESSAGE_RES_QUORUM_NOTIFICATION
+	MESSAGE_RES_QUORUM_NOTIFICATION,
+	MESSAGE_RES_QUORUM_GETTYPE
 };
 
 struct req_lib_quorum_trackstart {
@@ -57,7 +59,7 @@ struct req_lib_quorum_trackstart {
 
 
 struct res_lib_quorum_getquorate {
-        struct qb_ipc_response_header header __attribute__((aligned(8)));
+	struct qb_ipc_response_header header __attribute__((aligned(8)));
 	mar_uint32_t quorate;
 };
 
@@ -69,4 +71,9 @@ struct res_lib_quorum_notification {
 	mar_uint32_t view_list[];
 };
 
+struct res_lib_quorum_gettype {
+	struct qb_ipc_response_header header __attribute__((aligned(8)));
+	mar_uint32_t quorum_type;
+};
+
 #endif

include/corosync/quorum.h

@@ -59,13 +59,16 @@ typedef struct {
 	quorum_notification_fn_t quorum_notify_fn;
 } quorum_callbacks_t;
 
+#define QUORUM_FREE	0
+#define QUORUM_SET	1
 
 /**
  * Create a new quorum connection
  */
 cs_error_t quorum_initialize (
 	quorum_handle_t *handle,
-	quorum_callbacks_t *callbacks);
+	quorum_callbacks_t *callbacks,
+	uint32_t *quorum_type);
 
 /**
  * Close the quorum handle

lib/quorum.c

@@ -65,10 +65,14 @@ DECLARE_HDB_DATABASE(quorum_handle_t_db,NULL);
 
 cs_error_t quorum_initialize (
 	quorum_handle_t *handle,
-	quorum_callbacks_t *callbacks)
+	quorum_callbacks_t *callbacks,
+	uint32_t *quorum_type)
 {
 	cs_error_t error;
 	struct quorum_inst *quorum_inst;
+	struct iovec iov;
+	struct qb_ipc_request_header req;
+	struct res_lib_quorum_gettype res_lib_quorum_gettype;
 
 	error = hdb_error_to_cs(hdb_handle_create (&quorum_handle_t_db, sizeof (struct quorum_inst), handle));
 	if (error != CS_OK) {
@@ -87,6 +91,27 @@ cs_error_t quorum_initialize (
 		goto error_put_destroy;
 	}
 
+	req.size = sizeof (req);
+	req.id = MESSAGE_REQ_QUORUM_GETTYPE;
+
+	iov.iov_base = (char *)&req;
+	iov.iov_len = sizeof (req);
+
+	error = qb_to_cs_error(qb_ipcc_sendv_recv (
+		quorum_inst->c,
+		&iov,
+		1,
+		&res_lib_quorum_gettype,
+		sizeof (struct res_lib_quorum_gettype), -1));
+
+	if (error != CS_OK) {
+		goto error_put_destroy;
+	}
+
+	error = res_lib_quorum_gettype.header.error;
+
+	*quorum_type = res_lib_quorum_gettype.quorum_type;
+
 	if (callbacks)
 		memcpy(&quorum_inst->callbacks, callbacks, sizeof (*callbacks));
 	else
@@ -155,7 +180,7 @@ cs_error_t quorum_getquorate (
 	iov.iov_base = (char *)&req;
 	iov.iov_len = sizeof (req);
 
-       error = qb_to_cs_error(qb_ipcc_sendv_recv (
+	error = qb_to_cs_error(qb_ipcc_sendv_recv (
 		quorum_inst->c,
 		&iov,
 		1,

lib/sam.c

@@ -256,6 +256,7 @@ cs_error_t sam_initialize (
 	sam_recovery_policy_t recovery_policy)
 {
 	quorum_callbacks_t quorum_callbacks;
+	uint32_t quorum_type;
 	cs_error_t err;
 
 	if (sam_internal_data.internal_status != SAM_INTERNAL_STATUS_NOT_INITIALIZED) {
@@ -272,7 +273,7 @@ cs_error_t sam_initialize (
 		 * Initialize quorum
 		 */
 		quorum_callbacks.quorum_notify_fn = quorum_notification_fn;
-		if ((err = quorum_initialize (&sam_internal_data.quorum_handle, &quorum_callbacks)) != CS_OK) {
+		if ((err = quorum_initialize (&sam_internal_data.quorum_handle, &quorum_callbacks, &quorum_type)) != CS_OK) {
 			goto exit_error;
 		}
 

test/testquorum.c

@@ -35,10 +35,11 @@ int main(int argc, char *argv[])
 {
 	int quorate;
 	quorum_callbacks_t callbacks;
+	uint32_t quorum_type;
 	int err;
 
 	callbacks.quorum_notify_fn = quorum_notification_fn;
-	if ( (err=quorum_initialize(&g_handle, &callbacks)) != CS_OK)
+	if ( (err=quorum_initialize(&g_handle, &callbacks, &quorum_type)) != CS_OK)
 		fprintf(stderr, "quorum_initialize FAILED: %d\n", err);
 
 	if ( (err=quorum_trackstart(g_handle, CS_TRACK_CHANGES)) != CS_OK)

tools/corosync-notifyd.c

@@ -356,13 +356,15 @@ static void
 _cs_quorum_init(void)
 {
 	cs_error_t rc;
+	uint32_t quorum_type;
 	int fd;
 
 	quorum_callbacks_t quorum_callbacks = {
 		.quorum_notify_fn = _cs_quorum_notification,
 	};
 
-	rc = quorum_initialize (&quorum_handle, &quorum_callbacks);
+	rc = quorum_initialize (&quorum_handle, &quorum_callbacks,
+			        &quorum_type);
 	if (rc != CS_OK) {
 		qb_log(LOG_ERR, "Could not connect to corosync(quorum)");
 		return;

tools/corosync-quorumtool.c

@@ -95,6 +95,7 @@ static void quorum_notification_fn(
 	uint32_t *view_list);
 
 static quorum_handle_t q_handle;
+static uint32_t q_type;
 static quorum_callbacks_t q_callbacks = {
 	.quorum_notify_fn = quorum_notification_fn
 };
@@ -157,6 +158,10 @@ static int get_quorum_type(char *quorum_type, size_t quorum_type_len)
 		return -1;
 	}
 
+	if (q_type == QUORUM_FREE) {
+		return -1;
+	}
+
 	if ((err = cmap_get_string(cmap_handle, "quorum.provider", &str)) != CS_OK) {
 		goto out;
 	}
@@ -374,6 +379,11 @@ static int monitor_status(nodeid_format_t nodeid_format, name_format_t name_form
 
 	show_status();
 
+	if (q_type == QUORUM_FREE) {
+		printf("\nQuorum is not configured - cannot monitor\n");
+		return 0;
+	}
+
 	printf("starting monitoring loop\n");
 
 	err=quorum_trackstart(q_handle, CS_TRACK_CHANGES);
@@ -481,7 +491,7 @@ static int init_all(void) {
 		goto out;
 	}
 
-	if (quorum_initialize(&q_handle, &q_callbacks) != CS_OK) {
+	if (quorum_initialize(&q_handle, &q_callbacks, &q_type) != CS_OK) {
 		fprintf(stderr, "Cannot initialize QUORUM service\n");
 		q_handle = 0;
 		goto out;