Qnet improvements

- Support for membership node list
- Initial support for "pluggable" algorithms

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/Makefile.am

@@ -41,7 +41,7 @@ sbin_SCRIPTS             = corosync-qnetd-certutil
 corosync_qnetd_SOURCES	= corosync-qnetd.c dynar.c msg.c msgio.c nss-sock.c  \
 			    qnetd-client.c qnetd-client-list.c qnetd-log.c \
 			    qnetd-poll-array.c timer-list.c tlv.c send-buffer-list.c \
-			    node-list.c
+			    node-list.c qnetd-algo-test.c qnetd-algorithm.c
 
 corosync_qdevice_net_SOURCES	= corosync-qdevice-net.c dynar.c msg.c msgio.c nss-sock.c  \
 			    qnetd-client.c qnetd-client-list.c qnetd-log.c \

qdevices/corosync-qdevice-net.c

@@ -100,6 +100,7 @@ enum qdevice_net_state {
 	QDEVICE_NET_STATE_WAITING_STARTTLS_BEING_SENT,
 	QDEVICE_NET_STATE_WAITING_INIT_REPLY,
 	QDEVICE_NET_STATE_WAITING_SET_OPTION_REPLY,
+	QDEVICE_NET_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS,
 };
 
 struct qdevice_net_instance {
@@ -113,7 +114,7 @@ struct qdevice_net_instance {
 	int skipping_msg;
 	size_t msg_already_received_bytes;
 	enum qdevice_net_state state;
-	uint32_t expected_msg_seq_num;
+	uint32_t last_msg_seq_num;
 	uint32_t echo_request_expected_msg_seq_num;
 	uint32_t echo_reply_received_msg_seq_num;
 	enum tlv_tls_supported tls_supported;
@@ -278,7 +279,7 @@ qdevice_net_msg_check_seq_number(struct qdevice_net_instance *instance,
     const struct msg_decoded *msg)
 {
 
-	if (!msg->seq_number_set || msg->seq_number != instance->expected_msg_seq_num) {
+	if (!msg->seq_number_set || msg->seq_number != instance->last_msg_seq_num) {
 		qdevice_net_log(LOG_ERR, "Received message doesn't contain seq_number or "
 		    "it's not expected one.");
 
@@ -320,7 +321,7 @@ qdevice_net_send_init(struct qdevice_net_instance *instance)
 
 	tlv_get_supported_options(&supported_opts, &no_supported_opts);
 	msg_get_supported_messages(&supported_msgs, &no_supported_msgs);
-	instance->expected_msg_seq_num++;
+	instance->last_msg_seq_num++;
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
@@ -329,7 +330,8 @@ qdevice_net_send_init(struct qdevice_net_instance *instance)
 		return (-1);
 	}
 
-	if (msg_create_init(&send_buffer->buffer, 1, instance->expected_msg_seq_num,
+	if (msg_create_init(&send_buffer->buffer, 1, instance->last_msg_seq_num,
+	    instance->decision_algorithm,
 	    supported_msgs, no_supported_msgs, supported_opts, no_supported_opts,
 	    instance->node_id) == 0) {
 		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for init msg");
@@ -391,9 +393,9 @@ qdevice_net_msg_received_preinit_reply(struct qdevice_net_instance *instance,
 			return (-1);
 		}
 
-		instance->expected_msg_seq_num++;
+		instance->last_msg_seq_num++;
 		if (msg_create_starttls(&send_buffer->buffer, 1,
-		    instance->expected_msg_seq_num) == 0) {
+		    instance->last_msg_seq_num) == 0) {
 			qdevice_net_log(LOG_ERR, "Can't allocate send buffer for starttls msg");
 
 			return (-1);
@@ -430,6 +432,20 @@ qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
 		return (-1);
 	}
 
+	if (!msg->reply_error_code_set) {
+		qdevice_net_log(LOG_ERR, "Received init reply message without error code."
+		    "Disconnecting from server");
+
+		return (-1);
+	}
+
+	if (msg->reply_error_code != TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		qdevice_net_log(LOG_ERR, "Received init reply message with error code %"PRIu16". "
+		    "Disconnecting from server", msg->reply_error_code);
+
+		return (-1);
+	}
+
 	if (!msg->server_maximum_request_size_set || !msg->server_maximum_reply_size_set) {
 		qdevice_net_log(LOG_ERR, "Required maximum_request_size or maximum_reply_size "
 		    "option is unset");
@@ -501,10 +517,10 @@ qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
 		return (-1);
 	}
 
-	instance->expected_msg_seq_num++;
+	instance->last_msg_seq_num++;
 
-	if (msg_create_set_option(&send_buffer->buffer, 1, instance->expected_msg_seq_num,
-	    1, instance->decision_algorithm, 1, instance->heartbeat_interval) == 0) {
+	if (msg_create_set_option(&send_buffer->buffer, 1, instance->last_msg_seq_num,
+	    1, instance->heartbeat_interval) == 0) {
 		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for set option msg");
 
 		return (-1);
@@ -518,7 +534,7 @@ qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
 }
 
 static int
-qdevice_net_msg_received_stattls(struct qdevice_net_instance *instance,
+qdevice_net_msg_received_starttls(struct qdevice_net_instance *instance,
     const struct msg_decoded *msg)
 {
 
@@ -690,9 +706,28 @@ qdevice_net_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list)
 iter_finalize:
 	cmap_iter_finalize(cmap_handle, iter_handle);
 
+	if (ret_value != 0) {
+		node_list_free(list);
+	}
+
 	return (ret_value);
 }
 
+static
+int qdevice_net_get_cmap_config_version(cmap_handle_t cmap_handle, uint64_t *config_version)
+{
+	int res;
+
+	if (cmap_get_uint64(cmap_handle, "totem.config_version", config_version) == CS_OK) {
+		res = 1;
+	} else {
+		*config_version = 0;
+		res = 0;
+	}
+
+	return (res);
+}
+
 static int
 qdevice_net_send_config_node_list(struct qdevice_net_instance *instance, int initial)
 {
@@ -720,17 +755,14 @@ qdevice_net_send_config_node_list(struct qdevice_net_instance *instance, int ini
 		return (-1);
 	}
 
-	if (cmap_get_uint64(instance->cmap_handle, "totem.config_version",
-	    &config_version) == CS_OK) {
-		send_config_version = 1;
-	} else {
-		config_version = 0;
-		send_config_version = 0;
-	}
+	send_config_version = qdevice_net_get_cmap_config_version(instance->cmap_handle,
+	    &config_version);
 
-	if (msg_create_node_list(&send_buffer->buffer, 0, 0,
+	instance->last_msg_seq_num++;
+
+	if (msg_create_node_list(&send_buffer->buffer, 1, instance->last_msg_seq_num,
 	    (initial ? TLV_NODE_LIST_TYPE_INITIAL_CONFIG : TLV_NODE_LIST_TYPE_CHANGED_CONFIG),
-	    0, NULL, send_config_version, config_version, &nlist) == 0) {
+	    0, NULL, send_config_version, config_version, 0, TLV_QUORATE_INQUORATE, &nlist) == 0) {
 		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for config list msg");
 
 		node_list_free(&nlist);
@@ -743,6 +775,21 @@ qdevice_net_send_config_node_list(struct qdevice_net_instance *instance, int ini
 	return (0);
 }
 
+static int
+qdevice_net_register_votequorum_callbacks(struct qdevice_net_instance *instance)
+{
+	cs_error_t res;
+
+	if ((res = votequorum_trackstart(instance->votequorum_handle, 0, CS_TRACK_CHANGES)) != CS_OK) {
+		qdevice_net_log(LOG_ERR, "Can't start tracking votequorum changes. Error %s",
+		    cs_strerror(res));
+
+		return (-1);
+	}
+
+	return (0);
+}
+
 static int
 qdevice_net_msg_received_set_option_reply(struct qdevice_net_instance *instance,
     const struct msg_decoded *msg)
@@ -780,10 +827,19 @@ qdevice_net_msg_received_set_option_reply(struct qdevice_net_instance *instance,
 		}
 	}
 
+	/*
+	 * Now we can finally really send node list and initialize qdevice
+	 */
 	if (qdevice_net_send_config_node_list(instance, 1) != 0) {
 		return (-1);
 	}
 
+	if (qdevice_net_register_votequorum_callbacks(instance) != 0) {
+		return (-1);
+	}
+
+	instance->state = QDEVICE_NET_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
+
 	return (0);
 }
 
@@ -863,7 +919,7 @@ qdevice_net_msg_received(struct qdevice_net_instance *instance)
 		ret_val = qdevice_net_msg_received_preinit_reply(instance, &msg);
 		break;
 	case MSG_TYPE_STARTTLS:
-		ret_val = qdevice_net_msg_received_stattls(instance, &msg);
+		ret_val = qdevice_net_msg_received_starttls(instance, &msg);
 		break;
 	case MSG_TYPE_SERVER_ERROR:
 		ret_val = qdevice_net_msg_received_server_error(instance, &msg);
@@ -1141,7 +1197,8 @@ qdevice_net_poll(struct qdevice_net_instance *instance)
 
 	if (instance->schedule_disconnect) {
 		/*
-		 * Schedule disconnect can be set by this function or by some timer_list callback
+		 * Schedule disconnect can be set by this function, by some timer_list callback
+		 * or cmap/votequorum callbacks
 		 */
 		return (-1);
 	}
@@ -1359,14 +1416,119 @@ qdevice_net_instance_init_from_cmap(struct qdevice_net_instance *instance,
 	instance->cmap_handle = cmap_handle;
 }
 
-static void qdevice_net_votequorum_notification(votequorum_handle_t votequorum_handle,
+static enum tlv_node_state
+qdevice_net_convert_votequorum_to_tlv_node_state(uint32_t votequorum_node_state)
+{
+	enum tlv_node_state res;
+
+	switch (votequorum_node_state) {
+	case VOTEQUORUM_NODESTATE_MEMBER: res = TLV_NODE_STATE_MEMBER; break;
+	case VOTEQUORUM_NODESTATE_DEAD: res = TLV_NODE_STATE_DEAD; break;
+	case VOTEQUORUM_NODESTATE_LEAVING: res = TLV_NODE_STATE_LEAVING; break;
+	default:
+		errx(1, "qdevice_net_convert_votequorum_to_tlv_node_state: Unhandled votequorum "
+		    "node state %"PRIu32, votequorum_node_state);
+		break;
+	}
+
+	return (res);
+}
+
+static int
+qdevice_net_send_membership_node_list(struct qdevice_net_instance *instance,
+    enum tlv_quorate quorate, const struct tlv_ring_id *ring_id,
+    uint32_t node_list_entries, votequorum_node_t node_list[])
+{
+	struct node_list nlist;
+	struct send_buffer_list_entry *send_buffer;
+	uint64_t config_version;
+	int send_config_version;
+	uint32_t i;
+
+	node_list_init(&nlist);
+
+	for (i = 0; i < node_list_entries; i++) {
+		if (node_list[i].nodeid == 0) {
+			continue;
+		}
+
+		if (node_list_add(&nlist, node_list[i].nodeid, 0,
+		    qdevice_net_convert_votequorum_to_tlv_node_state(node_list[i].state)) == NULL) {
+			qdevice_net_log(LOG_ERR, "Can't allocate membership node list.");
+
+			node_list_free(&nlist);
+
+			return (-1);
+		}
+	}
+
+	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
+	if (send_buffer == NULL) {
+		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for config "
+		    "node list msg");
+
+		node_list_free(&nlist);
+
+		return (-1);
+	}
+
+	instance->last_msg_seq_num++;
+
+	send_config_version = qdevice_net_get_cmap_config_version(instance->cmap_handle,
+	    &config_version);
+
+	if (msg_create_node_list(&send_buffer->buffer, 1, instance->last_msg_seq_num,
+	    TLV_NODE_LIST_TYPE_MEMBERSHIP,
+	    1, ring_id, send_config_version, config_version, 0, TLV_QUORATE_INQUORATE, &nlist) == 0) {
+		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for config list msg");
+
+		node_list_free(&nlist);
+
+		return (-1);
+	}
+
+	send_buffer_list_put(&instance->send_buffer_list, send_buffer);
+
+	return (0);
+}
+
+static void
+qdevice_net_convert_votequorum_to_tlv_ring_id(struct tlv_ring_id *tlv_rid,
+    const votequorum_ring_id_t *votequorum_rid)
+{
+
+	tlv_rid->node_id = votequorum_rid->nodeid;
+	tlv_rid->seq = votequorum_rid->seq;
+}
+
+static void
+qdevice_net_votequorum_notification(votequorum_handle_t votequorum_handle,
     uint64_t context, uint32_t quorate,
-    votequorum_ring_id_t ring_id, uint32_t node_list_entries, votequorum_node_t node_list[])
+    votequorum_ring_id_t votequorum_ring_id,
+    uint32_t node_list_entries, votequorum_node_t node_list[])
 {
+	struct qdevice_net_instance *instance;
+	struct tlv_ring_id ring_id;
+
+	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
+		errx(1, "Fatal error. Can't get votequorum context");
+	}
+
+	qdevice_net_convert_votequorum_to_tlv_ring_id(&ring_id, &votequorum_ring_id);
+
+	if (qdevice_net_send_membership_node_list(instance,
+	    (quorate ? TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE),
+	    &ring_id, node_list_entries, node_list) != 0) {
+		/*
+		 * Fatal error -> schedule disconnect
+		 */
+		instance->schedule_disconnect = 1;
+	}
 
 	memcpy(&global_last_received_ring_id, &ring_id, sizeof(ring_id));
 }
 
+
 static void
 qdevice_net_init_votequorum(struct qdevice_net_instance *instance)
 {
@@ -1390,22 +1552,21 @@ qdevice_net_init_votequorum(struct qdevice_net_instance *instance)
 		errx(1, "Failed to initialize the votequorum API. Error %s", cs_strerror(res));
 	}
 
-	if ((res = votequorum_trackstart(votequorum_handle, 0, CS_TRACK_CHANGES)) != CS_OK) {
-		errx(1, "Can't start tracking votequorum changes. Error %s", cs_strerror(res));
-	}
-
 	if ((res = votequorum_qdevice_register(votequorum_handle,
 	    QDEVICE_NET_VOTEQUORUM_DEVICE_NAME)) != CS_OK) {
 		errx(1, "Can't register votequorum device. Error %s", cs_strerror(res));
 	}
 
+	if ((res = votequorum_context_set(votequorum_handle, (void *)instance)) != CS_OK) {
+		errx(1, "Can't set votequorum context. Error %s", cs_strerror(res));
+	}
+
 	instance->votequorum_handle = votequorum_handle;
 
 	votequorum_fd_get(votequorum_handle, &fd);
 	if ((instance->votequorum_poll_fd = PR_CreateSocketPollFd(fd)) == NULL) {
 		err_nss();
 	}
-
 }
 
 
@@ -1453,9 +1614,9 @@ main(void)
 		errx(1, "Can't allocate send buffer list");
 	}
 
-	instance.expected_msg_seq_num = 1;
+	instance.last_msg_seq_num = 1;
 	if (msg_create_preinit(&send_buffer->buffer, instance.cluster_name, 1,
-	    instance.expected_msg_seq_num) == 0) {
+	    instance.last_msg_seq_num) == 0) {
 		errx(1, "Can't allocate buffer");
 	}
 

qdevices/corosync-qnetd.c

@@ -60,6 +60,7 @@
 #include "qnetd-log.h"
 #include "dynar.h"
 #include "timer-list.h"
+#include "qnetd-algorithm.h"
 
 #define QNETD_LISTEN_BACKLOG		10
 #define QNETD_MAX_CLIENT_SEND_BUFFERS	10
@@ -373,17 +374,21 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
     const struct msg_decoded *msg)
 {
 	int res;
+	size_t zi;
 	enum msg_type *supported_msgs;
 	size_t no_supported_msgs;
 	enum tlv_opt_type *supported_opts;
 	size_t no_supported_opts;
 	struct send_buffer_list_entry *send_buffer;
+	enum tlv_reply_error_code reply_error_code;
 
 	supported_msgs = NULL;
 	supported_opts = NULL;
 	no_supported_msgs = 0;
 	no_supported_opts = 0;
 
+	reply_error_code = TLV_REPLY_ERROR_CODE_NO_ERROR;
+
 	if ((res = qnetd_client_check_tls(instance, client, msg)) != 0) {
 		return (res == -1 ? -1 : 0);
 	}
@@ -391,24 +396,17 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 	if (!client->preinit_received) {
 		qnetd_log(LOG_ERR, "Received init before preinit message. Sending error reply.");
 
-		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-		    TLV_REPLY_ERROR_CODE_PREINIT_REQUIRED) != 0) {
-			return (-1);
-		}
-
-		return (0);
+		reply_error_code = TLV_REPLY_ERROR_CODE_PREINIT_REQUIRED;
 	}
 
-	if (!msg->node_id_set) {
+	if (reply_error_code == TLV_REPLY_ERROR_CODE_NO_ERROR && !msg->node_id_set) {
 		qnetd_log(LOG_ERR, "Received init message without node id set. "
 		    "Sending error reply.");
 
-		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-		    TLV_REPLY_ERROR_CODE_DOESNT_CONTAIN_REQUIRED_OPTION) != 0) {
-			return (-1);
-		}
-
-		return (0);
+		reply_error_code = TLV_REPLY_ERROR_CODE_DOESNT_CONTAIN_REQUIRED_OPTION;
+	} else {
+		client->node_id_set = 1;
+		client->node_id = msg->node_id;
 	}
 
 	if (msg->supported_messages != NULL) {
@@ -447,9 +445,45 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 		tlv_get_supported_options(&supported_opts, &no_supported_opts);
 	}
 
-	client->node_id_set = 1;
-	client->node_id = msg->node_id;
-	client->init_received = 1;
+	if (reply_error_code == TLV_REPLY_ERROR_CODE_NO_ERROR && !msg->decision_algorithm_set) {
+		qnetd_log(LOG_ERR, "Received init message without decision algorithm. "
+		    "Sending error reply.");
+
+		reply_error_code = TLV_REPLY_ERROR_CODE_DOESNT_CONTAIN_REQUIRED_OPTION;
+	} else {
+		/*
+		 * Check if decision algorithm requested by client is supported
+		 */
+		res = 0;
+
+		for (zi = 0; zi < QNETD_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE && !res; zi++) {
+			if (qnetd_static_supported_decision_algorithms[zi] ==
+			    msg->decision_algorithm) {
+				res = 1;
+			}
+		}
+
+		if (!res) {
+			qnetd_log(LOG_ERR, "Client requested unsupported decision algorithm %u. "
+			    "Sending error reply.", msg->decision_algorithm);
+
+			reply_error_code = TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM;
+		}
+
+		client->decision_algorithm = msg->decision_algorithm;
+	}
+
+
+	if (reply_error_code == TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		reply_error_code = qnetd_algorithm_client_init(client);
+	}
+
+	if (reply_error_code == TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		/*
+		 * Correct init received
+		 */
+		client->init_received = 1;
+	}
 
 	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
 	if (send_buffer == NULL) {
@@ -460,6 +494,7 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 	}
 
 	if (msg_create_init_reply(&send_buffer->buffer, msg->seq_number_set, msg->seq_number,
+	    reply_error_code,
 	    supported_msgs, no_supported_msgs, supported_opts, no_supported_opts,
 	    instance->max_client_receive_size, instance->max_client_send_size,
 	    qnetd_static_supported_decision_algorithms,
@@ -507,7 +542,6 @@ qnetd_client_msg_received_set_option(struct qnetd_instance *instance, struct qne
     const struct msg_decoded *msg)
 {
 	int res;
-	size_t zi;
 	struct send_buffer_list_entry *send_buffer;
 
 	if ((res = qnetd_client_check_tls(instance, client, msg)) != 0) {
@@ -526,34 +560,6 @@ qnetd_client_msg_received_set_option(struct qnetd_instance *instance, struct qne
 		return (0);
 	}
 
-	if (msg->decision_algorithm_set) {
-		/*
-		 * Check if decision algorithm requested by client is supported
-		 */
-		res = 0;
-
-		for (zi = 0; zi < QNETD_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE && !res; zi++) {
-			if (qnetd_static_supported_decision_algorithms[zi] ==
-			    msg->decision_algorithm) {
-				res = 1;
-			}
-		}
-
-		if (!res) {
-			qnetd_log(LOG_ERR, "Client requested unsupported decision algorithm %u. "
-			    "Sending error reply.", msg->decision_algorithm);
-
-			if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-			    TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM) != 0) {
-				return (-1);
-			}
-
-			return (0);
-		}
-
-		client->decision_algorithm = msg->decision_algorithm;
-	}
-
 	if (msg->heartbeat_interval_set) {
 		/*
 		 * Check if heartbeat interval is valid
@@ -658,6 +664,10 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 {
 	int res;
 	struct send_buffer_list_entry *send_buffer;
+	enum tlv_reply_error_code reply_error_code;
+	enum tlv_vote result_vote;
+
+	reply_error_code = TLV_REPLY_ERROR_CODE_NO_ERROR;
 
 	if ((res = qnetd_client_check_tls(instance, client, msg)) != 0) {
 		return (res == -1 ? -1 : 0);
@@ -687,6 +697,35 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 		return (0);
 	}
 
+	switch (msg->node_list_type) {
+	case TLV_NODE_LIST_TYPE_INITIAL_CONFIG:
+		reply_error_code = qnetd_algorithm_config_node_list_received(client,
+		    &msg->nodes, 1, &result_vote);
+		break;
+	case TLV_NODE_LIST_TYPE_CHANGED_CONFIG:
+		break;
+	case TLV_NODE_LIST_TYPE_MEMBERSHIP:
+		reply_error_code = qnetd_algorithm_membership_node_list_received(client,
+		    &msg->nodes, &result_vote);
+		break;
+	default:
+		errx(1, "qnetd_client_msg_received_node_list fatal error. "
+		    "Unhandled node_list_type");
+		break;
+	}
+
+	if (reply_error_code != TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		qnetd_log(LOG_ERR, "Algorithm returned error code. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    reply_error_code) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
 	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
 	if (send_buffer == NULL) {
 		qnetd_log(LOG_ERR, "Can't alloc node list reply msg from list. "
@@ -696,7 +735,7 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 	}
 
 	if (msg_create_node_list_reply(&send_buffer->buffer, msg->seq_number_set, msg->seq_number,
-	    TLV_VOTE_ACK) == -1) {
+	    result_vote) == -1) {
 		qnetd_log(LOG_ERR, "Can't alloc node list reply msg. "
 		    "Disconnecting client connection.");
 
@@ -952,6 +991,9 @@ qnetd_client_accept(struct qnetd_instance *instance)
 
 	if (nss_sock_set_nonblocking(client_socket) != 0) {
 		qnetd_log_nss(LOG_ERR, "Can't set client socket to non blocking mode");
+
+		PR_Close(client_socket);
+
 		return (-1);
 	}
 
@@ -960,6 +1002,9 @@ qnetd_client_accept(struct qnetd_instance *instance)
 	    instance->max_client_send_size);
 	if (client == NULL) {
 		qnetd_log(LOG_ERR, "Can't add client to list");
+
+		PR_Close(client_socket);
+
 		return (-2);
 	}
 

qdevices/msg.c

@@ -266,6 +266,7 @@ msg_convert_msg_type_array_to_u16_array(const enum msg_type *msg_type_array, siz
 
 size_t
 msg_create_init(struct dynar *msg, int add_msg_seq_number, uint32_t msg_seq_number,
+    enum tlv_decision_algorithm_type decision_algorithm,
     const enum msg_type *supported_msgs, size_t no_supported_msgs,
     const enum tlv_opt_type *supported_opts, size_t no_supported_opts, uint32_t node_id)
 {
@@ -311,6 +312,10 @@ msg_create_init(struct dynar *msg, int add_msg_seq_number, uint32_t msg_seq_numb
 		goto small_buf_err;
         }
 
+	if (tlv_add_decision_algorithm(msg, decision_algorithm) == -1) {
+		goto small_buf_err;
+	}
+
 	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
 
 	return (dynar_size(msg));
@@ -321,6 +326,7 @@ small_buf_err:
 
 size_t
 msg_create_init_reply(struct dynar *msg, int add_msg_seq_number, uint32_t msg_seq_number,
+    enum tlv_reply_error_code reply_error_code,
     const enum msg_type *supported_msgs, size_t no_supported_msgs,
     const enum tlv_opt_type *supported_opts, size_t no_supported_opts,
     size_t server_maximum_request_size, size_t server_maximum_reply_size,
@@ -337,6 +343,10 @@ msg_create_init_reply(struct dynar *msg, int add_msg_seq_number, uint32_t msg_se
 	msg_add_type(msg, MSG_TYPE_INIT_REPLY);
 	msg_add_len(msg);
 
+	if (tlv_add_reply_error_code(msg, reply_error_code) == -1) {
+		goto small_buf_err;
+	}
+
 	if (supported_msgs != NULL && no_supported_msgs > 0) {
 		u16a = msg_convert_msg_type_array_to_u16_array(supported_msgs, no_supported_msgs);
 
@@ -390,7 +400,6 @@ small_buf_err:
 
 size_t
 msg_create_set_option(struct dynar *msg, int add_msg_seq_number, uint32_t msg_seq_number,
-    int add_decision_algorithm, enum tlv_decision_algorithm_type decision_algorithm,
     int add_heartbeat_interval, uint32_t heartbeat_interval)
 {
 
@@ -405,12 +414,6 @@ msg_create_set_option(struct dynar *msg, int add_msg_seq_number, uint32_t msg_se
 		}
 	}
 
-	if (add_decision_algorithm) {
-		if (tlv_add_decision_algorithm(msg, decision_algorithm) == -1) {
-			goto small_buf_err;
-		}
-	}
-
 	if (add_heartbeat_interval) {
 		if (tlv_add_heartbeat_interval(msg, heartbeat_interval) == -1) {
 			goto small_buf_err;
@@ -502,7 +505,9 @@ size_t
 msg_create_node_list(struct dynar *msg, int add_msg_seq_number,
     uint32_t msg_seq_number, enum tlv_node_list_type node_list_type,
     int add_ring_id, const struct tlv_ring_id *ring_id,
-    int add_config_version, uint64_t config_version, const struct node_list *nodes)
+    int add_config_version, uint64_t config_version,
+    int add_quorate, enum tlv_quorate quorate,
+    const struct node_list *nodes)
 {
 	struct node_list_entry *node_info;
 	struct tlv_node_info tlv_ni;
@@ -818,6 +823,13 @@ msg_decode(const struct dynar *msg, struct msg_decoded *decoded_msg)
 
 			decoded_msg->vote_set = 1;
 			break;
+		case TLV_OPT_QUORATE:
+			if ((res = tlv_iter_decode_quorate(&tlv_iter, &decoded_msg->quorate)) != 0) {
+				return (res);
+			}
+
+			decoded_msg->quorate_set = 1;
+			break;
 		default:
 			/*
 			 * Unknown option

qdevices/msg.h

@@ -104,6 +104,8 @@ struct msg_decoded {
 	enum tlv_node_list_type node_list_type;	// Valid only if node_list_type_set != 0
 	int vote_set;
 	enum tlv_vote vote;	// Valid only if vote_set != 0
+	int quorate_set;
+	enum tlv_quorate quorate;	// Valid only if quorate_set != 0
 };
 
 extern size_t		msg_create_preinit(struct dynar *msg, const char *cluster_name,
@@ -116,14 +118,16 @@ extern size_t		msg_create_starttls(struct dynar *msg, int add_msg_seq_number,
     uint32_t msg_seq_number);
 
 extern size_t		msg_create_init(struct dynar *msg, int add_msg_seq_number,
-    uint32_t msg_seq_number, const enum msg_type *supported_msgs, size_t no_supported_msgs,
+    uint32_t msg_seq_number, enum tlv_decision_algorithm_type decision_algorithm,
+    const enum msg_type *supported_msgs, size_t no_supported_msgs,
     const enum tlv_opt_type *supported_opts, size_t no_supported_opts, uint32_t node_id);
 
 extern size_t		msg_create_server_error(struct dynar *msg, int add_msg_seq_number,
     uint32_t msg_seq_number, enum tlv_reply_error_code reply_error_code);
 
 extern size_t		msg_create_init_reply(struct dynar *msg, int add_msg_seq_number,
-    uint32_t msg_seq_number, const enum msg_type *supported_msgs, size_t no_supported_msgs,
+    uint32_t msg_seq_number, enum tlv_reply_error_code reply_error_code,
+    const enum msg_type *supported_msgs, size_t no_supported_msgs,
     const enum tlv_opt_type *supported_opts, size_t no_supported_opts,
     size_t server_maximum_request_size, size_t server_maximum_reply_size,
     const enum tlv_decision_algorithm_type *supported_decision_algorithms,
@@ -131,7 +135,6 @@ extern size_t		msg_create_init_reply(struct dynar *msg, int add_msg_seq_number,
 
 extern size_t		msg_create_set_option(struct dynar *msg,
     int add_msg_seq_number, uint32_t msg_seq_number,
-    int add_decision_algorithm, enum tlv_decision_algorithm_type decision_algorithm,
     int add_heartbeat_interval, uint32_t heartbeat_interval);
 
 extern size_t		msg_create_set_option_reply(struct dynar *msg,
@@ -148,6 +151,7 @@ extern size_t		msg_create_node_list(struct dynar *msg,
     int add_msg_seq_number, uint32_t msg_seq_number, enum tlv_node_list_type node_list_type,
     int add_ring_id, const struct tlv_ring_id *ring_id,
     int add_config_version, uint64_t config_version,
+    int add_quorate, enum tlv_quorate quorate,
     const struct node_list *nodes);
 
 extern size_t		msg_create_node_list_reply(struct dynar *msg, int add_msg_seq_number,

qdevices/qnetd-algo-test.c

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+
+#include <string.h>
+
+#include "qnetd-algo-test.h"
+#include "qnetd-log.h"
+
+/*
+ * Called right after client sent init message. This happens after initial accept of client,
+ * tls handshake and sending basic information about cluster/client.
+ * Known information:
+ * - client->cluster_name (client->cluster_name_len)
+ * - client->node_id (client->node_id_set = 1)
+ * - client->decision_algorithm
+ *
+ * Callback is designed mainly for allocating client->algorithm_data.
+ *
+ * client is initialized qnetd_client structure.
+ *
+ * Return TLV_REPLY_ERROR_CODE_NO_ERROR on success, different TLV_REPLY_ERROR_CODE_*
+ * on failure (error is send back to client)
+ */
+enum tlv_reply_error_code
+qnetd_algo_test_client_init(struct qnetd_client *client)
+{
+	int *algo_data;
+
+	qnetd_log(LOG_INFO, "algo-test: New client connected");
+	qnetd_log(LOG_INFO, "algo-test:   cluster name = %s", client->cluster_name);
+	qnetd_log(LOG_INFO, "algo-test:   tls started = %u", client->tls_started);
+	qnetd_log(LOG_INFO, "algo-test:   tls peer certificate verified = %u",
+	    client->tls_peer_certificate_verified);
+	qnetd_log(LOG_INFO, "algo-test:   node_id = %"PRIx32, client->node_id);
+	qnetd_log(LOG_INFO, "algo-test:   pointer = 0x%p", client);
+
+	client->algorithm_data = malloc(sizeof(int));
+	if (client->algorithm_data == NULL) {
+		return (-1);
+	}
+
+	algo_data = client->algorithm_data;
+	*algo_data = 42;
+
+	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+}
+
+static const char *
+qnetd_algo_test_node_state_to_str(enum tlv_node_state node_state)
+{
+	switch (node_state) {
+	case TLV_NODE_STATE_NOT_SET: return ("not set"); break;
+	case TLV_NODE_STATE_MEMBER: return ("member"); break;
+	case TLV_NODE_STATE_DEAD: return ("dead"); break;
+	case TLV_NODE_STATE_LEAVING: return ("leaving"); break;
+	default: return ("unhandled"); break;
+	}
+
+	return ("");
+}
+
+
+static void
+qnetd_algo_dump_node_list(struct qnetd_client *client, const struct node_list *nodes)
+{
+	int *algo_data;
+	struct node_list_entry *node_info;
+
+	algo_data = client->algorithm_data;
+
+	qnetd_log(LOG_INFO, "algo-test:   algo data = %u", *algo_data);
+
+	TAILQ_FOREACH(node_info, nodes, entries) {
+		qnetd_log(LOG_INFO, "algo-test:   node_id = %"PRIx32", "
+		    "data_center_id = %"PRIx32", "
+		    "node_state = %s", node_info->node_id, node_info->data_center_id,
+		    qnetd_algo_test_node_state_to_str(node_info->node_state));
+	}
+}
+
+/*
+ * Called after client sent configuration node list
+ * All client fields are already set. Nodes is actual node list, initial is used
+ * for distrinquish between initial node list and changed node list.
+ *
+ * Function has to return result_vote. This can be one of ack/nack, ask_later (client
+ * should ask later for a vote) or wait_for_reply (client should wait for reply).
+ *
+ * Return TLV_REPLY_ERROR_CODE_NO_ERROR on success, different TLV_REPLY_ERROR_CODE_*
+ * on failure (error is send back to client)
+ */
+enum tlv_reply_error_code
+qnetd_algo_test_config_node_list_received(struct qnetd_client *client,
+    const struct node_list *nodes, int initial, enum tlv_vote *result_vote)
+{
+
+	qnetd_log(LOG_INFO, "algo-test: Client %p (cluster %s, node_id %"PRIx32") "
+	    "sent %s node list.", client, client->cluster_name, client->node_id,
+	    (initial ? "initial" : "changed"));
+
+	qnetd_algo_dump_node_list(client, nodes);
+
+	*result_vote = TLV_VOTE_ACK;
+
+	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+}
+
+enum tlv_reply_error_code
+qnetd_algo_test_membership_node_list_received(struct qnetd_client *client,
+    const struct node_list *nodes, enum tlv_vote *result_vote)
+{
+
+	qnetd_log(LOG_INFO, "algo-test: Client %p (cluster %s, node_id %"PRIx32") "
+	    "sent membership node list.", client, client->cluster_name, client->node_id);
+
+	qnetd_algo_dump_node_list(client, nodes);
+
+	*result_vote = TLV_VOTE_ACK;
+
+	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+}

qdevices/qnetd-algo-test.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QNETD_ALGO_TEST_H_
+#define _QNETD_ALGO_TEST_H_
+
+#include "qnetd-algorithm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern enum tlv_reply_error_code	qnetd_algo_test_client_init(struct qnetd_client *client);
+
+extern enum tlv_reply_error_code	qnetd_algo_test_config_node_list_received(
+    struct qnetd_client *client, const struct node_list *nodes, int initial,
+    enum tlv_vote *result_vote);
+
+extern enum tlv_reply_error_code	qnetd_algo_test_membership_node_list_received(
+    struct qnetd_client *client, const struct node_list *nodes, enum tlv_vote *result_vote);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QNETD_ALGO_TEST_H_ */

qdevices/qnetd-algorithm.c

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+
+#include <err.h>
+
+#include "qnetd-algorithm.h"
+#include "qnetd-algo-test.h"
+
+enum tlv_reply_error_code
+qnetd_algorithm_client_init(struct qnetd_client *client)
+{
+
+	switch (client->decision_algorithm) {
+	case TLV_DECISION_ALGORITHM_TYPE_TEST:
+		return (qnetd_algo_test_client_init(client));
+		break;
+	default:
+		errx(1, "qnetd_algorithm_client_init unhandled decision algorithm");
+		break;
+	}
+
+	return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
+}
+
+enum tlv_reply_error_code
+qnetd_algorithm_config_node_list_received(struct qnetd_client *client,
+    const struct node_list *nodes, int initial, enum tlv_vote *result_vote)
+{
+
+	switch (client->decision_algorithm) {
+	case TLV_DECISION_ALGORITHM_TYPE_TEST:
+		return (qnetd_algo_test_config_node_list_received(client, nodes, initial,
+		    result_vote));
+		break;
+	default:
+		errx(1, "qnetd_algorithm_config_node_list_received unhandled "
+		    "decision algorithm");
+		break;
+	}
+
+	return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
+
+}
+
+enum tlv_reply_error_code
+qnetd_algorithm_membership_node_list_received(struct qnetd_client *client,
+    const struct node_list *nodes, enum tlv_vote *result_vote)
+{
+
+	switch (client->decision_algorithm) {
+	case TLV_DECISION_ALGORITHM_TYPE_TEST:
+		return (qnetd_algo_test_membership_node_list_received(client, nodes,
+		    result_vote));
+		break;
+	default:
+		errx(1, "qnetd_algorithm_membership_node_list_received unhandled "
+		    "decision algorithm");
+		break;
+	}
+
+	return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
+}

qdevices/qnetd-algorithm.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QNETD_ALGORITHM_H_
+#define _QNETD_ALGORITHM_H_
+
+#include <sys/types.h>
+#include <inttypes.h>
+
+#include "tlv.h"
+#include "qnetd-client.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern enum tlv_reply_error_code	qnetd_algorithm_client_init(struct qnetd_client *client);
+
+extern enum tlv_reply_error_code	qnetd_algorithm_config_node_list_received(
+    struct qnetd_client *client, const struct node_list *nodes, int initial,
+    enum tlv_vote *result_vote);
+
+extern enum tlv_reply_error_code	qnetd_algorithm_membership_node_list_received(
+    struct qnetd_client *client, const struct node_list *nodes, enum tlv_vote *result_vote);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QNETD_ALGORITHM_H_ */

qdevices/qnetd-client.h

@@ -44,6 +44,7 @@
 #include "dynar.h"
 #include "tlv.h"
 #include "send-buffer-list.h"
+#include "node-list.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -67,6 +68,9 @@ struct qnetd_client {
 	enum tlv_decision_algorithm_type decision_algorithm;
 	uint32_t heartbeat_interval;
 	enum tlv_reply_error_code skipping_msg_reason;
+	void *algorithm_data;
+	struct node_list configuration_nodes;
+	struct node_list membership_nodes;
 	TAILQ_ENTRY(qnetd_client) entries;
 };
 

qdevices/tlv.c

@@ -56,7 +56,7 @@
 #define TLV_TYPE_LENGTH		2
 #define TLV_LENGTH_LENGTH	2
 
-#define TLV_STATIC_SUPPORTED_OPTIONS_SIZE      20
+#define TLV_STATIC_SUPPORTED_OPTIONS_SIZE      21
 
 enum tlv_opt_type tlv_static_supported_options[TLV_STATIC_SUPPORTED_OPTIONS_SIZE] = {
     TLV_OPT_MSG_SEQ_NUMBER,
@@ -79,6 +79,7 @@ enum tlv_opt_type tlv_static_supported_options[TLV_STATIC_SUPPORTED_OPTIONS_SIZE
     TLV_OPT_NODE_INFO,
     TLV_OPT_NODE_LIST_TYPE,
     TLV_OPT_VOTE,
+    TLV_OPT_QUORATE,
 };
 
 int
@@ -381,6 +382,13 @@ tlv_add_vote(struct dynar *msg, enum tlv_vote vote)
 	return (tlv_add_u8(msg, TLV_OPT_VOTE, vote));
 }
 
+int
+tlv_add_quorate(struct dynar *msg, enum tlv_quorate quorate)
+{
+
+	return (tlv_add_u8(msg, TLV_OPT_QUORATE, quorate));
+}
+
 void
 tlv_iter_init_str(const char *msg, size_t msg_len, size_t msg_header_len,
     struct tlv_iterator *tlv_iter)
@@ -857,9 +865,10 @@ tlv_iter_decode_vote(struct tlv_iterator *tlv_iter, enum tlv_vote *vote)
 
 	tmp_vote = u8;
 
-	if (tmp_vote != TLV_VOTE_UNDECIDED &&
-	    tmp_vote != TLV_VOTE_ACK &&
-	    tmp_vote != TLV_VOTE_NACK) {
+	if (tmp_vote != TLV_VOTE_ACK &&
+	    tmp_vote != TLV_VOTE_NACK &&
+	    tmp_vote != TLV_VOTE_ASK_LATER &&
+	    tmp_vote != TLV_VOTE_WAIT_FOR_REPLY) {
 		return (-4);
 	}
 
@@ -868,6 +877,28 @@ tlv_iter_decode_vote(struct tlv_iterator *tlv_iter, enum tlv_vote *vote)
 	return (0);
 }
 
+int
+tlv_iter_decode_quorate(struct tlv_iterator *tlv_iter, enum tlv_quorate *quorate)
+{
+	uint8_t u8;
+	enum tlv_quorate tmp_quorate;
+
+	if (tlv_iter_decode_u8(tlv_iter, &u8) != 0) {
+		return (-1);
+	}
+
+	tmp_quorate = u8;
+
+	if (tmp_quorate != TLV_QUORATE_QUORATE &&
+	    tmp_quorate != TLV_QUORATE_INQUORATE) {
+		return (-4);
+	}
+
+	*quorate = tmp_quorate;
+
+	return (0);
+}
+
 void
 tlv_get_supported_options(enum tlv_opt_type **supported_options, size_t *no_supported_options)
 {

qdevices/tlv.h

@@ -65,6 +65,7 @@ enum tlv_opt_type {
 	TLV_OPT_NODE_INFO = 17,
 	TLV_OPT_NODE_LIST_TYPE = 18,
 	TLV_OPT_VOTE = 19,
+	TLV_OPT_QUORATE = 20,
 };
 
 enum tlv_tls_supported {
@@ -113,9 +114,15 @@ enum tlv_node_list_type {
 };
 
 enum tlv_vote {
-	TLV_VOTE_UNDECIDED = 0,
 	TLV_VOTE_ACK = 1,
 	TLV_VOTE_NACK = 2,
+	TLV_VOTE_ASK_LATER = 3,
+	TLV_VOTE_WAIT_FOR_REPLY = 4,
+};
+
+enum tlv_quorate {
+	TLV_QUORATE_INQUORATE = 0,
+	TLV_QUORATE_QUORATE = 1,
 };
 
 struct tlv_node_info {
@@ -207,6 +214,8 @@ extern int			 tlv_add_node_list_type(struct dynar *msg,
 
 extern int			 tlv_add_vote(struct dynar *msg, enum tlv_vote vote);
 
+extern int			 tlv_add_quorate(struct dynar *msg, enum tlv_quorate quorate);
+
 extern void			 tlv_iter_init_str(const char *msg, size_t msg_len,
     size_t msg_header_len, struct tlv_iterator *tlv_iter);
 
@@ -273,6 +282,9 @@ extern int			 tlv_iter_decode_node_list_type(struct tlv_iterator *tlv_iter,
 extern int			 tlv_iter_decode_vote(struct tlv_iterator *tlv_iter,
     enum tlv_vote *vote);
 
+extern int			 tlv_iter_decode_quorate(struct tlv_iterator *tlv_iter,
+    enum tlv_quorate *quorate);
+
 extern void			 tlv_get_supported_options(enum tlv_opt_type **supported_options,
     size_t *no_supported_options);