qnetd: Improvements

- Move adding client to cluster to init phase instead of preinit
- Implement missing ask for vote and vote info messages
- Fix cluster name memory leak
- Refactor unexpected message handler to one generic function
- Move qnetd_client_send_err to new file
- Add qnetd_client_send_vote_info

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/Makefile.am

@@ -42,7 +42,7 @@ corosync_qnetd_SOURCES	= corosync-qnetd.c dynar.c msg.c msgio.c nss-sock.c  \
 			    qnetd-client.c qnetd-client-list.c qnetd-log.c \
 			    qnetd-poll-array.c timer-list.c tlv.c send-buffer-list.c \
 			    node-list.c qnetd-algo-test.c qnetd-algorithm.c \
-			    qnetd-algo-ffsplit.c qnetd-cluster-list.c
+			    qnetd-algo-ffsplit.c qnetd-cluster-list.c qnetd-client-send.c
 
 corosync_qdevice_net_SOURCES	= corosync-qdevice-net.c dynar.c msg.c msgio.c nss-sock.c  \
 			    qnetd-client.c qnetd-client-list.c qnetd-log.c \

qdevices/corosync-qdevice-net.c

@@ -672,7 +672,8 @@ qdevice_net_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list)
 			 */
 			clear_node_high_byte = 0;
 
-			if (cmap_get_string(cmap_handle, "totem.clear_node_high_bit", &tmp_str) == CS_OK) {
+			if (cmap_get_string(cmap_handle, "totem.clear_node_high_bit",
+			    &tmp_str) == CS_OK) {
 				if (strcmp (tmp_str, "yes") == 0) {
 					clear_node_high_byte = 1;
 				}
@@ -782,7 +783,8 @@ qdevice_net_register_votequorum_callbacks(struct qdevice_net_instance *instance)
 {
 	cs_error_t res;
 
-	if ((res = votequorum_trackstart(instance->votequorum_handle, 0, CS_TRACK_CHANGES)) != CS_OK) {
+	if ((res = votequorum_trackstart(instance->votequorum_handle, 0,
+	    CS_TRACK_CHANGES)) != CS_OK) {
 		qdevice_net_log(LOG_ERR, "Can't start tracking votequorum changes. Error %s",
 		    cs_strerror(res));
 

qdevices/corosync-qnetd.c

@@ -62,6 +62,7 @@
 #include "timer-list.h"
 #include "qnetd-algorithm.h"
 #include "qnetd-cluster-list.h"
+#include "qnetd-client-send.h"
 
 #define QNETD_LISTEN_BACKLOG		10
 #define QNETD_MAX_CLIENT_SEND_BUFFERS	10
@@ -148,39 +149,12 @@ qnetd_client_log_msg_decode_error(int ret)
 	}
 }
 
-static int
-qnetd_client_send_err(struct qnetd_client *client, int add_msg_seq_number, uint32_t msg_seq_number,
-    enum tlv_reply_error_code reply)
-{
-	struct send_buffer_list_entry *send_buffer;
-
-	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
-	if (send_buffer == NULL) {
-		qnetd_log(LOG_ERR, "Can't alloc server error msg from list. "
-		    "Disconnecting client connection.");
-
-		return (-1);
-	}
-
-	if (msg_create_server_error(&send_buffer->buffer, add_msg_seq_number,
-	    msg_seq_number, reply) == 0) {
-		qnetd_log(LOG_ERR, "Can't alloc server error msg. "
-		    "Disconnecting client connection.");
-
-		return (-1);
-	};
-
-	send_buffer_list_put(&client->send_buffer_list, send_buffer);
-
-	return (0);
-}
 
 static int
 qnetd_client_msg_received_preinit(struct qnetd_instance *instance, struct qnetd_client *client,
     const struct msg_decoded *msg)
 {
 	struct send_buffer_list_entry *send_buffer;
-	struct qnetd_cluster *cluster;
 
 	if (msg->cluster_name == NULL) {
 		qnetd_log(LOG_ERR, "Received preinit message without cluster name. "
@@ -211,19 +185,6 @@ qnetd_client_msg_received_preinit(struct qnetd_instance *instance, struct qnetd_
 	client->cluster_name_len = msg->cluster_name_len;
 	client->preinit_received = 1;
 
-	cluster = qnetd_cluster_list_add_client(&instance->clusters, client);
-	if (cluster == NULL) {
-		qnetd_log(LOG_ERR, "Can't add client to cluster list. Sending error reply.");
-
-		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-		    TLV_REPLY_ERROR_CODE_INTERNAL_ERROR) != 0) {
-			return (-1);
-		}
-
-		return (0);
-	}
-	client->cluster = cluster;
-
 	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
 	if (send_buffer == NULL) {
 		qnetd_log(LOG_ERR, "Can't alloc preinit reply msg from list. "
@@ -246,11 +207,11 @@ qnetd_client_msg_received_preinit(struct qnetd_instance *instance, struct qnetd_
 }
 
 static int
-qnetd_client_msg_received_preinit_reply(struct qnetd_instance *instance,
-    struct qnetd_client *client, const struct msg_decoded *msg)
+qnetd_client_msg_received_unexpected_msg(struct qnetd_client *client,
+    const struct msg_decoded *msg, const char *msg_str)
 {
 
-	qnetd_log(LOG_ERR, "Received preinit reply. Sending back error message");
+	qnetd_log(LOG_ERR, "Received %s message. Sending back error message", msg_str);
 
 	if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
 	    TLV_REPLY_ERROR_CODE_UNEXPECTED_MESSAGE) != 0) {
@@ -260,6 +221,14 @@ qnetd_client_msg_received_preinit_reply(struct qnetd_instance *instance,
 	return (0);
 }
 
+static int
+qnetd_client_msg_received_preinit_reply(struct qnetd_instance *instance,
+    struct qnetd_client *client, const struct msg_decoded *msg)
+{
+
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "preinit reply"));
+}
+
 static int
 qnetd_client_msg_received_starttls(struct qnetd_instance *instance, struct qnetd_client *client,
     const struct msg_decoded *msg)
@@ -297,14 +266,7 @@ qnetd_client_msg_received_server_error(struct qnetd_instance *instance, struct q
     const struct msg_decoded *msg)
 {
 
-	qnetd_log(LOG_ERR, "Received server error. Sending back error message");
-
-	if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-	    TLV_REPLY_ERROR_CODE_UNEXPECTED_MESSAGE) != 0) {
-		return (-1);
-	}
-
-	return (0);
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "server error"));
 }
 
 /*
@@ -399,6 +361,7 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 	size_t no_supported_opts;
 	struct send_buffer_list_entry *send_buffer;
 	enum tlv_reply_error_code reply_error_code;
+	struct qnetd_cluster *cluster;
 
 	supported_msgs = NULL;
 	supported_opts = NULL;
@@ -491,6 +454,17 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 		client->decision_algorithm = msg->decision_algorithm;
 	}
 
+	if (reply_error_code == TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		cluster = qnetd_cluster_list_add_client(&instance->clusters, client);
+		if (cluster == NULL) {
+			qnetd_log(LOG_ERR, "Can't add client to cluster list. "
+			    "Sending error reply.");
+
+			reply_error_code = TLV_REPLY_ERROR_CODE_INTERNAL_ERROR;
+		} else {
+			client->cluster = cluster;
+		}
+	}
 
 	if (reply_error_code == TLV_REPLY_ERROR_CODE_NO_ERROR) {
 		reply_error_code = qnetd_algorithm_client_init(client);
@@ -531,28 +505,16 @@ static int
 qnetd_client_msg_received_init_reply(struct qnetd_instance *instance, struct qnetd_client *client,
     const struct msg_decoded *msg)
 {
-	qnetd_log(LOG_ERR, "Received init reply. Sending back error message");
 
-	if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-	    TLV_REPLY_ERROR_CODE_UNEXPECTED_MESSAGE) != 0) {
-		return (-1);
-	}
-
-	return (0);
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "init reply"));
 }
 
 static int
 qnetd_client_msg_received_set_option_reply(struct qnetd_instance *instance,
     struct qnetd_client *client, const struct msg_decoded *msg)
 {
-	qnetd_log(LOG_ERR, "Received set option reply. Sending back error message");
 
-	if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-	    TLV_REPLY_ERROR_CODE_UNEXPECTED_MESSAGE) != 0) {
-		return (-1);
-	}
-
-	return (0);
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "set option reply"));
 }
 
 static int
@@ -622,21 +584,15 @@ qnetd_client_msg_received_set_option(struct qnetd_instance *instance, struct qne
 
 static int
 qnetd_client_msg_received_echo_reply(struct qnetd_instance *instance, struct qnetd_client *client,
-	const struct msg_decoded *msg)
+    const struct msg_decoded *msg)
 {
-	qnetd_log(LOG_ERR, "Received echo reply. Sending back error message");
 
-	if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-	    TLV_REPLY_ERROR_CODE_UNEXPECTED_MESSAGE) != 0) {
-		return (-1);
-	}
-
-	return (0);
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "echo reply"));
 }
 
 static int
 qnetd_client_msg_received_echo_request(struct qnetd_instance *instance, struct qnetd_client *client,
-	const struct msg_decoded *msg, const struct dynar *msg_orig)
+    const struct msg_decoded *msg, const struct dynar *msg_orig)
 {
 	int res;
 	struct send_buffer_list_entry *send_buffer;
@@ -692,7 +648,7 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 	}
 
 	if (!client->init_received) {
-		qnetd_log(LOG_ERR, "Received set option message before init message. "
+		qnetd_log(LOG_ERR, "Received node list message before init message. "
 		    "Sending error reply.");
 
 		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
@@ -833,16 +789,158 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 }
 
 static int
-qnetd_client_msg_received_node_list_reply(struct qnetd_instance *instance, struct qnetd_client *client,
-	const struct msg_decoded *msg)
+qnetd_client_msg_received_node_list_reply(struct qnetd_instance *instance,
+    struct qnetd_client *client, const struct msg_decoded *msg)
 {
-	qnetd_log(LOG_ERR, "Received node list reply. Sending back error message");
 
-	if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
-	    TLV_REPLY_ERROR_CODE_UNEXPECTED_MESSAGE) != 0) {
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "node list reply"));
+}
+
+static int
+qnetd_client_msg_received_ask_for_vote(struct qnetd_instance *instance, struct qnetd_client *client,
+    const struct msg_decoded *msg)
+{
+	int res;
+	struct send_buffer_list_entry *send_buffer;
+	enum tlv_reply_error_code reply_error_code;
+	enum tlv_vote result_vote;
+
+	reply_error_code = TLV_REPLY_ERROR_CODE_NO_ERROR;
+
+	if ((res = qnetd_client_check_tls(instance, client, msg)) != 0) {
+		return (res == -1 ? -1 : 0);
+	}
+
+	if (!client->init_received) {
+		qnetd_log(LOG_ERR, "Received ask for vote message before init message. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    TLV_REPLY_ERROR_CODE_INIT_REQUIRED) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
+	if (!msg->seq_number_set) {
+		qnetd_log(LOG_ERR, "Received ask for vote message without seq number set. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    TLV_REPLY_ERROR_CODE_DOESNT_CONTAIN_REQUIRED_OPTION) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
+	reply_error_code = qnetd_algorithm_ask_for_vote_received(client, msg->seq_number,
+	    &result_vote);
+
+	if (reply_error_code != TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		qnetd_log(LOG_ERR, "Algorithm returned error code. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    reply_error_code) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
+	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
+	if (send_buffer == NULL) {
+		qnetd_log(LOG_ERR, "Can't alloc ask for vote reply msg from list. "
+		    "Disconnecting client connection.");
+
+		return (-1);
+	}
+
+	if (msg_create_ask_for_vote_reply(&send_buffer->buffer, msg->seq_number,
+	    result_vote) == -1) {
+		qnetd_log(LOG_ERR, "Can't alloc ask for vote reply msg. "
+		    "Disconnecting client connection.");
+
 		return (-1);
 	}
 
+	send_buffer_list_put(&client->send_buffer_list, send_buffer);
+
+	return (0);
+}
+
+static int
+qnetd_client_msg_received_ask_for_vote_reply(struct qnetd_instance *instance,
+    struct qnetd_client *client, const struct msg_decoded *msg)
+{
+
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "ask for vote reply"));
+}
+
+static int
+qnetd_client_msg_received_vote_info(struct qnetd_instance *instance, struct qnetd_client *client,
+    const struct msg_decoded *msg)
+{
+
+	return (qnetd_client_msg_received_unexpected_msg(client, msg, "vote info"));
+}
+
+static int
+qnetd_client_msg_received_vote_info_reply(struct qnetd_instance *instance,
+    struct qnetd_client *client, const struct msg_decoded *msg)
+{
+	int res;
+	enum tlv_reply_error_code reply_error_code;
+	enum tlv_vote result_vote;
+
+	reply_error_code = TLV_REPLY_ERROR_CODE_NO_ERROR;
+
+	if ((res = qnetd_client_check_tls(instance, client, msg)) != 0) {
+		return (res == -1 ? -1 : 0);
+	}
+
+	if (!client->init_received) {
+		qnetd_log(LOG_ERR, "Received vote info reply before init message. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    TLV_REPLY_ERROR_CODE_INIT_REQUIRED) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
+	if (!msg->seq_number_set) {
+		qnetd_log(LOG_ERR, "Received vote info reply message without seq number set. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    TLV_REPLY_ERROR_CODE_DOESNT_CONTAIN_REQUIRED_OPTION) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
+	reply_error_code = qnetd_algorithm_membership_node_list_received(client,
+	    msg->seq_number, msg->config_version_set, msg->config_version,
+	    &msg->ring_id, msg->quorate, &msg->nodes, &result_vote);
+
+	if (reply_error_code != TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		qnetd_log(LOG_ERR, "Algorithm returned error code. "
+		    "Sending error reply.");
+
+		if (qnetd_client_send_err(client, msg->seq_number_set, msg->seq_number,
+		    reply_error_code) != 0) {
+			return (-1);
+		}
+
+		return (0);
+	}
+
 	return (0);
 }
 
@@ -911,6 +1009,18 @@ qnetd_client_msg_received(struct qnetd_instance *instance, struct qnetd_client *
 	case MSG_TYPE_NODE_LIST_REPLY:
 		ret_val = qnetd_client_msg_received_node_list_reply(instance, client, &msg);
 		break;
+	case MSG_TYPE_ASK_FOR_VOTE:
+		ret_val = qnetd_client_msg_received_ask_for_vote(instance, client, &msg);
+		break;
+	case MSG_TYPE_ASK_FOR_VOTE_REPLY:
+		ret_val = qnetd_client_msg_received_ask_for_vote_reply(instance, client, &msg);
+		break;
+	case MSG_TYPE_VOTE_INFO:
+		ret_val = qnetd_client_msg_received_vote_info(instance, client, &msg);
+		break;
+	case MSG_TYPE_VOTE_INFO_REPLY:
+		ret_val = qnetd_client_msg_received_vote_info_reply(instance, client, &msg);
+		break;
 	default:
 		qnetd_log(LOG_ERR, "Unsupported message %u received from client. "
 		    "Sending back error message", msg.type);
@@ -1103,7 +1213,9 @@ qnetd_client_disconnect(struct qnetd_instance *instance, struct qnetd_client *cl
 
 	qnetd_algorithm_client_disconnect(client, server_going_down);
 	PR_Close(client->socket);
-	qnetd_cluster_list_del_client(&instance->clusters, client->cluster, client);
+	if (client->cluster != NULL) {
+		qnetd_cluster_list_del_client(&instance->clusters, client->cluster, client);
+	}
 	qnetd_client_list_del(&instance->clients, client);
 }
 

qdevices/msg.c

@@ -44,7 +44,7 @@
 #define MSG_TYPE_LENGTH		2
 #define MSG_LENGTH_LENGTH	4
 
-#define MSG_STATIC_SUPPORTED_MESSAGES_SIZE	12
+#define MSG_STATIC_SUPPORTED_MESSAGES_SIZE	16
 
 enum msg_type msg_static_supported_messages[MSG_STATIC_SUPPORTED_MESSAGES_SIZE] = {
     MSG_TYPE_PREINIT,
@@ -59,6 +59,10 @@ enum msg_type msg_static_supported_messages[MSG_STATIC_SUPPORTED_MESSAGES_SIZE]
     MSG_TYPE_ECHO_REPLY,
     MSG_TYPE_NODE_LIST,
     MSG_TYPE_NODE_LIST_REPLY,
+    MSG_TYPE_ASK_FOR_VOTE,
+    MSG_TYPE_ASK_FOR_VOTE_REPLY,
+    MSG_TYPE_VOTE_INFO,
+    MSG_TYPE_VOTE_INFO_REPLY,
 };
 
 size_t
@@ -585,6 +589,98 @@ small_buf_err:
 	return (0);
 }
 
+size_t
+msg_create_ask_for_vote(struct dynar *msg, uint32_t msg_seq_number)
+{
+
+	dynar_clean(msg);
+
+	msg_add_type(msg, MSG_TYPE_ASK_FOR_VOTE);
+	msg_add_len(msg);
+
+	if (tlv_add_msg_seq_number(msg, msg_seq_number) == -1) {
+		goto small_buf_err;
+	}
+
+	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
+
+	return (dynar_size(msg));
+
+small_buf_err:
+	return (0);
+}
+
+size_t
+msg_create_ask_for_vote_reply(struct dynar *msg, uint32_t msg_seq_number, enum tlv_vote vote)
+{
+
+	dynar_clean(msg);
+
+	msg_add_type(msg, MSG_TYPE_ASK_FOR_VOTE_REPLY);
+	msg_add_len(msg);
+
+	if (tlv_add_msg_seq_number(msg, msg_seq_number) == -1) {
+		goto small_buf_err;
+	}
+
+	if (tlv_add_vote(msg, vote) == -1) {
+		goto small_buf_err;
+	}
+
+	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
+
+	return (dynar_size(msg));
+
+small_buf_err:
+	return (0);
+}
+
+size_t
+msg_create_vote_info(struct dynar *msg, uint32_t msg_seq_number, enum tlv_vote vote)
+{
+
+	dynar_clean(msg);
+
+	msg_add_type(msg, MSG_TYPE_VOTE_INFO);
+	msg_add_len(msg);
+
+	if (tlv_add_msg_seq_number(msg, msg_seq_number) == -1) {
+		goto small_buf_err;
+	}
+
+	if (tlv_add_vote(msg, vote) == -1) {
+		goto small_buf_err;
+	}
+
+	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
+
+	return (dynar_size(msg));
+
+small_buf_err:
+	return (0);
+}
+
+size_t
+msg_create_vote_info_reply(struct dynar *msg, uint32_t msg_seq_number)
+{
+
+	dynar_clean(msg);
+
+	msg_add_type(msg, MSG_TYPE_VOTE_INFO_REPLY);
+	msg_add_len(msg);
+
+	if (tlv_add_msg_seq_number(msg, msg_seq_number) == -1) {
+		goto small_buf_err;
+	}
+
+	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
+
+	return (dynar_size(msg));
+
+small_buf_err:
+	return (0);
+}
+
 int
 msg_is_valid_msg_type(const struct dynar *msg)
 {
@@ -826,7 +922,8 @@ msg_decode(const struct dynar *msg, struct msg_decoded *decoded_msg)
 			decoded_msg->vote_set = 1;
 			break;
 		case TLV_OPT_QUORATE:
-			if ((res = tlv_iter_decode_quorate(&tlv_iter, &decoded_msg->quorate)) != 0) {
+			if ((res = tlv_iter_decode_quorate(&tlv_iter,
+			    &decoded_msg->quorate)) != 0) {
 				return (res);
 			}
 

qdevices/msg.h

@@ -59,6 +59,10 @@ enum msg_type {
 	MSG_TYPE_ECHO_REPLY = 9,
 	MSG_TYPE_NODE_LIST = 10,
 	MSG_TYPE_NODE_LIST_REPLY = 11,
+	MSG_TYPE_ASK_FOR_VOTE = 12,
+	MSG_TYPE_ASK_FOR_VOTE_REPLY = 13,
+	MSG_TYPE_VOTE_INFO = 14,
+	MSG_TYPE_VOTE_INFO_REPLY = 15,
 };
 
 struct msg_decoded {
@@ -157,6 +161,16 @@ extern size_t		msg_create_node_list(struct dynar *msg,
 extern size_t		msg_create_node_list_reply(struct dynar *msg, uint32_t msg_seq_number,
     enum tlv_vote vote);
 
+extern size_t		msg_create_ask_for_vote(struct dynar *msg, uint32_t msg_seq_number);
+
+extern size_t		msg_create_ask_for_vote_reply(struct dynar *msg, uint32_t msg_seq_number,
+    enum tlv_vote vote);
+
+extern size_t		msg_create_vote_info(struct dynar *msg, uint32_t msg_seq_number,
+    enum tlv_vote vote);
+
+extern size_t		msg_create_vote_info_reply(struct dynar *msg, uint32_t msg_seq_number);
+
 extern size_t		msg_get_header_length(void);
 
 extern uint32_t		msg_get_len(const struct dynar *msg);

qdevices/qnetd-algo-ffsplit.c

@@ -88,3 +88,20 @@ qnetd_algo_ffsplit_client_disconnect(struct qnetd_client *client, int server_goi
 {
 
 }
+
+enum tlv_reply_error_code
+qnetd_algo_ffsplit_ask_for_vote_received(struct qnetd_client *client, uint32_t msg_seq_num,
+    enum tlv_vote *result_vote)
+{
+
+	*result_vote = TLV_VOTE_ASK_LATER;
+
+	return (TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM_MESSAGE);
+}
+
+enum tlv_reply_error_code
+qnetd_algo_ffsplit_vote_info_reply_received(struct qnetd_client *client, uint32_t msg_seq_num)
+{
+
+	return (TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM_MESSAGE);
+}

qdevices/qnetd-algo-ffsplit.h

@@ -56,6 +56,12 @@ extern enum tlv_reply_error_code	qnetd_algo_ffsplit_membership_node_list_receive
 extern void				qnetd_algo_ffsplit_client_disconnect(
     struct qnetd_client *client, int server_going_down);
 
+extern enum tlv_reply_error_code	qnetd_algo_ffsplit_ask_for_vote_received(
+    struct qnetd_client *client, uint32_t msg_seq_num, enum tlv_vote *result_vote);
+
+extern enum tlv_reply_error_code	qnetd_algo_ffsplit_vote_info_reply_received(
+    struct qnetd_client *client, uint32_t msg_seq_num);
+
 #ifdef __cplusplus
 }
 #endif

qdevices/qnetd-algo-test.c

@@ -38,6 +38,19 @@
 
 #include "qnetd-algo-test.h"
 #include "qnetd-log.h"
+#include "qnetd-cluster-list.h"
+
+static void
+qnetd_algo_test_dump_cluster(struct qnetd_cluster *cluster)
+{
+	struct qnetd_client *client;
+
+	qnetd_log(LOG_INFO, "algo-test:   Cluster dump:");
+	TAILQ_FOREACH(client, &cluster->client_list, cluster_entries) {
+		qnetd_log(LOG_INFO, "algo-test:     client = %p, node_id = %"PRIx32,
+		    client, client->node_id);
+	}
+}
 
 /*
  * Called right after client sent init message. This happens after initial accept of client,
@@ -46,6 +59,7 @@
  * - client->cluster_name (client->cluster_name_len)
  * - client->node_id (client->node_id_set = 1)
  * - client->decision_algorithm
+ * - client->cluster
  *
  * Callback is designed mainly for allocating client->algorithm_data.
  *
@@ -65,7 +79,9 @@ qnetd_algo_test_client_init(struct qnetd_client *client)
 	qnetd_log(LOG_INFO, "algo-test:   tls peer certificate verified = %u",
 	    client->tls_peer_certificate_verified);
 	qnetd_log(LOG_INFO, "algo-test:   node_id = %"PRIx32, client->node_id);
-	qnetd_log(LOG_INFO, "algo-test:   pointer = 0x%p", client);
+	qnetd_log(LOG_INFO, "algo-test:   pointer = %p", client);
+
+	qnetd_algo_test_dump_cluster(client->cluster);
 
 	client->algorithm_data = malloc(sizeof(int));
 	if (client->algorithm_data == NULL) {
@@ -188,6 +204,11 @@ qnetd_algo_test_membership_node_list_received(struct qnetd_client *client,
 	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
 }
 
+/*
+ * Called after client disconnect. Client structure is still existing (and it's part
+ * of a client->cluster), but it is destroyed (and removed from cluster) right after
+ * this callback finishes. Callback is used mainly for destroing client->algorithm_data.
+ */
 void
 qnetd_algo_test_client_disconnect(struct qnetd_client *client, int server_going_down)
 {
@@ -199,3 +220,30 @@ qnetd_algo_test_client_disconnect(struct qnetd_client *client, int server_going_
 
 	free(client->algorithm_data);
 }
+
+/*
+ * Called after client sent ask for vote message. This is usually happening after server
+ * replied TLV_VOTE_ASK_LATER.
+ */
+enum tlv_reply_error_code
+qnetd_algo_test_ask_for_vote_received(struct qnetd_client *client, uint32_t msg_seq_num,
+    enum tlv_vote *result_vote)
+{
+
+	qnetd_log(LOG_INFO, "algo-test: Client %p (cluster %s, node_id %"PRIx32") "
+	    "asked for a vote", client, client->cluster_name, client->node_id);
+
+	*result_vote = TLV_VOTE_ACK;
+
+	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+}
+
+enum tlv_reply_error_code
+qnetd_algo_test_vote_info_reply_received(struct qnetd_client *client, uint32_t msg_seq_num)
+{
+
+	qnetd_log(LOG_INFO, "algo-test: Client %p (cluster %s, node_id %"PRIx32") "
+	    "replied back to vote info message", client, client->cluster_name, client->node_id);
+
+	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+}

qdevices/qnetd-algo-test.h

@@ -56,6 +56,12 @@ extern enum tlv_reply_error_code	qnetd_algo_test_membership_node_list_received(
 extern void				qnetd_algo_test_client_disconnect(
     struct qnetd_client *client, int server_going_down);
 
+extern enum tlv_reply_error_code	qnetd_algo_test_ask_for_vote_received(
+    struct qnetd_client *client, uint32_t msg_seq_num, enum tlv_vote *result_vote);
+
+extern enum tlv_reply_error_code	qnetd_algo_test_vote_info_reply_received(
+    struct qnetd_client *client, uint32_t msg_seq_num);
+
 #ifdef __cplusplus
 }
 #endif

qdevices/qnetd-algorithm.c

@@ -125,3 +125,42 @@ qnetd_algorithm_client_disconnect(struct qnetd_client *client, int server_going_
 		break;
 	}
 }
+
+enum tlv_reply_error_code
+qnetd_algorithm_ask_for_vote_received(struct qnetd_client *client, uint32_t msg_seq_num,
+    enum tlv_vote *result_vote)
+{
+
+	switch (client->decision_algorithm) {
+	case TLV_DECISION_ALGORITHM_TYPE_TEST:
+		return (qnetd_algo_test_ask_for_vote_received(client, msg_seq_num, result_vote));
+		break;
+	case TLV_DECISION_ALGORITHM_TYPE_FFSPLIT:
+		return (qnetd_algo_ffsplit_ask_for_vote_received(client, msg_seq_num, result_vote));
+		break;
+	default:
+		errx(1, "qnetd_algorithm_ask_for_vote_received unhandled decision algorithm");
+		break;
+	}
+
+	return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
+}
+
+enum tlv_reply_error_code
+qnetd_algorithm_vote_info_reply_received(struct qnetd_client *client, uint32_t msg_seq_num)
+{
+
+	switch (client->decision_algorithm) {
+	case TLV_DECISION_ALGORITHM_TYPE_TEST:
+		return (qnetd_algo_test_vote_info_reply_received(client, msg_seq_num));
+		break;
+	case TLV_DECISION_ALGORITHM_TYPE_FFSPLIT:
+		return (qnetd_algo_ffsplit_vote_info_reply_received(client, msg_seq_num));
+		break;
+	default:
+		errx(1, "qnetd_algorithm_vote_info_reply_received unhandled decision algorithm");
+		break;
+	}
+
+	return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
+}

qdevices/qnetd-algorithm.h

@@ -60,6 +60,12 @@ extern enum tlv_reply_error_code	qnetd_algorithm_membership_node_list_received(
 extern void				qnetd_algorithm_client_disconnect(
     struct qnetd_client *client, int server_going_down);
 
+extern enum tlv_reply_error_code	qnetd_algorithm_ask_for_vote_received(
+    struct qnetd_client *client, uint32_t msg_seq_num, enum tlv_vote *result_vote);
+
+extern enum tlv_reply_error_code	qnetd_algorithm_vote_info_reply_received(
+    struct qnetd_client *client, uint32_t msg_seq_num);
+
 #ifdef __cplusplus
 }
 #endif

qdevices/qnetd-client-send.c

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+
+#include <string.h>
+
+#include "qnetd-client-send.h"
+#include "qnetd-log.h"
+#include "msg.h"
+
+int
+qnetd_client_send_err(struct qnetd_client *client, int add_msg_seq_number, uint32_t msg_seq_number,
+    enum tlv_reply_error_code reply)
+{
+	struct send_buffer_list_entry *send_buffer;
+
+	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
+	if (send_buffer == NULL) {
+		qnetd_log(LOG_ERR, "Can't alloc server error msg from list. "
+		    "Disconnecting client connection.");
+
+		return (-1);
+	}
+
+	if (msg_create_server_error(&send_buffer->buffer, add_msg_seq_number,
+	    msg_seq_number, reply) == 0) {
+		qnetd_log(LOG_ERR, "Can't alloc server error msg. "
+		    "Disconnecting client connection.");
+
+		return (-1);
+	};
+
+	send_buffer_list_put(&client->send_buffer_list, send_buffer);
+
+	return (0);
+}
+
+int
+qnetd_client_send_vote_info(struct qnetd_client *client, uint32_t msg_seq_number,
+    enum tlv_vote vote)
+{
+	struct send_buffer_list_entry *send_buffer;
+
+	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
+	if (send_buffer == NULL) {
+		qnetd_log(LOG_ERR, "Can't alloc vote info msg from list. "
+		    "Disconnecting client connection.");
+
+		return (-1);
+	}
+
+	if (msg_create_vote_info(&send_buffer->buffer, msg_seq_number, vote) == 0) {
+		qnetd_log(LOG_ERR, "Can't alloc vote info msg. "
+		    "Disconnecting client connection.");
+
+		return (-1);
+	};
+
+	send_buffer_list_put(&client->send_buffer_list, send_buffer);
+
+	return (0);
+}

qdevices/qnetd-client-send.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QNETD_CLIENT_SEND_H_
+#define _QNETD_CLIENT_SEND_H_
+
+#include <sys/types.h>
+
+#include "qnetd-client.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int		qnetd_client_send_err(struct qnetd_client *client,
+    int add_msg_seq_number, uint32_t msg_seq_number, enum tlv_reply_error_code reply);
+
+extern int		qnetd_client_send_vote_info(struct qnetd_client *client,
+    uint32_t msg_seq_number, enum tlv_vote vote);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QNETD_CLIENT_SEND_H_ */

qdevices/qnetd-client.c

@@ -56,6 +56,7 @@ void
 qnetd_client_destroy(struct qnetd_client *client)
 {
 
+	free(client->cluster_name);
 	node_list_free(&client->last_membership_node_list);
 	node_list_free(&client->configuration_node_list);
 	send_buffer_list_free(&client->send_buffer_list);

qdevices/tlv.h

@@ -89,6 +89,7 @@ enum tlv_reply_error_code {
 	TLV_REPLY_ERROR_CODE_INIT_REQUIRED = 11,
 	TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM = 12,
 	TLV_REPLY_ERROR_CODE_INVALID_HEARTBEAT_INTERVAL = 13,
+	TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM_MESSAGE = 14,
 };
 
 enum tlv_decision_algorithm_type {