Qnetd: ffsplit: Enhance ffsplit

50:50 split algorithm now works in following way:
- On client configuration change, membership change or disconnect wait
  till membership is stable (= all client configuration node list are
  equal, and all partitions has equal information).
- Choose best partition >= 50%
  - If no such partition exists, send NACK to all clients
- Send NACK to all clients who should receive NACK
- After all clients who should receive NACK confirm vote reception, send
  ACK to all clients who should get ACK

This ensures that there are never two partitions with ACK and it has
much better behavior than previous version, because if tie-breaker
partition is not connected, other partition gets ACK.

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/qdevice-ipc-cmd.c

@@ -191,7 +191,7 @@ qdevice_ipc_cmd_status_add_quorum_node_list(struct qdevice_instance *instance, s
 		node = &instance->vq_quorum_node_list[u32];
 
 		if (node->nodeid == 0) {
-			continue ;
+			continue;
 		}
 
 		if (dynar_str_catf(outbuf, "    %"PRIu32"\tNode ID = "UTILS_PRI_NODE_ID

qdevices/qdevice-net-algo-ffsplit.c

@@ -60,7 +60,7 @@ check_vqinfo_validity(struct qdevice_net_instance *instance)
 		cs_res = votequorum_getinfo(qdev_instance->votequorum_handle, node_id, &vq_info);
 
 		if (cs_res == CS_ERR_NOT_EXIST) {
-			continue ;
+			continue;
 		} else if (cs_res != CS_OK) {
 			qdevice_log(LOG_CRIT, "Can't get votequorum information for node "
 			    UTILS_PRI_NODE_ID ". Error %s", node_id, cs_strerror(cs_res));

qdevices/qnet-config.h

@@ -53,7 +53,7 @@ extern "C" {
 #define QNETD_DEFAULT_HOST_PORT				5403
 #define QNETD_DEFAULT_LISTEN_BACKLOG			10
 #define QNETD_MIN_LISTEN_BACKLOG			1
-#define QNETD_DEFAULT_MAX_CLIENT_SEND_BUFFERS		10
+#define QNETD_DEFAULT_MAX_CLIENT_SEND_BUFFERS		32
 #define QNETD_MIN_CLIENT_SEND_BUFFERS			2
 #define QNETD_DEFAULT_MAX_CLIENT_SEND_SIZE		(1 << 15)
 #define QNETD_DEFAULT_MAX_CLIENT_RECEIVE_SIZE		(1 << 15)

qdevices/qnetd-algo-ffsplit.c

@@ -38,21 +38,42 @@
 
 #include "qnetd-algo-ffsplit.h"
 #include "qnetd-log.h"
+#include "qnetd-log-debug.h"
 #include "qnetd-cluster-list.h"
 #include "qnetd-cluster.h"
+#include "qnetd-client-send.h"
 
-struct ffsplit_cluster_data {
-	uint8_t leader_set;
-	uint32_t leader_id;
+enum qnetd_algo_ffsplit_cluster_state {
+	QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_CHANGE,
+	QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_STABLE_MEMBERSHIP,
+	QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_NACKS,
+	QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_ACKS,
+};
+
+struct qnetd_algo_ffsplit_cluster_data {
+	enum qnetd_algo_ffsplit_cluster_state cluster_state;
+	const struct node_list *quorate_partition_node_list;
+};
+
+enum qnetd_algo_ffsplit_client_state {
+	QNETD_ALGO_FFSPLIT_CLIENT_STATE_WAITING_FOR_CHANGE,
+	QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_NACK,
+	QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_ACK,
+};
+
+struct qnetd_algo_ffsplit_client_data {
+	enum qnetd_algo_ffsplit_client_state client_state;
+	uint32_t vote_info_expected_seq_num;
 };
 
 enum tlv_reply_error_code
 qnetd_algo_ffsplit_client_init(struct qnetd_client *client)
 {
-	struct ffsplit_cluster_data *cluster_data;
+	struct qnetd_algo_ffsplit_cluster_data *cluster_data;
+	struct qnetd_algo_ffsplit_client_data *client_data;
 
 	if (qnetd_cluster_size(client->cluster) == 1) {
-		cluster_data = malloc(sizeof(struct ffsplit_cluster_data));
+		cluster_data = malloc(sizeof(*cluster_data));
 		if (cluster_data == NULL) {
 			qnetd_log(LOG_ERR, "ffsplit: Can't initialize cluster data for client %s",
 			    client->addr_str);
@@ -60,15 +81,28 @@ qnetd_algo_ffsplit_client_init(struct qnetd_client *client)
 			return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
 		}
 		memset(cluster_data, 0, sizeof(*cluster_data));
+		cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_CHANGE;
+		cluster_data->quorate_partition_node_list = NULL;
 
 		client->cluster->algorithm_data = cluster_data;
 	}
 
+	client_data = malloc(sizeof(*client_data));
+	if (client_data == NULL) {
+		qnetd_log(LOG_ERR, "ffsplit: Can't initialize node data for client %s",
+		    client->addr_str);
+
+		return (TLV_REPLY_ERROR_CODE_INTERNAL_ERROR);
+	}
+	memset(client_data, 0, sizeof(*client_data));
+	client_data->client_state = QNETD_ALGO_FFSPLIT_CLIENT_STATE_WAITING_FOR_CHANGE;
+	client->algorithm_data = client_data;
+
 	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
 }
 
 static int
-qnetd_algo_ffsplit_is_prefered_partition(struct qnetd_client *client,
+qnetd_algo_ffsplit_is_prefered_partition(const struct qnetd_client *client,
     const struct node_list *config_node_list, const struct node_list *membership_node_list)
 {
 	uint32_t prefered_node_id;
@@ -105,40 +139,397 @@ qnetd_algo_ffsplit_is_prefered_partition(struct qnetd_client *client,
 	return (node_list_find_node_id(membership_node_list, prefered_node_id) != NULL);
 }
 
-static enum tlv_vote
-qnetd_algo_ffsplit_do(struct qnetd_client *client, const struct node_list *config_node_list,
+static int
+qnetd_algo_ffsplit_is_membership_stable(const struct qnetd_client *client, int client_leaving,
+    const struct tlv_ring_id *ring_id, const struct node_list *config_node_list,
     const struct node_list *membership_node_list)
 {
-	struct ffplist_cluster_data *cluster_data;
+	const struct qnetd_client *iter_client1, *iter_client2;
+	const struct node_list *config_node_list1, *config_node_list2;
+	const struct node_list *membership_node_list1, *membership_node_list2;
+	const struct node_list_entry *iter_node1, *iter_node2;
+	const struct node_list_entry *iter_node3, *iter_node4;
+	const struct tlv_ring_id *ring_id1, *ring_id2;
+
+	/*
+	 * Test if all active clients share same config list.
+	 */
+	TAILQ_FOREACH(iter_client1, &client->cluster->client_list, cluster_entries) {
+		TAILQ_FOREACH(iter_client2, &client->cluster->client_list, cluster_entries) {
+			if (iter_client1 == iter_client2) {
+				continue;
+			}
+
+			if (iter_client1->node_id == client->node_id) {
+				if (client_leaving) {
+					continue;
+				}
+
+				config_node_list1 = config_node_list;
+			} else {
+				config_node_list1 = &iter_client1->configuration_node_list;
+			}
+
+			if (iter_client2->node_id == client->node_id) {
+				if (client_leaving) {
+					continue;
+				}
+
+				config_node_list2 = config_node_list;
+			} else {
+				config_node_list2 = &iter_client2->configuration_node_list;
+			}
 
-	cluster_data = (struct ffplist_cluster_data *)client->cluster->algorithm_data;
+			/*
+			 * Walk thru all node ids in given config node list...
+			 */
+			TAILQ_FOREACH(iter_node1, config_node_list1, entries) {
+				/*
+				 * ... and try to find given node id in other list
+				 */
+				iter_node2 = node_list_find_node_id(config_node_list2, iter_node1->node_id);
+
+				if (iter_node2 == NULL) {
+					/*
+					 * Node with iter_node1->node_id was not found in
+					 * config_node_list2 -> lists doesn't match
+					 */
+					return (0);
+				}
+			}
+		}
+	}
+
+	/*
+	 * Test if same partitions share same ring ids and membership node list
+	 */
+	TAILQ_FOREACH(iter_client1, &client->cluster->client_list, cluster_entries) {
+		if (iter_client1->node_id == client->node_id) {
+			if (client_leaving) {
+				continue;
+			}
+
+			membership_node_list1 = membership_node_list;
+			ring_id1 = ring_id;
+		} else {
+			membership_node_list1 = &iter_client1->last_membership_node_list;
+			ring_id1 = &iter_client1->last_ring_id;
+		}
+
+		/*
+		 * Walk thru all memberships nodes
+		 */
+		TAILQ_FOREACH(iter_node1, membership_node_list1, entries) {
+			/*
+			 * try to find client with given node id
+			 */
+			iter_client2 = qnetd_cluster_find_client_by_node_id(client->cluster,
+			    iter_node1->node_id);
+			if (iter_client2 == NULL) {
+				/*
+				 * Client with given id is not connected
+				 */
+				continue;
+			}
+
+			if (iter_client2->node_id == client->node_id) {
+				if (client_leaving) {
+					continue;
+				}
+
+				membership_node_list2 = membership_node_list;
+				ring_id2 = ring_id;
+			} else {
+				membership_node_list2 = &iter_client2->last_membership_node_list;
+				ring_id2 = &iter_client2->last_ring_id;
+			}
+
+			/*
+			 * Compare ring ids
+			 */
+			if (!tlv_ring_id_eq(ring_id1, ring_id2)) {
+				return (0);
+			}
+
+			/*
+			 * Now compare that membership node list equals, so walk thru all
+			 * members ...
+			 */
+			TAILQ_FOREACH(iter_node3, membership_node_list1, entries) {
+				/*
+				 * ... and try to find given node id in other membership node list
+				 */
+				iter_node4 = node_list_find_node_id(membership_node_list2, iter_node3->node_id);
+
+				if (iter_node4 == NULL) {
+					/*
+					 * Node with iter_node3->node_id was not found in
+					 * membership_node_list2 -> lists doesn't match
+					 */
+					return (0);
+				}
+			}
+		}
+	}
+
+	return (1);
+}
+
+/*
+ * 0 - Not quarate
+ * 1 - Quorate but not fulfilling tie breaker or quorate with more than 50%
+ * 2 - Quorate and fulfilling tie breaker
+ */
+static unsigned int
+qnetd_algo_ffsplit_get_partition_score(const struct qnetd_client *client,
+    const struct node_list *config_node_list, const struct node_list *membership_node_list)
+{
 
 	if (node_list_size(config_node_list) % 2 != 0) {
 		/*
 		 * Odd clusters never split into 50:50.
 		 */
 		if (node_list_size(membership_node_list) > node_list_size(config_node_list) / 2) {
-			return (TLV_VOTE_ACK);
+			return (1);
 		} else {
-			return (TLV_VOTE_NACK);
+			return (0);
 		}
 	} else {
 		if (node_list_size(membership_node_list) > node_list_size(config_node_list) / 2) {
-			return (TLV_VOTE_ACK);
+			return (1);
 		} else if (node_list_size(membership_node_list) < node_list_size(config_node_list) / 2) {
-			return (TLV_VOTE_NACK);
+			return (0);
 		} else {
 			/*
 			 * 50:50 split
 			 */
 			if (qnetd_algo_ffsplit_is_prefered_partition(client, config_node_list,
 			    membership_node_list)) {
-				return (TLV_VOTE_ACK);
+				return (2);
 			} else {
-				return (TLV_VOTE_NACK);
+				return (1);
+			}
+		}
+	}
+
+	qnetd_log(LOG_CRIT, "qnetd_algo_ffsplit_get_partition_score unhandled case");
+	exit(1);
+
+	return (-1);
+}
+
+static const struct node_list *
+qnetd_algo_ffsplit_select_partition(const struct qnetd_client *client, int client_leaving,
+    const struct node_list *config_node_list, const struct node_list *membership_node_list)
+{
+	unsigned int highest_score, iter_score;
+	const struct qnetd_client *iter_client;
+	const struct node_list *result_node_list;
+	const struct node_list *iter_config_node_list;
+	const struct node_list *iter_membership_node_list;
+
+	highest_score = 0;
+	result_node_list = NULL;
+
+	/*
+	 * Get highest score
+	 */
+	TAILQ_FOREACH(iter_client, &client->cluster->client_list, cluster_entries) {
+		if (iter_client->node_id == client->node_id) {
+			if (client_leaving) {
+				continue;
+			}
+
+			iter_config_node_list = config_node_list;
+			iter_membership_node_list = membership_node_list;
+		} else {
+			iter_config_node_list = &iter_client->configuration_node_list;
+			iter_membership_node_list = &iter_client->last_membership_node_list;
+		}
+
+		if ((iter_score = qnetd_algo_ffsplit_get_partition_score(iter_client, iter_config_node_list,
+		    iter_membership_node_list)) > highest_score) {
+			highest_score = iter_score;
+			result_node_list = iter_membership_node_list;
+		}
+	}
+
+	return (result_node_list);
+}
+
+/*
+ * Update state of all nodes to match quorate_partition_node_list
+ */
+static void
+qnetd_algo_ffsplit_update_nodes_state(struct qnetd_client *client, int client_leaving,
+    const struct node_list *quorate_partition_node_list)
+{
+	const struct qnetd_client *iter_client;
+	struct qnetd_algo_ffsplit_client_data *iter_client_data;
+
+	TAILQ_FOREACH(iter_client, &client->cluster->client_list, cluster_entries) {
+		iter_client_data = (struct qnetd_algo_ffsplit_client_data *)iter_client->algorithm_data;
+
+		if (iter_client->node_id == client->node_id && client_leaving) {
+			iter_client_data->client_state = QNETD_ALGO_FFSPLIT_CLIENT_STATE_WAITING_FOR_CHANGE;
+
+			continue;
+		}
+
+		if (quorate_partition_node_list == NULL ||
+		    node_list_find_node_id(quorate_partition_node_list, iter_client->node_id) == NULL) {
+			iter_client_data->client_state = QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_NACK;
+		} else {
+			iter_client_data->client_state = QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_ACK;
+		}
+	}
+}
+
+/*
+ * Send vote info. If client_leaving is set, client is ignored. if send_acks
+ * is set, only ACK votes are send (nodes in QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_ACK state),
+ * otherwise only NACK votes are send (nodes in QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_NACK state)
+ *
+ * Returns number of send votes
+ */
+static size_t
+qnetd_algo_ffsplit_send_votes(struct qnetd_client *client, int client_leaving,
+    const struct tlv_ring_id *ring_id, int send_acks)
+{
+	size_t sent_votes;
+	struct qnetd_client *iter_client;
+	struct qnetd_algo_ffsplit_client_data *iter_client_data;
+	const struct tlv_ring_id *ring_id_to_send;
+	enum tlv_vote vote_to_send;
+
+	sent_votes = 0;
+
+	TAILQ_FOREACH(iter_client, &client->cluster->client_list, cluster_entries) {
+		if (iter_client->node_id == client->node_id) {
+			if (client_leaving) {
+				continue;
 			}
+
+			ring_id_to_send = ring_id;
+		} else {
+			ring_id_to_send = &iter_client->last_ring_id;
+		}
+
+		iter_client_data = (struct qnetd_algo_ffsplit_client_data *)iter_client->algorithm_data;
+		vote_to_send = TLV_VOTE_UNDEFINED;
+
+		if (send_acks) {
+			if (iter_client_data->client_state == QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_ACK) {
+				vote_to_send = TLV_VOTE_ACK;
+			}
+		} else {
+			if (iter_client_data->client_state == QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_NACK) {
+				vote_to_send = TLV_VOTE_NACK;
+			}
+		}
+
+		if (vote_to_send != TLV_VOTE_UNDEFINED) {
+			iter_client_data->vote_info_expected_seq_num++;
+			sent_votes++;
+
+			if (qnetd_client_send_vote_info(iter_client,
+			    iter_client_data->vote_info_expected_seq_num, ring_id_to_send,
+			    vote_to_send) == -1) {
+				client->schedule_disconnect = 1;
+			}
+		}
+	}
+
+	return (sent_votes);
+}
+
+/*
+ * Return number of clients in QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_ACK state if sending_acks is
+ * set or number of nodes in QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_NACK state if sending_acks is
+ * not set
+ */
+static size_t
+qnetd_algo_ffsplit_no_clients_in_sending_state(struct qnetd_client *client, int sending_acks)
+{
+	size_t no_clients;
+	struct qnetd_client *iter_client;
+	struct qnetd_algo_ffsplit_client_data *iter_client_data;
+
+	no_clients = 0;
+
+	TAILQ_FOREACH(iter_client, &client->cluster->client_list, cluster_entries) {
+		iter_client_data = (struct qnetd_algo_ffsplit_client_data *)iter_client->algorithm_data;
+
+		if (sending_acks &&
+		    iter_client_data->client_state == QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_ACK) {
+			no_clients++;
+		}
+
+		if (!sending_acks &&
+		    iter_client_data->client_state == QNETD_ALGO_FFSPLIT_CLIENT_STATE_SENDING_NACK) {
+			no_clients++;
 		}
 	}
+
+	return (no_clients);
+}
+
+static enum tlv_vote
+qnetd_algo_ffsplit_do(struct qnetd_client *client, int client_leaving,
+    const struct tlv_ring_id *ring_id, const struct node_list *config_node_list,
+    const struct node_list *membership_node_list)
+{
+	struct qnetd_algo_ffsplit_cluster_data *cluster_data;
+	const struct node_list *quorate_partition_node_list;
+
+	cluster_data = (struct qnetd_algo_ffsplit_cluster_data *)client->cluster->algorithm_data;
+
+	cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_STABLE_MEMBERSHIP;
+
+	if (!qnetd_algo_ffsplit_is_membership_stable(client, client_leaving,
+	    ring_id, config_node_list, membership_node_list)) {
+		/*
+		 * Wait until membership is stable
+		 */
+		qnetd_log(LOG_DEBUG, "ffsplit: Membership for cluster %s is not yet stable", client->cluster_name);
+
+		return (TLV_VOTE_WAIT_FOR_REPLY);
+	}
+
+	qnetd_log(LOG_DEBUG, "ffsplit: Membership for cluster %s is now stable", client->cluster_name);
+
+	quorate_partition_node_list = qnetd_algo_ffsplit_select_partition(client, client_leaving,
+	    config_node_list, membership_node_list);
+	cluster_data->quorate_partition_node_list = quorate_partition_node_list;
+
+	if (quorate_partition_node_list == NULL) {
+		qnetd_log(LOG_DEBUG, "ffsplit: No quorate partition was selected");
+	} else {
+		qnetd_log(LOG_DEBUG, "ffsplit: Quorate partition selected");
+		qnetd_log_debug_dump_node_list(client, quorate_partition_node_list);
+	}
+
+	qnetd_algo_ffsplit_update_nodes_state(client, client_leaving, quorate_partition_node_list);
+
+	cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_NACKS;
+
+	if (qnetd_algo_ffsplit_send_votes(client, client_leaving, ring_id, 0) == 0) {
+		qnetd_log(LOG_DEBUG, "ffsplit: No client gets NACK");
+		/*
+		 * No one gets nack -> send acks
+		 */
+		cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_ACKS;
+
+		if (qnetd_algo_ffsplit_send_votes(client, client_leaving, ring_id, 1) == 0) {
+			qnetd_log(LOG_DEBUG, "ffsplit: No client gets ACK");
+			/*
+			 * No one gets acks -> finished
+			 */
+			cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_CHANGE;
+		}
+	}
+
+	return (TLV_VOTE_NO_CHANGE);
 }
 
 enum tlv_reply_error_code
@@ -173,7 +564,8 @@ qnetd_algo_ffsplit_config_node_list_received(struct qnetd_client *client,
 		 */
 		*result_vote = TLV_VOTE_ASK_LATER;
 	} else {
-		*result_vote = qnetd_algo_ffsplit_do(client, nodes, &client->last_membership_node_list);
+		*result_vote = qnetd_algo_ffsplit_do(client, 0, &client->last_ring_id,
+		    nodes, &client->last_membership_node_list);
 	}
 
 	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
@@ -225,7 +617,8 @@ qnetd_algo_ffsplit_membership_node_list_received(struct qnetd_client *client,
 		 */
 		*result_vote = TLV_VOTE_ASK_LATER;
 	} else {
-		*result_vote = qnetd_algo_ffsplit_do(client, &client->configuration_node_list, nodes);
+		*result_vote = qnetd_algo_ffsplit_do(client, 0, ring_id,
+		    &client->configuration_node_list, nodes);
 	}
 
 	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
@@ -249,6 +642,11 @@ void
 qnetd_algo_ffsplit_client_disconnect(struct qnetd_client *client, int server_going_down)
 {
 
+	(void)qnetd_algo_ffsplit_do(client, 1, &client->last_ring_id,
+	    &client->configuration_node_list, &client->last_membership_node_list);
+
+	free(client->algorithm_data);
+
 	if (qnetd_cluster_size(client->cluster) == 1) {
 		/*
 		 * Last client in the cluster
@@ -262,14 +660,60 @@ qnetd_algo_ffsplit_ask_for_vote_received(struct qnetd_client *client, uint32_t m
     enum tlv_vote *result_vote)
 {
 
+	/*
+	 * Ask for vote is not supported in current algorithm
+	 */
 	return (TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM_MESSAGE);
 }
 
 enum tlv_reply_error_code
 qnetd_algo_ffsplit_vote_info_reply_received(struct qnetd_client *client, uint32_t msg_seq_num)
 {
+	struct qnetd_algo_ffsplit_cluster_data *cluster_data;
+	struct qnetd_algo_ffsplit_client_data *client_data;
 
-	return (TLV_REPLY_ERROR_CODE_UNSUPPORTED_DECISION_ALGORITHM_MESSAGE);
+	cluster_data = (struct qnetd_algo_ffsplit_cluster_data *)client->cluster->algorithm_data;
+	client_data = (struct qnetd_algo_ffsplit_client_data *)client->algorithm_data;
+
+	if (client_data->vote_info_expected_seq_num != msg_seq_num) {
+		qnetd_log(LOG_DEBUG, "ffsplit: Received old vote info reply from client %s",
+		    client->addr_str);
+
+		return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+	}
+
+	client_data->client_state = QNETD_ALGO_FFSPLIT_CLIENT_STATE_WAITING_FOR_CHANGE;
+
+	if (cluster_data->cluster_state != QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_NACKS &&
+	    cluster_data->cluster_state != QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_ACKS) {
+		return (TLV_REPLY_ERROR_CODE_NO_ERROR);
+	}
+
+	if (cluster_data->cluster_state == QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_NACKS) {
+		if (qnetd_algo_ffsplit_no_clients_in_sending_state(client, 0) == 0) {
+			qnetd_log(LOG_DEBUG, "ffsplit: All NACK votes sent for cluster %s",
+			     client->cluster_name);
+
+			cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_SENDING_ACKS;
+
+			if (qnetd_algo_ffsplit_send_votes(client, 0, &client->last_ring_id, 1) == 0) {
+				qnetd_log(LOG_DEBUG, "ffsplit: No client gets ACK");
+				/*
+				 * No one gets acks -> finished
+				 */
+				cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_CHANGE;
+			}
+		}
+	} else {
+		if (qnetd_algo_ffsplit_no_clients_in_sending_state(client, 1) == 0) {
+			qnetd_log(LOG_DEBUG, "ffsplit: All ACK votes sent for cluster %s",
+			     client->cluster_name);
+
+			cluster_data->cluster_state = QNETD_ALGO_FFSPLIT_CLUSTER_STATE_WAITING_FOR_CHANGE;
+		}
+	}
+
+	return (TLV_REPLY_ERROR_CODE_NO_ERROR);
 }
 
 enum tlv_reply_error_code

qdevices/qnetd-client-algo-timer.c

@@ -78,7 +78,8 @@ qnetd_client_algo_timer_callback(void *data1, void *data2)
 		client->algo_timer_vote_info_msq_seq_number++;
 
 		if (qnetd_client_send_vote_info(client,
-		    client->algo_timer_vote_info_msq_seq_number, result_vote) != 0) {
+		    client->algo_timer_vote_info_msq_seq_number, &client->last_ring_id,
+		    result_vote) != 0) {
 			client->schedule_disconnect = 1;
 			return (0);
 		}

qdevices/qnetd-client-msg-received.c

@@ -772,13 +772,6 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 		qnetd_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(result_vote));
 	}
 
-	if (msg->node_list_type == TLV_NODE_LIST_TYPE_MEMBERSHIP &&
-	    result_vote == TLV_VOTE_NO_CHANGE) {
-		qnetd_log(LOG_ERR, "qnetd_client_msg_received_node_list fatal error. "
-		    "node_list_type is membership and algorithm result vote is no_change");
-		exit(1);
-	}
-
 	/*
 	 * Store node list for future use
 	 */

qdevices/qnetd-client-send.c

@@ -71,7 +71,7 @@ qnetd_client_send_err(struct qnetd_client *client, int add_msg_seq_number, uint3
 
 int
 qnetd_client_send_vote_info(struct qnetd_client *client, uint32_t msg_seq_number,
-    enum tlv_vote vote)
+    const struct tlv_ring_id *ring_id, enum tlv_vote vote)
 {
 	struct send_buffer_list_entry *send_buffer;
 
@@ -93,8 +93,7 @@ qnetd_client_send_vote_info(struct qnetd_client *client, uint32_t msg_seq_number
 		return (-1);
 	}
 
-	if (msg_create_vote_info(&send_buffer->buffer, msg_seq_number,
-	    &client->last_ring_id, vote) == 0) {
+	if (msg_create_vote_info(&send_buffer->buffer, msg_seq_number, ring_id, vote) == 0) {
 		qnetd_log(LOG_ERR, "Can't alloc vote info msg. "
 		    "Disconnecting client connection.");
 

qdevices/qnetd-client-send.h

@@ -47,7 +47,7 @@ extern int		qnetd_client_send_err(struct qnetd_client *client,
     int add_msg_seq_number, uint32_t msg_seq_number, enum tlv_reply_error_code reply);
 
 extern int		qnetd_client_send_vote_info(struct qnetd_client *client,
-    uint32_t msg_seq_number, enum tlv_vote vote);
+    uint32_t msg_seq_number, const struct tlv_ring_id *ring_id, enum tlv_vote vote);
 
 #ifdef __cplusplus
 }

qdevices/qnetd-cluster.c

@@ -79,3 +79,17 @@ qnetd_cluster_size(const struct qnetd_cluster *cluster)
 
 	return (res);
 }
+
+struct qnetd_client *
+qnetd_cluster_find_client_by_node_id(const struct qnetd_cluster *cluster, uint32_t node_id)
+{
+	struct qnetd_client *client;
+
+	TAILQ_FOREACH(client, &cluster->client_list, cluster_entries) {
+		if (client->node_id == node_id) {
+			return (client);
+		}
+	}
+
+	return (NULL);
+}

qdevices/qnetd-cluster.h

@@ -55,12 +55,15 @@ struct qnetd_cluster {
 	TAILQ_ENTRY(qnetd_cluster) entries;
 };
 
-extern int		qnetd_cluster_init(struct qnetd_cluster *cluster,
+extern int			qnetd_cluster_init(struct qnetd_cluster *cluster,
     const char *cluster_name, size_t cluster_name_len);
 
-extern void		qnetd_cluster_destroy(struct qnetd_cluster *cluster);
+extern void			qnetd_cluster_destroy(struct qnetd_cluster *cluster);
 
-extern size_t		qnetd_cluster_size(const struct qnetd_cluster *cluster);
+extern size_t			qnetd_cluster_size(const struct qnetd_cluster *cluster);
+
+extern struct qnetd_client	*qnetd_cluster_find_client_by_node_id(
+    const struct qnetd_cluster *cluster, uint32_t node_id);
 
 #ifdef __cplusplus
 }

qdevices/qnetd-ipc-cmd.c

@@ -251,7 +251,7 @@ qnetd_ipc_cmd_list(struct qnetd_instance *instance, struct dynar *outbuf, int ve
 	TAILQ_FOREACH(cluster, &instance->clusters, entries) {
 		if (cluster_name != NULL && strcmp(cluster_name, "") != 0 &&
 		    strcmp(cluster_name, cluster->cluster_name) != 0) {
-			continue ;
+			continue;
 		}
 
 		if (dynar_str_catf(outbuf, "Cluster \"%s\":\n", cluster->cluster_name) == -1) {