qdevice: Add tie_breaker option

Tie-breaker can be used in algorithm to decide if algorithm should
prefer highest, lowest or some concrete node id.

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/corosync-qdevice-net.c

@@ -392,7 +392,7 @@ qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
 	instance->last_msg_seq_num++;
 
 	if (msg_create_set_option(&send_buffer->buffer, 1, instance->last_msg_seq_num,
-	    1, instance->heartbeat_interval) == 0) {
+	    1, instance->heartbeat_interval, 1, &instance->tie_breaker) == 0) {
 		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for set option msg");
 
 		return (-1);
@@ -1030,8 +1030,10 @@ qdevice_net_instance_init_from_cmap(struct qdevice_net_instance *instance,
 	uint32_t node_id;
 	enum tlv_tls_supported tls_supported;
 	int i;
+	long int li;
 	char *str;
 	enum tlv_decision_algorithm_type decision_algorithm;
+	struct tlv_tie_breaker tie_breaker;
 	uint32_t heartbeat_interval;
 	uint32_t sync_heartbeat_interval;
 	uint32_t cast_vote_timer_interval;
@@ -1145,6 +1147,31 @@ qdevice_net_instance_init_from_cmap(struct qdevice_net_instance *instance,
 		free(str);
 	}
 
+	/*
+	 * Load tie_breaker mode
+	 */
+	memset(&tie_breaker, 0, sizeof(tie_breaker));
+
+	if (cmap_get_string(cmap_handle, "quorum.device.net.tie_breaker", &str) != CS_OK) {
+		tie_breaker.mode = QDEVICE_NET_DEFAULT_TIE_BREAKER_MODE;
+	} else {
+		if (strcmp(str, "lowest") == 0) {
+			tie_breaker.mode = TLV_TIE_BREAKER_MODE_LOWEST;
+		} else if (strcmp(str, "highest") == 0) {
+			tie_breaker.mode = TLV_TIE_BREAKER_MODE_HIGHEST;
+		} else {
+			li = strtol(str, &ep, 10);
+			if (li <= 0 || li > ((uint32_t)~0) || *ep != '\0') {
+				errx(1, "tie_breaker must be lowest|highest|valid_node_id");
+			}
+
+			tie_breaker.mode = TLV_TIE_BREAKER_MODE_NODE_ID;
+			tie_breaker.node_id = li;
+		}
+
+		free(str);
+	}
+
 	/*
 	 * Really initialize instance
 	 */
@@ -1154,7 +1181,7 @@ qdevice_net_instance_init_from_cmap(struct qdevice_net_instance *instance,
 	    QDEVICE_NET_MAX_MSG_RECEIVE_SIZE,
 	    tls_supported, node_id, decision_algorithm,
 	    heartbeat_interval, sync_heartbeat_interval, cast_vote_timer_interval,
-	    host_addr, host_port, cluster_name) == -1) {
+	    host_addr, host_port, cluster_name, &tie_breaker) == -1) {
 		errx(1, "Can't initialize qdevice-net");
 	}
 

qdevices/corosync-qnetd.c

@@ -454,6 +454,7 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 			reply_error_code = TLV_REPLY_ERROR_CODE_INTERNAL_ERROR;
 		} else {
 			client->cluster = cluster;
+			client->cluster_list = &instance->clusters;
 		}
 	}
 
@@ -552,6 +553,10 @@ qnetd_client_msg_received_set_option(struct qnetd_instance *instance, struct qne
 		client->heartbeat_interval = msg->heartbeat_interval;
 	}
 
+	if (msg->tie_breaker_set) {
+		memcpy(&client->tie_breaker, &msg->tie_breaker, sizeof(msg->tie_breaker));
+	}
+
 	send_buffer = send_buffer_list_get_new(&client->send_buffer_list);
 	if (send_buffer == NULL) {
 		qnetd_log(LOG_ERR, "Can't alloc set option reply msg from list. "
@@ -561,7 +566,8 @@ qnetd_client_msg_received_set_option(struct qnetd_instance *instance, struct qne
 	}
 
 	if (msg_create_set_option_reply(&send_buffer->buffer, msg->seq_number_set, msg->seq_number,
-	    client->decision_algorithm, client->heartbeat_interval) == -1) {
+	    client->decision_algorithm, client->heartbeat_interval,
+	    msg->tie_breaker_set, &msg->tie_breaker) == -1) {
 		qnetd_log(LOG_ERR, "Can't alloc set option reply msg. "
 		    "Disconnecting client connection.");
 

qdevices/msg.c

@@ -404,7 +404,8 @@ small_buf_err:
 
 size_t
 msg_create_set_option(struct dynar *msg, int add_msg_seq_number, uint32_t msg_seq_number,
-    int add_heartbeat_interval, uint32_t heartbeat_interval)
+    int add_heartbeat_interval, uint32_t heartbeat_interval,
+    int add_tie_breaker, const struct tlv_tie_breaker *tie_breaker)
 {
 
 	dynar_clean(msg);
@@ -424,6 +425,12 @@ msg_create_set_option(struct dynar *msg, int add_msg_seq_number, uint32_t msg_se
 		}
 	}
 
+	if (add_tie_breaker) {
+		if (tlv_add_tie_breaker(msg, tie_breaker) == -1) {
+			goto small_buf_err;
+		}
+	}
+
 	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
 
 	return (dynar_size(msg));
@@ -434,7 +441,8 @@ small_buf_err:
 
 size_t
 msg_create_set_option_reply(struct dynar *msg, int add_msg_seq_number, uint32_t msg_seq_number,
-    enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval)
+    enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
+    int add_tie_breaker, const struct tlv_tie_breaker *tie_breaker)
 {
 
 	dynar_clean(msg);
@@ -456,6 +464,12 @@ msg_create_set_option_reply(struct dynar *msg, int add_msg_seq_number, uint32_t
 		goto small_buf_err;
 	}
 
+	if (add_tie_breaker) {
+		if (tlv_add_tie_breaker(msg, tie_breaker) == -1) {
+			goto small_buf_err;
+		}
+	}
+
 	msg_set_len(msg, dynar_size(msg) - (MSG_TYPE_LENGTH + MSG_LENGTH_LENGTH));
 
 	return (dynar_size(msg));
@@ -735,6 +749,7 @@ msg_decode(const struct dynar *msg, struct msg_decoded *decoded_msg)
 	uint64_t u64;
 	struct tlv_ring_id ring_id;
 	struct tlv_node_info node_info;
+	struct tlv_tie_breaker tie_breaker;
 	size_t zi;
 	enum tlv_opt_type opt_type;
 	int iter_res;
@@ -929,6 +944,14 @@ msg_decode(const struct dynar *msg, struct msg_decoded *decoded_msg)
 
 			decoded_msg->quorate_set = 1;
 			break;
+		case TLV_OPT_TIE_BREAKER:
+			if ((res = tlv_iter_decode_tie_breaker(&tlv_iter, &tie_breaker)) != 0) {
+				return (res);
+			}
+
+			decoded_msg->tie_breaker_set = 1;
+			memcpy(&decoded_msg->tie_breaker, &tie_breaker, sizeof(tie_breaker));
+			break;
 		default:
 			/*
 			 * Unknown option

qdevices/msg.h

@@ -110,6 +110,8 @@ struct msg_decoded {
 	enum tlv_vote vote;	// Valid only if vote_set != 0
 	int quorate_set;
 	enum tlv_quorate quorate;	// Valid only if quorate_set != 0
+	int tie_breaker_set;
+	struct tlv_tie_breaker tie_breaker;
 };
 
 extern size_t		msg_create_preinit(struct dynar *msg, const char *cluster_name,
@@ -139,11 +141,13 @@ extern size_t		msg_create_init_reply(struct dynar *msg, int add_msg_seq_number,
 
 extern size_t		msg_create_set_option(struct dynar *msg,
     int add_msg_seq_number, uint32_t msg_seq_number,
-    int add_heartbeat_interval, uint32_t heartbeat_interval);
+    int add_heartbeat_interval, uint32_t heartbeat_interval,
+    int add_tie_breaker, const struct tlv_tie_breaker *tie_breaker);
 
 extern size_t		msg_create_set_option_reply(struct dynar *msg,
     int add_msg_seq_number, uint32_t msg_seq_number,
-    enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval);
+    enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
+    int add_tie_breaker, const struct tlv_tie_breaker *tie_breaker);
 
 extern size_t		msg_create_echo_request(struct dynar *msg, int add_msg_seq_number,
     uint32_t msg_seq_number);

qdevices/qdevice-net-instance.c

@@ -41,7 +41,8 @@ qdevice_net_instance_init(struct qdevice_net_instance *instance, size_t initial_
     enum tlv_tls_supported tls_supported, uint32_t node_id,
     enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
     uint32_t sync_heartbeat_interval, uint32_t cast_vote_timer_interval,
-    const char *host_addr, uint16_t host_port, const char *cluster_name)
+    const char *host_addr, uint16_t host_port, const char *cluster_name,
+    const struct tlv_tie_breaker *tie_breaker)
 {
 
 	memset(instance, 0, sizeof(*instance));
@@ -59,6 +60,7 @@ qdevice_net_instance_init(struct qdevice_net_instance *instance, size_t initial_
 	instance->host_addr = host_addr;
 	instance->host_port = host_port;
 	instance->cluster_name = cluster_name;
+	memcpy(&instance->tie_breaker, tie_breaker, sizeof(*tie_breaker));
 
 	dynar_init(&instance->receive_buffer, initial_receive_size);
 

qdevices/qdevice-net-instance.h

@@ -95,6 +95,7 @@ struct qdevice_net_instance {
 	votequorum_handle_t votequorum_handle;
 	PRFileDesc *votequorum_poll_fd;
 	votequorum_ring_id_t last_received_votequorum_ring_id;
+	struct tlv_tie_breaker tie_breaker;
 };
 
 extern int		qdevice_net_instance_init(struct qdevice_net_instance *instance,
@@ -103,7 +104,8 @@ extern int		qdevice_net_instance_init(struct qdevice_net_instance *instance,
     enum tlv_tls_supported tls_supported, uint32_t node_id,
     enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
     uint32_t sync_heartbeat_interval, uint32_t cast_vote_timer_interval,
-    const char *host_addr, uint16_t host_port, const char *cluster_name);
+    const char *host_addr, uint16_t host_port, const char *cluster_name,
+    const struct tlv_tie_breaker *tie_breaker);
 
 extern int		qdevice_net_instance_destroy(struct qdevice_net_instance *instance);
 

qdevices/qnet-config.h

@@ -76,6 +76,9 @@ extern "C" {
 
 #define QDEVICE_NET_DEFAULT_ALGORITHM		TLV_DECISION_ALGORITHM_TYPE_TEST
 
+#define QNETD_DEFAULT_TIE_BREAKER_MODE		TLV_TIE_BREAKER_MODE_LOWEST
+#define QDEVICE_NET_DEFAULT_TIE_BREAKER_MODE	QNETD_DEFAULT_TIE_BREAKER_MODE
+
 #define QDEVICE_NET_MAX_CS_TRY_AGAIN		10
 
 #ifdef __cplusplus

qdevices/qnetd-client.c

@@ -36,6 +36,7 @@
 
 #include <string.h>
 
+#include "qnet-config.h"
 #include "qnetd-client.h"
 
 void
@@ -51,6 +52,7 @@ qnetd_client_init(struct qnetd_client *client, PRFileDesc *sock, PRNetAddr *addr
 	node_list_init(&client->configuration_node_list);
 	node_list_init(&client->last_membership_node_list);
 	node_list_init(&client->last_quorum_node_list);
+	client->tie_breaker.mode = QNETD_DEFAULT_TIE_BREAKER_MODE;
 }
 
 void

qdevices/qnetd-client.h

@@ -66,6 +66,7 @@ struct qnetd_client {
 	uint8_t node_id_set;
 	uint32_t node_id;
 	enum tlv_decision_algorithm_type decision_algorithm;
+	struct tlv_tie_breaker tie_breaker;
 	uint32_t heartbeat_interval;
 	enum tlv_reply_error_code skipping_msg_reason;
 	void *algorithm_data;
@@ -73,6 +74,7 @@ struct qnetd_client {
 	struct node_list last_membership_node_list;
 	struct node_list last_quorum_node_list;
 	struct qnetd_cluster *cluster;
+	struct qnetd_cluster_list *cluster_list;
 	TAILQ_ENTRY(qnetd_client) entries;
 	TAILQ_ENTRY(qnetd_client) cluster_entries;
 };

qdevices/tlv.c

@@ -56,7 +56,7 @@
 #define TLV_TYPE_LENGTH		2
 #define TLV_LENGTH_LENGTH	2
 
-#define TLV_STATIC_SUPPORTED_OPTIONS_SIZE      21
+#define TLV_STATIC_SUPPORTED_OPTIONS_SIZE      22
 
 enum tlv_opt_type tlv_static_supported_options[TLV_STATIC_SUPPORTED_OPTIONS_SIZE] = {
     TLV_OPT_MSG_SEQ_NUMBER,
@@ -80,6 +80,7 @@ enum tlv_opt_type tlv_static_supported_options[TLV_STATIC_SUPPORTED_OPTIONS_SIZE
     TLV_OPT_NODE_LIST_TYPE,
     TLV_OPT_VOTE,
     TLV_OPT_QUORATE,
+    TLV_OPT_TIE_BREAKER,
 };
 
 int
@@ -308,6 +309,23 @@ tlv_add_ring_id(struct dynar *msg, const struct tlv_ring_id *ring_id)
 	return (tlv_add(msg, TLV_OPT_RING_ID, sizeof(tmp_buf), tmp_buf));
 }
 
+int
+tlv_add_tie_breaker(struct dynar *msg, const struct tlv_tie_breaker *tie_breaker)
+{
+	uint32_t nu32;
+	uint8_t u8;
+	char tmp_buf[5];
+
+	u8 = tie_breaker->mode;
+	nu32 = (tie_breaker->mode == TLV_TIE_BREAKER_MODE_NODE_ID ?
+	    htonl(tie_breaker->node_id) : 0);
+
+	memcpy(tmp_buf, &u8, sizeof(u8));
+	memcpy(tmp_buf + sizeof(u8), &nu32, sizeof(nu32));
+
+	return (tlv_add(msg, TLV_OPT_TIE_BREAKER, sizeof(tmp_buf), tmp_buf));
+}
+
 int
 tlv_add_config_version(struct dynar *msg, uint64_t config_version)
 {
@@ -749,6 +767,41 @@ tlv_iter_decode_ring_id(struct tlv_iterator *tlv_iter, struct tlv_ring_id *ring_
 	return (0);
 }
 
+int
+tlv_iter_decode_tie_breaker(struct tlv_iterator *tlv_iter, struct tlv_tie_breaker *tie_breaker)
+{
+	const char *opt_data;
+	uint16_t opt_len;
+	uint32_t nu32;
+	uint8_t u8;
+	enum tlv_tie_breaker_mode tie_breaker_mode;
+	char tmp_buf[5];
+
+	opt_len = tlv_iter_get_len(tlv_iter);
+	opt_data = tlv_iter_get_data(tlv_iter);
+
+	if (opt_len != sizeof(tmp_buf)) {
+		return (-1);
+	}
+
+	memcpy(&u8, opt_data, sizeof(u8));
+	tie_breaker_mode = u8;
+
+	if (tie_breaker_mode != TLV_TIE_BREAKER_MODE_LOWEST &&
+	    tie_breaker_mode != TLV_TIE_BREAKER_MODE_HIGHEST &&
+	    tie_breaker_mode != TLV_TIE_BREAKER_MODE_NODE_ID) {
+		return (-4);
+	}
+
+	memcpy(&nu32, opt_data + sizeof(u8), sizeof(nu32));
+
+	tie_breaker->mode = tie_breaker_mode;
+	tie_breaker->node_id = (tie_breaker->mode == TLV_TIE_BREAKER_MODE_NODE_ID ?
+	    ntohl(nu32) : 0);
+
+	return (0);
+}
+
 int
 tlv_iter_decode_node_state(struct tlv_iterator *tlv_iter, enum tlv_node_state *node_state)
 {

qdevices/tlv.h

@@ -66,6 +66,7 @@ enum tlv_opt_type {
 	TLV_OPT_NODE_LIST_TYPE = 18,
 	TLV_OPT_VOTE = 19,
 	TLV_OPT_QUORATE = 20,
+	TLV_OPT_TIE_BREAKER = 21,
 };
 
 enum tlv_tls_supported {
@@ -132,6 +133,17 @@ enum tlv_quorate {
 	TLV_QUORATE_QUORATE = 1,
 };
 
+enum tlv_tie_breaker_mode {
+	TLV_TIE_BREAKER_MODE_LOWEST = 1,
+	TLV_TIE_BREAKER_MODE_HIGHEST = 2,
+	TLV_TIE_BREAKER_MODE_NODE_ID = 3,
+};
+
+struct tlv_tie_breaker {
+	enum tlv_tie_breaker_mode mode;
+	uint32_t node_id;
+};
+
 struct tlv_node_info {
 	uint32_t node_id;
 	uint32_t data_center_id;		// 0 - data center id was not set
@@ -204,6 +216,9 @@ extern int			 tlv_add_heartbeat_interval(struct dynar *msg,
 extern int			 tlv_add_ring_id(struct dynar *msg,
     const struct tlv_ring_id *ring_id);
 
+extern int			 tlv_add_tie_breaker(struct dynar *msg,
+    const struct tlv_tie_breaker *tie_breaker);
+
 extern int			 tlv_add_config_version(struct dynar *msg,
     uint64_t config_version);
 
@@ -277,6 +292,9 @@ extern int			 tlv_iter_decode_decision_algorithm(struct tlv_iterator *tlv_iter,
 extern int			 tlv_iter_decode_ring_id(struct tlv_iterator *tlv_iter,
     struct tlv_ring_id *ring_id);
 
+extern int			 tlv_iter_decode_tie_breaker(struct tlv_iterator *tlv_iter,
+    struct tlv_tie_breaker *tie_breaker);
+
 extern int			 tlv_iter_decode_node_state(struct tlv_iterator *tlv_iter,
     enum tlv_node_state *node_state);