knet: Fix MTU sizes & allow transport config in corosync.conf

Corosync layers don't need to know the knet MTU size - this way
corosync fragments buffers only when they get larger than the
KNET buffer size (64K) and knet fragments below that based on
the actual MTU and transport considerations.

It is also now possible to configure knet to use UDP or SCTP
transports in corosync.conf. This is currently done per-link
so if you have more than 1 link you need several interface{}
stanzas inside totem{} to make it use other than the default
of UDP. if it's useful I might add the option of a global
default.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>

exec/coroparse.c

@@ -120,6 +120,7 @@ struct main_cp_cb_data {
 	int knet_ping_precision;
 	int knet_pong_count;
 	int knet_pmtud_interval;
+	char *knet_transport;
 
 	struct qb_list_head logger_subsys_items_head;
 	char *subsys;
@@ -766,6 +767,11 @@ static int main_config_parser_cb(const char *path,
 				data->knet_pong_count = val;
 				add_as_string = 0;
 			}
+			if (strcmp(path, "totem.interface.knet_transport") == 0) {
+				val_type = ICMAP_VALUETYPE_STRING;
+				data->knet_transport = strdup(value);
+				add_as_string = 0;
+			}
 			break;
 		case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS:
 			if (strcmp(key, "subsys") == 0) {
@@ -958,6 +964,7 @@ static int main_config_parser_cb(const char *path,
 			data->knet_ping_timeout = -1;
 			data->knet_ping_precision = -1;
 			data->knet_pong_count = -1;
+			data->knet_transport = NULL;
 			qb_list_init(&data->member_items_head);
 		};
 		if (strcmp(path, "totem") == 0) {
@@ -1082,6 +1089,12 @@ static int main_config_parser_cb(const char *path,
 						data->linknumber);
 				icmap_set_uint32_r(config_map, key_name, data->knet_pong_count);
 			}
+			if (data->knet_transport) {
+				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport",
+						data->linknumber);
+				icmap_set_string_r(config_map, key_name, data->knet_transport);
+				free(data->knet_transport);
+			}
 
 			ii = 0;
 

exec/totemconfig.c

@@ -52,6 +52,7 @@
 #include <corosync/swab.h>
 #include <qb/qblist.h>
 #include <qb/qbdefs.h>
+#include <libknet.h>
 #include <corosync/totem/totem.h>
 #include <corosync/config.h>
 #include <corosync/logsys.h>
@@ -80,6 +81,7 @@
 #define KNET_PING_PRECISION                     2048
 #define KNET_PONG_COUNT                         2
 #define KNET_PMTUD_INTERVAL                     30
+#define KNET_DEFAULT_TRANSPORT                  KNET_TRANSPORT_UDP
 
 #define DEFAULT_PORT				5405
 
@@ -1137,6 +1139,21 @@ extern int totem_config_read (
 			totem_config->interfaces[linknumber].knet_pong_count = u32;
 		}
 
+		totem_config->interfaces[linknumber].knet_transport = KNET_DEFAULT_TRANSPORT;
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport", linknumber);
+		if (icmap_get_string(tmp_key, &str) == CS_OK) {
+			if (strcmp(str, "sctp") == 0) {
+				totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_SCTP;
+			}
+			else if (strcmp(str, "udp") == 0) {
+				totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_UDP;
+			}
+			else {
+				*error_string = "Unrecognised knet_transport. expected 'udp' or 'sctp'";
+				return -1;
+			}
+		}
+
 		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.", linknumber);
 		member_iter = icmap_iter_init(tmp_key);
 		while ((member_iter_key = icmap_iter_next(member_iter, NULL, NULL)) != NULL) {
@@ -1400,7 +1417,12 @@ int totem_config_validate (
 	}
 
 	if (totem_config->net_mtu == 0) {
-		totem_config->net_mtu = 1500;
+		if (totem_config->transport_number == TOTEM_TRANSPORT_KNET) {
+			totem_config->net_mtu = KNET_MAX_PACKET_SIZE;
+		}
+		else {
+			totem_config->net_mtu = 1500;
+		}
 	}
 
 	return 0;

exec/totemknet.c

@@ -282,8 +282,8 @@ static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
 	struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
 	knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
 
-	// TODO: Check this
-	instance->totemknet_mtu_changed(instance->context, data_mtu);
+	/* We don't need to tell corosync the actual knet MTU */
+//	instance->totemknet_mtu_changed(instance->context, data_mtu);
 }
 
 int totemknet_crypto_set (
@@ -918,7 +918,7 @@ int totemknet_initialize (
 
 void *totemknet_buffer_alloc (void)
 {
-	return malloc (FRAME_SIZE_MAX);
+	return malloc(KNET_MAX_PACKET_SIZE);
 }
 
 void totemknet_buffer_release (void *ptr)
@@ -1111,7 +1111,9 @@ int totemknet_member_add (
 	/* Casts to remove const */
 	totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port+link_no, &remote_ss, &addrlen);
 	totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port+link_no, &local_ss, &addrlen);
-	err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no, KNET_TRANSPORT_UDP, &local_ss, &remote_ss);
+	err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
+				   instance->totem_config->interfaces[link_no].knet_transport,
+				   &local_ss, &remote_ss);
 	if (err) {
 		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
 		return -1;

include/corosync/totem/totem.h

@@ -74,6 +74,7 @@ struct totem_interface {
 	int knet_ping_timeout;
 	int knet_ping_precision;
 	int knet_pong_count;
+	int knet_transport;
 	struct totem_ip_address member_list[PROCESSOR_COUNT_MAX];
 };
 

man/corosync.conf.5

@@ -71,9 +71,6 @@ The
 For knet, multiple interface subsections define parameters for each knet link on the
 system.
 
-For UDP, there should be just one interface section that defines the multicast or
-broadcast options for the link.
-
 For UDPU an interface section is not needed and it is recommended that the nodelist
 is used to define cluster nodes.
 
@@ -107,6 +104,10 @@ the average link latency. (default 2048 samples)
 .TP
 knet_pong_count
 How many valid ping/pongs before a link is marked UP. (default 5)
+.TP
+
+knet_transport
+Which IP transport knet should use. valid values are "sctp" or "udp". (default: udp)
 
 .TP
 bindnetaddr (udp only)