totemconfig: Add support for knet_mtu

totem.knet_mtu is new configuration option which allows setting
of automatic or manual knet MTU.

Also reload of totem.knet_pmtud_interval is fixed now, so it works when
key is deleted (and set back default value).

Signed-off-by: Jan Friesse <jfriesse@redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>

exec/coroparse.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006-2019 Red Hat, Inc.
+ * Copyright (c) 2006-2022 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -121,6 +121,7 @@ struct main_cp_cb_data {
 	int knet_ping_precision;
 	int knet_pong_count;
 	int knet_pmtud_interval;
+	unsigned int knet_mtu;
 	char *knet_transport;
 
 	struct qb_list_head logger_subsys_items_head;
@@ -729,6 +730,7 @@ static int main_config_parser_cb(const char *path,
 			    (strcmp(path, "totem.max_messages") == 0) ||
 			    (strcmp(path, "totem.miss_count_const") == 0) ||
 			    (strcmp(path, "totem.knet_pmtud_interval") == 0) ||
+			    (strcmp(path, "totem.knet_mtu") == 0) ||
 			    (strcmp(path, "totem.knet_compression_threshold") == 0) ||
 			    (strcmp(path, "totem.netmtu") == 0)) {
 				val_type = ICMAP_VALUETYPE_UINT32;

exec/totemconfig.c

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2002-2005 MontaVista Software, Inc.
- * Copyright (c) 2006-2019 Red Hat, Inc.
+ * Copyright (c) 2006-2022 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -91,6 +91,7 @@
 #define KNET_PING_PRECISION                     2048
 #define KNET_PONG_COUNT                         2
 #define KNET_PMTUD_INTERVAL                     30
+#define KNET_MTU                                0
 #define KNET_DEFAULT_TRANSPORT                  KNET_TRANSPORT_UDP
 
 #define DEFAULT_PORT				5405
@@ -137,6 +138,8 @@ static void *totem_get_param_by_name(struct totem_config *totem_config, const ch
 		return &totem_config->miss_count_const;
 	if (strcmp(param_name, "totem.knet_pmtud_interval") == 0)
 		return &totem_config->knet_pmtud_interval;
+	if (strcmp(param_name, "totem.knet_mtu") == 0)
+		return &totem_config->knet_mtu;
 	if (strcmp(param_name, "totem.knet_compression_threshold") == 0)
 		return &totem_config->knet_compression_threshold;
 	if (strcmp(param_name, "totem.knet_compression_level") == 0)
@@ -335,6 +338,7 @@ void totem_volatile_config_read (struct totem_config *totem_config, icmap_map_t
 
 	totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.miss_count_const", deleted_key, MISS_COUNT_CONST, 0);
 	totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.knet_pmtud_interval", deleted_key, KNET_PMTUD_INTERVAL, 0);
+	totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.knet_mtu", deleted_key, KNET_MTU, 0);
 
 	totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.token_retransmit", deleted_key,
 	   (int)(totem_config->token_timeout / (totem_config->token_retransmits_before_loss_const + 0.2)), 0);

exec/totemknet.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2020 Red Hat, Inc.
+ * Copyright (c) 2016-2022 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -925,7 +925,7 @@ static void totemknet_refresh_config(
 	void *user_data)
 {
 	uint8_t reloading;
-	uint32_t value;
+	int after_reload;
 	uint32_t link_no;
 	size_t num_nodes;
 	knet_node_id_t host_ids[KNET_MAX_HOST];
@@ -943,18 +943,27 @@ static void totemknet_refresh_config(
 		return;
 	}
 
-	knet_set_access_list_config(instance);
+	after_reload = (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0);
 
-	if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
+	knet_set_access_list_config(instance);
 
-		instance->totem_config->knet_pmtud_interval = value;
-		knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
+	if (strcmp(key_name, "totem.knet_pmtud_interval") == 0 || after_reload) {
+		knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %u",
+		    instance->totem_config->knet_pmtud_interval);
 		err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
 		if (err) {
 			KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
 		}
 	}
 
+	if (strcmp(key_name, "totem.knet_mtu") == 0 || after_reload) {
+		knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_mtu now %u", instance->totem_config->knet_mtu);
+		err = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
+		if (err) {
+			KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud failed");
+		}
+	}
+
 	/* Configure link parameters for each node */
 	err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
 	if (err != 0) {
@@ -1213,6 +1222,10 @@ int totemknet_initialize (
 	if (res) {
 		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
 	}
+	res = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
+	if (res) {
+		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_set failed");
+	}
 	res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
 	if (res) {
 		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");

include/corosync/totem/totem.h

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005 MontaVista Software, Inc.
- * Copyright (c) 2006-2012 Red Hat, Inc.
+ * Copyright (c) 2006-2022 Red Hat, Inc.
  *
  * Author: Steven Dake (sdake@redhat.com)
  *
@@ -167,6 +167,7 @@ struct totem_config {
 	unsigned int node_id;
 	unsigned int clear_node_high_bit;
 	unsigned int knet_pmtud_interval;
+	unsigned int knet_mtu;
 
 	/*
 	 * key information

man/corosync.conf.5

@@ -1,6 +1,6 @@
 .\"/*
 .\" * Copyright (c) 2005 MontaVista Software, Inc.
-.\" * Copyright (c) 2006-2021 Red Hat, Inc.
+.\" * Copyright (c) 2006-2022 Red Hat, Inc.
 .\" *
 .\" * All rights reserved.
 .\" *
@@ -32,7 +32,7 @@
 .\" * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 .\" * THE POSSIBILITY OF SUCH DAMAGE.
 .\" */
-.TH COROSYNC_CONF 5 2021-08-11 "corosync Man Page" "Corosync Cluster Engine Programmer's Manual"
+.TH COROSYNC_CONF 5 2022-10-20 "corosync Man Page" "Corosync Cluster Engine Programmer's Manual"
 .SH NAME
 corosync.conf - corosync executive configuration file
 
@@ -273,8 +273,15 @@ is 8. For other transports it is 1.
 
 .TP
 netmtu
-This specifies the network maximum transmit unit.  To set this value beyond
-1500, the regular frame MTU, requires ethernet devices that support large, or
+This specifies maximum packet length sent by corosync. It's mainly for the UDPU
+(and UDP) transport, where it specifies the network maximum transmit size, but
+can be used also with the KNET transport, where it defines the maximum length of packets
+passed to the knet layer. To specify the network MTU manually for KNET, use the
+.B knet_mtu
+option.
+
+For UDPU (and UDP), setting this value beyond 1500, the regular frame MTU,
+requires ethernet devices that support large, or
 also called jumbo, frames.  If any device in the network doesn't support large
 frames, the protocol will not operate properly.  The hosts must also have their
 mtu size set from 1500 to whatever frame size is specified here.
@@ -293,7 +300,7 @@ that some device in the network doesn't support large frames.
 
 Choose hardware carefully if intending to use large frame support.
 
-The default is 1500.
+The default is 1500 for UDPU (and UDP) and 65536 for the KNET transport.
 
 .TP
 transport
@@ -568,6 +575,15 @@ knet_pmtud_interval
 How often the knet PMTUd runs to look for network MTU changes.
 Value in seconds, default: 30
 
+.TP
+knet_mtu
+Switch between manual and automatic MTU discovery. A value of 0 means
+automatic, other values set a manual MTU.
+In a setup with multiple interfaces, please specify
+the lowest MTU of the selected interfaces.
+
+The default value is 0.
+
 .TP
 block_unlisted_ips
 Allow UDPU and KNET to drop packets from IP addresses that are not known