Add totem/interface/ttl config option.

This adds a per-interface config option to
adjust the TTL.

Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
Reviewed-by: Steven Dake <sdake@redhat.com>

conf/corosync.conf.example

@@ -10,6 +10,7 @@ totem {
 		bindnetaddr: 192.168.1.1
 		mcastaddr: 226.94.1.1
 		mcastport: 5405
+		ttl: 1
 	}
 }
 

conf/corosync.conf.example.udpu

@@ -56,6 +56,7 @@ totem {
 		ringnumber: 0
 		bindnetaddr: 10.16.35.0
 		mcastport: 5405
+		ttl: 1
 	}
 	transport: udpu
 }

conf/lenses/corosync.aug

@@ -43,6 +43,7 @@ let interface =
   let setting =
     kv "ringnumber" Rx.integer
     |kv "mcastport" Rx.integer
+    |kv "ttl" Rx.integer
     |qstr /bindnetaddr|mcastaddr/ in
   section "interface" setting
 

conf/lenses/tests/test_corosync.aug

@@ -6,17 +6,18 @@ compatibility: whitetank
 totem {
 	version: 2
 	secauth: off
-    crypto_type: nss
-    crypto_accept: new
+	crypto_type: nss
+	crypto_accept: new
 	threads: 0
-    clear_node_high_bit: no
-    rrp_mode: none
-    transport: udp
-    token: 1000
+	clear_node_high_bit: no
+	rrp_mode: none
+	transport: udp
+	token: 1000
 	interface {
 		ringnumber: 0
 		bindnetaddr: 192.168.122.1
 		mcastaddr: 226.94.1.1
+		ttl: 45
 		mcastport: 5405
 	}
 }
@@ -95,6 +96,7 @@ test Corosync.lns get conf =
 		{ "ringnumber" = "0" }
 		{ "bindnetaddr" = "192.168.122.1" }
 		{ "mcastaddr" = "226.94.1.1" }
+		{ "ttl" = "45" }
 		{ "mcastport" = "5405" } } }
   { }
   { "logging"

exec/totemconfig.c

@@ -370,8 +370,6 @@ printf ("couldn't find totem handle\n");
 					&totem_config->interfaces[ringnumber].mcast_addr,
 					"255.255.255.255", 0);
 			}
-
-
 		}
 
 		/*
@@ -389,6 +387,19 @@ printf ("couldn't find totem handle\n");
 			res = totemip_parse (&totem_config->interfaces[ringnumber].bindnet, str,
 					     totem_config->interfaces[ringnumber].mcast_addr.family);
 		}
+
+		/*
+		 * Get the TTL
+		 */
+		if (totem_config->interfaces[ringnumber].mcast_addr.family == AF_INET6) {
+			totem_config->interfaces[ringnumber].ttl = 255;
+		} else {
+			totem_config->interfaces[ringnumber].ttl = 1;
+		}
+		if (!objdb_get_string (objdb, object_interface_handle, "ttl", &str)) {
+			totem_config->interfaces[ringnumber].ttl = atoi (str);
+		}
+
 		objdb->object_find_create (
 			object_interface_handle,
 			"member",
@@ -463,6 +474,11 @@ int totem_config_validate (
 			goto parse_error;
 		}
 
+		if (totem_config->interfaces[i].ttl > 255 || totem_config->interfaces[i].ttl < 1) {
+			error_reason = "Invalid TTL (should be 1..255)";
+			goto parse_error;
+		}
+
 		if (totem_config->interfaces[i].mcast_addr.family == AF_INET6 &&
 			totem_config->node_id == 0) {
 

exec/totemudp.c

@@ -1642,14 +1642,19 @@ static int totemudp_build_sockets_ip (
 	/*
 	 * Set multicast packets TTL
 	 */
-
-	if ( bindnet_address->family == AF_INET6 )
-	{
-		flag = 255;
+	flag = instance->totem_interface->ttl;
+	if (bindnet_address->family == AF_INET6) {
 		res = setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_HOPS,
 			&flag, sizeof (flag));
 		if (res == -1) {
-			perror ("setp mcast hops");
+			perror ("set mcast v6 TTL");
+			return (-1);
+		}
+	} else {
+		res = setsockopt(sockets->mcast_send, IPPROTO_IP, IP_MULTICAST_TTL,
+			&flag, sizeof(flag));
+		if (res == -1) {
+			perror ("set mcast v4 TTL");
 			return (-1);
 		}
 	}

exec/totemudpu.c

@@ -1316,6 +1316,7 @@ static int totemudpu_build_sockets_ip (
 	struct sockaddr_storage sockaddr;
 	int addrlen;
 	int res;
+	int flag;
 
 	/*
 	 * Setup unicast socket
@@ -1336,6 +1337,26 @@ static int totemudpu_build_sockets_ip (
 		return (-1);
 	}
 
+	/*
+	 * Set packets TTL
+	 */
+	flag = instance->totem_interface->ttl;
+	if (bindnet_address->family == AF_INET6) {
+		res = setsockopt (instance->token_socket, IPPROTO_IPV6,
+			IPV6_UNICAST_HOPS, &flag, sizeof (flag));
+		if (res == -1) {
+			perror ("set udpu v6 TTL");
+			return (-1);
+		}
+	} else {
+		res = setsockopt(instance->token_socket, IPPROTO_IP, IP_TTL,
+			&flag, sizeof(flag));
+		if (res == -1) {
+			perror ("set udpu v4 TTL");
+			return (-1);
+		}
+	}
+
 	/*
 	 * Bind to unicast socket used for token send/receives
 	 * This has the side effect of binding to the correct interface

include/corosync/totem/totem.h

@@ -57,6 +57,7 @@ struct totem_interface {
 	struct totem_ip_address boundto;
 	struct totem_ip_address mcast_addr;
 	uint16_t ip_port;
+	uint16_t ttl;
 	int member_count;
 	struct totem_ip_address member_list[PROCESSOR_COUNT_MAX];
 	

man/corosync.conf.5

@@ -127,6 +127,12 @@ mcastport - 1 (for mcast sends).
 If you have multiple clusters on the same network using the same mcastaddr 
 please configure the mcastports with a gap.
 
+.TP
+ttl
+This specifies the Time To Live (TTL). If you run your cluster on a routed
+network then the default of "1" will be too small. This option provides
+a way to increase this up to 255.
+
 .TP
 member
 This specifies a member on the interface and used with the udpu transport only.