Add totem/interface/ttl config option.

This adds a per-interface config option to
adjust the TTL.

Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
Reviewed-by: Steven Dake <sdake@redhat.com>

conf/corosync.conf.example

@@ -10,6 +10,7 @@ totem {
 		bindnetaddr: 192.168.1.1
 		mcastaddr: 226.94.1.1
 		mcastport: 5405
+		ttl: 1
 	}
 }
 

conf/corosync.conf.example.udpu

@@ -56,6 +56,7 @@ totem {
 		ringnumber: 0
 		bindnetaddr: 10.16.35.0
 		mcastport: 5405
+		ttl: 1
 	}
 	transport: udpu
 }

exec/totemconfig.c

@@ -373,8 +373,6 @@ printf ("couldn't find totem handle\n");
 					&totem_config->interfaces[ringnumber].mcast_addr,
 					"255.255.255.255", 0);
 			}
-
-
 		}
 
 		/*
@@ -392,6 +390,19 @@ printf ("couldn't find totem handle\n");
 			res = totemip_parse (&totem_config->interfaces[ringnumber].bindnet, str,
 					     totem_config->interfaces[ringnumber].mcast_addr.family);
 		}
+
+		/*
+		 * Get the TTL
+		 */
+		if (totem_config->interfaces[ringnumber].mcast_addr.family == AF_INET6) {
+			totem_config->interfaces[ringnumber].ttl = 255;
+		} else {
+			totem_config->interfaces[ringnumber].ttl = 1;
+		}
+		if (!objdb_get_string (objdb, object_interface_handle, "ttl", &str)) {
+			totem_config->interfaces[ringnumber].ttl = atoi (str);
+		}
+
 		objdb->object_find_create (
 			object_interface_handle,
 			"member",
@@ -466,6 +477,11 @@ int totem_config_validate (
 			goto parse_error;
 		}
 
+		if (totem_config->interfaces[i].ttl > 255 || totem_config->interfaces[i].ttl < 1) {
+			error_reason = "Invalid TTL (should be 1..255)";
+			goto parse_error;
+		}
+
 		if (totem_config->interfaces[i].mcast_addr.family == AF_INET6 &&
 			totem_config->node_id == 0) {
 

exec/totemudp.c

@@ -1642,14 +1642,19 @@ static int totemudp_build_sockets_ip (
 	/*
 	 * Set multicast packets TTL
 	 */
-
-	if ( bindnet_address->family == AF_INET6 )
-	{
-		flag = 255;
+	flag = instance->totem_interface->ttl;
+	if (bindnet_address->family == AF_INET6) {
 		res = setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_HOPS,
 			&flag, sizeof (flag));
 		if (res == -1) {
-			perror ("setp mcast hops");
+			perror ("set mcast v6 TTL");
+			return (-1);
+		}
+	} else {
+		res = setsockopt(sockets->mcast_send, IPPROTO_IP, IP_MULTICAST_TTL,
+			&flag, sizeof(flag));
+		if (res == -1) {
+			perror ("set mcast v4 TTL");
 			return (-1);
 		}
 	}

exec/totemudpu.c

@@ -1304,6 +1304,7 @@ static int totemudpu_build_sockets_ip (
 	int res;
 	unsigned int recvbuf_size;
 	unsigned int optlen = sizeof (recvbuf_size);
+	int flag;
 
 	/*
 	 * Setup unicast socket
@@ -1325,13 +1326,24 @@ static int totemudpu_build_sockets_ip (
 	}
 
 	/*
-	 * Force reuse
+	 * Set packets TTL
 	 */
-//	 flag = 1;
-//	 if ( setsockopt(instance->token_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
-//	 	perror("setsockopt reuseaddr");
-//		return (-1);
-//	}
+	flag = instance->totem_interface->ttl;
+	if (bindnet_address->family == AF_INET6) {
+		res = setsockopt (instance->token_socket, IPPROTO_IPV6,
+			IPV6_UNICAST_HOPS, &flag, sizeof (flag));
+		if (res == -1) {
+			perror ("set udpu v6 TTL");
+			return (-1);
+		}
+	} else {
+		res = setsockopt(instance->token_socket, IPPROTO_IP, IP_TTL,
+			&flag, sizeof(flag));
+		if (res == -1) {
+			perror ("set udpu v4 TTL");
+			return (-1);
+		}
+	}
 
 	/*
 	 * Bind to unicast socket used for token send/receives

include/corosync/totem/totem.h

@@ -57,6 +57,7 @@ struct totem_interface {
 	struct totem_ip_address boundto;
 	struct totem_ip_address mcast_addr;
 	uint16_t ip_port;
+	uint16_t ttl;
 	int member_count;
 	struct totem_ip_address member_list[PROCESSOR_COUNT_MAX];
 	

man/corosync.conf.5

@@ -127,6 +127,12 @@ mcastport - 1 (for mcast sends).
 If you have multiple clusters on the same network using the same mcastaddr 
 please configure the mcastports with a gap.
 
+.TP
+ttl
+This specifies the Time To Live (TTL). If you run your cluster on a routed
+network then the default of "1" will be too small. This option provides
+a way to increase this up to 255.
+
 .TP
 member
 This specifies a member on the interface and used with the udpu transport only.