totem: Use nodeid ONLY in srp_addr

This shrinks the srp_addr (and consequently every packet sent by
corosync) so that instead of containing loads of IP addresses to
identify a node, it just sends the nodeid.

This then allows us to make ring0 optional and replaceable when running
knet.

It also means that we need some other way of identifying the local
node in corosync.conf, so the nodelist.node.name entry is now mandatory
and is mapped to the local host using the same algorithm as used in
cman.

This code needs LOTS of testing as it touches a huge amount of totemsrp
and totemconfig.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

exec/coroparse.c

@@ -128,7 +128,6 @@ struct main_cp_cb_data {
 	struct qb_list_head member_items_head;
 
 	int node_number;
-	int ring0_addr_added;
 };
 
 static int read_config_file_into_icmap(
@@ -917,10 +916,6 @@ static int main_config_parser_cb(const char *path,
 				add_as_string = 0;
 			}
 
-			if (strcmp(key, "ring0_addr") == 0) {
-				data->ring0_addr_added = 1;
-			}
-
 			if (add_as_string) {
 				icmap_set_string_r(config_map, key_name, value);
 				add_as_string = 0;
@@ -1011,7 +1006,6 @@ static int main_config_parser_cb(const char *path,
 		}
 		if (strcmp(path, "nodelist.node") == 0) {
 			*state = MAIN_CP_CB_DATA_STATE_NODELIST_NODE;
-			data->ring0_addr_added = 0;
 		}
 		if (strcmp(path, "resources") == 0) {
 			*state = MAIN_CP_CB_DATA_STATE_RESOURCES;
@@ -1214,11 +1208,6 @@ static int main_config_parser_cb(const char *path,
 
 			break;
 		case MAIN_CP_CB_DATA_STATE_NODELIST_NODE:
-			if (!data->ring0_addr_added) {
-				*error_string = "No ring0_addr specified for node";
-
-				return (0);
-			}
 			data->node_number++;
 			break;
 		case MAIN_CP_CB_DATA_STATE_NORMAL:

exec/cpg.c

@@ -564,7 +564,7 @@ static void cpg_sync_init (
 		sizeof (unsigned int));
 	my_member_list_entries = member_list_entries;
 
-	last_sync_ring_id.nodeid = ring_id->rep.nodeid;
+	last_sync_ring_id.nodeid = ring_id->nodeid;
 	last_sync_ring_id.seq = ring_id->seq;
 
 	downlist_state = CPG_DOWNLIST_WAITING_FOR_MESSAGES;

exec/main.c

@@ -624,14 +624,14 @@ int main_mcast (
 
 static void corosync_ring_id_create_or_load (
 	struct memb_ring_id *memb_ring_id,
-	const struct totem_ip_address *addr)
+	unsigned int nodeid)
 {
 	int fd;
 	int res = 0;
 	char filename[PATH_MAX];
 
-	snprintf (filename, sizeof(filename), "%s/ringid_%s",
-		get_run_dir(), totemip_print (addr));
+	snprintf (filename, sizeof(filename), "%s/ringid_%u",
+		get_run_dir(), nodeid);
 	fd = open (filename, O_RDONLY, 0700);
 	/*
 	 * If file can be opened and read, read the ring id
@@ -664,20 +664,19 @@ static void corosync_ring_id_create_or_load (
 		}
 	}
 
-	totemip_copy(&memb_ring_id->rep, addr);
-	assert (!totemip_zero_check(&memb_ring_id->rep));
+	memb_ring_id->rep = nodeid;
 }
 
 static void corosync_ring_id_store (
 	const struct memb_ring_id *memb_ring_id,
-	const struct totem_ip_address *addr)
+	unsigned int nodeid)
 {
 	char filename[PATH_MAX];
 	int fd;
 	int res;
 
-	snprintf (filename, sizeof(filename), "%s/ringid_%s",
-		get_run_dir(), totemip_print (addr));
+	snprintf (filename, sizeof(filename), "%s/ringid_%u",
+		get_run_dir(), nodeid);
 
 	fd = open (filename, O_WRONLY, 0700);
 	if (fd == -1) {

exec/totemconfig.c

@@ -45,9 +45,12 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <ifaddrs.h>
+#include <netdb.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <sys/param.h>
+#include <sys/utsname.h>
 
 #include <corosync/swab.h>
 #include <qb/qblist.h>
@@ -462,6 +465,267 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro
 	return 0;
 }
 
+static int nodelist_byname(const char *find_name, int strip_domain)
+{
+	icmap_iter_t iter;
+	const char *iter_key;
+	char name_str[ICMAP_KEYNAME_MAXLEN];
+	int res = 0;
+	unsigned int node_pos;
+	char *name;
+	unsigned int namelen;
+
+	iter = icmap_iter_init("nodelist.node.");
+	while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
+		res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str);
+		if (res != 2) {
+			continue;
+		}
+		if (strcmp(name_str, "name")) {
+			continue;
+		}
+		if (icmap_get_string(iter_key, &name) != CS_OK) {
+			continue;
+		}
+		namelen = strlen(name);
+
+		if (strip_domain) {
+			char *dot;
+			dot = strchr(name, '.');
+			if (dot) {
+				namelen = name - dot - 1;
+			}
+		}
+		if (strncmp(find_name, name, namelen) == 0 &&
+		    strlen(find_name) == strlen(name)) {
+			icmap_iter_finalize(iter);
+			return node_pos;
+		}
+	}
+	icmap_iter_finalize(iter);
+	return -1;
+}
+
+/* Compare two addresses */
+static int ipaddr_equal(struct sockaddr_storage *addr1, struct sockaddr_storage *addr2)
+{
+	int addrlen = 0;
+
+	if (addr1->ss_family != addr2->ss_family)
+		return 0;
+
+	if (addr1->ss_family == AF_INET) {
+		addrlen = sizeof(struct sockaddr_in);
+	}
+	if (addr1->ss_family == AF_INET6) {
+		addrlen = sizeof(struct sockaddr_in6);
+	}
+	assert(addrlen);
+
+	if (memcmp(addr1, addr2, addrlen) == 0)
+		return 1;
+	else
+		return 0;
+
+}
+
+
+/* Finds the local node and returns its position in the nodelist.
+ * Uses nodelist.local_node_pos as a cache to save effort
+ */
+static int find_local_node(int use_cache)
+{
+	char nodename2[PATH_MAX];
+	char name_str[ICMAP_KEYNAME_MAXLEN];
+	icmap_iter_t iter;
+	const char *iter_key;
+	unsigned int cached_pos;
+	char *dot = NULL;
+	const char *node;
+	struct ifaddrs *ifa, *ifa_list;
+	struct sockaddr *sa;
+	int found = 0;
+	int node_pos = -1;
+	int res;
+	struct utsname utsname;
+
+	/* Check for cached value first */
+	if (use_cache) {
+		if (icmap_get_uint32("nodelist.local_node_pos", &cached_pos) == CS_OK) {
+			return cached_pos;
+		}
+	}
+
+	res = uname(&utsname);
+	if (res) {
+		return -1;
+	}
+	node = utsname.nodename;
+
+	/* 1. Exact match */
+	node_pos = nodelist_byname(node, 0);
+	if (node_pos > -1) {
+		found = 1;
+		goto ret_found;
+	}
+
+	/* 2. Try to match with increasingly more
+	 * specific versions of it
+	 */
+	strcpy(nodename2, node);
+	dot = strrchr(nodename2, '.');
+	while (dot) {
+		*dot = '\0';
+
+		node_pos = nodelist_byname(nodename2, 0);
+		if (node_pos > -1) {
+			found = 1;
+			goto ret_found;
+		}
+		dot = strrchr(nodename2, '.');
+	}
+
+	node_pos = nodelist_byname(nodename2, 1);
+	if (node_pos > -1) {
+		found = 1;
+		goto ret_found;
+	}
+
+	/*
+	 * The corosync.conf name may not be related to uname at all,
+	 * they may match a hostname on some network interface.
+	 */
+	if (getifaddrs(&ifa_list))
+		return -1;
+
+	for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) {
+		socklen_t salen = 0;
+
+		/* Restore this */
+		strcpy(nodename2, node);
+		sa = ifa->ifa_addr;
+		if (!sa) {
+			continue;
+		}
+		if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6) {
+			continue;
+		}
+
+		if (sa->sa_family == AF_INET) {
+			salen = sizeof(struct sockaddr_in);
+		}
+		if (sa->sa_family == AF_INET6) {
+			salen = sizeof(struct sockaddr_in6);
+		}
+
+		if (getnameinfo(sa, salen,
+				nodename2, sizeof(nodename2),
+				NULL, 0, 0) == 0) {
+
+			node_pos = nodelist_byname(nodename2, 0);
+			if (node_pos > -1) {
+				found = 1;
+				goto out;
+			}
+
+			/* Truncate this name and try again */
+			dot = strchr(nodename2, '.');
+			if (dot) {
+				*dot = '\0';
+
+				node_pos = nodelist_byname(nodename2, 0);
+				if (node_pos > -1) {
+					found = 1;
+					goto out;
+				}
+			}
+		}
+
+		/* See if it's the IP address that's in corosync.conf */
+		if (getnameinfo(sa, sizeof(*sa),
+				nodename2, sizeof(nodename2),
+				NULL, 0, NI_NUMERICHOST))
+			continue;
+
+		node_pos = nodelist_byname(nodename2, 0);
+		if (node_pos > -1) {
+			found = 1;
+			goto out;
+		}
+	}
+
+ out:
+	if (found) {
+		freeifaddrs(ifa_list);
+		goto ret_found;
+	}
+
+	/*
+	 * This section covers the usecase where the nodename specified in cluster.conf
+	 * is an alias specified in /etc/hosts. For example:
+	 * <ipaddr> hostname alias1 alias2
+	 * and <clusternode name="alias2">
+	 * the above calls use uname and getnameinfo does not return aliases.
+	 * here we take the name specified in cluster.conf, resolve it to an address
+	 * and then compare against all known local ip addresses.
+	 * if we have a match, we found our nodename. In theory this chunk of code
+	 * could replace all the checks above, but let's avoid any possible regressions
+	 * and use it as last.
+	 */
+
+	iter = icmap_iter_init("nodelist.node.");
+	while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
+		char *dbnodename = NULL;
+		struct addrinfo hints;
+		struct addrinfo *result = NULL, *rp = NULL;
+
+		res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str);
+		if (res != 2) {
+			continue;
+		}
+		if (strcmp(name_str, "name")) {
+			continue;
+		}
+		if (icmap_get_string(iter_key, &dbnodename) != CS_OK) {
+			continue;
+		}
+
+		memset(&hints, 0, sizeof(struct addrinfo));
+		hints.ai_family = AF_UNSPEC;
+		hints.ai_socktype = SOCK_DGRAM;
+		hints.ai_flags = 0;
+		hints.ai_protocol = IPPROTO_UDP;
+
+		if (getaddrinfo(dbnodename, NULL, &hints, &result)) {
+			continue;
+		}
+
+		for (rp = result; rp != NULL; rp = rp->ai_next) {
+			for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) {
+				if (ifa->ifa_addr &&
+				    ipaddr_equal((struct sockaddr_storage *)rp->ai_addr,
+						 (struct sockaddr_storage *)ifa->ifa_addr)) {
+					freeaddrinfo(result);
+					found = 1;
+					goto out2;
+				}
+			}
+		}
+
+		freeaddrinfo(result);
+	}
+out2:
+	icmap_iter_finalize(iter);
+	freeifaddrs(ifa_list);
+
+ret_found:
+	if (found) {
+		res = icmap_set_uint32("nodelist.local_node_pos", node_pos);
+	}
+
+	return node_pos;
+}
+
 static int totem_config_get_ip_version(struct totem_config *totem_config)
 {
 	int res;
@@ -583,7 +847,7 @@ static int check_for_duplicate_nodeids(
 			continue;
 		}
 
-		if (strcmp(tmp_key, "ring0_addr") != 0) {
+		if (strcmp(tmp_key, "nodeid") != 0) {
 			continue;
 		}
 
@@ -612,7 +876,7 @@ static int check_for_duplicate_nodeids(
 				continue;
 			}
 
-			if (strcmp(tmp_key, "ring0_addr") != 0) {
+			if (strcmp(tmp_key, "nodeid") != 0) {
 				continue;
 			}
 
@@ -648,57 +912,6 @@ static int check_for_duplicate_nodeids(
 }
 
 
-static int find_local_node_in_nodelist(struct totem_config *totem_config)
-{
-	icmap_iter_t iter;
-	const char *iter_key;
-	int res = 0;
-	unsigned int node_pos;
-	int local_node_pos = -1;
-	struct totem_ip_address bind_addr;
-	int interface_up, interface_num;
-	char tmp_key[ICMAP_KEYNAME_MAXLEN];
-	char *node_addr_str;
-	struct totem_ip_address node_addr;
-
-	res = totemip_iface_check(&totem_config->interfaces[0].bindnet,
-		&bind_addr, &interface_up, &interface_num,
-		totem_config->clear_node_high_bit);
-	if (res == -1) {
-		return (-1);
-	}
-
-	iter = icmap_iter_init("nodelist.node.");
-	while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
-		res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key);
-		if (res != 2) {
-			continue;
-		}
-
-		if (strcmp(tmp_key, "ring0_addr") != 0) {
-			continue;
-		}
-
-		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos);
-		if (icmap_get_string(tmp_key, &node_addr_str) != CS_OK) {
-			continue;
-		}
-
-		res = totemip_parse (&node_addr, node_addr_str, totem_config->ip_version);
-		free(node_addr_str);
-		if (res == -1) {
-			continue ;
-		}
-
-		if (totemip_equal(&bind_addr, &node_addr)) {
-			local_node_pos = node_pos;
-		}
-	}
-	icmap_iter_finalize(iter);
-
-	return (local_node_pos);
-}
-
 /*
  * This needs to be done last of all. It would be nice to do it when reading the
  * interface params, but the totem params need to have them to be read first. We
@@ -813,7 +1026,7 @@ static void reconfigure_links(struct totem_config *totem_config)
 	char *addr_string;
 	struct totem_ip_address local_ip;
 	int err;
-	unsigned int local_node_pos = find_local_node_in_nodelist(totem_config);
+	int local_node_pos = find_local_node(0);
 
 	for (i = 0; i<INTERFACE_MAX; i++) {
 		if (!totem_config->interfaces[i].configured) {
@@ -890,8 +1103,7 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in
 		if (res != 2) {
 			continue;
 		}
-
-		if (strcmp(tmp_key, "ring0_addr") != 0) {
+		if (strcmp(tmp_key, "nodeid") != 0) {
 			continue;
 		}
 
@@ -914,14 +1126,12 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in
 			}
 
 			member_count = totem_config->interfaces[linknumber].member_count;
-
 			res = totemip_parse(&totem_config->interfaces[linknumber].member_list[member_count],
 						node_addr_str, totem_config->ip_version);
 			if (res != -1) {
 				totem_config->interfaces[linknumber].member_list[member_count].nodeid = nodeid;
 				totem_config->interfaces[linknumber].member_count++;
 			}
-
 			totem_config->interfaces[linknumber].configured = 1;
 			free(node_addr_str);
 		}
@@ -977,112 +1187,11 @@ static void nodelist_dynamic_notify(
 }
 
 
-/*
- * Tries to find node (node_pos) in config nodelist which address matches any
- * local interface. Address can be stored in ring0_addr or if ipaddr_key_prefix is not NULL
- * key with prefix ipaddr_key is used (there can be multiuple of them)
- * This function differs  * from find_local_node_in_nodelist because it doesn't need bindnetaddr,
- * but doesn't work when bind addr is network address (so IP must be exact
- * match).
- *
- * Returns 1 on success (address was found, node_pos is then correctly set) or 0 on failure.
- */
-int totem_config_find_local_addr_in_nodelist(struct totem_config *totem_config, const char *ipaddr_key_prefix, unsigned int *node_pos)
-{
-	struct qb_list_head addrs;
-	struct totem_ip_if_address *if_addr;
-	icmap_iter_t iter, iter2;
-	const char *iter_key, *iter_key2;
-	struct qb_list_head *list;
-	const char *ipaddr_key;
-	int ip_version;
-	struct totem_ip_address node_addr;
-	char *node_addr_str;
-	int node_found = 0;
-	int res = 0;
-	char tmp_key[ICMAP_KEYNAME_MAXLEN];
-
-	if (totemip_getifaddrs(&addrs) == -1) {
-		return 0;
-	}
-
-	ip_version = totem_config_get_ip_version(totem_config);
-
-	iter = icmap_iter_init("nodelist.node.");
-
-	while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
-		res = sscanf(iter_key, "nodelist.node.%u.%s", node_pos, tmp_key);
-		if (res != 2) {
-			continue;
-		}
-
-		if (strcmp(tmp_key, "ring0_addr") != 0) {
-			continue;
-		}
-
-		if (icmap_get_string(iter_key, &node_addr_str) != CS_OK) {
-			continue ;
-		}
-
-		free(node_addr_str);
-
-		/*
-		 * ring0_addr found -> let's iterate thru ipaddr_key_prefix
-		 */
-		snprintf(tmp_key, sizeof(tmp_key), "nodelist.node.%u.%s", *node_pos,
-		    (ipaddr_key_prefix != NULL ? ipaddr_key_prefix : "ring0_addr"));
-
-		iter2 = icmap_iter_init(tmp_key);
-		while ((iter_key2 = icmap_iter_next(iter2, NULL, NULL)) != NULL) {
-			/*
-			 * ring0_addr must be exact match, not prefix
-			 */
-			ipaddr_key = (ipaddr_key_prefix != NULL ? iter_key2 : tmp_key);
-			if (icmap_get_string(ipaddr_key, &node_addr_str) != CS_OK) {
-				continue ;
-			}
-
-			if (totemip_parse(&node_addr, node_addr_str, ip_version) == -1) {
-				free(node_addr_str);
-				continue ;
-			}
-			free(node_addr_str);
-
-			/*
-			 * Try to match ip with if_addrs
-			 */
-			node_found = 0;
-			qb_list_for_each(list, &(addrs)) {
-				if_addr = qb_list_entry(list, struct totem_ip_if_address, list);
-
-				if (totemip_equal(&node_addr, &if_addr->ip_addr)) {
-					node_found = 1;
-					break;
-				}
-			}
-
-			if (node_found) {
-				break ;
-			}
-		}
-
-		icmap_iter_finalize(iter2);
-
-		if (node_found) {
-			break ;
-		}
-	}
-
-	icmap_iter_finalize(iter);
-	totemip_freeifaddrs(&addrs);
-
-	return (node_found);
-}
 
 static void config_convert_nodelist_to_interface(struct totem_config *totem_config)
 {
 	int res = 0;
-	unsigned int node_pos;
+	int node_pos;
 	char tmp_key[ICMAP_KEYNAME_MAXLEN];
 	char tmp_key2[ICMAP_KEYNAME_MAXLEN];
 	char *node_addr_str;
@@ -1090,7 +1199,8 @@ static void config_convert_nodelist_to_interface(struct totem_config *totem_conf
 	icmap_iter_t iter;
 	const char *iter_key;
 
-	if (totem_config_find_local_addr_in_nodelist(totem_config, NULL, &node_pos)) {
+	node_pos = find_local_node(1);
+	if (node_pos > -1) {
 		/*
 		 * We found node, so create interface section
 		 */
@@ -1180,7 +1290,9 @@ static int get_interface_params(struct totem_config *totem_config,
 			/*
 			 * Get the bind net address
 			 */
-			if (icmap_get_string(iter_key, &str) == CS_OK) {
+			snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber);
+
+			if (icmap_get_string(tmp_key, &str) == CS_OK) {
 				res = totemip_parse (&totem_config->interfaces[linknumber].bindnet, str,
 						     totem_config->ip_version);
 				free(str);
@@ -1462,14 +1574,13 @@ extern int totem_config_read (
 	/*
 	 * Check existence of nodelist
 	 */
-	if (icmap_get_string("nodelist.node.0.ring0_addr", &str) == CS_OK) {
+	if (icmap_get_string("nodelist.node.0.name", &str) == CS_OK) {
 		free(str);
 		/*
 		 * find local node
 		 */
-		local_node_pos = find_local_node_in_nodelist(totem_config);
+		local_node_pos = find_local_node(1);
 		if (local_node_pos != -1) {
-			icmap_set_uint32("nodelist.local_node_pos", local_node_pos);
 
 			snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", local_node_pos);
 
@@ -1482,13 +1593,7 @@ extern int totem_config_read (
 				return -1;
 			}
 
-			/*
-			 * Make localnode ring0_addr read only, so we can be sure that local
-			 * node never changes. If rebinding to other IP would be in future
-			 * supported, this must be changed and handled properly!
-			 */
-			snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", local_node_pos);
-			icmap_set_ro_access(tmp_key, 0, 1);
+			/* Users must not change this */
 			icmap_set_ro_access("nodelist.local_node_pos", 0, 1);
 		}
 
@@ -1522,8 +1627,6 @@ int totem_config_validate (
 	int num_configured = 0;
 	unsigned int interface_max = INTERFACE_MAX;
 
-
-
 	for (i = 0; i < INTERFACE_MAX; i++) {
 		if (totem_config->interfaces[i].configured) {
 			num_configured++;
@@ -1883,7 +1986,6 @@ static void totem_reload_notify(
 	void *user_data)
 {
 	struct totem_config *totem_config = (struct totem_config *)user_data;
-	uint32_t local_node_pos;
 	const char *error_string;
 	uint64_t warnings;
 
@@ -1911,10 +2013,7 @@ static void totem_reload_notify(
 		}
 
 		/* Reinstate the local_node_pos */
-		local_node_pos = find_local_node_in_nodelist(totem_config);
-		if (local_node_pos != -1) {
-			icmap_set_uint32("nodelist.local_node_pos", local_node_pos);
-		}
+		(void)find_local_node(0);
 
 		/* Reconfigure network params as appropriate */
 		totempg_reconfigure();

exec/totemknet.c

@@ -145,6 +145,8 @@ struct totemknet_instance {
 
 	int our_nodeid;
 
+	int loopback_link;
+
 	struct totem_config *totem_config;
 
 	struct totem_ip_address token_target;
@@ -832,6 +834,8 @@ int totemknet_initialize (
 
 	instance->totemknet_target_set_completed = target_set_completed;
 
+	instance->loopback_link = -1;
+
 	res = pipe(instance->logpipes);
 	if (res == -1) {
 	    KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
@@ -1043,13 +1047,12 @@ extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *t
 
 int totemknet_token_target_set (
 	void *knet_context,
-	const struct totem_ip_address *token_target)
+	unsigned int nodeid)
 {
 	struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
 	int res = 0;
 
-	memcpy (&instance->token_target, token_target,
-		sizeof (struct totem_ip_address));
+	instance->token_target.nodeid = nodeid;
 
 	instance->totemknet_target_set_completed (instance->context);
 
@@ -1139,18 +1142,23 @@ int totemknet_member_add (
 	int addrlen;
 
 	/* Only create 1 loopback link */
-	if (member->nodeid == instance->our_nodeid && link_no > 0) {
+	// NOTE: THis depends on member_remove being run before member_add when reeconfiguring
+	// otherwise we could be left with no loopback.
+	if (member->nodeid == instance->our_nodeid && instance->loopback_link > -1) {
 		return 0;
 	}
 
 	knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: %d (%s), link=%d", member->nodeid, totemip_print(member), link_no);
 	knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet:      local: %d (%s)", local->nodeid, totemip_print(local));
-	if (link_no == 0) {
-		if (knet_host_add(instance->knet_handle, member->nodeid)) {
-			KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
-			return -1;
-		}
 
+	// TODO FIXME - prints errors when host already exists
+	err = knet_host_add(instance->knet_handle, member->nodeid);
+	if (err != 0 && errno != EEXIST) {
+		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
+		return -1;
+	}
+
+	if (err == 0) {
 		if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
 			KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
 			return -1;
@@ -1161,7 +1169,11 @@ int totemknet_member_add (
 	/* Casts to remove const */
 	totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
 	totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
+
 	if (member->nodeid == instance->our_nodeid) {
+		knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
+
+		instance->loopback_link = link_no;
 		err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
 					   KNET_TRANSPORT_LOOPBACK,
 					   &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
@@ -1221,9 +1233,9 @@ int totemknet_member_remove (
 
 	knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: %d, link=%d", token_target->nodeid, link_no);
 
-	/* Only link 0 is valid for localhost */
-	if (token_target->nodeid == instance->our_nodeid && link_no > 0) {
-		return 0;
+	/* Removing link with the loopback on it */
+	if (token_target->nodeid == instance->our_nodeid && link_no == instance->loopback_link) {
+		instance->loopback_link= -1;
 	}
 
 	/* Tidy stats */

exec/totemknet.h

@@ -112,7 +112,7 @@ extern int totemknet_iface_set (void *net_context,
 
 extern int totemknet_token_target_set (
 	void *knet_context,
-	const struct totem_ip_address *token_target);
+	unsigned int nodeid);
 
 extern int totemknet_crypto_set (
 	void *knet_context,

exec/totemnet.c

@@ -116,7 +116,7 @@ struct transport {
 
 	int (*token_target_set) (
 		void *transport_context,
-		const struct totem_ip_address *token_target);
+		unsigned int nodeid);
 
 	int (*crypto_set) (
 		void *transport_context,
@@ -176,6 +176,8 @@ struct transport transport_entries[] = {
 		.token_target_set = totemudp_token_target_set,
 		.crypto_set = totemudp_crypto_set,
 		.recv_mcast_empty = totemudp_recv_mcast_empty,
+		.member_add = totemudp_member_add,
+		.member_remove = totemudp_member_remove,
 		.reconfigure = totemudp_reconfigure
 	},
 	{
@@ -476,12 +478,12 @@ int totemnet_ifaces_get (
 
 int totemnet_token_target_set (
 	void *net_context,
-	const struct totem_ip_address *token_target)
+	unsigned int nodeid)
 {
 	struct totemnet_instance *instance = (struct totemnet_instance *)net_context;
 	unsigned int res;
 
-	res = instance->transport->token_target_set (instance->transport_context, token_target);
+	res = instance->transport->token_target_set (instance->transport_context, nodeid);
 
 	return (res);
 }

exec/totemnet.h

@@ -129,7 +129,7 @@ extern int totemnet_ifaces_get (
 
 extern int totemnet_token_target_set (
 	void *net_context,
-	const struct totem_ip_address *token_target);
+	unsigned int target_nodeid);
 
 extern int totemnet_crypto_set (
 	void *net_context,

exec/totemsrp.c

@@ -102,11 +102,9 @@
 
 /*
  * SRP address.
- * CC: TODO: Can we remove IP address from this and just use nodeids?
  */
 struct srp_addr {
-	uint8_t no_addrs;
-	struct totem_ip_address addr[INTERFACE_MAX];
+	unsigned int nodeid;
 };
 
 /*
@@ -300,8 +298,12 @@ struct totemsrp_instance {
 
 	int consensus_list_entries;
 
+	int lowest_active_if;
+
 	struct srp_addr my_id;
 
+	struct totem_ip_address my_addrs[INTERFACE_MAX];
+
 	struct srp_addr my_proc_list[PROCESSOR_COUNT_MAX];
 
 	struct srp_addr my_failed_list[PROCESSOR_COUNT_MAX];
@@ -466,11 +468,11 @@ struct totemsrp_instance {
 
 	void (*memb_ring_id_create_or_load) (
 		struct memb_ring_id *memb_ring_id,
-		const struct totem_ip_address *addr);
+		unsigned int nodeid);
 
 	void (*memb_ring_id_store) (
 		const struct memb_ring_id *memb_ring_id,
-		const struct totem_ip_address *addr);
+		unsigned int nodeid);
 
 	int global_seqno;
 
@@ -617,6 +619,7 @@ static void totemsrp_instance_initialize (struct totemsrp_instance *instance);
 static void srp_addr_copy (struct srp_addr *dest, const struct srp_addr *src);
 
 static void srp_addr_to_nodeid (
+	struct totemsrp_instance *instance,
 	unsigned int *nodeid_out,
 	struct srp_addr *srp_addr_in,
 	unsigned int entries);
@@ -736,8 +739,6 @@ static void totemsrp_instance_initialize (struct totemsrp_instance *instance)
 
 	instance->commit_token = (struct memb_commit_token *)instance->commit_token_storage;
 
-	instance->my_id.no_addrs = INTERFACE_MAX;
-
 	instance->waiting_trans_ack = 1;
 }
 
@@ -873,7 +874,7 @@ int totemsrp_initialize (
 	/*
 	 * Initialize local variables for totemsrp
 	 */
-	totemip_copy (&instance->mcast_address, &totem_config->interfaces[0].mcast_addr);
+	totemip_copy (&instance->mcast_address, &totem_config->interfaces[instance->lowest_active_if].mcast_addr);
 
 	/*
 	 * Display totem configuration
@@ -974,6 +975,8 @@ int totemsrp_initialize (
 		goto error_exit;
 	}
 
+	instance->my_id.nodeid = instance->totem_config->interfaces[instance->lowest_active_if].boundto.nodeid;
+
 	/*
 	 * Must have net_mtu adjusted by totemnet_initialize first
 	 */
@@ -1037,54 +1040,18 @@ int totemsrp_ifaces_get (
 {
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context;
 	int res = 0;
-	unsigned int found = 0;
-	int i;
 
 	memset(interfaces, 0, sizeof(struct totem_ip_address) * interfaces_size);
 
-	for (i = 0; i < instance->my_memb_entries; i++) {
-		if (instance->my_memb_list[i].addr[0].nodeid == nodeid) {
-			found = 1;
-			break;
-		}
-	}
-
-	if (found) {
-		*iface_count = INTERFACE_MAX;
-
-		if (interfaces_size >= *iface_count) {
-			memcpy (interfaces, instance->my_memb_list[i].addr,
-				sizeof (struct totem_ip_address) * *iface_count);
-		} else {
-			res = -2;
-		}
-
-		goto finish;
-	}
-
-	for (i = 0; i < instance->my_left_memb_entries; i++) {
-		if (instance->my_left_memb_list[i].addr[0].nodeid == nodeid) {
-			found = 1;
-			break;
-		}
-	}
-
-	if (found) {
-		*iface_count = INTERFACE_MAX;
-
-		if (interfaces_size >= *iface_count) {
-			memcpy (interfaces, instance->my_left_memb_list[i].addr,
-				sizeof (struct totem_ip_address) * *iface_count);
-		} else {
-			res = -2;
-		}
+	if (interfaces_size >= *iface_count) {
+		memcpy (interfaces, instance->my_addrs,
+			sizeof (struct totem_ip_address) * *iface_count);
 	} else {
-		res = -1;
+		res = -2;
 	}
 
 	*iface_count = INTERFACE_MAX;
 
-finish:
 	totemnet_ifaces_get(instance->totemnet_context, status, iface_count);
 	return (res);
 }
@@ -1109,7 +1076,7 @@ unsigned int totemsrp_my_nodeid_get (
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context;
 	unsigned int res;
 
-	res = instance->totem_config->interfaces[0].boundto.nodeid;
+	res = instance->my_id.nodeid;
 
 	return (res);
 }
@@ -1120,7 +1087,7 @@ int totemsrp_my_family_get (
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context;
 	int res;
 
-	res = instance->totem_config->interfaces[0].boundto.family;
+	res = instance->totem_config->interfaces[instance->lowest_active_if].boundto.family;
 
 	return (res);
 }
@@ -1131,30 +1098,19 @@ int totemsrp_my_family_get (
  */
 static int srp_addr_equal (const struct srp_addr *a, const struct srp_addr *b)
 {
-	unsigned int i;
-	unsigned int res;
-
-	for (i = 0; i < 1; i++) {
-		res = totemip_equal (&a->addr[i], &b->addr[i]);
-		if (res == 0) {
-			return (0);
-		}
+	if (a->nodeid == b->nodeid) {
+		return 1;
 	}
-	return (1);
+	return 0;
 }
 
 static void srp_addr_copy (struct srp_addr *dest, const struct srp_addr *src)
 {
-	unsigned int i;
-
-	dest->no_addrs = src->no_addrs;
-
-	for (i = 0; i < INTERFACE_MAX; i++) {
-		totemip_copy (&dest->addr[i], &src->addr[i]);
-	}
+	dest->nodeid = src->nodeid;
 }
 
 static void srp_addr_to_nodeid (
+	struct totemsrp_instance *instance,
 	unsigned int *nodeid_out,
 	struct srp_addr *srp_addr_in,
 	unsigned int entries)
@@ -1162,17 +1118,13 @@ static void srp_addr_to_nodeid (
 	unsigned int i;
 
 	for (i = 0; i < entries; i++) {
-		nodeid_out[i] = srp_addr_in[i].addr[0].nodeid;
+		nodeid_out[i] = srp_addr_in[i].nodeid;
 	}
 }
 
 static void srp_addr_copy_endian_convert (struct srp_addr *out, const struct srp_addr *in)
 {
-	int i;
-
-	for (i = 0; i < INTERFACE_MAX; i++) {
-		totemip_copy_endian_convert (&out->addr[i], &in->addr[i]);
-	}
+	out->nodeid = swab32 (in->nodeid);
 }
 
 static void memb_consensus_reset (struct totemsrp_instance *instance)
@@ -1216,7 +1168,7 @@ static void memb_consensus_set (
 	int found = 0;
 	int i;
 
-	if (addr->addr[0].nodeid == LEAVE_DUMMY_NODEID)
+	if (addr->nodeid == LEAVE_DUMMY_NODEID)
 	        return;
 
 	for (i = 0; i < instance->consensus_list_entries; i++) {
@@ -1927,9 +1879,9 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 	/*
 	 * Deliver transitional configuration to application
 	 */
-	srp_addr_to_nodeid (left_list, instance->my_left_memb_list,
+	srp_addr_to_nodeid (instance, left_list, instance->my_left_memb_list,
 		instance->my_left_memb_entries);
-	srp_addr_to_nodeid (trans_memb_list_totemip,
+	srp_addr_to_nodeid (instance, trans_memb_list_totemip,
 		instance->my_trans_memb_list, instance->my_trans_memb_entries);
 	instance->totemsrp_confchg_fn (TOTEM_CONFIGURATION_TRANSITIONAL,
 		trans_memb_list_totemip, instance->my_trans_memb_entries,
@@ -1947,9 +1899,9 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 	/*
 	 * Deliver regular configuration to application
 	 */
-	srp_addr_to_nodeid (new_memb_list_totemip,
+	srp_addr_to_nodeid (instance, new_memb_list_totemip,
 		instance->my_new_memb_list, instance->my_new_memb_entries);
-	srp_addr_to_nodeid (joined_list_totemip, joined_list,
+	srp_addr_to_nodeid (instance, joined_list_totemip, joined_list,
 		joined_list_entries);
 	instance->totemsrp_confchg_fn (TOTEM_CONFIGURATION_REGULAR,
 		new_memb_list_totemip, instance->my_new_memb_entries,
@@ -2063,8 +2015,8 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 	log_printf (instance->totemsrp_log_level_debug,
 		"entering OPERATIONAL state.");
 	log_printf (instance->totemsrp_log_level_notice,
-		"A new membership (%s:%lld) was formed. Members%s%s",
-		totemip_print (&instance->my_ring_id.rep),
+		"A new membership (%u:%lld) was formed. Members%s%s",
+		instance->my_ring_id.rep,
 		instance->my_ring_id.seq,
 		joined_node_msg,
 		left_node_msg);
@@ -2199,7 +2151,8 @@ static void memb_state_commit_enter (
 	instance->memb_timer_state_gather_consensus_timeout = 0;
 
 	memb_ring_id_set (instance, &instance->commit_token->ring_id);
-	instance->memb_ring_id_store (&instance->my_ring_id, &instance->my_id.addr[0]);
+
+	instance->memb_ring_id_store (&instance->my_ring_id, instance->my_id.nodeid);
 
 	instance->token_ring_id_seq = instance->my_ring_id.seq;
 
@@ -2283,15 +2236,15 @@ static void memb_state_recovery_enter (
 
 	for (i = 0; i < instance->my_trans_memb_entries; i++) {
 		log_printf (instance->totemsrp_log_level_debug,
-			"TRANS [%d] member %s:", i, totemip_print (&instance->my_trans_memb_list[i].addr[0]));
+			"TRANS [%d] member %u:", i, instance->my_trans_memb_list[i].nodeid);
 	}
 	for (i = 0; i < instance->my_new_memb_entries; i++) {
 		log_printf (instance->totemsrp_log_level_debug,
-			"position [%d] member %s:", i, totemip_print (&addr[i].addr[0]));
+			"position [%d] member %u:", i, addr[i].nodeid);
 		log_printf (instance->totemsrp_log_level_debug,
-			"previous ring seq %llx rep %s",
+			"previous ring seq %llx rep %u",
 			memb_list[i].ring_id.seq,
-			totemip_print (&memb_list[i].ring_id.rep));
+			memb_list[i].ring_id.rep);
 
 		log_printf (instance->totemsrp_log_level_debug,
 			"aru %x high delivered %x received flag %d",
@@ -2378,7 +2331,8 @@ static void memb_state_recovery_enter (
 		message_item.mcast->header.type = MESSAGE_TYPE_MCAST;
 		srp_addr_copy (&message_item.mcast->system_from, &instance->my_id);
 		message_item.mcast->header.encapsulated = MESSAGE_ENCAPSULATED;
-		message_item.mcast->header.nodeid = instance->my_id.addr[0].nodeid;
+
+		message_item.mcast->header.nodeid = instance->my_id.nodeid;
 		assert (message_item.mcast->header.nodeid);
 		message_item.mcast->header.endian_detector = ENDIAN_LOCAL;
 		memcpy (&message_item.mcast->ring_id, &instance->my_ring_id,
@@ -2465,7 +2419,8 @@ int totemsrp_mcast (
 	message_item.mcast->header.type = MESSAGE_TYPE_MCAST;
 	message_item.mcast->header.endian_detector = ENDIAN_LOCAL;
 	message_item.mcast->header.encapsulated = MESSAGE_NOT_ENCAPSULATED;
-	message_item.mcast->header.nodeid = instance->my_id.addr[0].nodeid;
+
+	message_item.mcast->header.nodeid = instance->my_id.nodeid;
 	assert (message_item.mcast->header.nodeid);
 
 	message_item.mcast->guarantee = guarantee;
@@ -2911,7 +2866,7 @@ static void timer_function_merge_detect_timeout(void *data)
 
 	switch (instance->memb_state) {
 	case MEMB_STATE_OPERATIONAL:
-		if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0])) {
+		if (instance->my_ring_id.rep == instance->my_id.nodeid) {
 			memb_merge_detect_transmit (instance);
 		}
 		break;
@@ -2936,7 +2891,7 @@ static int token_send (
 	orf_token_size = sizeof (struct orf_token) +
 		(orf_token->rtr_list_entries * sizeof (struct rtr_item));
 
-	orf_token->header.nodeid = instance->my_id.addr[0].nodeid;
+	orf_token->header.nodeid = instance->my_id.nodeid;
 	memcpy (instance->orf_token_retransmit, orf_token, orf_token_size);
 	instance->orf_token_retransmit_size = orf_token_size;
 	assert (orf_token->header.nodeid);
@@ -2970,7 +2925,7 @@ static int token_hold_cancel_send (struct totemsrp_instance *instance)
 	token_hold_cancel.header.type = MESSAGE_TYPE_TOKEN_HOLD_CANCEL;
 	token_hold_cancel.header.endian_detector = ENDIAN_LOCAL;
 	token_hold_cancel.header.encapsulated = 0;
-	token_hold_cancel.header.nodeid = instance->my_id.addr[0].nodeid;
+	token_hold_cancel.header.nodeid = instance->my_id.nodeid;
 	memcpy (&token_hold_cancel.ring_id, &instance->my_ring_id,
 		sizeof (struct memb_ring_id));
 	assert (token_hold_cancel.header.nodeid);
@@ -2991,7 +2946,7 @@ static int orf_token_send_initial (struct totemsrp_instance *instance)
 	orf_token.header.type = MESSAGE_TYPE_ORF_TOKEN;
 	orf_token.header.endian_detector = ENDIAN_LOCAL;
 	orf_token.header.encapsulated = 0;
-	orf_token.header.nodeid = instance->my_id.addr[0].nodeid;
+	orf_token.header.nodeid = instance->my_id.nodeid;
 	assert (orf_token.header.nodeid);
 	orf_token.seq = SEQNO_START_MSG;
 	orf_token.token_seq = SEQNO_START_TOKEN;
@@ -3009,7 +2964,7 @@ static int orf_token_send_initial (struct totemsrp_instance *instance)
 
 	orf_token.aru = 0;
 	orf_token.aru = SEQNO_START_MSG - 1;
-	orf_token.aru_addr = instance->my_id.addr[0].nodeid;
+	orf_token.aru_addr = instance->my_id.nodeid;
 
 	memcpy (&orf_token.ring_id, &instance->my_ring_id, sizeof (struct memb_ring_id));
 	orf_token.fcc = 0;
@@ -3083,7 +3038,7 @@ static void memb_state_commit_token_update (
 		}
 	}
 
-	instance->commit_token->header.nodeid = instance->my_id.addr[0].nodeid;
+	instance->commit_token->header.nodeid = instance->my_id.nodeid;
 	instance->commit_token->memb_index += 1;
 	assert (instance->commit_token->memb_index <= instance->commit_token->addr_entries);
 	assert (instance->commit_token->header.nodeid);
@@ -3099,8 +3054,8 @@ static void memb_state_commit_token_target_set (
 	/* Totemnet just looks at the node id */
 	totemnet_token_target_set (
 		instance->totemnet_context,
-		&addr[instance->commit_token->memb_index %
-		      instance->commit_token->addr_entries].addr[0]);
+		addr[instance->commit_token->memb_index %
+		      instance->commit_token->addr_entries].nodeid);
 }
 
 static int memb_state_commit_token_send_recovery (
@@ -3110,7 +3065,7 @@ static int memb_state_commit_token_send_recovery (
 	unsigned int commit_token_size;
 
 	commit_token->token_seq++;
-	commit_token->header.nodeid = instance->my_id.addr[0].nodeid;
+	commit_token->header.nodeid = instance->my_id.nodeid;
 	commit_token_size = sizeof (struct memb_commit_token) +
 		((sizeof (struct srp_addr) +
 			sizeof (struct memb_commit_token_memb_entry)) * commit_token->addr_entries);
@@ -3139,7 +3094,7 @@ static int memb_state_commit_token_send (
 	unsigned int commit_token_size;
 
 	instance->commit_token->token_seq++;
-	instance->commit_token->header.nodeid = instance->my_id.addr[0].nodeid;
+	instance->commit_token->header.nodeid = instance->my_id.nodeid;
 	commit_token_size = sizeof (struct memb_commit_token) +
 		((sizeof (struct srp_addr) +
 			sizeof (struct memb_commit_token_memb_entry)) * instance->commit_token->addr_entries);
@@ -3168,7 +3123,7 @@ static int memb_lowest_in_config (struct totemsrp_instance *instance)
 	struct srp_addr token_memb[PROCESSOR_COUNT_MAX];
 	int token_memb_entries = 0;
 	int i;
-	struct totem_ip_address *lowest_addr;
+	unsigned int lowest_nodeid;
 
 	memb_set_subtract (token_memb, &token_memb_entries,
 		instance->my_proc_list, instance->my_proc_list_entries,
@@ -3178,13 +3133,13 @@ static int memb_lowest_in_config (struct totemsrp_instance *instance)
 	 * find representative by searching for smallest identifier
 	 */
 
-	lowest_addr = &token_memb[0].addr[0];
+	lowest_nodeid = token_memb[0].nodeid;
 	for (i = 1; i < token_memb_entries; i++) {
-		if (totemip_compare(lowest_addr, &token_memb[i].addr[0]) > 0) {
-			totemip_copy (lowest_addr, &token_memb[i].addr[0]);
+		if (lowest_nodeid > token_memb[i].nodeid) {
+			lowest_nodeid = token_memb[i].nodeid;
 		}
 	}
-	return (totemip_compare (lowest_addr, &instance->my_id.addr[0]) == 0);
+	return (lowest_nodeid == instance->my_id.nodeid);
 }
 
 static int srp_addr_compare (const void *a, const void *b)
@@ -3192,7 +3147,7 @@ static int srp_addr_compare (const void *a, const void *b)
 	const struct srp_addr *srp_a = (const struct srp_addr *)a;
 	const struct srp_addr *srp_b = (const struct srp_addr *)b;
 
-	return (totemip_compare (&srp_a->addr[0], &srp_b->addr[0]));
+	return (srp_a->nodeid == srp_b->nodeid);
 }
 
 static void memb_state_commit_token_create (
@@ -3214,11 +3169,10 @@ static void memb_state_commit_token_create (
 	instance->commit_token->header.type = MESSAGE_TYPE_MEMB_COMMIT_TOKEN;
 	instance->commit_token->header.endian_detector = ENDIAN_LOCAL;
 	instance->commit_token->header.encapsulated = 0;
-	instance->commit_token->header.nodeid = instance->my_id.addr[0].nodeid;
+	instance->commit_token->header.nodeid = instance->my_id.nodeid;
 	assert (instance->commit_token->header.nodeid);
 
-	totemip_copy(&instance->commit_token->ring_id.rep, &instance->my_id.addr[0]);
-
+	instance->commit_token->ring_id.rep = instance->my_id.nodeid;
 	instance->commit_token->ring_id.seq = instance->token_ring_id_seq + 4;
 
 	/*
@@ -3250,7 +3204,7 @@ static void memb_join_message_send (struct totemsrp_instance *instance)
 	memb_join->header.type = MESSAGE_TYPE_MEMB_JOIN;
 	memb_join->header.endian_detector = ENDIAN_LOCAL;
 	memb_join->header.encapsulated = 0;
-	memb_join->header.nodeid = instance->my_id.addr[0].nodeid;
+	memb_join->header.nodeid = instance->my_id.nodeid;
 	assert (memb_join->header.nodeid);
 
 	memb_join->ring_seq = instance->my_ring_id.seq;
@@ -3326,7 +3280,7 @@ static void memb_leave_message_send (struct totemsrp_instance *instance)
 	memb_join->proc_list_entries = active_memb_entries;
 	memb_join->failed_list_entries = instance->my_failed_list_entries;
 	srp_addr_copy (&memb_join->system_from, &instance->my_id);
-	memb_join->system_from.addr[0].nodeid = LEAVE_DUMMY_NODEID;
+	memb_join->system_from.nodeid = LEAVE_DUMMY_NODEID;
 
 	// TODO: CC Maybe use the actual join send routine.
 	/*
@@ -3369,7 +3323,7 @@ static void memb_merge_detect_transmit (struct totemsrp_instance *instance)
 	memb_merge_detect.header.type = MESSAGE_TYPE_MEMB_MERGE_DETECT;
 	memb_merge_detect.header.endian_detector = ENDIAN_LOCAL;
 	memb_merge_detect.header.encapsulated = 0;
-	memb_merge_detect.header.nodeid = instance->my_id.addr[0].nodeid;
+	memb_merge_detect.header.nodeid = instance->my_id.nodeid;
 	srp_addr_copy (&memb_merge_detect.system_from, &instance->my_id);
 	memcpy (&memb_merge_detect.ring_id, &instance->my_ring_id,
 		sizeof (struct memb_ring_id));
@@ -3655,13 +3609,14 @@ static int message_handler_orf_token (
 	 * Determine if we should hold (in reality drop) the token
 	 */
 	instance->my_token_held = 0;
-	if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0]) &&
+	if (instance->my_ring_id.rep == instance->my_id.nodeid &&
 		instance->my_seq_unchanged > instance->totem_config->seqno_unchanged_const) {
 		instance->my_token_held = 1;
-	} else
-		if (!totemip_equal(&instance->my_ring_id.rep,  &instance->my_id.addr[0]) &&
-		instance->my_seq_unchanged >= instance->totem_config->seqno_unchanged_const) {
-		instance->my_token_held = 1;
+	} else {
+		if (instance->my_ring_id.rep != instance->my_id.nodeid &&
+		    instance->my_seq_unchanged >= instance->totem_config->seqno_unchanged_const) {
+			instance->my_token_held = 1;
+		}
 	}
 
 	/*
@@ -3669,7 +3624,7 @@ static int message_handler_orf_token (
 	 * this processor is the ring rep
 	 */
 	forward_token = 1;
-	if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0])) {
+	if (instance->my_ring_id.rep == instance->my_id.nodeid) {
 		if (instance->my_token_held) {
 			forward_token = 0;
 		}
@@ -3741,14 +3696,14 @@ printf ("token seq %d\n", token->seq);
 			mcasted_regular);
 
 		if (sq_lt_compare (instance->my_aru, token->aru) ||
-			instance->my_id.addr[0].nodeid ==  token->aru_addr ||
+			instance->my_id.nodeid == token->aru_addr ||
 			token->aru_addr == 0) {
 
 			token->aru = instance->my_aru;
 			if (token->aru == token->seq) {
 				token->aru_addr = 0;
 			} else {
-				token->aru_addr = instance->my_id.addr[0].nodeid;
+				token->aru_addr = instance->my_id.nodeid;
 			}
 		}
 		if (token->aru == last_aru && token->aru_addr != 0) {
@@ -3763,7 +3718,7 @@ printf ("token seq %d\n", token->seq);
 		 * to failed list (so node never mark itself as failed)
 		 */
 		if (instance->my_aru_count > instance->totem_config->fail_to_recv_const &&
-			token->aru_addr == instance->my_id.addr[0].nodeid) {
+		    token->aru_addr == instance->my_id.nodeid) {
 
 			log_printf (instance->totemsrp_log_level_error,
 				"FAILED TO RECEIVE");
@@ -3860,7 +3815,7 @@ printf ("token seq %d\n", token->seq);
 			 */
 			reset_token_timeout (instance); // REVIEWED
 			reset_token_retransmit_timeout (instance); // REVIEWED
-			if (totemip_equal(&instance->my_id.addr[0], &instance->my_ring_id.rep) &&
+			if (instance->my_id.nodeid == instance->my_ring_id.rep &&
 				instance->my_token_held == 1) {
 
 				start_token_hold_retransmit_timeout (instance);
@@ -4060,8 +4015,8 @@ static int message_handler_mcast (
 	}
 
 	log_printf (instance->totemsrp_log_level_trace,
-		"Received ringid(%s:%lld) seq %x",
-		totemip_print (&mcast_header.ring_id.rep),
+		"Received ringid(%u:%lld) seq %x",
+		mcast_header.ring_id.rep,
 		mcast_header.ring_id.seq,
 		mcast_header.seq);
 
@@ -4186,9 +4141,9 @@ static void memb_join_process (
 			if (memb_join->header.nodeid == LEAVE_DUMMY_NODEID) {
 				log_printf (instance->totemsrp_log_level_warning,
 			    		"Discarding LEAVE message during flush, nodeid=%u",
-						memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid : LEAVE_DUMMY_NODEID);
+						memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].nodeid : LEAVE_DUMMY_NODEID);
 				if (memb_join->failed_list_entries > 0) {
-					my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid);
+					my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].nodeid);
 				}
 			} else {
 				log_printf (instance->totemsrp_log_level_warning,
@@ -4198,9 +4153,9 @@ static void memb_join_process (
 		} else {
 			if (memb_join->header.nodeid == LEAVE_DUMMY_NODEID) {
 				log_printf (instance->totemsrp_log_level_debug,
-				    "Received LEAVE message from %u", memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid : LEAVE_DUMMY_NODEID);
+				    "Received LEAVE message from %u", memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].nodeid : LEAVE_DUMMY_NODEID);
 				if (memb_join->failed_list_entries > 0) {
-					my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid);
+					my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].nodeid);
 				}
 			}
 		}
@@ -4351,7 +4306,7 @@ static void memb_commit_token_endian_convert (const struct memb_commit_token *in
 	out->header.endian_detector = ENDIAN_LOCAL;
 	out->header.nodeid = swab32 (in->header.nodeid);
 	out->token_seq = swab32 (in->token_seq);
-	totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep);
+	out->ring_id.rep = swab32(in->ring_id.rep);
 	out->ring_id.seq = swab64 (in->ring_id.seq);
 	out->retrans_flg = swab32 (in->retrans_flg);
 	out->memb_index = swab32 (in->memb_index);
@@ -4365,9 +4320,8 @@ static void memb_commit_token_endian_convert (const struct memb_commit_token *in
 		/*
 		 * Only convert the memb entry if it has been set
 		 */
-		if (in_memb_list[i].ring_id.rep.family != 0) {
-			totemip_copy_endian_convert (&out_memb_list[i].ring_id.rep,
-				     &in_memb_list[i].ring_id.rep);
+		if (in_memb_list[i].ring_id.rep != 0) {
+			out_memb_list[i].ring_id.rep = swab32(in_memb_list[i].ring_id.rep);
 
 			out_memb_list[i].ring_id.seq =
 				swab64 (in_memb_list[i].ring_id.seq);
@@ -4388,7 +4342,7 @@ static void orf_token_endian_convert (const struct orf_token *in, struct orf_tok
 	out->seq = swab32 (in->seq);
 	out->token_seq = swab32 (in->token_seq);
 	out->aru = swab32 (in->aru);
-	totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep);
+	out->ring_id.rep = swab32(in->ring_id.rep);
 	out->aru_addr = swab32(in->aru_addr);
 	out->ring_id.seq = swab64 (in->ring_id.seq);
 	out->fcc = swab32 (in->fcc);
@@ -4396,7 +4350,7 @@ static void orf_token_endian_convert (const struct orf_token *in, struct orf_tok
 	out->retrans_flg = swab32 (in->retrans_flg);
 	out->rtr_list_entries = swab32 (in->rtr_list_entries);
 	for (i = 0; i < out->rtr_list_entries; i++) {
-		totemip_copy_endian_convert(&out->rtr_list[i].ring_id.rep, &in->rtr_list[i].ring_id.rep);
+		out->rtr_list[i].ring_id.rep = swab32(in->rtr_list[i].ring_id.rep);
 		out->rtr_list[i].ring_id.seq = swab64 (in->rtr_list[i].ring_id.seq);
 		out->rtr_list[i].seq = swab32 (in->rtr_list[i].seq);
 	}
@@ -4411,7 +4365,7 @@ static void mcast_endian_convert (const struct mcast *in, struct mcast *out)
 
 	out->seq = swab32 (in->seq);
 	out->this_seqno = swab32 (in->this_seqno);
-	totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep);
+	out->ring_id.rep = swab32(in->ring_id.rep);
 	out->ring_id.seq = swab64 (in->ring_id.seq);
 	out->node_id = swab32 (in->node_id);
 	out->guarantee = swab32 (in->guarantee);
@@ -4425,7 +4379,7 @@ static void memb_merge_detect_endian_convert (
 	out->header.type = in->header.type;
 	out->header.endian_detector = ENDIAN_LOCAL;
 	out->header.nodeid = swab32 (in->header.nodeid);
-	totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep);
+	out->ring_id.rep = swab32(in->ring_id.rep);
 	out->ring_id.seq = swab64 (in->ring_id.seq);
 	srp_addr_copy_endian_convert (&out->system_from, &in->system_from);
 }
@@ -4594,7 +4548,7 @@ static int message_handler_memb_commit_token (
 			break;
 
 		case MEMB_STATE_RECOVERY:
-			if (totemip_equal (&instance->my_id.addr[0], &instance->my_ring_id.rep)) {
+			if (instance->my_id.nodeid == instance->my_ring_id.rep) {
 
 				/* Filter out duplicated tokens */
 				if (instance->originated_orf_token) {
@@ -4628,7 +4582,7 @@ static int message_handler_token_hold_cancel (
 		sizeof (struct memb_ring_id)) == 0) {
 
 		instance->my_seq_unchanged = 0;
-		if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0])) {
+		if (instance->my_ring_id.rep == instance->my_id.nodeid) {
 			timer_function_token_retransmit_timeout (instance);
 		}
 	}
@@ -4695,7 +4649,7 @@ int totemsrp_iface_set (
 	struct totemsrp_instance *instance = context;
 	int res;
 
-	totemip_copy(&instance->my_id.addr[iface_no], interface_addr);
+	totemip_copy(&instance->my_addrs[iface_no], interface_addr);
 
 	res = totemnet_iface_set (
 		instance->totemnet_context,
@@ -4716,20 +4670,19 @@ void main_iface_change_fn (
 	int num_interfaces;
 	int i;
 
-	totemip_copy (&instance->my_id.addr[iface_no], iface_addr);
-	assert (instance->my_id.addr[iface_no].nodeid);
-
-	totemip_copy (&instance->my_memb_list[0].addr[iface_no], iface_addr);
+	if (!instance->my_id.nodeid) {
+		instance->my_id.nodeid = iface_addr->nodeid;
+	}
+	totemip_copy (&instance->my_addrs[iface_no], iface_addr);
 
 	if (instance->iface_changes++ == 0) {
-		instance->memb_ring_id_create_or_load (&instance->my_ring_id,
-		    &instance->my_id.addr[0]);
+		instance->memb_ring_id_create_or_load (&instance->my_ring_id, instance->my_id.nodeid);
 		instance->token_ring_id_seq = instance->my_ring_id.seq;
 		log_printf (
 			instance->totemsrp_log_level_debug,
-			"Created or loaded sequence id %llx.%s for this ring.",
+			"Created or loaded sequence id %llx.%u for this ring.",
 			instance->my_ring_id.seq,
-			totemip_print (&instance->my_ring_id.rep));
+			instance->my_ring_id.rep);
 
 		if (instance->totemsrp_service_ready_fn) {
 			instance->totemsrp_service_ready_fn ();
@@ -4750,7 +4703,6 @@ void main_iface_change_fn (
 		}
 	}
 
-
 	if (instance->iface_changes >= num_interfaces) {
 		memb_state_gather_enter (instance, TOTEMSRP_GSFROM_INTERFACE_CHANGE);
 	}
@@ -4777,7 +4729,7 @@ int totemsrp_member_add (
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)context;
 	int res;
 
-	res = totemnet_member_add (instance->totemnet_context, &instance->my_id.addr[iface_no], member, iface_no);
+	res = totemnet_member_add (instance->totemnet_context, &instance->my_addrs[iface_no], member, iface_no);
 
 	return (res);
 }

exec/totemudp.c

@@ -81,6 +81,11 @@
 #define BIND_STATE_REGULAR	1
 #define BIND_STATE_LOOPBACK	2
 
+struct totemudp_member {
+	struct qb_list_head list;
+	struct totem_ip_address member;
+};
+
 struct totemudp_socket {
 	int mcast_recv;
 	int mcast_send;
@@ -143,6 +148,8 @@ struct totemudp_instance {
 
 	void *udp_context;
 
+	struct qb_list_head member_list;
+
 	char iov_buffer[UDP_RECEIVE_FRAME_SIZE_MAX];
 
 	char iov_buffer_flush[UDP_RECEIVE_FRAME_SIZE_MAX];
@@ -216,6 +223,8 @@ static void totemudp_instance_initialize (struct totemudp_instance *instance)
 	 * There is always atleast 1 processor
 	 */
 	instance->my_memb_entries = 1;
+
+	qb_list_init (&instance->member_list);
 }
 
 #define log_printf(level, format, args...)				\
@@ -1344,16 +1353,26 @@ extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *tot
 
 int totemudp_token_target_set (
 	void *udp_context,
-	const struct totem_ip_address *token_target)
+	unsigned int nodeid)
 {
 	struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
+	struct qb_list_head *list;
+	struct totemudp_member *member;
 	int res = 0;
 
-	memcpy (&instance->token_target, token_target,
-		sizeof (struct totem_ip_address));
+	qb_list_for_each(list, &(instance->member_list)) {
+		member = qb_list_entry (list,
+			struct totemudp_member,
+			list);
 
-	instance->totemudp_target_set_completed (instance->context);
+		if (member->member.nodeid == nodeid) {
+			memcpy (&instance->token_target, &member->member,
+				sizeof (struct totem_ip_address));
 
+			instance->totemudp_target_set_completed (instance->context);
+			break;
+		}
+	}
 	return (res);
 }
 
@@ -1421,6 +1440,65 @@ extern int totemudp_recv_mcast_empty (
 	return (msg_processed);
 }
 
+
+int totemudp_member_add (
+	void *udp_context,
+	const struct totem_ip_address *local,
+	const struct totem_ip_address *member,
+	int ring_no)
+{
+	struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
+
+	struct totemudp_member *new_member;
+
+	new_member = malloc (sizeof (struct totemudp_member));
+	if (new_member == NULL) {
+		return (-1);
+	}
+
+	memset(new_member, 0, sizeof(*new_member));
+
+	qb_list_init (&new_member->list);
+	qb_list_add_tail (&new_member->list, &instance->member_list);
+	memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
+
+	return (0);
+}
+
+int totemudp_member_remove (
+	void *udp_context,
+	const struct totem_ip_address *token_target,
+	int ring_no)
+{
+	int found = 0;
+	struct qb_list_head *list;
+	struct totemudp_member *member;
+	struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
+
+	/*
+	 * Find the member to remove and close its socket
+	 */
+	qb_list_for_each(list, &(instance->member_list)) {
+		member = qb_list_entry (list,
+			struct totemudp_member,
+			list);
+
+		if (totemip_compare (token_target, &member->member)==0) {
+			found = 1;
+			break;
+		}
+	}
+
+	/*
+	 * Delete the member from the list
+	 */
+	if (found) {
+		qb_list_del (list);
+	}
+
+	return (0);
+}
+
 int totemudp_iface_set (void *net_context,
 	const struct totem_ip_address *local_addr,
 	unsigned short ip_port,

exec/totemudp.h

@@ -112,7 +112,7 @@ extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *tot
 
 extern int totemudp_token_target_set (
 	void *udp_context,
-	const struct totem_ip_address *token_target);
+	unsigned int nodeid);
 
 extern int totemudp_crypto_set (
 	void *udp_context,
@@ -122,6 +122,17 @@ extern int totemudp_crypto_set (
 extern int totemudp_recv_mcast_empty (
 	void *udp_context);
 
+extern int totemudp_member_add (
+	void *udpu_context,
+	const struct totem_ip_address *local,
+	const struct totem_ip_address *member,
+	int ring_no);
+
+extern int totemudp_member_remove (
+	void *udpu_context,
+	const struct totem_ip_address *member,
+	int ring_no);
+
 extern int totemudp_reconfigure (
 	void *udp_context,
 	struct totem_config *totem_config);

exec/totemudpu.c

@@ -957,16 +957,27 @@ extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *t
 
 int totemudpu_token_target_set (
 	void *udpu_context,
-	const struct totem_ip_address *token_target)
+	unsigned int nodeid)
 {
+
 	struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
+	struct qb_list_head *list;
+	struct totemudpu_member *member;
 	int res = 0;
 
-	memcpy (&instance->token_target, token_target,
-		sizeof (struct totem_ip_address));
+	qb_list_for_each(list, &(instance->member_list)) {
+		member = qb_list_entry (list,
+			struct totemudpu_member,
+			list);
 
-	instance->totemudpu_target_set_completed (instance->context);
+		if (member->member.nodeid == nodeid) {
+			memcpy (&instance->token_target, &member->member,
+				sizeof (struct totem_ip_address));
 
+			instance->totemudpu_target_set_completed (instance->context);
+			break;
+		}
+	}
 	return (res);
 }
 

exec/totemudpu.h

@@ -112,7 +112,7 @@ extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *t
 
 extern int totemudpu_token_target_set (
 	void *udpu_context,
-	const struct totem_ip_address *token_target);
+	unsigned int nodeid);
 
 extern int totemudpu_crypto_set (
 	void *udpu_context,

exec/votequorum.c

@@ -1172,7 +1172,7 @@ static int votequorum_read_nodelist_configuration(uint32_t *votes,
 			continue;
 		}
 
-		if (strcmp(tmp_key, "ring0_addr") != 0) {
+		if (strcmp(tmp_key, "nodeid") != 0) {
 			continue;
 		}
 
@@ -1813,13 +1813,13 @@ static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t conte
 
 	ENTER();
 
-	log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = %d/%lld", quorum_ringid.rep.nodeid, quorum_ringid.seq);
+	log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = %d/%lld", quorum_ringid.nodeid, quorum_ringid.seq);
 
 	size = sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(uint32_t) * quorum_members_entries;
 
 	res_lib_votequorum_notification = (struct res_lib_votequorum_nodelist_notification *)&buf;
 	res_lib_votequorum_notification->node_list_entries = quorum_members_entries;
-	res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.rep.nodeid;
+	res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.nodeid;
 	res_lib_votequorum_notification->ring_id.seq = quorum_ringid.seq;
 	res_lib_votequorum_notification->context = context;
 
@@ -2945,12 +2945,12 @@ static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
 	}
 
 	if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
-		if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.rep.nodeid &&
+		if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.nodeid &&
 		      req_lib_votequorum_qdevice_poll->ring_id.seq == quorum_ringid.seq)) {
 			log_printf(LOGSYS_LEVEL_DEBUG, "Received poll ring id (%u.%"PRIu64") != last sync "
 			    "ring id (%u.%"PRIu64"). Ignoring poll call.",
 			    req_lib_votequorum_qdevice_poll->ring_id.nodeid, req_lib_votequorum_qdevice_poll->ring_id.seq,
-			    quorum_ringid.rep.nodeid, quorum_ringid.seq);
+			    quorum_ringid.nodeid, quorum_ringid.seq);
 			error = CS_ERR_MESSAGE_ERROR;
 			goto out;
 		}

include/corosync/coroapi.h

@@ -120,7 +120,7 @@ struct totem_ip_address {
  * @brief The memb_ring_id struct
  */
 struct memb_ring_id {
-	struct totem_ip_address rep;
+	unsigned int nodeid;
 	unsigned long long seq;
 } __attribute__((packed));
 #endif

include/corosync/totem/totem.h

@@ -137,7 +137,7 @@ typedef enum {
 
 #define MEMB_RING_ID
 struct memb_ring_id {
-	struct totem_ip_address rep;
+	unsigned int rep;
 	unsigned long long seq;
 } __attribute__((packed));
 
@@ -225,11 +225,11 @@ struct totem_config {
 
 	void (*totem_memb_ring_id_create_or_load) (
 	    struct memb_ring_id *memb_ring_id,
-	    const struct totem_ip_address *addr);
+	    unsigned int nodeid);
 
 	void (*totem_memb_ring_id_store) (
 	    const struct memb_ring_id *memb_ring_id,
-	    const struct totem_ip_address *addr);
+	    unsigned int nodeid);
 };
 
 #define TOTEM_CONFIGURATION_TYPE

tools/corosync-cfgtool.c

@@ -110,6 +110,7 @@ linkstatusget_do (char *interface_name, int brief)
 		printf ("Could not get the link status, the error is: %d\n", result);
 	} else {
 		for (i = 0; i < interface_count; i++) {
+			s = 0;
 			if ( (interface_name &&
 			      interface_names[i][0] != '\0' &&
 			      (interface_name[0]=='\0' ||

tools/corosync-quorumtool.c

@@ -279,7 +279,7 @@ static const char *node_name_by_nodelist(uint32_t nodeid)
 			continue;
 		}
 
-		if (strcmp(tmp_key, "ring0_addr") != 0) {
+		if (strcmp(tmp_key, "nodeid") != 0) {
 			continue;
 		}
 
@@ -290,7 +290,7 @@ static const char *node_name_by_nodelist(uint32_t nodeid)
 		if (cur_nodeid != nodeid) {
 			continue;
 		}
-		snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos);
+		snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.name", node_pos);
 		if (cmap_get_string(cmap_handle, tmp_key, &str) != CS_OK) {
 			continue;
 		}