totem: Add Kronosnet transport.

This is a big update that removes RRP & MRP from the codebase
and makes knet the default transport for corosync. UDP & UDPU
are still (currently) supported but are deprecated. Also crypto
and mutiple interfaces are only supported over knet.

To compile this codebase you will need to install libknet from
https://github.com/fabbione/kronosnet

The corosync.conf(5) man page has been updated with info on the new
options. Older config files should still work but many options
have changed because of the knet implementation so configs should
be checked carefully. In particular any cluster using using RRP
over UDP or UDPU will not start as RRP is no longer present. If you
need multiple interface support then you should be using the knet transport.

Knet brings many benefits to the corosync codebase, it provides support
for more interfaces than RRP (up to 8), will be more reliable in the event
of network outages and allows dynamic reconfiguration of interfaces.
It also fixes the ifup/ifdown and 127.0.0.1 binding problems that have
plagued corosync/openais from day 1

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>

configure.ac

@@ -163,6 +163,7 @@ CPPFLAGS="$SAVE_CPPFLAGS"
 LIBS="$SAVE_LIBS"
 AC_CHECK_LIB([pthread], [pthread_create])
 AC_CHECK_LIB([socket], [socket])
+AC_CHECK_LIB([knet], [knet_handle_new])
 AC_CHECK_LIB([nsl], [t_open])
 AC_CHECK_LIB([rt], [sched_getscheduler])
 AC_CHECK_LIB([z], [crc32],
@@ -453,14 +454,6 @@ if test "x${enable_testagents}" = xyes; then
 	WITH_LIST="$WITH_LIST --with testagents"
 fi
 
-if test "x${enable_rdma}" = xyes; then
-	PKG_CHECK_MODULES([rdmacm],[rdmacm])
-	PKG_CHECK_MODULES([ibverbs],[ibverbs])
-	AC_DEFINE_UNQUOTED([HAVE_RDMA], 1, [have rdmacm])
-	PACKAGE_FEATURES="$PACKAGE_FEATURES rdma"
-	WITH_LIST="$WITH_LIST --with rdma"
-fi
-
 if test "x${enable_monitoring}" = xyes; then
 	PKG_CHECK_MODULES([statgrab], [libstatgrab])
 	PKG_CHECK_MODULES([statgrabge090], [libstatgrab >= 0.90],

exec/Makefile.am

@@ -33,25 +33,22 @@ MAINTAINERCLEANFILES	= Makefile.in
 
 noinst_HEADERS		= apidef.h cs_queue.h logconfig.h main.h \
 			  quorum.h service.h timer.h totemconfig.h \
-			  totemmrp.h totemnet.h totemudp.h totemiba.h \
-			  totemrrp.h totemudpu.h totemsrp.h util.h vsf.h \
+			  totemnet.h totemudp.h \
+			  totemudpu.h totemsrp.h util.h vsf.h \
 			  schedwrk.h sync.h fsm.h votequorum.h vsf_ykd.h \
-			  totemcrypto.h
+			  totemknet.h
 
 TOTEM_SRC		= totemip.c totemnet.c totemudp.c \
-			  totemudpu.c totemrrp.c totemsrp.c totemmrp.c \
-			  totempg.c totemcrypto.c
+			  totemudpu.c totemsrp.c \
+			  totempg.c totemknet.c
 
-if BUILD_RDMA
-TOTEM_SRC		+= totemiba.c
-endif
 
 lib_LTLIBRARIES		= libtotem_pg.la
 libtotem_pg_la_SOURCES	= $(TOTEM_SRC)
 libtotem_pg_la_CFLAGS	= $(nss_CFLAGS) $(rdmacm_CFLAGS) $(ibverbs_CFLAGS)
 libtotem_pg_la_LDFLAGS	= -version-number $(subst .,:,$(SONAME))
 libtotem_pg_la_LIBADD	= -lpthread $(LIBQB_LIBS) $(nss_LIBS) \
-			  $(rdmacm_LIBS) $(ibverbs_LIBS)
+			  $(rdmacm_LIBS) $(ibverbs_LIBS) -lknet
 
 sbin_PROGRAMS		= corosync
 

exec/coroparse.c

@@ -108,12 +108,16 @@ struct key_value_list_item {
 };
 
 struct main_cp_cb_data {
-	int ringnumber;
+	int linknumber;
 	char *bindnetaddr;
 	char *mcastaddr;
 	char *broadcast;
 	int mcastport;
 	int ttl;
+	int knet_link_priority;
+	int knet_ping_interval;
+	int knet_ping_timeout;
+	int knet_ping_precision;
 
 	struct list_head logger_subsys_items_head;
 	char *subsys;
@@ -682,13 +686,13 @@ static int main_config_parser_cb(const char *path,
 			break;
 
 		case MAIN_CP_CB_DATA_STATE_INTERFACE:
-			if (strcmp(path, "totem.interface.ringnumber") == 0) {
+			if (strcmp(path, "totem.interface.linknumber") == 0) {
 				val_type = ICMAP_VALUETYPE_UINT8;
 				if (safe_atoq(value, &val, val_type) != 0) {
 					goto atoi_error;
 				}
 
-				data->ringnumber = val;
+				data->linknumber = val;
 				add_as_string = 0;
 			}
 			if (strcmp(path, "totem.interface.bindnetaddr") == 0) {
@@ -719,6 +723,38 @@ static int main_config_parser_cb(const char *path,
 				data->ttl = val;
 				add_as_string = 0;
 			}
+			if (strcmp(path, "totem.interface.knet_link_priority") == 0) {
+				val_type = ICMAP_VALUETYPE_UINT8;
+				if (safe_atoq(value, &val, val_type) != 0) {
+					goto atoi_error;
+				}
+				data->knet_link_priority = val;
+				add_as_string = 0;
+			}
+			if (strcmp(path, "totem.interface.knet_ping_interval") == 0) {
+				val_type = ICMAP_VALUETYPE_UINT32;
+				if (safe_atoq(value, &val, val_type) != 0) {
+					goto atoi_error;
+				}
+				data->knet_ping_interval = val;
+				add_as_string = 0;
+			}
+			if (strcmp(path, "totem.interface.knet_ping_timeout") == 0) {
+				val_type = ICMAP_VALUETYPE_UINT32;
+				if (safe_atoq(value, &val, val_type) != 0) {
+					goto atoi_error;
+				}
+				data->knet_ping_timeout = val;
+				add_as_string = 0;
+			}
+			if (strcmp(path, "totem.interface.knet_ping_precision") == 0) {
+				val_type = ICMAP_VALUETYPE_UINT32;
+				if (safe_atoq(value, &val, val_type) != 0) {
+					goto atoi_error;
+				}
+				data->knet_ping_precision = val;
+				add_as_string = 0;
+			}
 			break;
 		case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS:
 			if (strcmp(key, "subsys") == 0) {
@@ -903,9 +939,13 @@ static int main_config_parser_cb(const char *path,
 	case PARSER_CB_SECTION_START:
 		if (strcmp(path, "totem.interface") == 0) {
 			*state = MAIN_CP_CB_DATA_STATE_INTERFACE;
-			data->ringnumber = 0;
+			data->linknumber = 0;
 			data->mcastport = -1;
 			data->ttl = -1;
+			data->knet_link_priority = -1;
+			data->knet_ping_interval = -1;
+			data->knet_ping_timeout = -1;
+			data->knet_ping_precision = -1;
 			list_init(&data->member_items_head);
 		};
 		if (strcmp(path, "totem") == 0) {
@@ -969,7 +1009,7 @@ static int main_config_parser_cb(const char *path,
 			 */
 			if (data->bindnetaddr != NULL) {
 				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr",
-						data->ringnumber);
+						data->linknumber);
 				icmap_set_string_r(config_map, key_name, data->bindnetaddr);
 
 				free(data->bindnetaddr);
@@ -978,7 +1018,7 @@ static int main_config_parser_cb(const char *path,
 
 			if (data->mcastaddr != NULL) {
 				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr",
-						data->ringnumber);
+						data->linknumber);
 				icmap_set_string_r(config_map, key_name, data->mcastaddr);
 
 				free(data->mcastaddr);
@@ -987,7 +1027,7 @@ static int main_config_parser_cb(const char *path,
 
 			if (data->broadcast != NULL) {
 				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast",
-						data->ringnumber);
+						data->linknumber);
 				icmap_set_string_r(config_map, key_name, data->broadcast);
 
 				free(data->broadcast);
@@ -996,15 +1036,35 @@ static int main_config_parser_cb(const char *path,
 
 			if (data->mcastport > -1) {
 				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport",
-						data->ringnumber);
+						data->linknumber);
 				icmap_set_uint16_r(config_map, key_name, data->mcastport);
 			}
 
 			if (data->ttl > -1) {
 				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl",
-						data->ringnumber);
+						data->linknumber);
 				icmap_set_uint8_r(config_map, key_name, data->ttl);
 			}
+			if (data->knet_link_priority > -1) {
+				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_link_priority",
+						data->linknumber);
+				icmap_set_uint8_r(config_map, key_name, data->knet_link_priority);
+			}
+			if (data->knet_ping_interval > -1) {
+				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_interval",
+						data->linknumber);
+				icmap_set_uint32_r(config_map, key_name, data->knet_ping_interval);
+			}
+			if (data->knet_ping_timeout > -1) {
+				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_timeout",
+						data->linknumber);
+				icmap_set_uint32_r(config_map, key_name, data->knet_ping_timeout);
+			}
+			if (data->knet_ping_precision > -1) {
+				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_precision",
+						data->linknumber);
+				icmap_set_uint32_r(config_map, key_name, data->knet_ping_precision);
+			}
 
 			ii = 0;
 			for (iter = data->member_items_head.next;
@@ -1012,7 +1072,7 @@ static int main_config_parser_cb(const char *path,
 				kv_item = list_entry(iter, struct key_value_list_item, list);
 
 				snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.%u",
-						data->ringnumber, ii);
+						data->linknumber, ii);
 				icmap_set_string_r(config_map, key_name, kv_item->value);
 
 				iter_next = iter->next;

exec/main.c

@@ -495,86 +495,81 @@ static void corosync_totem_stats_updater (void *data)
 	uint32_t total_mtt_rx_token;
 	uint32_t total_backlog_calc;
 	uint32_t total_token_holdtime;
-	int t, prev, i;
+	int t, prev;
 	int32_t token_count;
-	char key_name[ICMAP_KEYNAME_MAXLEN];
 
 	stats = api->totem_get_stats();
 
 	icmap_set_uint32("runtime.totem.pg.msg_reserved", stats->msg_reserved);
 	icmap_set_uint32("runtime.totem.pg.msg_queue_avail", stats->msg_queue_avail);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_tx", stats->mrp->srp->orf_token_tx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_rx", stats->mrp->srp->orf_token_rx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_tx", stats->mrp->srp->memb_merge_detect_tx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_rx", stats->mrp->srp->memb_merge_detect_rx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_tx", stats->mrp->srp->memb_join_tx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_rx", stats->mrp->srp->memb_join_rx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_tx", stats->mrp->srp->mcast_tx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_retx", stats->mrp->srp->mcast_retx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_rx", stats->mrp->srp->mcast_rx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_tx", stats->mrp->srp->memb_commit_token_tx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_rx", stats->mrp->srp->memb_commit_token_rx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_tx", stats->mrp->srp->token_hold_cancel_tx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_rx", stats->mrp->srp->token_hold_cancel_rx);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_entered", stats->mrp->srp->operational_entered);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_token_lost", stats->mrp->srp->operational_token_lost);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_entered", stats->mrp->srp->gather_entered);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_token_lost", stats->mrp->srp->gather_token_lost);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_entered", stats->mrp->srp->commit_entered);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_token_lost", stats->mrp->srp->commit_token_lost);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_entered", stats->mrp->srp->recovery_entered);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_token_lost", stats->mrp->srp->recovery_token_lost);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.consensus_timeouts", stats->mrp->srp->consensus_timeouts);
-	icmap_set_uint64("runtime.totem.pg.mrp.srp.rx_msg_dropped", stats->mrp->srp->rx_msg_dropped);
-	icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_gather", stats->mrp->srp->continuous_gather);
-	icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_sendmsg_failures",
-	    stats->mrp->srp->continuous_sendmsg_failures);
-
-	icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure",
-		stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0);
-
-	if (stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ||
-	    stats->mrp->srp->continuous_sendmsg_failures > MAX_NO_CONT_SENDMSG_FAILURES) {
+	icmap_set_uint64("runtime.totem.pg.srp.orf_token_tx", stats->srp->orf_token_tx);
+	icmap_set_uint64("runtime.totem.pg.srp.orf_token_rx", stats->srp->orf_token_rx);
+	icmap_set_uint64("runtime.totem.pg.srp.memb_merge_detect_tx", stats->srp->memb_merge_detect_tx);
+	icmap_set_uint64("runtime.totem.pg.srp.memb_merge_detect_rx", stats->srp->memb_merge_detect_rx);
+	icmap_set_uint64("runtime.totem.pg.srp.memb_join_tx", stats->srp->memb_join_tx);
+	icmap_set_uint64("runtime.totem.pg.srp.memb_join_rx", stats->srp->memb_join_rx);
+	icmap_set_uint64("runtime.totem.pg.srp.mcast_tx", stats->srp->mcast_tx);
+	icmap_set_uint64("runtime.totem.pg.srp.mcast_retx", stats->srp->mcast_retx);
+	icmap_set_uint64("runtime.totem.pg.srp.mcast_rx", stats->srp->mcast_rx);
+	icmap_set_uint64("runtime.totem.pg.srp.memb_commit_token_tx", stats->srp->memb_commit_token_tx);
+	icmap_set_uint64("runtime.totem.pg.srp.memb_commit_token_rx", stats->srp->memb_commit_token_rx);
+	icmap_set_uint64("runtime.totem.pg.srp.token_hold_cancel_tx", stats->srp->token_hold_cancel_tx);
+	icmap_set_uint64("runtime.totem.pg.srp.token_hold_cancel_rx", stats->srp->token_hold_cancel_rx);
+	icmap_set_uint64("runtime.totem.pg.srp.operational_entered", stats->srp->operational_entered);
+	icmap_set_uint64("runtime.totem.pg.srp.operational_token_lost", stats->srp->operational_token_lost);
+	icmap_set_uint64("runtime.totem.pg.srp.gather_entered", stats->srp->gather_entered);
+	icmap_set_uint64("runtime.totem.pg.srp.gather_token_lost", stats->srp->gather_token_lost);
+	icmap_set_uint64("runtime.totem.pg.srp.commit_entered", stats->srp->commit_entered);
+	icmap_set_uint64("runtime.totem.pg.srp.commit_token_lost", stats->srp->commit_token_lost);
+	icmap_set_uint64("runtime.totem.pg.srp.recovery_entered", stats->srp->recovery_entered);
+	icmap_set_uint64("runtime.totem.pg.srp.recovery_token_lost", stats->srp->recovery_token_lost);
+	icmap_set_uint64("runtime.totem.pg.srp.consensus_timeouts", stats->srp->consensus_timeouts);
+	icmap_set_uint64("runtime.totem.pg.srp.rx_msg_dropped", stats->srp->rx_msg_dropped);
+	icmap_set_uint32("runtime.totem.pg.srp.continuous_gather", stats->srp->continuous_gather);
+	icmap_set_uint32("runtime.totem.pg.srp.continuous_sendmsg_failures",
+	    stats->srp->continuous_sendmsg_failures);
+
+	icmap_set_uint8("runtime.totem.pg.srp.firewall_enabled_or_nic_failure",
+		stats->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0);
+
+	if (stats->srp->continuous_gather > MAX_NO_CONT_GATHER ||
+	    stats->srp->continuous_sendmsg_failures > MAX_NO_CONT_SENDMSG_FAILURES) {
 		log_printf (LOGSYS_LEVEL_WARNING,
 			"Totem is unable to form a cluster because of an "
 			"operating system or network fault. The most common "
 			"cause of this message is that the local firewall is "
 			"configured improperly.");
-		icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 1);
+		icmap_set_uint8("runtime.totem.pg.srp.firewall_enabled_or_nic_failure", 1);
 	} else {
-		icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 0);
+		icmap_set_uint8("runtime.totem.pg.srp.firewall_enabled_or_nic_failure", 0);
 	}
 
-	for (i = 0; i < stats->mrp->srp->rrp->interface_count; i++) {
-		snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.totem.pg.mrp.rrp.%u.faulty", i);
-		icmap_set_uint8(key_name, stats->mrp->srp->rrp->faulty[i]);
-	}
 	total_mtt_rx_token = 0;
 	total_token_holdtime = 0;
 	total_backlog_calc = 0;
 	token_count = 0;
-	t = stats->mrp->srp->latest_token;
+	t = stats->srp->latest_token;
 	while (1) {
 		if (t == 0)
 			prev = TOTEM_TOKEN_STATS_MAX - 1;
 		else
 			prev = t - 1;
-		if (prev == stats->mrp->srp->earliest_token)
+		if (prev == stats->srp->earliest_token)
 			break;
 		/* if tx == 0, then dropped token (not ours) */
-		if (stats->mrp->srp->token[t].tx != 0 ||
-			(stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx) > 0 ) {
-			total_mtt_rx_token += (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx);
-			total_token_holdtime += (stats->mrp->srp->token[t].tx - stats->mrp->srp->token[t].rx);
-			total_backlog_calc += stats->mrp->srp->token[t].backlog_calc;
+		if (stats->srp->token[t].tx != 0 ||
+			(stats->srp->token[t].rx - stats->srp->token[prev].rx) > 0 ) {
+			total_mtt_rx_token += (stats->srp->token[t].rx - stats->srp->token[prev].rx);
+			total_token_holdtime += (stats->srp->token[t].tx - stats->srp->token[t].rx);
+			total_backlog_calc += stats->srp->token[t].backlog_calc;
 			token_count++;
 		}
 		t = prev;
 	}
 	if (token_count) {
-		icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
-		icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", (total_token_holdtime / token_count));
-		icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", (total_backlog_calc / token_count));
+		icmap_set_uint32("runtime.totem.pg.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
+		icmap_set_uint32("runtime.totem.pg.srp.avg_token_workload", (total_token_holdtime / token_count));
+		icmap_set_uint32("runtime.totem.pg.srp.avg_backlog_calc", (total_backlog_calc / token_count));
 	}
 
 	cs_ipcs_stats_update();
@@ -586,9 +581,9 @@ static void corosync_totem_stats_updater (void *data)
 
 static void corosync_totem_stats_init (void)
 {
-	icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", 0);
-	icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", 0);
-	icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", 0);
+	icmap_set_uint32("runtime.totem.pg.srp.mtt_rx_token", 0);
+	icmap_set_uint32("runtime.totem.pg.srp.avg_token_workload", 0);
+	icmap_set_uint32("runtime.totem.pg.srp.avg_backlog_calc", 0);
 
 	/* start stats timer */
 	api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
@@ -1312,7 +1307,7 @@ int main (int argc, char **argv, char **envp)
 	totem_config.totem_logging_configuration = totem_logging_configuration;
 	totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
 			"totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
-			"totempg.c,totemiba.c,totemudp.c,totemudpu.c,totemnet.c");
+			"totempg.c,totemiba.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c");
 
 	totem_config.totem_logging_configuration.log_level_security = LOGSYS_LEVEL_WARNING;
 	totem_config.totem_logging_configuration.log_level_error = LOGSYS_LEVEL_ERROR;
@@ -1321,6 +1316,7 @@ int main (int argc, char **argv, char **envp)
 	totem_config.totem_logging_configuration.log_level_debug = LOGSYS_LEVEL_DEBUG;
 	totem_config.totem_logging_configuration.log_level_trace = LOGSYS_LEVEL_TRACE;
 	totem_config.totem_logging_configuration.log_printf = _logsys_log_printf;
+
 	logsys_config_apply();
 
 	/*

exec/totemconfig.c

@@ -73,10 +73,11 @@
 #define WINDOW_SIZE				50
 #define MAX_MESSAGES				17
 #define MISS_COUNT_CONST			5
-#define RRP_PROBLEM_COUNT_TIMEOUT		2000
-#define RRP_PROBLEM_COUNT_THRESHOLD_DEFAULT	10
-#define RRP_PROBLEM_COUNT_THRESHOLD_MIN		2
-#define RRP_AUTORECOVERY_CHECK_TIMEOUT		1000
+
+/* These currently match the defaults in libknet.h */
+#define KNET_PING_INTERVAL                      1000
+#define KNET_PING_TIMEOUT                       2000
+#define KNET_PING_PRECISION                     2048
 
 #define DEFAULT_PORT				5405
 
@@ -110,16 +111,6 @@ static uint32_t *totem_get_param_by_name(struct totem_config *totem_config, cons
 		return &totem_config->fail_to_recv_const;
 	if (strcmp(param_name, "totem.seqno_unchanged_const") == 0)
 		return &totem_config->seqno_unchanged_const;
-	if (strcmp(param_name, "totem.rrp_token_expired_timeout") == 0)
-		return &totem_config->rrp_token_expired_timeout;
-	if (strcmp(param_name, "totem.rrp_problem_count_timeout") == 0)
-		return &totem_config->rrp_problem_count_timeout;
-	if (strcmp(param_name, "totem.rrp_problem_count_threshold") == 0)
-		return &totem_config->rrp_problem_count_threshold;
-	if (strcmp(param_name, "totem.rrp_problem_count_mcast_threshold") == 0)
-		return &totem_config->rrp_problem_count_mcast_threshold;
-	if (strcmp(param_name, "totem.rrp_autorecovery_check_timeout") == 0)
-		return &totem_config->rrp_autorecovery_check_timeout;
 	if (strcmp(param_name, "totem.heartbeat_failures_allowed") == 0)
 		return &totem_config->heartbeat_failures_allowed;
 	if (strcmp(param_name, "totem.max_network_delay") == 0)
@@ -222,21 +213,6 @@ static void totem_volatile_config_read (struct totem_config *totem_config, const
 
 	totem_volatile_config_set_value(totem_config, "totem.send_join", deleted_key, 0, 1);
 
-	totem_volatile_config_set_value(totem_config, "totem.rrp_problem_count_timeout", deleted_key,
-	    RRP_PROBLEM_COUNT_TIMEOUT, 0);
-
-	totem_volatile_config_set_value(totem_config, "totem.rrp_problem_count_threshold", deleted_key,
-	    RRP_PROBLEM_COUNT_THRESHOLD_DEFAULT, 0);
-
-	totem_volatile_config_set_value(totem_config, "totem.rrp_problem_count_mcast_threshold", deleted_key,
-	    totem_config->rrp_problem_count_threshold * 10, 0);
-
-	totem_volatile_config_set_value(totem_config, "totem.rrp_token_expired_timeout", deleted_key,
-	    totem_config->token_retransmit_timeout, 0);
-
-	totem_volatile_config_set_value(totem_config, "totem.rrp_autorecovery_check_timeout", deleted_key,
-	    RRP_AUTORECOVERY_CHECK_TIMEOUT, 0);
-
 	totem_volatile_config_set_value(totem_config, "totem.heartbeat_failures_allowed", deleted_key, 0, 1);
 }
 
@@ -310,33 +286,6 @@ static int totem_volatile_config_validate (
 		goto parse_error;
 	}
 
-	if (totem_config->rrp_problem_count_timeout < MINIMUM_TIMEOUT) {
-		snprintf (local_error_reason, sizeof(local_error_reason),
-			"The RRP problem count timeout parameter (%d ms) may not be less than (%d ms).",
-			totem_config->rrp_problem_count_timeout, MINIMUM_TIMEOUT);
-		goto parse_error;
-	}
-
-	if (totem_config->rrp_problem_count_threshold < RRP_PROBLEM_COUNT_THRESHOLD_MIN) {
-		snprintf (local_error_reason, sizeof(local_error_reason),
-			"The RRP problem count threshold (%d problem count) may not be less than (%d problem count).",
-			totem_config->rrp_problem_count_threshold, RRP_PROBLEM_COUNT_THRESHOLD_MIN);
-		goto parse_error;
-	}
-	if (totem_config->rrp_problem_count_mcast_threshold < RRP_PROBLEM_COUNT_THRESHOLD_MIN) {
-		snprintf (local_error_reason, sizeof(local_error_reason),
-			"The RRP multicast problem count threshold (%d problem count) may not be less than (%d problem count).",
-			totem_config->rrp_problem_count_mcast_threshold, RRP_PROBLEM_COUNT_THRESHOLD_MIN);
-		goto parse_error;
-	}
-
-	if (totem_config->rrp_token_expired_timeout < MINIMUM_TIMEOUT) {
-		snprintf (local_error_reason, sizeof(local_error_reason),
-			"The RRP token expired timeout parameter (%d ms) may not be less than (%d ms).",
-			totem_config->rrp_token_expired_timeout, MINIMUM_TIMEOUT);
-		goto parse_error;
-	}
-
 	return 0;
 
 parse_error:
@@ -353,16 +302,8 @@ static int totem_get_crypto(struct totem_config *totem_config)
 	const char *tmp_cipher;
 	const char *tmp_hash;
 
-	tmp_hash = "sha1";
-	tmp_cipher = "aes256";
-
-	if (icmap_get_string("totem.secauth", &str) == CS_OK) {
-		if (strcmp (str, "off") == 0) {
-			tmp_hash = "none";
-			tmp_cipher = "none";
-		}
-		free(str);
-	}
+	tmp_hash = "none";
+	tmp_cipher = "none";
 
 	if (icmap_get_string("totem.crypto_cipher", &str) == CS_OK) {
 		if (strcmp(str, "none") == 0) {
@@ -377,6 +318,9 @@ static int totem_get_crypto(struct totem_config *totem_config)
 		if (strcmp(str, "aes128") == 0) {
 			tmp_cipher = "aes128";
 		}
+		if (strcmp(str, "aes256") == 0) {
+			tmp_cipher = "aes256";
+		}
 		if (strcmp(str, "3des") == 0) {
 			tmp_cipher = "3des";
 		}
@@ -453,7 +397,7 @@ static uint16_t generate_cluster_id (const char *cluster_name)
 
 static int get_cluster_mcast_addr (
 		const char *cluster_name,
-		unsigned int ringnumber,
+		unsigned int linknumber,
 		int ip_version,
 		struct totem_ip_address *res)
 {
@@ -465,7 +409,7 @@ static int get_cluster_mcast_addr (
 		return (-1);
 	}
 
-	clusterid = generate_cluster_id(cluster_name) + ringnumber;
+	clusterid = generate_cluster_id(cluster_name) + linknumber;
 	memset (res, 0, sizeof(*res));
 
 	switch (ip_version) {
@@ -728,7 +672,7 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in
 	char tmp_key2[ICMAP_KEYNAME_MAXLEN];
 	char *node_addr_str;
 	int member_count;
-	unsigned int ringnumber = 0;
+	unsigned int linknumber = 0;
 	int i, j;
 	struct totem_interface *orig_interfaces = NULL;
 	struct totem_interface *new_interfaces = NULL;
@@ -769,7 +713,13 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in
 		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.", node_pos);
 		iter2 = icmap_iter_init(tmp_key);
 		while ((iter_key2 = icmap_iter_next(iter2, NULL, NULL)) != NULL) {
-			res = sscanf(iter_key2, "nodelist.node.%u.ring%u%s", &node_pos, &ringnumber, tmp_key2);
+			unsigned int nodeid;
+
+			snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos);
+			if (icmap_get_uint32(tmp_key, &nodeid) != CS_OK) {
+			}
+
+			res = sscanf(iter_key2, "nodelist.node.%u.ring%u%s", &node_pos, &linknumber, tmp_key2);
 			if (res != 3 || strcmp(tmp_key2, "_addr") != 0) {
 				continue;
 			}
@@ -778,12 +728,13 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in
 				continue;
 			}
 
-			member_count = totem_config->interfaces[ringnumber].member_count;
+			member_count = totem_config->interfaces[linknumber].member_count;
 
-			res = totemip_parse(&totem_config->interfaces[ringnumber].member_list[member_count],
+			res = totemip_parse(&totem_config->interfaces[linknumber].member_list[member_count],
 						node_addr_str, totem_config->ip_version);
 			if (res != -1) {
-				totem_config->interfaces[ringnumber].member_count++;
+				totem_config->interfaces[linknumber].member_list[member_count].nodeid = nodeid;
+				totem_config->interfaces[linknumber].member_count++;
 			}
 			free(node_addr_str);
 		}
@@ -946,7 +897,7 @@ static void config_convert_nodelist_to_interface(struct totem_config *totem_conf
 	char tmp_key[ICMAP_KEYNAME_MAXLEN];
 	char tmp_key2[ICMAP_KEYNAME_MAXLEN];
 	char *node_addr_str;
-	unsigned int ringnumber = 0;
+	unsigned int linknumber = 0;
 	icmap_iter_t iter;
 	const char *iter_key;
 
@@ -957,7 +908,7 @@ static void config_convert_nodelist_to_interface(struct totem_config *totem_conf
 		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.", node_pos);
 		iter = icmap_iter_init(tmp_key);
 		while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
-			res = sscanf(iter_key, "nodelist.node.%u.ring%u%s", &node_pos, &ringnumber, tmp_key2);
+			res = sscanf(iter_key, "nodelist.node.%u.ring%u%s", &node_pos, &linknumber, tmp_key2);
 			if (res != 3 || strcmp(tmp_key2, "_addr") != 0) {
 				continue ;
 			}
@@ -966,7 +917,7 @@ static void config_convert_nodelist_to_interface(struct totem_config *totem_conf
 				continue;
 			}
 
-			snprintf(tmp_key2, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", ringnumber);
+			snprintf(tmp_key2, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber);
 			icmap_set_string(tmp_key2, node_addr_str);
 			free(node_addr_str);
 		}
@@ -982,15 +933,16 @@ extern int totem_config_read (
 {
 	int res = 0;
 	char *str;
-	unsigned int ringnumber = 0;
+	unsigned int linknumber = 0;
 	int member_count = 0;
 	icmap_iter_t iter, member_iter;
 	const char *iter_key;
 	const char *member_iter_key;
-	char ringnumber_key[ICMAP_KEYNAME_MAXLEN];
+	char linknumber_key[ICMAP_KEYNAME_MAXLEN];
 	char tmp_key[ICMAP_KEYNAME_MAXLEN];
 	uint8_t u8;
 	uint16_t u16;
+	uint32_t u32;
 	char *cluster_name = NULL;
 	int i;
 	int local_node_pos;
@@ -1008,7 +960,7 @@ extern int totem_config_read (
 	memset (totem_config->interfaces, 0,
 		sizeof (struct totem_interface) * INTERFACE_MAX);
 
-	strcpy (totem_config->rrp_mode, "none");
+	strcpy (totem_config->link_mode, "passive");
 
 	icmap_get_uint32("totem.version", (uint32_t *)&totem_config->version);
 
@@ -1017,14 +969,14 @@ extern int totem_config_read (
 		return -1;
 	}
 
-	if (icmap_get_string("totem.rrp_mode", &str) == CS_OK) {
-		if (strlen(str) >= TOTEM_RRP_MODE_BYTES) {
-			*error_string = "totem.rrp_mode is too long";
+	if (icmap_get_string("totem.link_mode", &str) == CS_OK) {
+		if (strlen(str) >= TOTEM_LINK_MODE_BYTES) {
+			*error_string = "totem.link_mode is too long";
 			free(str);
 
 			return -1;
 		}
-		strcpy (totem_config->rrp_mode, str);
+		strcpy (totem_config->link_mode, str);
 		free(str);
 	}
 
@@ -1065,7 +1017,7 @@ extern int totem_config_read (
 
 	iter = icmap_iter_init("totem.interface.");
 	while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
-		res = sscanf(iter_key, "totem.interface.%[^.].%s", ringnumber_key, tmp_key);
+		res = sscanf(iter_key, "totem.interface.%[^.].%s", linknumber_key, tmp_key);
 		if (res != 2) {
 			continue;
 		}
@@ -1076,14 +1028,14 @@ extern int totem_config_read (
 
 		member_count = 0;
 
-		ringnumber = atoi(ringnumber_key);
+		linknumber = atoi(linknumber_key);
 
-		if (ringnumber >= INTERFACE_MAX) {
+		if (linknumber >= INTERFACE_MAX) {
 			free(cluster_name);
 
 			snprintf (error_string_response, sizeof(error_string_response),
 			    "parse error in config: interface ring number %u is bigger than allowed maximum %u\n",
-			    ringnumber, INTERFACE_MAX - 1);
+			    linknumber, INTERFACE_MAX - 1);
 
 			*error_string = error_string_response;
 			return -1;
@@ -1093,7 +1045,7 @@ extern int totem_config_read (
 		 * Get the bind net address
 		 */
 		if (icmap_get_string(iter_key, &str) == CS_OK) {
-			res = totemip_parse (&totem_config->interfaces[ringnumber].bindnet, str,
+			res = totemip_parse (&totem_config->interfaces[linknumber].bindnet, str,
 			    totem_config->ip_version);
 			free(str);
 		}
@@ -1101,9 +1053,9 @@ extern int totem_config_read (
 		/*
 		 * Get interface multicast address
 		 */
-		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", ringnumber);
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", linknumber);
 		if (icmap_get_string(tmp_key, &str) == CS_OK) {
-			res = totemip_parse (&totem_config->interfaces[ringnumber].mcast_addr, str, totem_config->ip_version);
+			res = totemip_parse (&totem_config->interfaces[linknumber].mcast_addr, str, totem_config->ip_version);
 			free(str);
 		} else {
 			/*
@@ -1113,12 +1065,12 @@ extern int totem_config_read (
 			 * checked later anyway.
 			 */
 			(void)get_cluster_mcast_addr (cluster_name,
-					ringnumber,
+					linknumber,
 					totem_config->ip_version,
-					&totem_config->interfaces[ringnumber].mcast_addr);
+					&totem_config->interfaces[linknumber].mcast_addr);
 		}
 
-		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast", ringnumber);
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast", linknumber);
 		if (icmap_get_string(tmp_key, &str) == CS_OK) {
 			if (strcmp (str, "yes") == 0) {
 				totem_config->broadcast_use = 1;
@@ -1129,27 +1081,53 @@ extern int totem_config_read (
 		/*
 		 * Get mcast port
 		 */
-		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", ringnumber);
-		if (icmap_get_uint16(tmp_key, &totem_config->interfaces[ringnumber].ip_port) != CS_OK) {
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", linknumber);
+		if (icmap_get_uint16(tmp_key, &totem_config->interfaces[linknumber].ip_port) != CS_OK) {
 			if (totem_config->broadcast_use) {
-				totem_config->interfaces[ringnumber].ip_port = DEFAULT_PORT + (2 * ringnumber);
+				totem_config->interfaces[linknumber].ip_port = DEFAULT_PORT + (2 * linknumber);
 			} else {
-				totem_config->interfaces[ringnumber].ip_port = DEFAULT_PORT;
+				totem_config->interfaces[linknumber].ip_port = DEFAULT_PORT;
 			}
 		}
 
 		/*
 		 * Get the TTL
 		 */
-		totem_config->interfaces[ringnumber].ttl = 1;
+		totem_config->interfaces[linknumber].ttl = 1;
+
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl", linknumber);
+
+		if (icmap_get_uint8(tmp_key, &u8) == CS_OK) {
+			totem_config->interfaces[linknumber].ttl = u8;
+		}
 
-		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl", ringnumber);
+		/*
+		 * Get the knet link params
+		 */
+		totem_config->interfaces[linknumber].knet_link_priority = 1;
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_link_priority", linknumber);
 
 		if (icmap_get_uint8(tmp_key, &u8) == CS_OK) {
-			totem_config->interfaces[ringnumber].ttl = u8;
+			totem_config->interfaces[linknumber].knet_link_priority = u8;
+		}
+
+		totem_config->interfaces[linknumber].knet_ping_interval = KNET_PING_INTERVAL;
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_interval", linknumber);
+		if (icmap_get_uint32(tmp_key, &u32) == CS_OK) {
+			totem_config->interfaces[linknumber].knet_ping_interval = u32;
+		}
+		totem_config->interfaces[linknumber].knet_ping_timeout = KNET_PING_TIMEOUT;
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_timeout", linknumber);
+		if (icmap_get_uint32(tmp_key, &u32) == CS_OK) {
+			totem_config->interfaces[linknumber].knet_ping_timeout = u32;
+		}
+		totem_config->interfaces[linknumber].knet_ping_precision = KNET_PING_PRECISION;
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_precision", linknumber);
+		if (icmap_get_uint32(tmp_key, &u32) == CS_OK) {
+			totem_config->interfaces[linknumber].knet_ping_precision = u32;
 		}
 
-		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.", ringnumber);
+		snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.", linknumber);
 		member_iter = icmap_iter_init(tmp_key);
 		while ((member_iter_key = icmap_iter_next(member_iter, NULL, NULL)) != NULL) {
 			if (member_count == 0) {
@@ -1163,13 +1141,13 @@ extern int totem_config_read (
 			}
 
 			if (icmap_get_string(member_iter_key, &str) == CS_OK) {
-				res = totemip_parse (&totem_config->interfaces[ringnumber].member_list[member_count++],
+				res = totemip_parse (&totem_config->interfaces[linknumber].member_list[member_count++],
 						str, totem_config->ip_version);
 			}
 		}
 		icmap_iter_finalize(member_iter);
 
-		totem_config->interfaces[ringnumber].member_count = member_count;
+		totem_config->interfaces[linknumber].member_count = member_count;
 		totem_config->interface_count++;
 	}
 	icmap_iter_finalize(iter);
@@ -1178,8 +1156,8 @@ extern int totem_config_read (
 	 * Use broadcast is global, so if set, make sure to fill mcast addr correctly
 	 */
 	if (totem_config->broadcast_use) {
-		for (ringnumber = 0; ringnumber < totem_config->interface_count; ringnumber++) {
-			totemip_parse (&totem_config->interfaces[ringnumber].mcast_addr,
+		for (linknumber = 0; linknumber < totem_config->interface_count; linknumber++) {
+			totemip_parse (&totem_config->interfaces[linknumber].mcast_addr,
 				"255.255.255.255", 0);
 		}
 	}
@@ -1202,15 +1180,20 @@ extern int totem_config_read (
 		}
 	}
 
-	totem_config->transport_number = TOTEM_TRANSPORT_UDP;
+	totem_config->transport_number = TOTEM_TRANSPORT_KNET;
 	if (icmap_get_string("totem.transport", &str) == CS_OK) {
 		if (strcmp (str, "udpu") == 0) {
 			totem_config->transport_number = TOTEM_TRANSPORT_UDPU;
 		}
 
-		if (strcmp (str, "iba") == 0) {
-			totem_config->transport_number = TOTEM_TRANSPORT_RDMA;
+		if (strcmp (str, "udp") == 0) {
+			totem_config->transport_number = TOTEM_TRANSPORT_UDP;
+		}
+
+		if (strcmp (str, "knet") == 0) {
+			totem_config->transport_number = TOTEM_TRANSPORT_KNET;
 		}
+
 		free(str);
 	}
 
@@ -1306,6 +1289,15 @@ int totem_config_validate (
 			error_reason = "Can only set ttl on multicast transport types";
 			goto parse_error;
 		}
+		if (totem_config->interfaces[i].knet_link_priority > 255) {
+			error_reason = "Invalid link priority (should be 0..255)";
+			goto parse_error;
+		}
+		if (totem_config->transport_number != TOTEM_TRANSPORT_KNET &&
+		    totem_config->interfaces[i].knet_link_priority != 1) {
+			error_reason = "Can only set link priority on knet transport type";
+			goto parse_error;
+		}
 
 		if (totem_config->interfaces[i].mcast_addr.family == AF_INET6 &&
 			totem_config->node_id == 0) {
@@ -1362,28 +1354,41 @@ int totem_config_validate (
 	}
 
 	/*
-	 * RRP values validation
+	 * KNET Link values validation
 	 */
-	if (strcmp (totem_config->rrp_mode, "none") &&
-		strcmp (totem_config->rrp_mode, "active") &&
-		strcmp (totem_config->rrp_mode, "passive")) {
+	if (strcmp (totem_config->link_mode, "active") &&
+	        strcmp (totem_config->link_mode, "rr") &&
+		strcmp (totem_config->link_mode, "passive")) {
 		snprintf (local_error_reason, sizeof(local_error_reason),
-			"The RRP mode \"%s\" specified is invalid.  It must be none, active, or passive.\n", totem_config->rrp_mode);
+			"The Knet link mode \"%s\" specified is invalid.  It must be active, passive or rr.\n", totem_config->link_mode);
 		goto parse_error;
 	}
 
-	if (strcmp (totem_config->rrp_mode, "none") == 0) {
+	/* Only Knet does multiple interfaces */
+	if (totem_config->transport_number != TOTEM_TRANSPORT_KNET) {
 		interface_max = 1;
 	}
+
 	if (interface_max < totem_config->interface_count) {
 		snprintf (parse_error, sizeof(parse_error),
-			"%d is too many configured interfaces for the rrp_mode setting %s.",
-			totem_config->interface_count,
-			totem_config->rrp_mode);
+			  "%d is too many configured interfaces for non-Knet transport.",
+			  totem_config->interface_count);
 		error_reason = parse_error;
 		goto parse_error;
 	}
 
+	/* Only knet allows crypto */
+	if (totem_config->transport_number != TOTEM_TRANSPORT_KNET) {
+		if ((strcmp(totem_config->crypto_cipher_type, "none") != 0) ||
+		    (strcmp(totem_config->crypto_hash_type, "none") != 0)) {
+
+			snprintf (parse_error, sizeof(parse_error),
+				  "crypto_cipher & crypto_hash are only valid for the Knet transport.");
+			error_reason = parse_error;
+			goto parse_error;
+		}
+	}
+
 	if (totem_config->net_mtu == 0) {
 		totem_config->net_mtu = 1500;
 	}
@@ -1526,18 +1531,6 @@ static void debug_dump_totem_config(const struct totem_config *totem_config)
 	    "window size per rotation (%d messages) maximum messages per rotation (%d messages)",
 	    totem_config->window_size, totem_config->max_messages);
 	log_printf(LOGSYS_LEVEL_DEBUG, "missed count const (%d messages)", totem_config->miss_count_const);
-	log_printf(LOGSYS_LEVEL_DEBUG, "RRP token expired timeout (%d ms)",
-	    totem_config->rrp_token_expired_timeout);
-	log_printf(LOGSYS_LEVEL_DEBUG, "RRP token problem counter (%d ms)",
-	    totem_config->rrp_problem_count_timeout);
-	log_printf(LOGSYS_LEVEL_DEBUG, "RRP threshold (%d problem count)",
-	    totem_config->rrp_problem_count_threshold);
-	log_printf(LOGSYS_LEVEL_DEBUG, "RRP multicast threshold (%d problem count)",
-	    totem_config->rrp_problem_count_mcast_threshold);
-	log_printf(LOGSYS_LEVEL_DEBUG, "RRP automatic recovery check timeout (%d ms)",
-	    totem_config->rrp_autorecovery_check_timeout);
-	log_printf(LOGSYS_LEVEL_DEBUG, "RRP mode set to %s.",
-	    totem_config->rrp_mode);
 	log_printf(LOGSYS_LEVEL_DEBUG, "heartbeat_failures_allowed (%d)",
 	    totem_config->heartbeat_failures_allowed);
 	log_printf(LOGSYS_LEVEL_DEBUG, "max_network_delay (%d ms)", totem_config->max_network_delay);

exec/totemcrypto.c

@@ -1,831 +0,0 @@
-/*
- * Copyright (c) 2006-2012 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *         Christine Caulfield (ccaulfie@redhat.com)
- *         Jan Friesse (jfriesse@redhat.com)
- *         Fabio M. Di Nitto (fdinitto@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <nss.h>
-#include <pk11pub.h>
-#include <pkcs11.h>
-#include <prerror.h>
-#include <blapit.h>
-#include <hasht.h>
-
-#define LOGSYS_UTILS_ONLY 1
-#include <corosync/logsys.h>
-#include <corosync/totem/totem.h>
-#include "totemcrypto.h"
-
-/*
- * define onwire crypto header
- */
-
-struct crypto_config_header {
-	uint8_t	crypto_cipher_type;
-	uint8_t	crypto_hash_type;
-	uint8_t __pad0;
-	uint8_t __pad1;
-} __attribute__((packed));
-
-/*
- * crypto definitions and conversion tables
- */
-
-#define SALT_SIZE 16
-
-/*
- * This are defined in new NSS. For older one, we will define our own
- */
-#ifndef AES_256_KEY_LENGTH
-#define AES_256_KEY_LENGTH 32
-#endif
-
-#ifndef AES_192_KEY_LENGTH
-#define AES_192_KEY_LENGTH 24
-#endif
-
-#ifndef AES_128_KEY_LENGTH
-#define AES_128_KEY_LENGTH 16
-#endif
-
-/*
- * while CRYPTO_CIPHER_TYPE_2_X are not a real cipher at all,
- * we still allocate a value for them because we use crypto_crypt_t
- * internally and we don't want overlaps
- */
-
-enum crypto_crypt_t {
-	CRYPTO_CIPHER_TYPE_NONE = 0,
-	CRYPTO_CIPHER_TYPE_AES256 = 1,
-	CRYPTO_CIPHER_TYPE_AES192 = 2,
-	CRYPTO_CIPHER_TYPE_AES128 = 3,
-	CRYPTO_CIPHER_TYPE_3DES = 4,
-	CRYPTO_CIPHER_TYPE_2_3 = UINT8_MAX - 1,
-	CRYPTO_CIPHER_TYPE_2_2 = UINT8_MAX
-};
-
-CK_MECHANISM_TYPE cipher_to_nss[] = {
-	0,				/* CRYPTO_CIPHER_TYPE_NONE */
-	CKM_AES_CBC_PAD,		/* CRYPTO_CIPHER_TYPE_AES256 */
-	CKM_AES_CBC_PAD,		/* CRYPTO_CIPHER_TYPE_AES192 */
-	CKM_AES_CBC_PAD,		/* CRYPTO_CIPHER_TYPE_AES128 */
-	CKM_DES3_CBC_PAD		/* CRYPTO_CIPHER_TYPE_3DES */
-};
-
-size_t cipher_key_len[] = {
-	0,				/* CRYPTO_CIPHER_TYPE_NONE */
-	AES_256_KEY_LENGTH,		/* CRYPTO_CIPHER_TYPE_AES256 */
-	AES_192_KEY_LENGTH,		/* CRYPTO_CIPHER_TYPE_AES192 */
-	AES_128_KEY_LENGTH,		/* CRYPTO_CIPHER_TYPE_AES128 */
-	24				/* CRYPTO_CIPHER_TYPE_3DES - no magic in nss headers */
-};
-
-size_t cypher_block_len[] = {
-	0,				/* CRYPTO_CIPHER_TYPE_NONE */
-	AES_BLOCK_SIZE,			/* CRYPTO_CIPHER_TYPE_AES256 */
-	AES_BLOCK_SIZE,			/* CRYPTO_CIPHER_TYPE_AES192 */
-	AES_BLOCK_SIZE,			/* CRYPTO_CIPHER_TYPE_AES128 */
-	0				/* CRYPTO_CIPHER_TYPE_3DES */
-};
-
-/*
- * hash definitions and conversion tables
- */
-
-/*
- * while CRYPTO_HASH_TYPE_2_X are not a real hash mechanism at all,
- * we still allocate a value for them because we use crypto_hash_t
- * internally and we don't want overlaps
- */
-
-enum crypto_hash_t {
-	CRYPTO_HASH_TYPE_NONE	= 0,
-	CRYPTO_HASH_TYPE_MD5	= 1,
-	CRYPTO_HASH_TYPE_SHA1	= 2,
-	CRYPTO_HASH_TYPE_SHA256	= 3,
-	CRYPTO_HASH_TYPE_SHA384	= 4,
-	CRYPTO_HASH_TYPE_SHA512	= 5,
-	CRYPTO_HASH_TYPE_2_3	= UINT8_MAX - 1,
-	CRYPTO_HASH_TYPE_2_2	= UINT8_MAX
-};
-
-CK_MECHANISM_TYPE hash_to_nss[] = {
-	0,				/* CRYPTO_HASH_TYPE_NONE */
-	CKM_MD5_HMAC,			/* CRYPTO_HASH_TYPE_MD5 */
-	CKM_SHA_1_HMAC,			/* CRYPTO_HASH_TYPE_SHA1 */
-	CKM_SHA256_HMAC,		/* CRYPTO_HASH_TYPE_SHA256 */
-	CKM_SHA384_HMAC,		/* CRYPTO_HASH_TYPE_SHA384 */
-	CKM_SHA512_HMAC			/* CRYPTO_HASH_TYPE_SHA512 */
-};
-
-size_t hash_len[] = {
-	0,				/* CRYPTO_HASH_TYPE_NONE */
-	MD5_LENGTH,			/* CRYPTO_HASH_TYPE_MD5 */
-	SHA1_LENGTH,			/* CRYPTO_HASH_TYPE_SHA1 */
-	SHA256_LENGTH,			/* CRYPTO_HASH_TYPE_SHA256 */
-	SHA384_LENGTH,			/* CRYPTO_HASH_TYPE_SHA384 */
-	SHA512_LENGTH			/* CRYPTO_HASH_TYPE_SHA512 */
-};
-
-size_t hash_block_len[] = {
-	0,				/* CRYPTO_HASH_TYPE_NONE */
-	MD5_BLOCK_LENGTH,		/* CRYPTO_HASH_TYPE_MD5 */
-	SHA1_BLOCK_LENGTH,		/* CRYPTO_HASH_TYPE_SHA1 */
-	SHA256_BLOCK_LENGTH,		/* CRYPTO_HASH_TYPE_SHA256 */
-	SHA384_BLOCK_LENGTH,		/* CRYPTO_HASH_TYPE_SHA384 */
-	SHA512_BLOCK_LENGTH		/* CRYPTO_HASH_TYPE_SHA512 */
-};
-
-struct crypto_instance {
-	PK11SymKey   *nss_sym_key;
-	PK11SymKey   *nss_sym_key_sign;
-
-	unsigned char private_key[1024];
-
-	unsigned int private_key_len;
-
-	enum crypto_crypt_t crypto_cipher_type;
-
-	enum crypto_hash_t crypto_hash_type;
-
-	unsigned int crypto_header_size;
-
-	void (*log_printf_func) (
-		int level,
-		int subsys,
-		const char *function,
-		const char *file,
-		int line,
-		const char *format,
-		...)__attribute__((format(printf, 6, 7)));
-
-	int log_level_security;
-	int log_level_notice;
-	int log_level_error;
-	int log_subsys_id;
-};
-
-#define log_printf(level, format, args...)				\
-do {									\
-	instance->log_printf_func (					\
-		level, instance->log_subsys_id,				\
-		__FUNCTION__, __FILE__, __LINE__,			\
-		(const char *)format, ##args);				\
-} while (0);
-
-/*
- * crypt/decrypt functions
- */
-
-static int string_to_crypto_cipher_type(const char* crypto_cipher_type)
-{
-	if (strcmp(crypto_cipher_type, "none") == 0) {
-		return CRYPTO_CIPHER_TYPE_NONE;
-	} else if (strcmp(crypto_cipher_type, "aes256") == 0) {
-		return CRYPTO_CIPHER_TYPE_AES256;
-	} else if (strcmp(crypto_cipher_type, "aes192") == 0) {
-		return CRYPTO_CIPHER_TYPE_AES192;
-	} else if (strcmp(crypto_cipher_type, "aes128") == 0) {
-		return CRYPTO_CIPHER_TYPE_AES128;
-	} else if (strcmp(crypto_cipher_type, "3des") == 0) {
-		return CRYPTO_CIPHER_TYPE_3DES;
-	}
-	return CRYPTO_CIPHER_TYPE_AES256;
-}
-
-static int init_nss_crypto(struct crypto_instance *instance)
-{
-	PK11SlotInfo*	crypt_slot = NULL;
-	SECItem		crypt_param;
-
-	if (!cipher_to_nss[instance->crypto_cipher_type]) {
-		return 0;
-	}
-
-	crypt_param.type = siBuffer;
-	crypt_param.data = instance->private_key;
-	crypt_param.len = cipher_key_len[instance->crypto_cipher_type];
-
-	crypt_slot = PK11_GetBestSlot(cipher_to_nss[instance->crypto_cipher_type], NULL);
-	if (crypt_slot == NULL) {
-		log_printf(instance->log_level_security, "Unable to find security slot (err %d)",
-			   PR_GetError());
-		return -1;
-	}
-
-	instance->nss_sym_key = PK11_ImportSymKey(crypt_slot,
-						  cipher_to_nss[instance->crypto_cipher_type],
-						  PK11_OriginUnwrap, CKA_ENCRYPT|CKA_DECRYPT,
-						  &crypt_param, NULL);
-	if (instance->nss_sym_key == NULL) {
-		log_printf(instance->log_level_security, "Failure to import key into NSS (err %d)",
-			   PR_GetError());
-		return -1;
-	}
-
-	PK11_FreeSlot(crypt_slot);
-
-	return 0;
-}
-
-static int encrypt_nss(
-	struct crypto_instance *instance,
-	const unsigned char *buf_in,
-	const size_t buf_in_len,
-	unsigned char *buf_out,
-	size_t *buf_out_len)
-{
-	PK11Context*	crypt_context = NULL;
-	SECItem		crypt_param;
-	SECItem		*nss_sec_param = NULL;
-	int		tmp1_outlen = 0;
-	unsigned int	tmp2_outlen = 0;
-	unsigned char	*salt = buf_out;
-	unsigned char	*data = buf_out + SALT_SIZE;
-	int		err = -1;
-
-	if (!cipher_to_nss[instance->crypto_cipher_type]) {
-		memcpy(buf_out, buf_in, buf_in_len);
-		*buf_out_len = buf_in_len;
-		return 0;
-	}
-
-	if (PK11_GenerateRandom (salt, SALT_SIZE) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			"Failure to generate a random number %d",
-			PR_GetError());
-		goto out;
-	}
-
-	crypt_param.type = siBuffer;
-	crypt_param.data = salt;
-	crypt_param.len = SALT_SIZE;
-
-	nss_sec_param = PK11_ParamFromIV (cipher_to_nss[instance->crypto_cipher_type],
-					  &crypt_param);
-	if (nss_sec_param == NULL) {
-		log_printf(instance->log_level_security,
-			   "Failure to set up PKCS11 param (err %d)",
-			   PR_GetError());
-		goto out;
-	}
-
-	/*
-	 * Create cipher context for encryption
-	 */
-	crypt_context = PK11_CreateContextBySymKey (cipher_to_nss[instance->crypto_cipher_type],
-						    CKA_ENCRYPT,
-						    instance->nss_sym_key,
-						    nss_sec_param);
-	if (!crypt_context) {
-		log_printf(instance->log_level_security,
-			   "PK11_CreateContext failed (encrypt) crypt_type=%d (err %d)",
-			   (int)cipher_to_nss[instance->crypto_cipher_type],
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_CipherOp(crypt_context, data,
-			  &tmp1_outlen,
-			  FRAME_SIZE_MAX - instance->crypto_header_size,
-			  (unsigned char *)buf_in, buf_in_len) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_CipherOp failed (encrypt) crypt_type=%d (err %d)",
-			   (int)cipher_to_nss[instance->crypto_cipher_type],
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_DigestFinal(crypt_context, data + tmp1_outlen,
-			     &tmp2_outlen, FRAME_SIZE_MAX - tmp1_outlen) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_DigestFinal failed (encrypt) crypt_type=%d (err %d)",
-			   (int)cipher_to_nss[instance->crypto_cipher_type],
-			   PR_GetError());
-		goto out;
-
-	}
-
-	*buf_out_len = tmp1_outlen + tmp2_outlen + SALT_SIZE;
-
-	err = 0;
-
-out:
-	if (crypt_context) {
-		PK11_DestroyContext(crypt_context, PR_TRUE);
-	}
-	if (nss_sec_param) {
-		SECITEM_FreeItem(nss_sec_param, PR_TRUE);
-	}
-	return err;
-}
-
-static int decrypt_nss (
-	struct crypto_instance *instance,
-	unsigned char *buf,
-	int *buf_len)
-{
-	PK11Context*	decrypt_context = NULL;
-	SECItem		decrypt_param;
-	int		tmp1_outlen = 0;
-	unsigned int	tmp2_outlen = 0;
-	unsigned char	*salt = buf;
-	unsigned char	*data = salt + SALT_SIZE;
-	int		datalen = *buf_len - SALT_SIZE;
-	unsigned char	outbuf[FRAME_SIZE_MAX];
-	int		outbuf_len;
-	int		err = -1;
-
-	if (!cipher_to_nss[instance->crypto_cipher_type]) {
-		return 0;
-	}
-
-	/* Create cipher context for decryption */
-	decrypt_param.type = siBuffer;
-	decrypt_param.data = salt;
-	decrypt_param.len = SALT_SIZE;
-
-	decrypt_context = PK11_CreateContextBySymKey(cipher_to_nss[instance->crypto_cipher_type],
-						     CKA_DECRYPT,
-						     instance->nss_sym_key, &decrypt_param);
-	if (!decrypt_context) {
-		log_printf(instance->log_level_security,
-			   "PK11_CreateContext (decrypt) failed (err %d)",
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_CipherOp(decrypt_context, outbuf, &tmp1_outlen,
-			  sizeof(outbuf), data, datalen) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_CipherOp (decrypt) failed (err %d)",
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_DigestFinal(decrypt_context, outbuf + tmp1_outlen, &tmp2_outlen,
-			     sizeof(outbuf) - tmp1_outlen) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_DigestFinal (decrypt) failed (err %d)",
-			   PR_GetError()); 
-		goto out;
-	}
-
-	outbuf_len = tmp1_outlen + tmp2_outlen;
-
-	memset(buf, 0, *buf_len);
-	memcpy(buf, outbuf, outbuf_len);
-
-	*buf_len = outbuf_len;
-
-	err = 0;
-
-out:
-	if (decrypt_context) {
-		PK11_DestroyContext(decrypt_context, PR_TRUE);
-	}
-
-	return err;
-}
-
-
-/*
- * hash/hmac/digest functions
- */
-
-static int string_to_crypto_hash_type(const char* crypto_hash_type)
-{
-	if (strcmp(crypto_hash_type, "none") == 0) {
-		return CRYPTO_HASH_TYPE_NONE;
-	} else if (strcmp(crypto_hash_type, "md5") == 0) {
-		return CRYPTO_HASH_TYPE_MD5;
-	} else if (strcmp(crypto_hash_type, "sha1") == 0) {
-		return CRYPTO_HASH_TYPE_SHA1;
-	} else if (strcmp(crypto_hash_type, "sha256") == 0) {
-		return CRYPTO_HASH_TYPE_SHA256;
-	} else if (strcmp(crypto_hash_type, "sha384") == 0) {
-		return CRYPTO_HASH_TYPE_SHA384;
-	} else if (strcmp(crypto_hash_type, "sha512") == 0) {
-		return CRYPTO_HASH_TYPE_SHA512;
-	}
-
-	return CRYPTO_HASH_TYPE_SHA1;
-}
-
-static int init_nss_hash(struct crypto_instance *instance)
-{
-	PK11SlotInfo*	hash_slot = NULL;
-	SECItem		hash_param;
-
-	if (!hash_to_nss[instance->crypto_hash_type]) {
-		return 0;
-	}
-
-	hash_param.type = siBuffer;
-	hash_param.data = instance->private_key;
-	hash_param.len = instance->private_key_len;
-
-	hash_slot = PK11_GetBestSlot(hash_to_nss[instance->crypto_hash_type], NULL);
-	if (hash_slot == NULL) {
-		log_printf(instance->log_level_security, "Unable to find security slot (err %d)",
-			   PR_GetError());
-		return -1;
-	}
-
-	instance->nss_sym_key_sign = PK11_ImportSymKey(hash_slot,
-						       hash_to_nss[instance->crypto_hash_type],
-						       PK11_OriginUnwrap, CKA_SIGN,
-						       &hash_param, NULL);
-	if (instance->nss_sym_key_sign == NULL) {
-		log_printf(instance->log_level_security, "Failure to import key into NSS (err %d)",
-			   PR_GetError());
-		return -1;
-	}
-
-	PK11_FreeSlot(hash_slot);
-
-	return 0;
-}
-
-static int calculate_nss_hash(
-	struct crypto_instance *instance,
-	const unsigned char *buf,
-	const size_t buf_len,
-	unsigned char *hash)
-{
-	PK11Context*	hash_context = NULL;
-	SECItem		hash_param;
-	unsigned int	hash_tmp_outlen = 0;
-	unsigned char	hash_block[hash_block_len[instance->crypto_hash_type]];
-	int		err = -1;
-
-	/* Now do the digest */
-	hash_param.type = siBuffer;
-	hash_param.data = 0;
-	hash_param.len = 0;
-
-	hash_context = PK11_CreateContextBySymKey(hash_to_nss[instance->crypto_hash_type],
-						 CKA_SIGN,
-						 instance->nss_sym_key_sign,
-						 &hash_param);
-
-	if (!hash_context) {
-		log_printf(instance->log_level_security,
-			   "PK11_CreateContext failed (hash) hash_type=%d (err %d)",
-			   (int)hash_to_nss[instance->crypto_hash_type],
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_DigestBegin(hash_context) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_DigestBegin failed (hash) hash_type=%d (err %d)",
-			   (int)hash_to_nss[instance->crypto_hash_type],
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_DigestOp(hash_context,
-			  buf,
-			  buf_len) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_DigestOp failed (hash) hash_type=%d (err %d)",
-			   (int)hash_to_nss[instance->crypto_hash_type],
-			   PR_GetError());
-		goto out;
-	}
-
-	if (PK11_DigestFinal(hash_context,
-			     hash_block,
-			     &hash_tmp_outlen,
-			     hash_block_len[instance->crypto_hash_type]) != SECSuccess) {
-		log_printf(instance->log_level_security,
-			   "PK11_DigestFinale failed (hash) hash_type=%d (err %d)",
-			   (int)hash_to_nss[instance->crypto_hash_type],
-			   PR_GetError());
-		goto out;
-	}
-
-	memcpy(hash, hash_block, hash_len[instance->crypto_hash_type]);
-	err = 0;
-
-out:
-	if (hash_context) {
-		PK11_DestroyContext(hash_context, PR_TRUE);
-	}
-
-	return err;
-}
-
-/*
- * global/glue nss functions
- */
-
-static int init_nss_db(struct crypto_instance *instance)
-{
-	if ((!cipher_to_nss[instance->crypto_cipher_type]) &&
-	    (!hash_to_nss[instance->crypto_hash_type])) {
-		return 0;
-	}
-
-	if (NSS_NoDB_Init(".") != SECSuccess) {
-		log_printf(instance->log_level_security, "NSS DB initialization failed (err %d)",
-			   PR_GetError());
-		return -1;
-	}
-
-	return 0;
-}
-
-static int init_nss(struct crypto_instance *instance,
-		    const char *crypto_cipher_type,
-		    const char *crypto_hash_type)
-{
-	log_printf(instance->log_level_notice,
-		   "Initializing transmit/receive security (NSS) crypto: %s hash: %s",
-		   crypto_cipher_type, crypto_hash_type);
-
-	if (init_nss_db(instance) < 0) {
-		return -1;
-	}
-
-	if (init_nss_crypto(instance) < 0) {
-		return -1;
-	}
-
-	if (init_nss_hash(instance) < 0) {
-		return -1;
-	}
-
-	return 0;
-}
-
-static int encrypt_and_sign_nss_2_3 (
-	struct crypto_instance *instance,
-	const unsigned char *buf_in,
-	const size_t buf_in_len,
-	unsigned char *buf_out,
-	size_t *buf_out_len)
-{
-	if (encrypt_nss(instance,
-			buf_in, buf_in_len,
-			buf_out + sizeof(struct crypto_config_header), buf_out_len) < 0) {
-		return -1;
-	}
-
-	*buf_out_len += sizeof(struct crypto_config_header);
-
-	if (hash_to_nss[instance->crypto_hash_type]) {
-		if (calculate_nss_hash(instance, buf_out, *buf_out_len, buf_out + *buf_out_len) < 0) {
-			return -1;
-		}
-		*buf_out_len += hash_len[instance->crypto_hash_type];
-	}
-
-	return 0;
-}
-
-static int authenticate_nss_2_3 (
-	struct crypto_instance *instance,
-	unsigned char *buf,
-	int *buf_len)
-{
-	if (hash_to_nss[instance->crypto_hash_type]) {
-		unsigned char	tmp_hash[hash_len[instance->crypto_hash_type]];
-		int             datalen = *buf_len - hash_len[instance->crypto_hash_type];
-
-		if (calculate_nss_hash(instance, buf, datalen, tmp_hash) < 0) {
-			return -1;
-		}
-
-		if (memcmp(tmp_hash, buf + datalen, hash_len[instance->crypto_hash_type]) != 0) {
-			log_printf(instance->log_level_error, "Digest does not match");
-			return -1;
-		}
-		*buf_len = datalen;
-	}
-
-	return 0;
-}
-
-static int decrypt_nss_2_3 (
-	struct crypto_instance *instance,
-	unsigned char *buf,
-	int *buf_len)
-{
-	*buf_len -= sizeof(struct crypto_config_header);
-
-	if (decrypt_nss(instance, buf + sizeof(struct crypto_config_header), buf_len) < 0) {
-		return -1;
-	}
-
-	return 0;
-}
-
-/*
- * exported API
- */
-
-size_t crypto_sec_header_size(
-	const char *crypto_cipher_type,
-	const char *crypto_hash_type)
-{
-	int crypto_cipher = string_to_crypto_cipher_type(crypto_cipher_type);
-	int crypto_hash = string_to_crypto_hash_type(crypto_hash_type);
-	size_t hdr_size = 0;
-	int block_size = 0;
-
-	hdr_size = sizeof(struct crypto_config_header);
-
-	if (crypto_hash) {
-		hdr_size += hash_len[crypto_hash];
-	}
-
-	if (crypto_cipher) {
-		hdr_size += SALT_SIZE;
-		if (cypher_block_len[crypto_cipher]) {
-			block_size = cypher_block_len[crypto_cipher];
-		} else {
-			block_size = PK11_GetBlockSize(crypto_cipher, NULL);
-			if (block_size < 0) {
-				/*
-				 * failsafe. we can potentially lose up to 63
-				 * byte per packet, but better than fragmenting
-				 */
-				block_size = 64;
-			}
-		}
-		hdr_size += (block_size * 2);
-	}
-
-	return hdr_size;
-}
-
-/*
- * 2.0 packet format:
- *   crypto_cipher_type | crypto_hash_type | __pad0 | __pad1 | hash | salt | data
- *   only data is encrypted, hash only covers salt + data
- *
- * 2.2/2.3 packet format
- *   fake_crypto_cipher_type | fake_crypto_hash_type | __pad0 | __pad1 | salt | data | hash
- *   only data is encrypted, hash covers the whole packet
- *
- *  we need to leave fake_* unencrypted for older versions of corosync to reject the packets,
- *  we need to leave __pad0|1 unencrypted for performance reasons (saves at least 2 memcpy and
- *  and extra buffer but values are hashed and verified.
- */
-
-int crypto_encrypt_and_sign (
-	struct crypto_instance *instance,
-	const unsigned char *buf_in,
-	const size_t buf_in_len,
-	unsigned char *buf_out,
-	size_t *buf_out_len)
-{
-	struct crypto_config_header *cch = (struct crypto_config_header *)buf_out;
-	int err;
-
-	cch->crypto_cipher_type = CRYPTO_CIPHER_TYPE_2_3;
-	cch->crypto_hash_type = CRYPTO_HASH_TYPE_2_3;
-	cch->__pad0 = 0;
-	cch->__pad1 = 0;
-
-	err = encrypt_and_sign_nss_2_3(instance,
-				       buf_in, buf_in_len,
-				       buf_out, buf_out_len);
-
-	return err;
-}
-
-int crypto_authenticate_and_decrypt (struct crypto_instance *instance,
-	unsigned char *buf,
-	int *buf_len)
-{
-	struct crypto_config_header *cch = (struct crypto_config_header *)buf;
-
-	if (cch->crypto_cipher_type != CRYPTO_CIPHER_TYPE_2_3) {
-		log_printf(instance->log_level_security,
-			   "Incoming packet has different crypto type. Rejecting");
-		return -1;
-	}
-
-	if (cch->crypto_hash_type != CRYPTO_HASH_TYPE_2_3) {
-		log_printf(instance->log_level_security,
-			   "Incoming packet has different hash type. Rejecting");
-		return -1;
-	}
-
-	/*
-	 * authenticate packet first
-	 */
-
-	if (authenticate_nss_2_3(instance, buf, buf_len) != 0) {
-		return -1;
-	}
-
-	/*
-	 * now we can "trust" the padding bytes/future features
-	 */
-
-	if ((cch->__pad0 != 0) || (cch->__pad1 != 0)) {
-		log_printf(instance->log_level_security,
-			   "Incoming packet appears to have features not supported by this version of corosync. Rejecting");
-		return -1;
-	}
-
-	/*
-	 * decrypt
-	 */
-
-	if (decrypt_nss_2_3(instance, buf, buf_len) != 0) {
-		return -1;
-	}
-
-	/*
-	 * invalidate config header and kill it
-	 */
-	cch = NULL;
-	memmove(buf, buf + sizeof(struct crypto_config_header), *buf_len);
-
-	return 0;
-}
-
-struct crypto_instance *crypto_init(
-	const unsigned char *private_key,
-	unsigned int private_key_len,
-	const char *crypto_cipher_type,
-	const char *crypto_hash_type,
-	void (*log_printf_func) (
-		int level,
-		int subsys,
-                const char *function,
-                const char *file,
-                int line,
-                const char *format,
-                ...)__attribute__((format(printf, 6, 7))),
-	int log_level_security,
-	int log_level_notice,
-	int log_level_error,
-	int log_subsys_id)
-{
-	struct crypto_instance *instance;
-	instance = malloc(sizeof(*instance));
-	if (instance == NULL) {
-		return (NULL);
-	}
-	memset(instance, 0, sizeof(struct crypto_instance));
-
-	memcpy(instance->private_key, private_key, private_key_len);
-	instance->private_key_len = private_key_len;
-
-	instance->crypto_cipher_type = string_to_crypto_cipher_type(crypto_cipher_type);
-	instance->crypto_hash_type = string_to_crypto_hash_type(crypto_hash_type);
-
-	instance->crypto_header_size = crypto_sec_header_size(crypto_cipher_type, crypto_hash_type);
-
-	instance->log_printf_func = log_printf_func;
-	instance->log_level_security = log_level_security;
-	instance->log_level_notice = log_level_notice;
-	instance->log_level_error = log_level_error;
-	instance->log_subsys_id = log_subsys_id;
-
-	if (init_nss(instance, crypto_cipher_type, crypto_hash_type) < 0) {
-		free(instance);
-		return(NULL);
-	}
-
-	return (instance);
-}

exec/totemcrypto.h

@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2006-2012 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *         Christine Caulfield (ccaulfie@redhat.com)
- *         Jan Friesse (jfriesse@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef TOTEMCRYPTO_H_DEFINED
-#define TOTEMCRYPTO_H_DEFINED
-
-#include <sys/types.h>
-
-struct crypto_instance;
-
-extern size_t crypto_sec_header_size(
-	const char *crypto_cipher_type,
-	const char *crypto_hash_type);
-
-extern int crypto_authenticate_and_decrypt (
-	struct crypto_instance *instance,
-	unsigned char *buf,
-	int *buf_len);
-
-extern int crypto_encrypt_and_sign (
-	struct crypto_instance *instance,
-	const unsigned char *buf_in,
-	const size_t buf_in_len,
-	unsigned char *buf_out, 
-	size_t *buf_out_len);
-
-extern struct crypto_instance *crypto_init(
-	const unsigned char *private_key,
-	unsigned int private_key_len,
-	const char *crypto_cipher_type,
-	const char *crypto_hash_type,
-	void (*log_printf_func) (
-		int level,
-		int subsys,
-		const char *function,
-		const char *file,
-		int line,
-		const char *format,
-		...)__attribute__((format(printf, 6, 7))),
-	int log_level_security,
-	int log_level_notice,
-	int log_level_error,
-	int log_subsys_id);
-
-#endif /* TOTEMCRYPTO_H_DEFINED */

exec/totemiba.c

@@ -1,1642 +0,0 @@
-/*
- * Copyright (c) 2009-2012 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
-
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include <assert.h>
-#include <pthread.h>
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <sys/un.h>
-#include <sys/ioctl.h>
-#include <sys/param.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <sched.h>
-#include <time.h>
-#include <sys/time.h>
-#include <sys/poll.h>
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <rdma/rdma_cma.h>
-#include <assert.h>
-#include <errno.h>
-
-#include <corosync/sq.h>
-#include <corosync/list.h>
-#include <corosync/hdb.h>
-#include <corosync/swab.h>
-
-#include <qb/qbdefs.h>
-#include <qb/qbloop.h>
-#define LOGSYS_UTILS_ONLY 1
-#include <corosync/logsys.h>
-#include "totemiba.h"
-
-#define COMPLETION_QUEUE_ENTRIES 100
-
-#define TOTAL_READ_POSTS 100
-
-#define MAX_MTU_SIZE 4096
-
-#define MCAST_REJOIN_MSEC	100
-
-struct totemiba_instance {
-	struct sockaddr bind_addr;
-
-	struct sockaddr send_token_bind_addr;
-
-	struct sockaddr mcast_addr;
-
-	struct sockaddr token_addr;
-
-	struct sockaddr local_mcast_bind_addr;
-
-	struct totem_interface *totem_interface;
-
-	struct totem_config *totem_config;
-
-	totemsrp_stats_t *stats;
-
-	void (*totemiba_iface_change_fn) (
-		void *context,
-		const struct totem_ip_address *iface_address);
-
-	void (*totemiba_deliver_fn) (
-		void *context,
-		const void *msg,
-		unsigned int msg_len);
-
-	void (*totemiba_target_set_completed) (
-		void *context);
-
-	void *rrp_context;
-
-	qb_loop_timer_handle timer_netif_check_timeout;
-
-	qb_loop_t *totemiba_poll_handle;
-
-	struct totem_ip_address my_id;
-
-	struct rdma_event_channel *mcast_channel;
-
-	struct rdma_cm_id *mcast_cma_id;
-
-	struct ibv_pd *mcast_pd;
-
-	struct sockaddr mcast_dest_addr;
-
-	uint32_t mcast_qpn;
-
-	uint32_t mcast_qkey;
-
-	struct ibv_ah *mcast_ah;
-
-	struct ibv_comp_channel *mcast_send_completion_channel;
-
-	struct ibv_comp_channel *mcast_recv_completion_channel;
-
-	struct ibv_cq *mcast_send_cq;
-
-	struct ibv_cq *mcast_recv_cq;
-
-	int recv_token_accepted;
-
-	struct rdma_event_channel *recv_token_channel;
-
-	struct rdma_event_channel *listen_recv_token_channel;
-
-	struct rdma_cm_id *listen_recv_token_cma_id;
-
-	struct rdma_cm_id *recv_token_cma_id;
-
-	struct ibv_pd *recv_token_pd;
-
-	struct sockaddr recv_token_dest_addr;
-
-	struct ibv_comp_channel *recv_token_send_completion_channel;
-
-	struct ibv_comp_channel *recv_token_recv_completion_channel;
-
-	struct ibv_cq *recv_token_send_cq;
-
-	struct ibv_cq *recv_token_recv_cq;
-
-	int send_token_bound;
-
-	struct rdma_event_channel *send_token_channel;
-
-	struct rdma_cm_id *send_token_cma_id;
-
-	struct ibv_pd *send_token_pd;
-
-	struct sockaddr send_token_dest_addr;
-
-	uint32_t send_token_qpn;
-
-	uint32_t send_token_qkey;
-
-	struct ibv_ah *send_token_ah;
-
-	struct ibv_comp_channel *send_token_send_completion_channel;
-
-	struct ibv_comp_channel *send_token_recv_completion_channel;
-
-	struct ibv_cq *send_token_send_cq;
-
-	struct ibv_cq *send_token_recv_cq;
-
-        void (*totemiba_log_printf) (
-		int level,
-		int subsys,
-		const char *function,
-		const char *file,
-		int line,
-		const char *format,
-		...)__attribute__((format(printf, 6, 7)));
-
-
-	int totemiba_subsys_id;
-
-	struct list_head mcast_send_buf_free;
-
-	struct list_head token_send_buf_free;
-
-	struct list_head mcast_send_buf_head;
-
-	struct list_head token_send_buf_head;
-
-	struct list_head recv_token_recv_buf_head;
-
-	int mcast_seen_joined;
-
-	qb_loop_timer_handle mcast_rejoin;
-};
-union u {
-	uint64_t wr_id;
-	void *v;
-};
-
-#define log_printf(level, format, args...)			\
-do {								\
-        instance->totemiba_log_printf (				\
-			level,					\
-			instance->totemiba_subsys_id,		\
-			__FUNCTION__, __FILE__, __LINE__,	\
-			(const char *)format, ##args);		\
-} while (0);
-
-struct recv_buf {
-	struct list_head list_all;
-	struct ibv_recv_wr recv_wr;
-	struct ibv_sge sge;
-	struct ibv_mr *mr;
-	char buffer[MAX_MTU_SIZE + sizeof (struct ibv_grh)];
-};
-
-struct send_buf {
-	struct list_head list_free;
-	struct list_head list_all;
-	struct ibv_mr *mr;
-	char buffer[MAX_MTU_SIZE];
-};
-
-static hdb_handle_t
-void2wrid (void *v) { union u u; u.v = v; return u.wr_id; }
-
-static void *
-wrid2void (uint64_t wr_id) { union u u; u.wr_id = wr_id; return u.v; }
-
-static void totemiba_instance_initialize (struct totemiba_instance *instance)
-{
-	memset (instance, 0, sizeof (struct totemiba_instance));
-	list_init (&instance->mcast_send_buf_free);
-	list_init (&instance->token_send_buf_free);
-	list_init (&instance->mcast_send_buf_head);
-	list_init (&instance->token_send_buf_head);
-	list_init (&instance->recv_token_recv_buf_head);
-}
-
-static inline struct send_buf *mcast_send_buf_get (
-	struct totemiba_instance *instance)
-{
-	struct send_buf *send_buf;
-
-	if (list_empty (&instance->mcast_send_buf_free) == 0) {
-		send_buf = list_entry (instance->mcast_send_buf_free.next, struct send_buf, list_free);
-		list_del (&send_buf->list_free);
-		return (send_buf);
-	}
-
-	send_buf = malloc (sizeof (struct send_buf));
-	if (send_buf == NULL) {
-		return (NULL);
-	}
-	send_buf->mr = ibv_reg_mr (instance->mcast_pd,
-		send_buf->buffer,
-		MAX_MTU_SIZE, IBV_ACCESS_LOCAL_WRITE);
-	if (send_buf->mr == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't register memory range");
-		free (send_buf);
-		return (NULL);
-	}
-	list_init (&send_buf->list_all);
-	list_add_tail (&send_buf->list_all, &instance->mcast_send_buf_head);
-		
-	return (send_buf);
-}
-
-static inline void mcast_send_buf_put (
-	struct totemiba_instance *instance,
-	struct send_buf *send_buf)
-{
-	list_init (&send_buf->list_free);
-	list_add_tail (&send_buf->list_free, &instance->mcast_send_buf_free);
-}
-
-static inline struct send_buf *token_send_buf_get (
-	struct totemiba_instance *instance)
-{
-	struct send_buf *send_buf;
-
-	if (list_empty (&instance->token_send_buf_free) == 0) {
-		send_buf = list_entry (instance->token_send_buf_free.next, struct send_buf, list_free);
-		list_del (&send_buf->list_free);
-		return (send_buf);
-	}
-
-	send_buf = malloc (sizeof (struct send_buf));
-	if (send_buf == NULL) {
-		return (NULL);
-	}
-	send_buf->mr = ibv_reg_mr (instance->send_token_pd,
-		send_buf->buffer,
-		MAX_MTU_SIZE, IBV_ACCESS_LOCAL_WRITE);
-	if (send_buf->mr == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't register memory range");
-		free (send_buf);
-		return (NULL);
-	}
-	list_init (&send_buf->list_all);
-	list_add_tail (&send_buf->list_all, &instance->token_send_buf_head);
-		
-	return (send_buf);
-}
-
-static inline void token_send_buf_destroy (struct totemiba_instance *instance)
-{
-	struct list_head *list;
-	struct send_buf *send_buf;
-
-        for (list = instance->token_send_buf_head.next; list != &instance->token_send_buf_head;) {
-                send_buf = list_entry (list, struct send_buf, list_all);
-		list = list->next;
-		ibv_dereg_mr (send_buf->mr);
-		free (send_buf);
-	}
-
-	list_init (&instance->token_send_buf_free);
-	list_init (&instance->token_send_buf_head);
-}
-
-static inline void token_send_buf_put (
-	struct totemiba_instance *instance,
-	struct send_buf *send_buf)
-{
-	list_init (&send_buf->list_free);
-	list_add_tail (&send_buf->list_free, &instance->token_send_buf_free);
-}
-
-static inline struct recv_buf *recv_token_recv_buf_create (
-	struct totemiba_instance *instance)
-{
-	struct recv_buf *recv_buf;
-
-	recv_buf = malloc (sizeof (struct recv_buf));
-	if (recv_buf == NULL) {
-		return (NULL);
-	}
-
-	recv_buf->mr = ibv_reg_mr (instance->recv_token_pd, &recv_buf->buffer,
-		MAX_MTU_SIZE + sizeof (struct ibv_grh),
-		IBV_ACCESS_LOCAL_WRITE);
-
-	recv_buf->recv_wr.next = NULL;
-	recv_buf->recv_wr.sg_list = &recv_buf->sge;
-	recv_buf->recv_wr.num_sge = 1;
-	recv_buf->recv_wr.wr_id = (uintptr_t)recv_buf;
-
-	recv_buf->sge.length = MAX_MTU_SIZE + sizeof (struct ibv_grh);
-	recv_buf->sge.lkey = recv_buf->mr->lkey;
-	recv_buf->sge.addr = (uintptr_t)recv_buf->buffer;
-
-	list_init (&recv_buf->list_all);
-	list_add (&recv_buf->list_all, &instance->recv_token_recv_buf_head);
-	return (recv_buf);
-}
-
-static inline int recv_token_recv_buf_post (struct totemiba_instance *instance, struct recv_buf *recv_buf)
-{
-	struct ibv_recv_wr *fail_recv;
-	int res;
-
-	res = ibv_post_recv (instance->recv_token_cma_id->qp, &recv_buf->recv_wr, &fail_recv);
-
-	return (res);
-}
-
-static inline void recv_token_recv_buf_post_initial (struct totemiba_instance *instance)
-{
-	struct recv_buf *recv_buf;
-	unsigned int i;
-
-	for (i = 0; i < TOTAL_READ_POSTS; i++) {
-		recv_buf = recv_token_recv_buf_create (instance);
-
-		recv_token_recv_buf_post (instance, recv_buf);
-	}
-}
-
-static inline void recv_token_recv_buf_post_destroy (
-	struct totemiba_instance *instance)
-{
-	struct recv_buf *recv_buf;
-	struct list_head *list;
-
-	for (list = instance->recv_token_recv_buf_head.next;
-		list != &instance->recv_token_recv_buf_head;) {
-
-		recv_buf = list_entry (list, struct recv_buf, list_all);
-		list = list->next;
-		ibv_dereg_mr (recv_buf->mr);
-		free (recv_buf);
-	}
-	list_init (&instance->recv_token_recv_buf_head);
-}
-
-static inline struct recv_buf *mcast_recv_buf_create (struct totemiba_instance *instance)
-{
-	struct recv_buf *recv_buf;
-	struct ibv_mr *mr;
-
-	recv_buf = malloc (sizeof (struct recv_buf));
-	if (recv_buf == NULL) {
-		return (NULL);
-	}
-
-	mr = ibv_reg_mr (instance->mcast_pd, &recv_buf->buffer,
-		MAX_MTU_SIZE + sizeof (struct ibv_grh),
-		IBV_ACCESS_LOCAL_WRITE);
-
-	recv_buf->recv_wr.next = NULL;
-	recv_buf->recv_wr.sg_list = &recv_buf->sge;
-	recv_buf->recv_wr.num_sge = 1;
-	recv_buf->recv_wr.wr_id = (uintptr_t)recv_buf;
-
-	recv_buf->sge.length = MAX_MTU_SIZE + sizeof (struct ibv_grh);
-	recv_buf->sge.lkey = mr->lkey;
-	recv_buf->sge.addr = (uintptr_t)recv_buf->buffer;
-
-	return (recv_buf);
-}
-
-static inline int mcast_recv_buf_post (struct totemiba_instance *instance, struct recv_buf *recv_buf)
-{
-	struct ibv_recv_wr *fail_recv;
-	int res;
-
-	res = ibv_post_recv (instance->mcast_cma_id->qp, &recv_buf->recv_wr, &fail_recv);
-
-	return (res);
-}
-
-static inline void mcast_recv_buf_post_initial (struct totemiba_instance *instance)
-{
-	struct recv_buf *recv_buf;
-	unsigned int i;
-
-	for (i = 0; i < TOTAL_READ_POSTS; i++) {
-		recv_buf = mcast_recv_buf_create (instance);
-
-		mcast_recv_buf_post (instance, recv_buf);
-	}
-}
-
-static inline void iba_deliver_fn (struct totemiba_instance *instance, uint64_t wr_id, uint32_t bytes)
-{
-	const char *addr;
-	const struct recv_buf *recv_buf;
-
-	recv_buf = wrid2void(wr_id);
-	addr = &recv_buf->buffer[sizeof (struct ibv_grh)];
-
-	bytes -= sizeof (struct ibv_grh);
-	instance->totemiba_deliver_fn (instance->rrp_context, addr, bytes);
-}
-
-static int mcast_cq_send_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct ibv_wc wc[32];
-	struct ibv_cq *ev_cq;
-	void *ev_ctx;
-	int res;
-	int i;
-
-	ibv_get_cq_event (instance->mcast_send_completion_channel, &ev_cq, &ev_ctx);
-	ibv_ack_cq_events (ev_cq, 1);
-	res = ibv_req_notify_cq (ev_cq, 0);
-
-	res = ibv_poll_cq (instance->mcast_send_cq, 32, wc);
-	if (res > 0) {
-		for (i = 0; i < res; i++) {
-			mcast_send_buf_put (instance, wrid2void(wc[i].wr_id));
-		}
-	}
-
-	return (0);
-}
-
-static int mcast_cq_recv_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct ibv_wc wc[64];
-	struct ibv_cq *ev_cq;
-	void *ev_ctx;
-	int res;
-	int i;
-
-	ibv_get_cq_event (instance->mcast_recv_completion_channel, &ev_cq, &ev_ctx);
-	ibv_ack_cq_events (ev_cq, 1);
-	res = ibv_req_notify_cq (ev_cq, 0);
-
-	res = ibv_poll_cq (instance->mcast_recv_cq, 64, wc);
-	if (res > 0) {
-		for (i = 0; i < res; i++) {
-			iba_deliver_fn (instance, wc[i].wr_id, wc[i].byte_len);
-			mcast_recv_buf_post (instance, wrid2void(wc[i].wr_id));
-		}
-	}
-
-	return (0);
-}
-
-static void mcast_rejoin (void *data)
-{
-	int res;
-	struct totemiba_instance *instance = (struct totemiba_instance *)data;
-
-	res = rdma_leave_multicast (instance->mcast_cma_id, &instance->mcast_addr);
-	if (instance->mcast_ah) {
-		ibv_destroy_ah (instance->mcast_ah);
-		instance->mcast_ah = 0;
-	}
-
-	res = rdma_join_multicast (instance->mcast_cma_id, &instance->mcast_addr, instance);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_DEBUG,
-		    "rdma_join_multicast failed, errno=%d, rejoining in %u ms",
-		    errno,
-		    MCAST_REJOIN_MSEC);
-		qb_loop_timer_add (instance->totemiba_poll_handle,
-			QB_LOOP_MED,
-			MCAST_REJOIN_MSEC * QB_TIME_NS_IN_MSEC,
-			(void *)instance,
-			mcast_rejoin,
-			&instance->mcast_rejoin);
-	}
-}
-
-static int mcast_rdma_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct rdma_cm_event *event;
-
-	int res;
-
-	res = rdma_get_cm_event (instance->mcast_channel, &event);
-	if (res != 0) {
-		return (0);
-	}
-
-	switch (event->event) {
-	/*
-	 * occurs when we resolve the multicast address
-	 */
-	case RDMA_CM_EVENT_ADDR_RESOLVED:
-		res = rdma_join_multicast (instance->mcast_cma_id, &instance->mcast_addr, instance);
-		usleep(1000);
-		if (res == 0) break;
-	case RDMA_CM_EVENT_MULTICAST_ERROR:
-		log_printf (LOGSYS_LEVEL_ERROR, "multicast error, trying to rejoin in %u ms", MCAST_REJOIN_MSEC);
-		qb_loop_timer_add (instance->totemiba_poll_handle,
-			QB_LOOP_MED,
-			MCAST_REJOIN_MSEC * QB_TIME_NS_IN_MSEC,
-			(void *)instance,
-			mcast_rejoin,
-			&instance->mcast_rejoin);
-		break;
-	/*
-	 * occurs when the CM joins the multicast group
-	 */
-	case RDMA_CM_EVENT_MULTICAST_JOIN:
-		instance->mcast_qpn = event->param.ud.qp_num;
-		instance->mcast_qkey = event->param.ud.qkey;
-		instance->mcast_ah = ibv_create_ah (instance->mcast_pd, &event->param.ud.ah_attr);
-
-		if (instance->mcast_seen_joined == 0) {
-			log_printf (LOGSYS_LEVEL_DEBUG, "joining mcast 1st time, running callbacks");
-			instance->totemiba_iface_change_fn (instance->rrp_context, &instance->my_id);
-			instance->mcast_seen_joined=1;
-		}
-		log_printf (LOGSYS_LEVEL_NOTICE, "Joined multicast!");
-		break;
-	case RDMA_CM_EVENT_ADDR_ERROR:
-	case RDMA_CM_EVENT_ROUTE_ERROR:
-	case RDMA_CM_EVENT_DEVICE_REMOVAL:
-		break;
-	default:
-		log_printf (LOGSYS_LEVEL_ERROR, "default %d", event->event);
-		break;
-	}
-
-	rdma_ack_cm_event (event);
-	return (0);
-}
-
-static int recv_token_cq_send_event_fn (
-	int fd,
-	int revents,
-	void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct ibv_wc wc[32];
-	struct ibv_cq *ev_cq;
-	void *ev_ctx;
-	int res;
-	int i;
-
-	ibv_get_cq_event (instance->recv_token_send_completion_channel, &ev_cq, &ev_ctx);
-	ibv_ack_cq_events (ev_cq, 1);
-	res = ibv_req_notify_cq (ev_cq, 0);
-
-	res = ibv_poll_cq (instance->recv_token_send_cq, 32, wc);
-	if (res > 0) {
-		for (i = 0; i < res; i++) {
-			iba_deliver_fn (instance, wc[i].wr_id, wc[i].byte_len);
-			ibv_dereg_mr (wrid2void(wc[i].wr_id));
-		}
-	}
-
-	return (0);
-}
-
-static int recv_token_cq_recv_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct ibv_wc wc[32];
-	struct ibv_cq *ev_cq;
-	void *ev_ctx;
-	int res;
-	int i;
-
-	ibv_get_cq_event (instance->recv_token_recv_completion_channel, &ev_cq, &ev_ctx);
-	ibv_ack_cq_events (ev_cq, 1);
-	res = ibv_req_notify_cq (ev_cq, 0);
-
-	res = ibv_poll_cq (instance->recv_token_recv_cq, 32, wc);
-	if (res > 0) {
-		for (i = 0; i < res; i++) {
-			iba_deliver_fn (instance, wc[i].wr_id, wc[i].byte_len);
-			recv_token_recv_buf_post (instance, wrid2void(wc[i].wr_id));
-		}
-	}
-
-	return (0);
-}
-
-static int recv_token_accept_destroy (struct totemiba_instance *instance)
-{
-	if (instance->recv_token_accepted == 0) {
-		return (0);
-	}
-
-	qb_loop_poll_del (
-		instance->totemiba_poll_handle,
-		instance->recv_token_recv_completion_channel->fd);
-
-	qb_loop_poll_del (
-		instance->totemiba_poll_handle,
-		instance->recv_token_send_completion_channel->fd);
-
-	rdma_destroy_qp (instance->recv_token_cma_id);
-
-	recv_token_recv_buf_post_destroy (instance);
-
-	ibv_destroy_cq (instance->recv_token_send_cq);
-
-	ibv_destroy_cq (instance->recv_token_recv_cq);
-
-	ibv_destroy_comp_channel (instance->recv_token_send_completion_channel);
-
-	ibv_destroy_comp_channel (instance->recv_token_recv_completion_channel);
-
-	ibv_dealloc_pd (instance->recv_token_pd);
-
-	rdma_destroy_id (instance->recv_token_cma_id);
-
-	return (0);
-}
-
-static int recv_token_accept_setup (struct totemiba_instance *instance)
-{
-	struct ibv_qp_init_attr init_qp_attr;
-	int res = 0;
-
-	/*
-	 * Allocate the protection domain
-	 */
-	instance->recv_token_pd = ibv_alloc_pd (instance->recv_token_cma_id->verbs);
-
-	/*
-	 * Create a completion channel
-	 */
-	instance->recv_token_recv_completion_channel = ibv_create_comp_channel (instance->recv_token_cma_id->verbs);
-	if (instance->recv_token_recv_completion_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion channel");
-		return (-1);
-	}
-
-	/*
-	 * Create the completion queue
-	 */
-	instance->recv_token_recv_cq = ibv_create_cq (instance->recv_token_cma_id->verbs,
-		COMPLETION_QUEUE_ENTRIES, instance,
-		instance->recv_token_recv_completion_channel, 0);
-	if (instance->recv_token_recv_cq == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion queue");
-		return (-1);
-	}
-	res = ibv_req_notify_cq (instance->recv_token_recv_cq, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't request notifications of the completion queue");
-		return (-1);
-	}
-
-	/*
-	 * Create a completion channel
-	 */
-	instance->recv_token_send_completion_channel = ibv_create_comp_channel (instance->recv_token_cma_id->verbs);
-	if (instance->recv_token_send_completion_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion channel");
-		return (-1);
-	}
-
-	/*
-	 * Create the completion queue
-	 */
-	instance->recv_token_send_cq = ibv_create_cq (instance->recv_token_cma_id->verbs,
-		COMPLETION_QUEUE_ENTRIES, instance,
-		instance->recv_token_send_completion_channel, 0);
-	if (instance->recv_token_send_cq == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion queue");
-		return (-1);
-	}
-	res = ibv_req_notify_cq (instance->recv_token_send_cq, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't request notifications of the completion queue");
-		return (-1);
-	}
-	memset (&init_qp_attr, 0, sizeof (struct ibv_qp_init_attr));
-	init_qp_attr.cap.max_send_wr = 50;
-	init_qp_attr.cap.max_recv_wr = TOTAL_READ_POSTS;
-	init_qp_attr.cap.max_send_sge = 1;
-	init_qp_attr.cap.max_recv_sge = 1;
-	init_qp_attr.qp_context = instance;
-	init_qp_attr.sq_sig_all = 0;
-	init_qp_attr.qp_type = IBV_QPT_UD;
-	init_qp_attr.send_cq = instance->recv_token_send_cq;
-	init_qp_attr.recv_cq = instance->recv_token_recv_cq;
-	res = rdma_create_qp (instance->recv_token_cma_id, instance->recv_token_pd,
-		&init_qp_attr);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create queue pair");
-		return (-1);
-	}
-	
-	recv_token_recv_buf_post_initial (instance);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->recv_token_recv_completion_channel->fd,
-		POLLIN, instance, recv_token_cq_recv_event_fn);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->recv_token_send_completion_channel->fd,
-		POLLIN, instance, recv_token_cq_send_event_fn);
-
-	instance->recv_token_accepted = 1;
-
-	return (res);
-};
-
-static int recv_token_rdma_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct rdma_cm_event *event;
-	struct rdma_conn_param conn_param;
-
-	int res;
-
-	res = rdma_get_cm_event (instance->listen_recv_token_channel, &event);
-	if (res != 0) {
-		return (0);
-	}
-
-	switch (event->event) {
-	case RDMA_CM_EVENT_CONNECT_REQUEST:
-		recv_token_accept_destroy (instance);
-
-		instance->recv_token_cma_id = event->id;
-		recv_token_accept_setup (instance);
-		memset (&conn_param, 0, sizeof (struct rdma_conn_param));
-		conn_param.qp_num = instance->recv_token_cma_id->qp->qp_num;
-		res = rdma_accept (instance->recv_token_cma_id, &conn_param);
-		break;
-	default:
-		log_printf (LOGSYS_LEVEL_ERROR, "default %d", event->event);
-		break;
-	}
-
-	res = rdma_ack_cm_event (event);
-	return (0);
-}
-
-static int send_token_cq_send_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct ibv_wc wc[32];
-	struct ibv_cq *ev_cq;
-	void *ev_ctx;
-	int res;
-	int i;
-
-	ibv_get_cq_event (instance->send_token_send_completion_channel, &ev_cq, &ev_ctx);
-	ibv_ack_cq_events (ev_cq, 1);
-	res = ibv_req_notify_cq (ev_cq, 0);
-
-	res = ibv_poll_cq (instance->send_token_send_cq, 32, wc);
-	if (res > 0) {
-		for (i = 0; i < res; i++) {
-			token_send_buf_put (instance, wrid2void(wc[i].wr_id));
-		}
-	}
-
-	return (0);
-}
-
-static int send_token_cq_recv_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct ibv_wc wc[32];
-	struct ibv_cq *ev_cq;
-	void *ev_ctx;
-	int res;
-	int i;
-
-	ibv_get_cq_event (instance->send_token_recv_completion_channel, &ev_cq, &ev_ctx);
-	ibv_ack_cq_events (ev_cq, 1);
-	res = ibv_req_notify_cq (ev_cq, 0);
-
-	res = ibv_poll_cq (instance->send_token_recv_cq, 32, wc);
-	if (res > 0) {
-		for (i = 0; i < res; i++) {
-			iba_deliver_fn (instance, wc[i].wr_id, wc[i].byte_len);
-		}
-	}
-
-	return (0);
-}
-
-static int send_token_rdma_event_fn (int fd, int events, void *context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)context;
-	struct rdma_cm_event *event;
-	struct rdma_conn_param conn_param;
-
-	int res;
-
-	res = rdma_get_cm_event (instance->send_token_channel, &event);
-	if (res != 0) {
-		return (0);
-	}
-
-	switch (event->event) {
-	/*
-	 * occurs when we resolve the multicast address
-	 */
-	case RDMA_CM_EVENT_ADDR_RESOLVED:
-		res = rdma_resolve_route (instance->send_token_cma_id, 2000);
-		break;
-	/*
-	 * occurs when the CM joins the multicast group
-	 */
-	case RDMA_CM_EVENT_ROUTE_RESOLVED:
-		memset (&conn_param, 0, sizeof (struct rdma_conn_param));
-		conn_param.private_data = NULL;
-		conn_param.private_data_len = 0;
-		res = rdma_connect (instance->send_token_cma_id, &conn_param);
-		break;
-	case RDMA_CM_EVENT_ESTABLISHED:
-		instance->send_token_qpn = event->param.ud.qp_num;
-		instance->send_token_qkey = event->param.ud.qkey;
-		instance->send_token_ah = ibv_create_ah (instance->send_token_pd, &event->param.ud.ah_attr);
-		instance->totemiba_target_set_completed (instance->rrp_context);
-		break;
-
-	case RDMA_CM_EVENT_ADDR_ERROR:
-	case RDMA_CM_EVENT_ROUTE_ERROR:
-	case RDMA_CM_EVENT_MULTICAST_ERROR:
-		log_printf (LOGSYS_LEVEL_ERROR,
-			"send_token_rdma_event_fn multicast error");
-		break;
-	case RDMA_CM_EVENT_DEVICE_REMOVAL:
-		break;
-	case RDMA_CM_EVENT_UNREACHABLE:
-		log_printf (LOGSYS_LEVEL_ERROR,
-			"send_token_rdma_event_fn unreachable");
-		break;
-	default:
-		log_printf (LOGSYS_LEVEL_ERROR,
-			"send_token_rdma_event_fn unknown event %d",
-			event->event);
-		break;
-	}
-
-	rdma_ack_cm_event (event);
-	return (0);
-}
-
-static int send_token_bind (struct totemiba_instance *instance)
-{
-	int res;
-	struct ibv_qp_init_attr init_qp_attr;
-
-	instance->send_token_channel = rdma_create_event_channel();
-	if (instance->send_token_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create rdma channel");
-		return (-1);
-	}
-
-	res = rdma_create_id (instance->send_token_channel,
-		&instance->send_token_cma_id, NULL, RDMA_PS_UDP);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error creating send_token_cma_id");
-		return (-1);
-	}
-
-	res = rdma_bind_addr (instance->send_token_cma_id,
-		&instance->send_token_bind_addr);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error doing rdma_bind_addr for send token");
-		return (-1);
-	}
-
-	/*
-	 * Resolve the send_token address into a GUID
-	 */
-	res = rdma_resolve_addr (instance->send_token_cma_id,
-		&instance->bind_addr, &instance->token_addr, 2000);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error resolving send token address %d %d", res, errno);
-		return (-1);
-	}
-
-	/*
-	 * Allocate the protection domain
-	 */
-	instance->send_token_pd = ibv_alloc_pd (instance->send_token_cma_id->verbs);
-	
-	/*
-	 * Create a completion channel
-	 */
-	instance->send_token_recv_completion_channel = ibv_create_comp_channel (instance->send_token_cma_id->verbs);
-	if (instance->send_token_recv_completion_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion channel");
-		return (-1);
-	}
-
-	/*
-	 * Create the completion queue
-	 */
-	instance->send_token_recv_cq = ibv_create_cq (instance->send_token_cma_id->verbs,
-		COMPLETION_QUEUE_ENTRIES, instance,
-		instance->send_token_recv_completion_channel, 0);
-	if (instance->send_token_recv_cq == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion queue");
-		return (-1);
-	}
-	res = ibv_req_notify_cq (instance->send_token_recv_cq, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR,
-			"couldn't request notifications of the completion queue");
-		return (-1);
-	}
-
-	/*
-	 * Create a completion channel
-	 */
-	instance->send_token_send_completion_channel =
-		ibv_create_comp_channel (instance->send_token_cma_id->verbs);
-
-	if (instance->send_token_send_completion_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion channel");
-		return (-1);
-	}
-	
-	/*
-	 * Create the completion queue
-	 */
-	instance->send_token_send_cq = ibv_create_cq (
-		instance->send_token_cma_id->verbs,
-		COMPLETION_QUEUE_ENTRIES, instance,
-		instance->send_token_send_completion_channel, 0);
-	if (instance->send_token_send_cq == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion queue");
-		return (-1);
-	}
-
-	res = ibv_req_notify_cq (instance->send_token_send_cq, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR,
-			"couldn't request notifications of the completion queue");
-		return (-1);
-	}
-	memset (&init_qp_attr, 0, sizeof (struct ibv_qp_init_attr));
-	init_qp_attr.cap.max_send_wr = 50;
-	init_qp_attr.cap.max_recv_wr = TOTAL_READ_POSTS;
-	init_qp_attr.cap.max_send_sge = 1;
-	init_qp_attr.cap.max_recv_sge = 1;
-	init_qp_attr.qp_context = instance;
-	init_qp_attr.sq_sig_all = 0;
-	init_qp_attr.qp_type = IBV_QPT_UD;
-	init_qp_attr.send_cq = instance->send_token_send_cq;
-	init_qp_attr.recv_cq = instance->send_token_recv_cq;
-	res = rdma_create_qp (instance->send_token_cma_id,
-		instance->send_token_pd, &init_qp_attr);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create queue pair");
-		return (-1);
-	}
-	
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->send_token_recv_completion_channel->fd,
-		POLLIN, instance, send_token_cq_recv_event_fn);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->send_token_send_completion_channel->fd,
-		POLLIN, instance, send_token_cq_send_event_fn);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->send_token_channel->fd,
-		POLLIN, instance, send_token_rdma_event_fn);
-
-	instance->send_token_bound = 1;
-	return (0);
-}
-
-static int send_token_unbind (struct totemiba_instance *instance)
-{
-	if (instance->send_token_bound == 0) {
-		return (0);
-	}
-
-	qb_loop_poll_del (
-		instance->totemiba_poll_handle,
-		instance->send_token_recv_completion_channel->fd);
-	qb_loop_poll_del (
-		instance->totemiba_poll_handle,
-		instance->send_token_send_completion_channel->fd);
-	qb_loop_poll_del (
-		instance->totemiba_poll_handle,
-		instance->send_token_channel->fd);
-
-	if(instance->send_token_ah)
-	{
-		ibv_destroy_ah(instance->send_token_ah);
-		instance->send_token_ah = 0;
-	}
-
-	rdma_destroy_qp (instance->send_token_cma_id);
-	ibv_destroy_cq (instance->send_token_send_cq);
-	ibv_destroy_cq (instance->send_token_recv_cq);
-	ibv_destroy_comp_channel (instance->send_token_send_completion_channel);
-	ibv_destroy_comp_channel (instance->send_token_recv_completion_channel);
-	token_send_buf_destroy (instance);
-	ibv_dealloc_pd (instance->send_token_pd);
-	rdma_destroy_id (instance->send_token_cma_id);
-	rdma_destroy_event_channel (instance->send_token_channel);
-	return (0);
-}
-
-static int recv_token_bind (struct totemiba_instance *instance)
-{
-	int res;
-	struct ibv_port_attr port_attr;
-
-	instance->listen_recv_token_channel = rdma_create_event_channel();
-	if (instance->listen_recv_token_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create rdma channel");
-		return (-1);
-	}
-
-	res = rdma_create_id (instance->listen_recv_token_channel,
-		&instance->listen_recv_token_cma_id, NULL, RDMA_PS_UDP);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error creating recv_token_cma_id");
-		return (-1);
-	}
-
-	res = rdma_bind_addr (instance->listen_recv_token_cma_id,
-		&instance->bind_addr);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error doing rdma_bind_addr for recv token");
-		return (-1);
-	}
-
-	/*
-	 * Determine active_mtu of port and compare it with the configured one (160 is aproximation of all totem
-	 * structures.
-	 *
-	 * TODO: Implement MTU discovery also for IP and handle MTU correctly for all structures inside totemsrp,
-	 *       crypto, ...
-	 */
-	res = ibv_query_port (instance->listen_recv_token_cma_id->verbs, instance->listen_recv_token_cma_id->port_num, &port_attr);
-	if ( (1 << (port_attr.active_mtu + 7)) < instance->totem_config->net_mtu + 160) {
-		log_printf (LOGSYS_LEVEL_ERROR, "requested net_mtu is %d and is larger than the active port mtu %d\n",\
-				instance->totem_config->net_mtu + 160, (1 << (port_attr.active_mtu + 7)));
-		return (-1);
-	}
-
-	/*
-	 * Resolve the recv_token address into a GUID
-	 */
-	res = rdma_listen (instance->listen_recv_token_cma_id, 10);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error listening %d %d", res, errno);
-		return (-1);
-	}
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->listen_recv_token_channel->fd,
-		POLLIN, instance, recv_token_rdma_event_fn);
-
-	return (0);
-}
-
-static int mcast_bind (struct totemiba_instance *instance)
-{
-	int res;
-	struct ibv_qp_init_attr init_qp_attr;
-
-	instance->mcast_channel = rdma_create_event_channel();
-	if (instance->mcast_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create rdma channel");
-		return (-1);
-	}
-
-	res = rdma_create_id (instance->mcast_channel, &instance->mcast_cma_id, NULL, RDMA_PS_UDP);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error creating mcast_cma_id");
-		return (-1);
-	}
-
-	res = rdma_bind_addr (instance->mcast_cma_id, &instance->local_mcast_bind_addr);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error doing rdma_bind_addr for mcast");
-		return (-1);
-	}
-
-	/*
-	 * Resolve the multicast address into a GUID
-	 */
-	res = rdma_resolve_addr (instance->mcast_cma_id, &instance->local_mcast_bind_addr,
-		&instance->mcast_addr, 5000);
-	if (res) {
-		log_printf (LOGSYS_LEVEL_ERROR, "error resolving multicast address %d %d", res, errno);
-		return (-1);
-	}
-
-	/*
-	 * Allocate the protection domain
-	 */
-	instance->mcast_pd = ibv_alloc_pd (instance->mcast_cma_id->verbs);
-	
-	/*
-	 * Create a completion channel
-	 */
-	instance->mcast_recv_completion_channel = ibv_create_comp_channel (instance->mcast_cma_id->verbs);
-	if (instance->mcast_recv_completion_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion channel");
-		return (-1);
-	}
-
-	/*
-	 * Create the completion queue
-	 */
-	instance->mcast_recv_cq = ibv_create_cq (instance->mcast_cma_id->verbs,
-		COMPLETION_QUEUE_ENTRIES, instance,
-		instance->mcast_recv_completion_channel, 0);
-	if (instance->mcast_recv_cq == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion queue");
-		return (-1);
-	}
-	res = ibv_req_notify_cq (instance->mcast_recv_cq, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't request notifications of the completion queue");
-		return (-1);
-	}
-
-	/*
-	 * Create a completion channel
-	 */
-	instance->mcast_send_completion_channel = ibv_create_comp_channel (instance->mcast_cma_id->verbs);
-	if (instance->mcast_send_completion_channel == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion channel");
-		return (-1);
-	}
-
-	/*
-	 * Create the completion queue
-	 */
-	instance->mcast_send_cq = ibv_create_cq (instance->mcast_cma_id->verbs,
-		COMPLETION_QUEUE_ENTRIES, instance,
-		instance->mcast_send_completion_channel, 0);
-	if (instance->mcast_send_cq == NULL) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create completion queue");
-		return (-1);
-	}
-	res = ibv_req_notify_cq (instance->mcast_send_cq, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't request notifications of the completion queue");
-		return (-1);
-	}
-	memset (&init_qp_attr, 0, sizeof (struct ibv_qp_init_attr));
-	init_qp_attr.cap.max_send_wr = 50;
-	init_qp_attr.cap.max_recv_wr = TOTAL_READ_POSTS;
-	init_qp_attr.cap.max_send_sge = 1;
-	init_qp_attr.cap.max_recv_sge = 1;
-	init_qp_attr.qp_context = instance;
-	init_qp_attr.sq_sig_all = 0;
-	init_qp_attr.qp_type = IBV_QPT_UD;
-	init_qp_attr.send_cq = instance->mcast_send_cq;
-	init_qp_attr.recv_cq = instance->mcast_recv_cq;
-	res = rdma_create_qp (instance->mcast_cma_id, instance->mcast_pd,
-		&init_qp_attr);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "couldn't create queue pair");
-		return (-1);
-	}
-	
-	mcast_recv_buf_post_initial (instance);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->mcast_recv_completion_channel->fd,
-		POLLIN, instance, mcast_cq_recv_event_fn);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->mcast_send_completion_channel->fd,
-		POLLIN, instance, mcast_cq_send_event_fn);
-
-	qb_loop_poll_add (
-		instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		instance->mcast_channel->fd,
-		POLLIN, instance, mcast_rdma_event_fn);
-
-	return (0);
-}
-
-static void timer_function_netif_check_timeout (
-      void *data)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)data;
-	int res;
-	int interface_up;
-	int interface_num;
-	int addr_len;
-
-	totemip_iface_check (&instance->totem_interface->bindnet,
-		&instance->totem_interface->boundto, &interface_up, &interface_num, instance->totem_config->clear_node_high_bit);
-
-	totemip_totemip_to_sockaddr_convert(&instance->totem_interface->boundto,
-		instance->totem_interface->ip_port, (struct sockaddr_storage *)&instance->bind_addr,
-		&addr_len);
-
-	totemip_totemip_to_sockaddr_convert(&instance->totem_interface->boundto,
-		0, (struct sockaddr_storage *)&instance->send_token_bind_addr,
-		&addr_len);
-
-	totemip_totemip_to_sockaddr_convert(&instance->totem_interface->boundto,
-		0, (struct sockaddr_storage *)&instance->local_mcast_bind_addr,
-		&addr_len);
-
-	totemip_totemip_to_sockaddr_convert(&instance->totem_interface->boundto,
-		instance->totem_interface->ip_port, (struct sockaddr_storage *)&instance->my_id,
-		&addr_len);
-
-	totemip_sockaddr_to_totemip_convert(
-		(const struct sockaddr_storage *)&instance->bind_addr,
-		&instance->my_id);
-
-	memcpy (&instance->my_id, &instance->totem_interface->boundto,
-		sizeof (struct totem_ip_address));
-
-	totemip_totemip_to_sockaddr_convert(&instance->totem_interface->mcast_addr,
-		instance->totem_interface->ip_port,
-		(struct sockaddr_storage *)&instance->mcast_addr, &addr_len);
-
-	res = recv_token_bind (instance);
-
-	res = mcast_bind (instance);
-}
-
-int totemiba_crypto_set (
-	void *iba_context,
-	const char *cipher_type,
-	const char *hash_type)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-
-int totemiba_finalize (
-	void *iba_context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-
-/*
- * Create an instance
- */
-int totemiba_initialize (
-	qb_loop_t *qb_poll_handle,
-	void **iba_context,
-	struct totem_config *totem_config,
-	totemsrp_stats_t *stats,
-	int interface_no,
-	void *context,
-
-	void (*deliver_fn) (
-		void *context,
-		const void *msg,
-		unsigned int msg_len),
-
-	void (*iface_change_fn) (
-		void *context,
-		const struct totem_ip_address *iface_address),
-
-	void (*target_set_completed) (
-		void *context))
-{
-	struct totemiba_instance *instance;
-	int res = 0;
-
-	instance = malloc (sizeof (struct totemiba_instance));
-	if (instance == NULL) {
-		return (-1);
-	}
-
-	totemiba_instance_initialize (instance);
-
-	instance->totem_interface = &totem_config->interfaces[interface_no];
-
-	instance->totemiba_poll_handle = qb_poll_handle;
-
-	instance->totem_interface->bindnet.nodeid = totem_config->node_id;
-
-	instance->totemiba_deliver_fn = deliver_fn;
-
-	instance->totemiba_target_set_completed = target_set_completed;
-
-	instance->totemiba_iface_change_fn = iface_change_fn;
-
-	instance->totem_config = totem_config;
-	instance->stats = stats;
-
-	instance->rrp_context = context;
-
-	qb_loop_timer_add (instance->totemiba_poll_handle,
-		QB_LOOP_MED,
-		100*QB_TIME_NS_IN_MSEC,
-		(void *)instance,
-		timer_function_netif_check_timeout,
-		&instance->timer_netif_check_timeout);
-
-	instance->totemiba_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
-	instance->totemiba_log_printf = totem_config->totem_logging_configuration.log_printf;
-
-	*iba_context = instance;
-	return (res);
-}
-
-void *totemiba_buffer_alloc (void)
-{
-	return malloc (MAX_MTU_SIZE);
-}
-
-void totemiba_buffer_release (void *ptr)
-{
-	return free (ptr);
-}
-
-int totemiba_processor_count_set (
-	void *iba_context,
-	int processor_count)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-
-int totemiba_recv_flush (void *iba_context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-
-int totemiba_send_flush (void *iba_context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-
-int totemiba_token_send (
-	void *iba_context,
-	const void *ms,
-	unsigned int msg_len)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-	struct ibv_send_wr send_wr, *failed_send_wr;
-	struct ibv_sge sge;
-	void *msg;
-	struct send_buf *send_buf;
-
-	send_buf = token_send_buf_get (instance);
-	if (send_buf == NULL) {
-		return (-1);
-	}
-	msg = send_buf->buffer;
-	memcpy (msg, ms, msg_len);
-
-	send_wr.next = NULL;
-	send_wr.sg_list = &sge;
-	send_wr.num_sge = 1;
-	send_wr.opcode = IBV_WR_SEND;
-	send_wr.send_flags = IBV_SEND_SIGNALED;
-	send_wr.wr_id = void2wrid(send_buf);
-	send_wr.imm_data = 0;
-	send_wr.wr.ud.ah = instance->send_token_ah;
-	send_wr.wr.ud.remote_qpn = instance->send_token_qpn;
-	send_wr.wr.ud.remote_qkey = instance->send_token_qkey;
-
-	sge.length = msg_len;
-	sge.lkey = send_buf->mr->lkey;
-	sge.addr = (uintptr_t)msg;
-
-	if(instance->send_token_ah != 0 && instance->send_token_bound)
-		res = ibv_post_send (instance->send_token_cma_id->qp, &send_wr, &failed_send_wr);
-
-	return (res);
-}
-
-int totemiba_mcast_flush_send (
-	void *iba_context,
-	const void *ms,
-	unsigned int msg_len)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-	struct ibv_send_wr send_wr, *failed_send_wr;
-	struct ibv_sge sge;
-	void *msg;
-	struct send_buf *send_buf;
-
-	send_buf = mcast_send_buf_get (instance);
-	if (send_buf == NULL) {
-		return (-1);
-	}
-
-	msg = send_buf->buffer;
-	memcpy (msg, ms, msg_len);
-	send_wr.next = NULL;
-	send_wr.sg_list = &sge;
-	send_wr.num_sge = 1;
-	send_wr.opcode = IBV_WR_SEND;
-	send_wr.send_flags = IBV_SEND_SIGNALED;
-	send_wr.wr_id = void2wrid(send_buf);
-	send_wr.imm_data = 0;
-	send_wr.wr.ud.ah = instance->mcast_ah;
-	send_wr.wr.ud.remote_qpn = instance->mcast_qpn;
-	send_wr.wr.ud.remote_qkey = instance->mcast_qkey;
-
-	sge.length = msg_len;
-	sge.lkey = send_buf->mr->lkey;
-	sge.addr = (uintptr_t)msg;
-
-	if (instance->mcast_ah != 0) {
-		res = ibv_post_send (instance->mcast_cma_id->qp, &send_wr, &failed_send_wr);
-	}
-
-	return (res);
-}
-
-int totemiba_mcast_noflush_send (
-	void *iba_context,
-	const void *ms,
-	unsigned int msg_len)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-	struct ibv_send_wr send_wr, *failed_send_wr;
-	struct ibv_sge sge;
-	void *msg;
-	struct send_buf *send_buf;
-
-	send_buf = mcast_send_buf_get (instance);
-	if (send_buf == NULL) {
-		return (-1);
-	}
-
-	msg = send_buf->buffer;
-	memcpy (msg, ms, msg_len);
-	send_wr.next = NULL;
-	send_wr.sg_list = &sge;
-	send_wr.num_sge = 1;
-	send_wr.opcode = IBV_WR_SEND;
-	send_wr.send_flags = IBV_SEND_SIGNALED;
-	send_wr.wr_id = void2wrid(send_buf);
-	send_wr.imm_data = 0;
-	send_wr.wr.ud.ah = instance->mcast_ah;
-	send_wr.wr.ud.remote_qpn = instance->mcast_qpn;
-	send_wr.wr.ud.remote_qkey = instance->mcast_qkey;
-
-	sge.length = msg_len;
-	sge.lkey = send_buf->mr->lkey;
-	sge.addr = (uintptr_t)msg;
-
-	if (instance->mcast_ah != 0) {
-		res = ibv_post_send (instance->mcast_cma_id->qp, &send_wr, &failed_send_wr);
-	}
-
-	return (res);
-}
-
-extern int totemiba_iface_check (void *iba_context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-
-extern void totemiba_net_mtu_adjust (void *iba_context, struct totem_config *totem_config)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	instance = NULL;
-}
-
-const char *totemiba_iface_print (void *iba_context)  {
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-
-        const char *ret_char;
-
-        ret_char = totemip_print (&instance->my_id);
-
-        return (ret_char);
-}
-
-int totemiba_iface_get (
-	void *iba_context,
-	struct totem_ip_address *addr)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
-
-	return (res);
-}
-
-int totemiba_token_target_set (
-	void *iba_context,
-	const struct totem_ip_address *token_target)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-	int addr_len = 16;
-
-	totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)token_target,
-		instance->totem_interface->ip_port, (struct sockaddr_storage *)&instance->token_addr,
-		&addr_len);
-
-	res = send_token_unbind (instance);
-
-	res = send_token_bind (instance);
-
-	return (res);
-}
-
-extern int totemiba_recv_mcast_empty (
-	void *iba_context)
-{
-	struct totemiba_instance *instance = (struct totemiba_instance *)iba_context;
-	int res = 0;
-
-	instance = NULL;
-
-	return (res);
-}
-

exec/totemiba.h

@@ -1,118 +0,0 @@
-/*
- * Copyright (c) 2009-2011 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef TOTEMIBA_H_DEFINED
-#define TOTEMIBA_H_DEFINED
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <corosync/hdb.h>
-#include <qb/qbloop.h>
-
-#include <corosync/totem/totem.h>
-
-/**
- * Create an instance
- */
-extern int totemiba_initialize (
-	qb_loop_t* qb_poll_handle,
-	void **iba_handle,
-	struct totem_config *totem_config,
-	totemsrp_stats_t *stats,
-	int interface_no,
-	void *context,
-
-	void (*deliver_fn) (
-		void *context,
-		const void *msg,
-		unsigned int msg_len),
-
-	void (*iface_change_fn) (
-		void *context,
-		const struct totem_ip_address *iface_address),
-
-	void (*target_set_completed) (
-		void *context));
-
-extern void *totemiba_buffer_alloc (void);
-
-extern void totemiba_buffer_release (void *ptr);
-
-extern int totemiba_processor_count_set (
-	void *iba_context,
-	int processor_count);
-
-extern int totemiba_token_send (
-	void *iba_context,
-	const void *msg,
-	unsigned int msg_len);
-
-extern int totemiba_mcast_flush_send (
-	void *iba_context,
-	const void *msg,
-	unsigned int msg_len);
-
-extern int totemiba_mcast_noflush_send (
-	void *iba_context,
-	const void *msg,
-	unsigned int msg_len);
-
-extern int totemiba_recv_flush (void *iba_context);
-
-extern int totemiba_send_flush (void *iba_context);
-
-extern int totemiba_iface_check (void *iba_context);
-
-extern int totemiba_finalize (void *iba_context);
-
-extern void totemiba_net_mtu_adjust (void *iba_context, struct totem_config *totem_config);
-
-extern const char *totemiba_iface_print (void *iba_context);
-
-extern int totemiba_iface_get (
-	void *iba_context,
-	struct totem_ip_address *addr);
-
-extern int totemiba_token_target_set (
-	void *iba_context,
-	const struct totem_ip_address *token_target);
-
-extern int totemiba_crypto_set (
-	void *iba_context,
-	const char *cipher_type,
-	const char *hash_type);
-
-extern int totemiba_recv_mcast_empty (
-	void *iba_context);
-
-#endif /* TOTEMIBA_H_DEFINED */

exec/totemmrp.c

@@ -1,286 +0,0 @@
-/*
- * Copyright (c) 2005 MontaVista Software, Inc.
- * Copyright (c) 2006-2007, 2009 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <sys/un.h>
-#include <sys/ioctl.h>
-#include <sys/param.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <sched.h>
-#include <time.h>
-#include <sys/time.h>
-#include <sys/poll.h>
-
-#include <corosync/totem/totem.h>
-#include <qb/qbloop.h>
-
-#include "totemmrp.h"
-#include "totemsrp.h"
-
-void *totemsrp_context;
-
-void totemmrp_deliver_fn (
-	unsigned int nodeid,
-	const void *msg,
-	unsigned int msg_len,
-	int endian_conversion_required);
-
-void totemmrp_confchg_fn (
-	enum totem_configuration_type configuration_type,
-	const unsigned int *member_list, size_t member_list_entries,
-	const unsigned int *left_list, size_t left_list_entries,
-	const unsigned int *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id);
-
-void (*pg_deliver_fn) (
-	unsigned int nodeid,
-	const void *msg,
-	unsigned int msg_len,
-	int endian_conversion_required) = 0;
-
-void (*pg_confchg_fn) (
-	enum totem_configuration_type configuration_type,
-	const unsigned int *member_list, size_t member_list_entries,
-	const unsigned int *left_list, size_t left_list_entries,
-	const unsigned int *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id) = 0;
-
-void totemmrp_deliver_fn (
-	unsigned int nodeid,
-	const void *msg,
-	unsigned int msg_len,
-	int endian_conversion_required)
-{
-	pg_deliver_fn (nodeid, msg, msg_len, endian_conversion_required);
-}
-
-void totemmrp_confchg_fn (
-	enum totem_configuration_type configuration_type,
-	const unsigned int *member_list, size_t member_list_entries,
-	const unsigned int *left_list, size_t left_list_entries,
-	const unsigned int *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id)
-{
-	pg_confchg_fn (configuration_type,
-		member_list, member_list_entries,
-		left_list, left_list_entries,
-		joined_list, joined_list_entries,
-		ring_id);
-}
-
-/*
- * Initialize the totem multiple ring protocol
- */
-int totemmrp_initialize (
-	qb_loop_t *poll_handle,
-	struct totem_config *totem_config,
-	totempg_stats_t *stats,
-
-	void (*deliver_fn) (
-		unsigned int nodeid,
-		const void *msg,
-		unsigned int msg_len,
-		int endian_conversion_required),
-	void (*confchg_fn) (
-		enum totem_configuration_type configuration_type,
-		const unsigned int *member_list, size_t member_list_entries,
-		const unsigned int *left_list, size_t left_list_entries,
-		const unsigned int *joined_list, size_t joined_list_entries,
-		const struct memb_ring_id *ring_id),
-	void (*waiting_trans_ack_cb_fn) (
-		int waiting_trans_ack))
-{
-	int result;
-	pg_deliver_fn = deliver_fn;
-	pg_confchg_fn = confchg_fn;
-
-	stats->mrp = calloc (sizeof(totemmrp_stats_t), 1);
-	result = totemsrp_initialize (
-		poll_handle,
-		&totemsrp_context,
-		totem_config,
-		stats->mrp,
-		totemmrp_deliver_fn,
-		totemmrp_confchg_fn,
-		waiting_trans_ack_cb_fn);
-
-	return (result);
-}
-
-void totemmrp_finalize (void)
-{
-	totemsrp_finalize (totemsrp_context);
-}
-
-/*
- * Multicast a message
- */
-int totemmrp_mcast (
-	struct iovec *iovec,
-	unsigned int iov_len,
-	int priority)
-{
-	return totemsrp_mcast (totemsrp_context, iovec, iov_len, priority);
-}
-
-/*
- * Return number of available messages that can be queued
- */
-int totemmrp_avail (void)
-{
-	return (totemsrp_avail (totemsrp_context));
-}
-
-int totemmrp_callback_token_create (
-	void **handle_out,
-	enum totem_callback_token_type type,
-	int delete,
-	int (*callback_fn) (enum totem_callback_token_type type, const void *),
-	const void *data)
-{
-	return totemsrp_callback_token_create (totemsrp_context, handle_out, type, delete, callback_fn, data);
-}
-
-void totemmrp_callback_token_destroy (
-	void *handle_out)
-{
-	totemsrp_callback_token_destroy (totemsrp_context, handle_out);
-}
-
-void totemmrp_event_signal (enum totem_event_type type, int value)
-{
-	totemsrp_event_signal (totemsrp_context, type, value);
-}
-
-int totemmrp_ifaces_get (
-	unsigned int nodeid,
-	struct totem_ip_address *interfaces,
-	unsigned int interfaces_size,
-	char ***status,
-	unsigned int *iface_count)
-{
-	int res;
-
-	res = totemsrp_ifaces_get (
-		totemsrp_context,
-		nodeid,
-		interfaces,
-		interfaces_size,
-		status,
-		iface_count);
-
-	return (res);
-}
-
-int totemmrp_crypto_set (
-	const char *cipher_type,
-	const char *hash_type)
-{
-	return totemsrp_crypto_set (totemsrp_context,
-				    cipher_type,
-				    hash_type);
-}
-
-unsigned int totemmrp_my_nodeid_get (void)
-{
-	return (totemsrp_my_nodeid_get (totemsrp_context));
-}
-
-int totemmrp_my_family_get (void)
-{
-	return (totemsrp_my_family_get (totemsrp_context));
-}
-
-extern int totemmrp_ring_reenable (void)
-{
-	int res;
-
-	res = totemsrp_ring_reenable (
-		totemsrp_context);
-
-	return (res);
-}
-
-extern void totemmrp_service_ready_register (
-        void (*totem_service_ready) (void))
-{
-	totemsrp_service_ready_register (
-		totemsrp_context,
-		totem_service_ready);
-}
-
-int totemmrp_member_add (
-        const struct totem_ip_address *member,
-        int ring_no)
-{
-	int res;
-
-	res = totemsrp_member_add (totemsrp_context, member, ring_no);
-
-	return (res);
-}
-
-int totemmrp_member_remove (
-       const struct totem_ip_address *member,
-        int ring_no)
-{
-	int res;
-
-	res = totemsrp_member_remove (totemsrp_context, member, ring_no);
-
-	return (res);
-}
-
-void totemmrp_threaded_mode_enable (void)
-{
-	totemsrp_threaded_mode_enable (totemsrp_context);
-}
-
-void totemmrp_trans_ack (void)
-{
-	totemsrp_trans_ack (totemsrp_context);
-}

exec/totemmrp.h

@@ -1,138 +0,0 @@
-/*
- * Copyright (c) 2005 MontaVista Software, Inc.
- * Copyright (c) 2006-2011 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/**
- * @file
- * Totem Single Ring Protocol
- *
- * depends on poll abstraction, POSIX, IPV4
- */
-
-#ifndef TOTEMMRP_H_DEFINED
-#define TOTEMMRP_H_DEFINED
-
-#include <corosync/totem/totem.h>
-
-/**
- * Initialize the logger
- */
-extern void totemmrp_log_printf_init (
-	void (*log_printf) (int , char *, ...),
-	int log_level_security,
-	int log_level_error,
-	int log_level_warning,
-	int log_level_notice,
-	int log_level_debug);
-
-/**
- * Initialize the group messaging interface
- */
-extern int totemmrp_initialize (
-	qb_loop_t *poll_handle,
-	struct totem_config *totem_config,
-	totempg_stats_t *stats,
-
-	void (*deliver_fn) (
-		unsigned int nodeid,
-		const void *msg,
-		unsigned int msg_len,
-		int endian_conversion_required),
-	void (*confchg_fn) (
-		enum totem_configuration_type configuration_type,
-		const unsigned int *member_list, size_t member_list_entries,
-		const unsigned int *left_list, size_t left_list_entries,
-		const unsigned int *joined_list, size_t joined_list_entries,
-		const struct memb_ring_id *ring_id),
-	void (*waiting_trans_ack_cb_fn) (
-		int waiting_trans_ack));
-
-extern void totemmrp_finalize (void);
-
-/**
- * Multicast a message
- */
-extern int totemmrp_mcast (
-	struct iovec *iovec,
-	unsigned int iov_len,
-	int priority);
-
-/**
- * Return number of available messages that can be queued
- */
-extern int totemmrp_avail (void);
-
-extern int totemmrp_callback_token_create (
-	void **handle_out,
-	enum totem_callback_token_type type,
-	int delete,
-	int (*callback_fn) (enum totem_callback_token_type type, const void *),
-	const void *data);
-
-extern void totemmrp_callback_token_destroy (
-	void *handle_out);
-
-extern void totemmrp_event_signal (enum totem_event_type type, int value);
-
-extern int totemmrp_ifaces_get (
-	unsigned int nodeid,
-	struct totem_ip_address *interfaces,
-	unsigned int interfaces_size,
-	char ***status,
-	unsigned int *iface_count);
-
-extern unsigned int totemmrp_my_nodeid_get (void);
-
-extern int totemmrp_my_family_get (void);
-
-extern int totemmrp_crypto_set (const char *cipher_type, const char *hash_type);
-
-extern int totemmrp_ring_reenable (void);
-
-extern void totemmrp_service_ready_register (
-        void (*totem_service_ready) (void));
-
-extern int totemmrp_member_add (
-	const struct totem_ip_address *member,
-	int ring_no);
-
-extern int totemmrp_member_remove (
-	const struct totem_ip_address *member,
-	int ring_no);
-
-void totemmrp_threaded_mode_enable (void);
-
-void totemmrp_trans_ack (void);
-
-#endif /* TOTEMMRP_H_DEFINED */

exec/totemnet.c

@@ -42,6 +42,7 @@
 #endif
 #include <totemudp.h>
 #include <totemudpu.h>
+#include <totemknet.h>
 #include <totemnet.h>
 #include <qb/qbloop.h>
 
@@ -50,13 +51,12 @@
 
 struct transport {
 	const char *name;
-	
+
 	int (*initialize) (
 		qb_loop_t *loop_pt,
 		void **transport_instance,
 		struct totem_config *totem_config,
 		totemsrp_stats_t *stats,
-		int interface_no,
 		void *context,
 
 		void (*deliver_fn) (
@@ -66,7 +66,12 @@ struct transport {
 
 		void (*iface_change_fn) (
 			void *context,
-			const struct totem_ip_address *iface_address),
+			const struct totem_ip_address *iface_address,
+			unsigned int ring_no),
+
+		void (*mtu_changed) (
+			void *context,
+			int net_mtu),
 
 		void (*target_set_completed) (
 			void *context));
@@ -107,9 +112,10 @@ struct transport {
 
 	const char *(*iface_print) (void *transport_context);
 
-	int (*iface_get) (
+	int (*ifaces_get) (
 		void *transport_context,
-		struct totem_ip_address *addr);
+		char ***status,
+		unsigned int *iface_count);
 
 	int (*token_target_set) (
 		void *transport_context,
@@ -125,11 +131,14 @@ struct transport {
 
 	int (*member_add) (
 		void *transport_context,
-		const struct totem_ip_address *member);
+		const struct totem_ip_address *local,
+		const struct totem_ip_address *member,
+		int ring_no);
 
 	int (*member_remove) (
 		void *transport_context,
-		const struct totem_ip_address *member);
+		const struct totem_ip_address *member,
+		int ring_no);
 
 	int (*member_set_active) (
 		void *transport_context,
@@ -152,8 +161,7 @@ struct transport transport_entries[] = {
 		.iface_check = totemudp_iface_check,
 		.finalize = totemudp_finalize,
 		.net_mtu_adjust = totemudp_net_mtu_adjust,
-		.iface_print = totemudp_iface_print,
-		.iface_get = totemudp_iface_get,
+		.ifaces_get = totemudp_ifaces_get,
 		.token_target_set = totemudp_token_target_set,
 		.crypto_set = totemudp_crypto_set,
 		.recv_mcast_empty = totemudp_recv_mcast_empty
@@ -172,45 +180,41 @@ struct transport transport_entries[] = {
 		.iface_check = totemudpu_iface_check,
 		.finalize = totemudpu_finalize,
 		.net_mtu_adjust = totemudpu_net_mtu_adjust,
-		.iface_print = totemudpu_iface_print,
-		.iface_get = totemudpu_iface_get,
+		.ifaces_get = totemudpu_ifaces_get,
 		.token_target_set = totemudpu_token_target_set,
 		.crypto_set = totemudpu_crypto_set,
 		.recv_mcast_empty = totemudpu_recv_mcast_empty,
 		.member_add = totemudpu_member_add,
 		.member_remove = totemudpu_member_remove,
-		.member_set_active = totemudpu_member_set_active
 	},
-#ifdef HAVE_RDMA
 	{
-		.name = "Infiniband/IP",
-		.initialize = totemiba_initialize,
-		.buffer_alloc = totemiba_buffer_alloc,
-		.buffer_release = totemiba_buffer_release,
-		.processor_count_set = totemiba_processor_count_set,
-		.token_send = totemiba_token_send,
-		.mcast_flush_send = totemiba_mcast_flush_send,
-		.mcast_noflush_send = totemiba_mcast_noflush_send,
-		.recv_flush = totemiba_recv_flush,
-		.send_flush = totemiba_send_flush,
-		.iface_check = totemiba_iface_check,
-		.finalize = totemiba_finalize,
-		.net_mtu_adjust = totemiba_net_mtu_adjust,
-		.iface_print = totemiba_iface_print,
-		.iface_get = totemiba_iface_get,
-		.token_target_set = totemiba_token_target_set,
-		.crypto_set = totemiba_crypto_set,
-		.recv_mcast_empty = totemiba_recv_mcast_empty
+		.name = "Kronosnet",
+		.initialize = totemknet_initialize,
+		.buffer_alloc = totemknet_buffer_alloc,
+		.buffer_release = totemknet_buffer_release,
+		.processor_count_set = totemknet_processor_count_set,
+		.token_send = totemknet_token_send,
+		.mcast_flush_send = totemknet_mcast_flush_send,
+		.mcast_noflush_send = totemknet_mcast_noflush_send,
+		.recv_flush = totemknet_recv_flush,
+		.send_flush = totemknet_send_flush,
+		.iface_check = totemknet_iface_check,
+		.finalize = totemknet_finalize,
+		.net_mtu_adjust = totemknet_net_mtu_adjust,
+		.ifaces_get = totemknet_ifaces_get,
+		.token_target_set = totemknet_token_target_set,
+		.crypto_set = totemknet_crypto_set,
+		.recv_mcast_empty = totemknet_recv_mcast_empty,
+		.member_add = totemknet_member_add,
+		.member_remove = totemknet_member_remove,
 
 	}
-#endif
 };
-	
+
 struct totemnet_instance {
 	void *transport_context;
 
 	struct transport *transport;
-
         void (*totemnet_log_printf) (
                 int level,
 		int subsys,
@@ -280,7 +284,6 @@ int totemnet_initialize (
 	void **net_context,
 	struct totem_config *totem_config,
 	totemsrp_stats_t *stats,
-	int interface_no,
 	void *context,
 
 	void (*deliver_fn) (
@@ -290,7 +293,12 @@ int totemnet_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address),
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no),
+
+	void (*mtu_changed) (
+		void *context,
+		int net_mtu),
 
 	void (*target_set_completed) (
 		void *context))
@@ -306,7 +314,7 @@ int totemnet_initialize (
 
 	res = instance->transport->initialize (loop_pt,
 		&instance->transport_context, totem_config, stats,
-		interface_no, context, deliver_fn, iface_change_fn, target_set_completed);
+		context, deliver_fn, iface_change_fn, mtu_changed, target_set_completed);
 
 	if (res == -1) {
 		goto error_destroy;
@@ -424,23 +432,16 @@ extern int totemnet_net_mtu_adjust (void *net_context, struct totem_config *tote
 	return (res);
 }
 
-const char *totemnet_iface_print (void *net_context)  {
-	struct totemnet_instance *instance = (struct totemnet_instance *)net_context;
-	const char *ret_char;
-
-	ret_char = instance->transport->iface_print (instance->transport_context);
-	return (ret_char);
-}
-
-int totemnet_iface_get (
+int totemnet_ifaces_get (
 	void *net_context,
-	struct totem_ip_address *addr)
+	char ***status,
+	unsigned int *iface_count)
 {
 	struct totemnet_instance *instance = (struct totemnet_instance *)net_context;
 	unsigned int res;
 
-	res = instance->transport->iface_get (instance->transport_context, addr);
-	
+	res = instance->transport->ifaces_get (instance->transport_context, status, iface_count);
+
 	return (res);
 }
 
@@ -469,7 +470,9 @@ extern int totemnet_recv_mcast_empty (
 
 extern int totemnet_member_add (
 	void *net_context,
-	const struct totem_ip_address *member)
+	const struct totem_ip_address *local,
+	const struct totem_ip_address *member,
+	int ring_no)
 {
 	struct totemnet_instance *instance = (struct totemnet_instance *)net_context;
 	unsigned int res = 0;
@@ -477,7 +480,9 @@ extern int totemnet_member_add (
 	if (instance->transport->member_add) {
 		res = instance->transport->member_add (
 			instance->transport_context,
-			member);
+			local,
+			member,
+			ring_no);
 	}
 
 	return (res);
@@ -485,7 +490,8 @@ extern int totemnet_member_add (
 
 extern int totemnet_member_remove (
 	void *net_context,
-	const struct totem_ip_address *member)
+	const struct totem_ip_address *member,
+	int ring_no)
 {
 	struct totemnet_instance *instance = (struct totemnet_instance *)net_context;
 	unsigned int res = 0;
@@ -493,7 +499,8 @@ extern int totemnet_member_remove (
 	if (instance->transport->member_remove) {
 		res = instance->transport->member_remove (
 			instance->transport_context,
-			member);
+			member,
+			ring_no);
 	}
 
 	return (res);

exec/totemnet.h

@@ -59,7 +59,6 @@ extern int totemnet_initialize (
 	void **net_context,
 	struct totem_config *totem_config,
 	totemsrp_stats_t *stats,
-	int interface_no,
 	void *context,
 
 	void (*deliver_fn) (
@@ -69,7 +68,12 @@ extern int totemnet_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address),
+		const struct totem_ip_address *iface_address,
+		unsigned int iface_no),
+
+	void (*mtu_changed) (
+		void *context,
+		int net_mtu),
 
 	void (*target_set_completed) (
 		void *context));
@@ -109,9 +113,10 @@ extern int totemnet_net_mtu_adjust (void *net_context, struct totem_config *tote
 
 extern const char *totemnet_iface_print (void *net_context);
 
-extern int totemnet_iface_get (
+extern int totemnet_ifaces_get (
 	void *net_context,
-	struct totem_ip_address *addr);
+	char ***status,
+	unsigned int *iface_count);
 
 extern int totemnet_token_target_set (
 	void *net_context,
@@ -127,11 +132,14 @@ extern int totemnet_recv_mcast_empty (
 
 extern int totemnet_member_add (
 	void *net_context,
-	const struct totem_ip_address *member);
+	const struct totem_ip_address *local,
+	const struct totem_ip_address *member,
+	int ring_no);
 
 extern int totemnet_member_remove (
 	void *net_context,
-	const struct totem_ip_address *member);
+	const struct totem_ip_address *member,
+	int ring_no);
 
 extern int totemnet_member_set_active (
 	void *net_context,

exec/totempg.c

@@ -108,7 +108,6 @@
 #define LOGSYS_UTILS_ONLY 1
 #include <corosync/logsys.h>
 
-#include "totemmrp.h"
 #include "totemsrp.h"
 
 #define min(a,b) ((a) < (b)) ? a : b
@@ -170,6 +169,8 @@ static totem_queue_level_changed_fn totem_queue_level_changed = NULL;
 
 static uint32_t totempg_threaded_mode = 0;
 
+static void *totemsrp_context;
+
 /*
  * Function and data used to log messages
  */
@@ -745,7 +746,7 @@ int callback_token_received_fn (enum totem_callback_token_type type,
 		}
 		return (0);
 	}
-	if (totemmrp_avail() == 0) {
+	if (totemsrp_avail(totemsrp_context) == 0) {
 		if (totempg_threaded_mode == 1) {
 			pthread_mutex_unlock (&mcast_msg_mutex);
 		}
@@ -770,7 +771,7 @@ int callback_token_received_fn (enum totem_callback_token_type type,
 	iovecs[1].iov_len = mcast_packed_msg_count * sizeof (unsigned short);
 	iovecs[2].iov_base = (void *)&fragmentation_data[0];
 	iovecs[2].iov_len = fragment_size;
-	(void)totemmrp_mcast (iovecs, 3, 0);
+	(void)totemsrp_mcast (totemsrp_context, iovecs, 3, 0);
 
 	mcast_packed_msg_count = 0;
 	fragment_size = 0;
@@ -806,22 +807,24 @@ int totempg_initialize (
 
 	totemsrp_net_mtu_adjust (totem_config);
 
-	res = totemmrp_initialize (
+	res = totemsrp_initialize (
 		poll_handle,
+		&totemsrp_context,
 		totem_config,
 		&totempg_stats,
 		totempg_deliver_fn,
 		totempg_confchg_fn,
 		totempg_waiting_trans_ack_cb);
 
-	totemmrp_callback_token_create (
+	totemsrp_callback_token_create (
+		totemsrp_context,
 		&callback_token_received_handle,
 		TOTEM_CALLBACK_TOKEN_RECEIVED,
 		0,
 		callback_token_received_fn,
 		0);
 
-	totempg_size_limit = (totemmrp_avail() - 1) *
+	totempg_size_limit = (totemsrp_avail(totemsrp_context) - 1) *
 		(totempg_totem_config->net_mtu -
 		sizeof (struct totempg_mcast) - 16);
 
@@ -835,7 +838,7 @@ void totempg_finalize (void)
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_lock (&totempg_mutex);
 	}
-	totemmrp_finalize ();
+	totemsrp_finalize (totemsrp_context);
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_unlock (&totempg_mutex);
 	}
@@ -863,7 +866,7 @@ static int mcast_msg (
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_lock (&mcast_msg_mutex);
 	}
-	totemmrp_event_signal (TOTEM_EVENT_NEW_MSG, 1);
+	totemsrp_event_signal (totemsrp_context, TOTEM_EVENT_NEW_MSG, 1);
 
 	/*
 	 * Remove zero length iovectors from the list
@@ -971,8 +974,8 @@ static int mcast_msg (
 				sizeof(unsigned short);
 			iovecs[2].iov_base = (void *)data_ptr;
 			iovecs[2].iov_len = max_packet_size;
-			assert (totemmrp_avail() > 0);
-			res = totemmrp_mcast (iovecs, 3, guarantee);
+			assert (totemsrp_avail(totemsrp_context) > 0);
+			res = totemsrp_mcast (totemsrp_context, iovecs, 3, guarantee);
 			if (res == -1) {
 				goto error_exit;
 			}
@@ -1026,7 +1029,7 @@ static int msg_count_send_ok (
 {
 	int avail = 0;
 
-	avail = totemmrp_avail ();
+	avail = totemsrp_avail (totemsrp_context);
 	totempg_stats.msg_queue_avail = avail;
 
 	return ((avail - totempg_reserved) > msg_count);
@@ -1038,7 +1041,7 @@ static int byte_count_send_ok (
 	unsigned int msg_count = 0;
 	int avail = 0;
 
-	avail = totemmrp_avail ();
+	avail = totemsrp_avail (totemsrp_context);
 
 	msg_count = (byte_count / (totempg_totem_config->net_mtu - sizeof (struct totempg_mcast) - 16)) + 1;
 
@@ -1071,7 +1074,7 @@ static void send_release (
 
 static uint32_t q_level_precent_used(void)
 {
-	return (100 - (((totemmrp_avail() - totempg_reserved) * 100) / MESSAGE_QUEUE_MAX));
+	return (100 - (((totemsrp_avail(totemsrp_context) - totempg_reserved) * 100) / MESSAGE_QUEUE_MAX));
 }
 
 int totempg_callback_token_create (
@@ -1085,7 +1088,7 @@ int totempg_callback_token_create (
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_lock (&callback_token_mutex);
 	}
-	res = totemmrp_callback_token_create (handle_out, type, delete,
+	res = totemsrp_callback_token_create (totemsrp_context, handle_out, type, delete,
 		callback_fn, data);
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_unlock (&callback_token_mutex);
@@ -1099,7 +1102,7 @@ void totempg_callback_token_destroy (
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_lock (&callback_token_mutex);
 	}
-	totemmrp_callback_token_destroy (handle_out);
+	totemsrp_callback_token_destroy (totemsrp_context, handle_out);
 	if (totempg_threaded_mode == 1) {
 		pthread_mutex_unlock (&callback_token_mutex);
 	}
@@ -1418,7 +1421,8 @@ int totempg_ifaces_get (
 {
 	int res;
 
-	res = totemmrp_ifaces_get (
+	res = totemsrp_ifaces_get (
+		totemsrp_context,
 		nodeid,
 		interfaces,
 		interfaces_size,
@@ -1430,7 +1434,7 @@ int totempg_ifaces_get (
 
 void totempg_event_signal (enum totem_event_type type, int value)
 {
-	totemmrp_event_signal (type, value);
+	totemsrp_event_signal (totemsrp_context, type, value);
 }
 
 void* totempg_get_stats (void)
@@ -1444,7 +1448,7 @@ int totempg_crypto_set (
 {
 	int res;
 
-	res = totemmrp_crypto_set (cipher_type, hash_type);
+	res = totemsrp_crypto_set (totemsrp_context, cipher_type, hash_type);
 
 	return (res);
 }
@@ -1453,7 +1457,7 @@ int totempg_ring_reenable (void)
 {
 	int res;
 
-	res = totemmrp_ring_reenable ();
+	res = totemsrp_ring_reenable (totemsrp_context);
 
 	return (res);
 }
@@ -1464,19 +1468,18 @@ const char *totempg_ifaces_print (unsigned int nodeid)
 	static char iface_string[256 * INTERFACE_MAX];
 	char one_iface[ONE_IFACE_LEN+1];
 	struct totem_ip_address interfaces[INTERFACE_MAX];
-	char **status;
 	unsigned int iface_count;
 	unsigned int i;
 	int res;
 
 	iface_string[0] = '\0';
 
-	res = totempg_ifaces_get (nodeid, interfaces, INTERFACE_MAX, &status, &iface_count);
+	res = totempg_ifaces_get (nodeid, interfaces, INTERFACE_MAX, NULL, &iface_count);
 	if (res == -1) {
 		return ("no interface found for nodeid");
 	}
 
-	res = totempg_ifaces_get (nodeid, interfaces, INTERFACE_MAX, &status, &iface_count);
+	res = totempg_ifaces_get (nodeid, interfaces, INTERFACE_MAX, NULL, &iface_count);
 
 	for (i = 0; i < iface_count; i++) {
 		snprintf (one_iface, ONE_IFACE_LEN,
@@ -1489,17 +1492,17 @@ const char *totempg_ifaces_print (unsigned int nodeid)
 
 unsigned int totempg_my_nodeid_get (void)
 {
-	return (totemmrp_my_nodeid_get());
+	return (totemsrp_my_nodeid_get(totemsrp_context));
 }
 
 int totempg_my_family_get (void)
 {
-	return (totemmrp_my_family_get());
+	return (totemsrp_my_family_get(totemsrp_context));
 }
 extern void totempg_service_ready_register (
 	void (*totem_service_ready) (void))
 {
-	totemmrp_service_ready_register (totem_service_ready);
+	totemsrp_service_ready_register (totemsrp_context, totem_service_ready);
 }
 
 void totempg_queue_level_register_callback (totem_queue_level_changed_fn fn)
@@ -1511,24 +1514,24 @@ extern int totempg_member_add (
 	const struct totem_ip_address *member,
 	int ring_no)
 {
-	return totemmrp_member_add (member, ring_no);
+	return totemsrp_member_add (totemsrp_context, member, ring_no);
 }
 
 extern int totempg_member_remove (
 	const struct totem_ip_address *member,
 	int ring_no)
 {
-	return totemmrp_member_remove (member, ring_no);
+	return totemsrp_member_remove (totemsrp_context, member, ring_no);
 }
 
 void totempg_threaded_mode_enable (void)
 {
 	totempg_threaded_mode = 1;
-	totemmrp_threaded_mode_enable ();
+	totemsrp_threaded_mode_enable (totemsrp_context);
 }
 
 void totempg_trans_ack (void)
 {
-	totemmrp_trans_ack ();
+	totemsrp_trans_ack (totemsrp_context);
 }
 

exec/totemrrp.c

@@ -1,2335 +0,0 @@
-/*
- * Copyright (c) 2005 MontaVista Software, Inc.
- * Copyright (c) 2006-2012 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include <assert.h>
-#include <pthread.h>
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <sys/un.h>
-#include <sys/ioctl.h>
-#include <sys/param.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <sched.h>
-#include <time.h>
-#include <sys/time.h>
-#include <sys/poll.h>
-#include <limits.h>
-
-#include <corosync/sq.h>
-#include <corosync/list.h>
-#include <corosync/swab.h>
-#include <qb/qbdefs.h>
-#include <qb/qbloop.h>
-#define LOGSYS_UTILS_ONLY 1
-#include <corosync/logsys.h>
-
-#include "totemnet.h"
-#include "totemrrp.h"
-
-void rrp_deliver_fn (
-	void *context,
-	const void *msg,
-	unsigned int msg_len);
-
-void rrp_iface_change_fn (
-	void *context,
-	const struct totem_ip_address *iface_addr);
-
-struct totemrrp_instance;
-struct passive_instance {
-	struct totemrrp_instance *rrp_instance;
-	unsigned int *faulty;
-	unsigned int *token_recv_count;
-	unsigned int *mcast_recv_count;
-	unsigned char token[15000];
-	unsigned int token_len;
-        qb_loop_timer_handle timer_expired_token;
-        qb_loop_timer_handle timer_problem_decrementer;
-	void *totemrrp_context;
-	unsigned int token_xmit_iface;
-	unsigned int msg_xmit_iface;
-};
-
-struct active_instance {
-	struct totemrrp_instance *rrp_instance;
-	unsigned int *faulty;
-	unsigned int *last_token_recv;
-	unsigned int *counter_problems;
-	unsigned char token[15000];
-	unsigned int token_len;
-	unsigned int last_token_seq;
-        qb_loop_timer_handle timer_expired_token;
-        qb_loop_timer_handle timer_problem_decrementer;
-	void *totemrrp_context;
-};
-
-struct rrp_algo {
-	const char *name;
-
-	void * (*initialize) (
-		struct totemrrp_instance *rrp_instance,
-		int interface_count);
-
-	void (*mcast_recv) (
-		struct totemrrp_instance *instance,
-		unsigned int iface_no,
-		void *context,
-		const void *msg,
-		unsigned int msg_len);
-
-	void (*mcast_noflush_send) (
-		struct totemrrp_instance *instance,
-		const void *msg,
-		unsigned int msg_len);
-
-	void (*mcast_flush_send) (
-		struct totemrrp_instance *instance,
-		const void *msg,
-		unsigned int msg_len);
-
-	void (*token_recv) (
-		struct totemrrp_instance *instance,
-		unsigned int iface_no,
-		void *context,
-		const void *msg,
-		unsigned int msg_len,
-		unsigned int token_seqid);
-
-	void (*token_send) (
-		struct totemrrp_instance *instance,
-		const void *msg,
-		unsigned int msg_len);
-
-	void (*recv_flush) (
-		struct totemrrp_instance *instance);
-
-	void (*send_flush) (
-		struct totemrrp_instance *instance);
-
-	void (*iface_check) (
-		struct totemrrp_instance *instance);
-
-	void (*processor_count_set) (
-		struct totemrrp_instance *instance,
-		unsigned int processor_count);
-
-	void (*token_target_set) (
-		struct totemrrp_instance *instance,
-		struct totem_ip_address *token_target,
-		unsigned int iface_no);
-
-	void (*ring_reenable) (
-		struct totemrrp_instance *instance,
-		unsigned int iface_no);
-
-	int (*mcast_recv_empty) (
-		struct totemrrp_instance *instance);
-
-	int (*member_add) (
-		struct totemrrp_instance *instance,
-		const struct totem_ip_address *member,
-		unsigned int iface_no);
-
-	int (*member_remove) (
-		struct totemrrp_instance *instance,
-		const struct totem_ip_address *member,
-		unsigned int iface_no);
-
-	void (*membership_changed) (
-		struct totemrrp_instance *instance,
-	        enum totem_configuration_type configuration_type,
-		const struct srp_addr *member_list, size_t member_list_entries,
-		const struct srp_addr *left_list, size_t left_list_entries,
-		const struct srp_addr *joined_list, size_t joined_list_entries,
-		const struct memb_ring_id *ring_id);
-};
-
-#define STATUS_STR_LEN 512
-struct totemrrp_instance {
-	qb_loop_t *poll_handle;
-
-	struct totem_interface *interfaces;
-
-	struct rrp_algo *rrp_algo;
-
-	void *context;
-
-	char *status[INTERFACE_MAX];
-
-	void (*totemrrp_deliver_fn) (
-		void *context,
-		const void *msg,
-		unsigned int msg_len);
-
-	void (*totemrrp_iface_change_fn) (
-		void *context,
-		const struct totem_ip_address *iface_addr,
-		unsigned int iface_no);
-
-	void (*totemrrp_token_seqid_get) (
-		const void *msg,
-		unsigned int *seqid,
-		unsigned int *token_is);
-
-	void (*totemrrp_target_set_completed) (
-		void *context);
-
-	unsigned int (*totemrrp_msgs_missing) (void);
-
-	/*
-	 * Function and data used to log messages
-	 */
-	int totemrrp_log_level_security;
-
-	int totemrrp_log_level_error;
-
-	int totemrrp_log_level_warning;
-
-	int totemrrp_log_level_notice;
-
-	int totemrrp_log_level_debug;
-
-	int totemrrp_subsys_id;
-
-	void (*totemrrp_log_printf) (
-		int level,
-		int subsys,
-		const char *function,
-		const char *file,
-		int line,
-		const char *format, ...)__attribute__((format(printf, 6, 7)));
-
-	void **net_handles;
-
-	void *rrp_algo_instance;
-
-	int interface_count;
-
-	int processor_count;
-
-	int my_nodeid;
-
-	struct totem_config *totem_config;
-
-	void *deliver_fn_context[INTERFACE_MAX];
-
-	qb_loop_timer_handle timer_active_test_ring_timeout[INTERFACE_MAX];
-
-	totemrrp_stats_t stats;
-};
-
-static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
-		unsigned int iface_no, int is_faulty);
-
-/*
- * None Replication Forward Declerations
- */
-static void none_mcast_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len);
-
-static void none_mcast_noflush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void none_mcast_flush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void none_token_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len,
-	unsigned int token_seqid);
-
-static void none_token_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void none_recv_flush (
-	struct totemrrp_instance *instance);
-
-static void none_send_flush (
-	struct totemrrp_instance *instance);
-
-static void none_iface_check (
-	struct totemrrp_instance *instance);
-
-static void none_processor_count_set (
-	struct totemrrp_instance *instance,
-	unsigned int processor_count_set);
-
-static void none_token_target_set (
-	struct totemrrp_instance *instance,
-	struct totem_ip_address *token_target,
-	unsigned int iface_no);
-
-static void none_ring_reenable (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no);
-
-static int none_mcast_recv_empty (
-	struct totemrrp_instance *instance);
-
-static int none_member_add (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no);
-
-static int none_member_remove (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no);
-
-static void none_membership_changed (
-	struct totemrrp_instance *instance,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id);
-
-/*
- * Passive Replication Forward Declerations
- */
-static void *passive_instance_initialize (
-	struct totemrrp_instance *rrp_instance,
-	int interface_count);
-
-static void passive_mcast_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len);
-
-static void passive_mcast_noflush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void passive_mcast_flush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void passive_monitor (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	int is_token_recv_count);
-
-static void passive_token_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len,
-	unsigned int token_seqid);
-
-static void passive_token_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void passive_recv_flush (
-	struct totemrrp_instance *instance);
-
-static void passive_send_flush (
-	struct totemrrp_instance *instance);
-
-static void passive_iface_check (
-	struct totemrrp_instance *instance);
-
-static void passive_processor_count_set (
-	struct totemrrp_instance *instance,
-	unsigned int processor_count_set);
-
-static void passive_token_target_set (
-	struct totemrrp_instance *instance,
-	struct totem_ip_address *token_target,
-	unsigned int iface_no);
-
-static void passive_ring_reenable (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no);
-
-static int passive_mcast_recv_empty (
-	struct totemrrp_instance *instance);
-
-static int passive_member_add (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no);
-
-static int passive_member_remove (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no);
-
-static void passive_membership_changed (
-	struct totemrrp_instance *instance,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id);
-
-/*
- * Active Replication Forward Definitions
- */
-static void *active_instance_initialize (
-	struct totemrrp_instance *rrp_instance,
-	int interface_count);
-
-static void active_mcast_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len);
-
-static void active_mcast_noflush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void active_mcast_flush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void active_token_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len,
-	unsigned int token_seqid);
-
-static void active_token_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len);
-
-static void active_recv_flush (
-	struct totemrrp_instance *instance);
-
-static void active_send_flush (
-	struct totemrrp_instance *instance);
-
-static void active_iface_check (
-	struct totemrrp_instance *instance);
-
-static void active_processor_count_set (
-	struct totemrrp_instance *instance,
-	unsigned int processor_count_set);
-
-static void active_token_target_set (
-	struct totemrrp_instance *instance,
-	struct totem_ip_address *token_target,
-	unsigned int iface_no);
-
-static void active_ring_reenable (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no);
-
-static int active_mcast_recv_empty (
-	struct totemrrp_instance *instance);
-
-static int active_member_add (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no);
-
-static int active_member_remove (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no);
-
-static void active_membership_changed (
-	struct totemrrp_instance *instance,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id);
-
-static void active_timer_expired_token_start (
-	struct active_instance *active_instance);
-
-static void active_timer_expired_token_cancel (
-	struct active_instance *active_instance);
-
-static void active_timer_problem_decrementer_start (
-	struct active_instance *active_instance);
-
-static void active_timer_problem_decrementer_cancel (
-	struct active_instance *active_instance);
-
-/*
- * 0-5 reserved for totemsrp.c
- */
-#define MESSAGE_TYPE_RING_TEST_ACTIVE		6
-#define MESSAGE_TYPE_RING_TEST_ACTIVATE		7
-
-#define ENDIAN_LOCAL				0xff22
-
-/*
- * Rollover handling:
- *
- * ARR_SEQNO_START_TOKEN is the starting sequence number of last seen sequence
- * for a token for active redundand ring.  This should remain zero, unless testing
- * overflow in which case 07fffff00 or 0xffffff00 are good starting values.
- * It should be same as on defined in totemsrp.c
- */
-
-#define ARR_SEQNO_START_TOKEN 0x0
-
-/*
- * These can be used ot test different rollover points
- * #define ARR_SEQNO_START_MSG 0xfffffe00
- */
-
-/*
- * Threshold value when recv_count for passive rrp should be adjusted.
- * Set this value to some smaller for testing of adjusting proper
- * functionality. Also keep in mind that this value must be smaller
- * then rrp_problem_count_threshold
- */
-#define PASSIVE_RECV_COUNT_THRESHOLD		(INT_MAX / 2)
-
-struct message_header {
-	char type;
-	char encapsulated;
-	unsigned short endian_detector;
-	int ring_number;
-	int nodeid_activator;
-} __attribute__((packed));
-
-struct deliver_fn_context {
-	struct totemrrp_instance *instance;
-	void *context;
-	int iface_no;
-};
-
-struct rrp_algo none_algo = {
-	.name			= "none",
-	.initialize		= NULL,
-	.mcast_recv		= none_mcast_recv,
-	.mcast_noflush_send	= none_mcast_noflush_send,
-	.mcast_flush_send	= none_mcast_flush_send,
-	.token_recv		= none_token_recv,
-	.token_send		= none_token_send,
-	.recv_flush		= none_recv_flush,
-	.send_flush		= none_send_flush,
-	.iface_check		= none_iface_check,
-	.processor_count_set	= none_processor_count_set,
-	.token_target_set	= none_token_target_set,
-	.ring_reenable		= none_ring_reenable,
-	.mcast_recv_empty	= none_mcast_recv_empty,
-	.member_add		= none_member_add,
-	.member_remove		= none_member_remove,
-	.membership_changed	= none_membership_changed
-};
-
-struct rrp_algo passive_algo = {
-	.name			= "passive",
-	.initialize		= passive_instance_initialize,
-	.mcast_recv		= passive_mcast_recv,
-	.mcast_noflush_send	= passive_mcast_noflush_send,
-	.mcast_flush_send	= passive_mcast_flush_send,
-	.token_recv		= passive_token_recv,
-	.token_send		= passive_token_send,
-	.recv_flush		= passive_recv_flush,
-	.send_flush		= passive_send_flush,
-	.iface_check		= passive_iface_check,
-	.processor_count_set	= passive_processor_count_set,
-	.token_target_set	= passive_token_target_set,
-	.ring_reenable		= passive_ring_reenable,
-	.mcast_recv_empty	= passive_mcast_recv_empty,
-	.member_add		= passive_member_add,
-	.member_remove		= passive_member_remove,
-	.membership_changed	= passive_membership_changed
-};
-
-struct rrp_algo active_algo = {
-	.name			= "active",
-	.initialize		= active_instance_initialize,
-	.mcast_recv		= active_mcast_recv,
-	.mcast_noflush_send	= active_mcast_noflush_send,
-	.mcast_flush_send	= active_mcast_flush_send,
-	.token_recv		= active_token_recv,
-	.token_send		= active_token_send,
-	.recv_flush		= active_recv_flush,
-	.send_flush		= active_send_flush,
-	.iface_check		= active_iface_check,
-	.processor_count_set	= active_processor_count_set,
-	.token_target_set	= active_token_target_set,
-	.ring_reenable		= active_ring_reenable,
-	.mcast_recv_empty	= active_mcast_recv_empty,
-	.member_add		= active_member_add,
-	.member_remove		= active_member_remove,
-	.membership_changed	= active_membership_changed
-};
-
-struct rrp_algo *rrp_algos[] = {
-	&none_algo,
-	&passive_algo,
-	&active_algo
-};
-
-#define RRP_ALGOS_COUNT 3
-
-#define log_printf(level, format, args...)			\
-do {								\
-	rrp_instance->totemrrp_log_printf (			\
-		level, rrp_instance->totemrrp_subsys_id,	\
-		__FUNCTION__, __FILE__, __LINE__,		\
-		format, ##args);				\
-} while (0);
-
-static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
-		unsigned int iface_no, int is_faulty)
-{
-	rrp_instance->stats.faulty[iface_no] = (is_faulty ? 1 : 0);
-}
-
-static void test_active_msg_endian_convert(const struct message_header *in, struct message_header *out)
-{
-	out->type = in->type;
-	out->encapsulated = in->encapsulated;
-	out->endian_detector = ENDIAN_LOCAL;
-	out->ring_number = swab32 (in->ring_number);
-	out->nodeid_activator = swab32(in->nodeid_activator);
-}
-
-static void timer_function_test_ring_timeout (void *context)
-{
-	struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
-	struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
-	unsigned int *faulty = NULL;
-	int iface_no = deliver_fn_context->iface_no;
-	struct message_header msg = {
-		.type = MESSAGE_TYPE_RING_TEST_ACTIVE,
-		.endian_detector = ENDIAN_LOCAL,
-	};
-
-	if (strcmp(rrp_instance->totem_config->rrp_mode, "active") == 0)
-		faulty = ((struct active_instance *)(rrp_instance->rrp_algo_instance))->faulty;
-	if (strcmp(rrp_instance->totem_config->rrp_mode, "passive") == 0)
-		faulty = ((struct passive_instance *)(rrp_instance->rrp_algo_instance))->faulty;
-
-	assert (faulty != NULL);
-
-	if (faulty[iface_no] == 1) {
-		msg.ring_number = iface_no;
-		msg.nodeid_activator = rrp_instance->my_nodeid;
-		totemnet_token_send (
-			rrp_instance->net_handles[iface_no],
-			&msg, sizeof (struct message_header));
-		qb_loop_timer_add (rrp_instance->poll_handle,
-			QB_LOOP_MED,
-			rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
-			(void *)deliver_fn_context,
-			timer_function_test_ring_timeout,
-			&rrp_instance->timer_active_test_ring_timeout[iface_no]);
-	}
-}
-
-/*
- * None Replication Implementation
- */
-
-static void none_mcast_recv (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	rrp_instance->totemrrp_deliver_fn (
-		context,
-		msg,
-		msg_len);
-}
-
-static void none_mcast_flush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	totemnet_mcast_flush_send (instance->net_handles[0], msg, msg_len);
-}
-
-static void none_mcast_noflush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	totemnet_mcast_noflush_send (instance->net_handles[0], msg, msg_len);
-}
-
-static void none_token_recv (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len,
-	unsigned int token_seq)
-{
-	rrp_instance->totemrrp_deliver_fn (
-		context,
-		msg,
-		msg_len);
-}
-
-static void none_token_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	totemnet_token_send (
-		instance->net_handles[0],
-		msg, msg_len);
-}
-
-static void none_recv_flush (struct totemrrp_instance *instance)
-{
-	totemnet_recv_flush (instance->net_handles[0]);
-}
-
-static void none_send_flush (struct totemrrp_instance *instance)
-{
-	totemnet_send_flush (instance->net_handles[0]);
-}
-
-static void none_iface_check (struct totemrrp_instance *instance)
-{
-	totemnet_iface_check (instance->net_handles[0]);
-}
-
-static void none_processor_count_set (
-	struct totemrrp_instance *instance,
-	unsigned int processor_count)
-{
-	totemnet_processor_count_set (instance->net_handles[0],
-		processor_count);
-}
-
-static void none_token_target_set (
-	struct totemrrp_instance *instance,
-	struct totem_ip_address *token_target,
-	unsigned int iface_no)
-{
-	totemnet_token_target_set (instance->net_handles[0], token_target);
-}
-
-static void none_ring_reenable (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no)
-{
-	/*
-	 * No operation
-	 */
-}
-
-static int none_mcast_recv_empty (
-	struct totemrrp_instance *instance)
-{
-	int res;
-
-	res = totemnet_recv_mcast_empty (instance->net_handles[0]);
-
-	return (res);
-}
-
-static int none_member_add (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no)
-{
-	int res;
-	res = totemnet_member_add (instance->net_handles[0], member);
-	return (res);
-}
-
-static int none_member_remove (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no)
-{
-	int res;
-	res = totemnet_member_remove (instance->net_handles[0], member);
-	return (res);
-}
-
-static void none_membership_changed (
-	struct totemrrp_instance *rrp_instance,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id)
-{
-	int i;
-
-	for (i = 0; i < left_list_entries; i++) {
-		if (left_list->no_addrs < 1 ||
-		    (left_list[i].addr[0].family != AF_INET && left_list[i].addr[0].family != AF_INET6)) {
-			log_printf(rrp_instance->totemrrp_log_level_error,
-				"Membership left list contains incorrect address. "
-				"This is sign of misconfiguration between nodes!");
-		} else {
-			totemnet_member_set_active(rrp_instance->net_handles[0],
-			    &left_list[i].addr[0], 0);
-		}
-	}
-
-	for (i = 0; i < joined_list_entries; i++) {
-		if (joined_list->no_addrs < 1 ||
-		    (joined_list[i].addr[0].family != AF_INET && joined_list[i].addr[0].family != AF_INET6)) {
-			log_printf(rrp_instance->totemrrp_log_level_error,
-				"Membership join list contains incorrect address. "
-				"This is sign of misconfiguration between nodes!");
-		} else {
-			totemnet_member_set_active(rrp_instance->net_handles[0],
-			    &joined_list[i].addr[0], 1);
-		}
-	}
-}
-
-/*
- * Passive Replication Implementation
- */
-void *passive_instance_initialize (
-	struct totemrrp_instance *rrp_instance,
-	int interface_count)
-{
-	struct passive_instance *instance;
-	int i;
-
-	instance = malloc (sizeof (struct passive_instance));
-	if (instance == 0) {
-		goto error_exit;
-	}
-	memset (instance, 0, sizeof (struct passive_instance));
-
-	instance->faulty = malloc (sizeof (int) * interface_count);
-	if (instance->faulty == 0) {
-		free (instance);
-		instance = 0;
-		goto error_exit;
-	}
-	memset (instance->faulty, 0, sizeof (int) * interface_count);
-
-	for (i = 0; i < interface_count; i++) {
-		stats_set_interface_faulty (rrp_instance, i, 0);
-	}
-
-	instance->token_recv_count = malloc (sizeof (int) * interface_count);
-	if (instance->token_recv_count == 0) {
-		free (instance->faulty);
-		free (instance);
-		instance = 0;
-		goto error_exit;
-	}
-	memset (instance->token_recv_count, 0, sizeof (int) * interface_count);
-
-	instance->mcast_recv_count = malloc (sizeof (int) * interface_count);
-	if (instance->mcast_recv_count == 0) {
-		free (instance->token_recv_count);
-		free (instance->faulty);
-		free (instance);
-		instance = 0;
-		goto error_exit;
-	}
-	memset (instance->mcast_recv_count, 0, sizeof (int) * interface_count);
-
-error_exit:
-	return ((void *)instance);
-}
-
-static void timer_function_passive_token_expired (void *context)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)context;
-	struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
-
-	rrp_instance->totemrrp_deliver_fn (
-		passive_instance->totemrrp_context,
-		passive_instance->token,
-		passive_instance->token_len);
-}
-
-/* TODO
-static void timer_function_passive_problem_decrementer (void *context)
-{
-//	struct passive_instance *passive_instance = (struct passive_instance *)context;
-//	struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
-
-}
-*/
-
-
-static void passive_timer_expired_token_start (
-	struct passive_instance *passive_instance)
-{
-        qb_loop_timer_add (
-		passive_instance->rrp_instance->poll_handle,
-		QB_LOOP_MED,
-		passive_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
-		(void *)passive_instance,
-		timer_function_passive_token_expired,
-		&passive_instance->timer_expired_token);
-}
-
-static void passive_timer_expired_token_cancel (
-	struct passive_instance *passive_instance)
-{
-        qb_loop_timer_del (
-		passive_instance->rrp_instance->poll_handle,
-		passive_instance->timer_expired_token);
-}
-
-/*
-static void passive_timer_problem_decrementer_start (
-	struct passive_instance *passive_instance)
-{
-        qb_loop_timer_add (
-		QB_LOOP_MED,
-		passive_instance->rrp_instance->poll_handle,
-		passive_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
-		(void *)passive_instance,
-		timer_function_passive_problem_decrementer,
-		&passive_instance->timer_problem_decrementer);
-}
-
-static void passive_timer_problem_decrementer_cancel (
-	struct passive_instance *passive_instance)
-{
-        qb_loop_timer_del (
-		passive_instance->rrp_instance->poll_handle,
-		passive_instance->timer_problem_decrementer);
-}
-*/
-
-/*
- * Monitor function implementation from rrp paper.
- * rrp_instance is passive rrp instance, iface_no is interface with received messgae/token and
- * is_token_recv_count is boolean variable which donates if message is token (>1) or regular
- * message (= 0)
- */
-static void passive_monitor (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	int is_token_recv_count)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
-	unsigned int *recv_count;
-	unsigned int max;
-	unsigned int i;
-	unsigned int min_all, min_active;
-	unsigned int threshold;
-
-	/*
-	 * Monitor for failures
-	 */
-	if (is_token_recv_count) {
-		recv_count = passive_instance->token_recv_count;
-		threshold = rrp_instance->totem_config->rrp_problem_count_threshold;
-	} else {
-		recv_count = passive_instance->mcast_recv_count;
-		threshold = rrp_instance->totem_config->rrp_problem_count_mcast_threshold;
-	}
-
-	recv_count[iface_no] += 1;
-
-	max = 0;
-	for (i = 0; i < rrp_instance->interface_count; i++) {
-		if (max < recv_count[i]) {
-			max = recv_count[i];
-		}
-	}
-
-	/*
-	 * Max is larger then threshold -> start adjusting process
-	 */
-	if (max > PASSIVE_RECV_COUNT_THRESHOLD) {
-		min_all = min_active = recv_count[iface_no];
-
-		for (i = 0; i < rrp_instance->interface_count; i++) {
-			if (recv_count[i] < min_all) {
-				min_all = recv_count[i];
-			}
-
-			if (passive_instance->faulty[i] == 0 &&
-			    recv_count[i] < min_active) {
-				min_active = recv_count[i];
-			}
-		}
-
-		if (min_all > 0) {
-			/*
-			 * There is one or more faulty device with recv_count > 0
-			 */
-			for (i = 0; i < rrp_instance->interface_count; i++) {
-				recv_count[i] -= min_all;
-			}
-		} else {
-			/*
-			 * No faulty device with recv_count > 0, adjust only active
-			 * devices
-			 */
-			for (i = 0; i < rrp_instance->interface_count; i++) {
-				if (passive_instance->faulty[i] == 0) {
-					recv_count[i] -= min_active;
-				}
-			}
-		}
-
-		/*
-		 * Find again max
-		 */
-		max = 0;
-
-		for (i = 0; i < rrp_instance->interface_count; i++) {
-			if (max < recv_count[i]) {
-				max = recv_count[i];
-			}
-		}
-	}
-
-	for (i = 0; i < rrp_instance->interface_count; i++) {
-		if ((passive_instance->faulty[i] == 0) &&
-		    (max - recv_count[i] > threshold)) {
-			passive_instance->faulty[i] = 1;
-
-			qb_loop_timer_add (rrp_instance->poll_handle,
-				QB_LOOP_MED,
-				rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
-				rrp_instance->deliver_fn_context[i],
-				timer_function_test_ring_timeout,
-				&rrp_instance->timer_active_test_ring_timeout[i]);
-
-			stats_set_interface_faulty (rrp_instance, i, passive_instance->faulty[i]);
-
-			snprintf (rrp_instance->status[i], STATUS_STR_LEN,
-				"Marking ringid %u interface %s FAULTY",
-				i,
-				totemnet_iface_print (rrp_instance->net_handles[i]));
-			log_printf (
-				rrp_instance->totemrrp_log_level_error,
-				"%s",
-				rrp_instance->status[i]);
-		}
-	}
-}
-
-static void passive_mcast_recv (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
-
-	rrp_instance->totemrrp_deliver_fn (
-		context,
-		msg,
-		msg_len);
-
-	if (rrp_instance->totemrrp_msgs_missing() == 0 &&
-		passive_instance->timer_expired_token) {
-		/*
-		 * Delivers the last token
-		 */
-		rrp_instance->totemrrp_deliver_fn (
-			passive_instance->totemrrp_context,
-			passive_instance->token,
-			passive_instance->token_len);
-		passive_timer_expired_token_cancel (passive_instance);
-	}
-
-	passive_monitor (rrp_instance, iface_no, 0);
-}
-
-static void passive_mcast_flush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	int i = 0;
-
-	do {
-		passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
-		i++;
-	} while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
-
-	if (i <= instance->interface_count) {
-		totemnet_mcast_flush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
-	}
-}
-
-static void passive_mcast_noflush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	int i = 0;
-
-	do {
-		passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
-		i++;
-	} while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
-
-	if (i <= instance->interface_count) {
-		totemnet_mcast_noflush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
-	}
-}
-
-static void passive_token_recv (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len,
-	unsigned int token_seq)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
-
-	passive_instance->totemrrp_context = context; // this should be in totemrrp_instance ? TODO
-
-	if (rrp_instance->totemrrp_msgs_missing() == 0) {
-		rrp_instance->totemrrp_deliver_fn (
-			context,
-			msg,
-			msg_len);
-	} else {
-		memcpy (passive_instance->token, msg, msg_len);
-		passive_timer_expired_token_start (passive_instance);
-
-	}
-
-	passive_monitor (rrp_instance, iface_no, 1);
-}
-
-static void passive_token_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	int i = 0;
-
-	do {
-		passive_instance->token_xmit_iface = (passive_instance->token_xmit_iface + 1) % instance->interface_count;
-		i++;
-	} while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->token_xmit_iface] == 1));
-
-	if (i <= instance->interface_count) {
-		totemnet_token_send (
-		    instance->net_handles[passive_instance->token_xmit_iface],
-		    msg, msg_len);
-	}
-
-}
-
-static void passive_recv_flush (struct totemrrp_instance *instance)
-{
-	struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_recv_flush (instance->net_handles[i]);
-		}
-	}
-}
-
-static void passive_send_flush (struct totemrrp_instance *instance)
-{
-	struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_send_flush (instance->net_handles[i]);
-		}
-	}
-}
-
-static void passive_iface_check (struct totemrrp_instance *instance)
-{
-	struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_iface_check (instance->net_handles[i]);
-		}
-	}
-}
-
-static void passive_processor_count_set (
-	struct totemrrp_instance *instance,
-	unsigned int processor_count)
-{
-	struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_processor_count_set (instance->net_handles[i],
-				processor_count);
-		}
-	}
-}
-
-static void passive_token_target_set (
-	struct totemrrp_instance *instance,
-	struct totem_ip_address *token_target,
-	unsigned int iface_no)
-{
-	totemnet_token_target_set (instance->net_handles[iface_no], token_target);
-}
-
-static int passive_mcast_recv_empty (
-	struct totemrrp_instance *instance)
-{
-	int res;
-	int msgs_emptied = 0;
-	int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		res = totemnet_recv_mcast_empty (instance->net_handles[i]);
-		if (res == -1) {
-			return (-1);
-		}
-		if (res == 1) {
-			msgs_emptied = 1;
-		}
-	}
-
-	return (msgs_emptied);
-}
-
-static int passive_member_add (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no)
-{
-	int res;
-	res = totemnet_member_add (instance->net_handles[iface_no], member);
-	return (res);
-}
-
-static int passive_member_remove (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no)
-{
-	int res;
-	res = totemnet_member_remove (instance->net_handles[iface_no], member);
-	return (res);
-}
-
-static void passive_membership_changed (
-	struct totemrrp_instance *rrp_instance,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id)
-{
-	int i;
-	int interface;
-
-	for (interface = 0; interface < rrp_instance->interface_count; interface++) {
-		for (i = 0; i < left_list_entries; i++) {
-			if (left_list->no_addrs < interface + 1 ||
-			    (left_list[i].addr[interface].family != AF_INET &&
-			     left_list[i].addr[interface].family != AF_INET6)) {
-				log_printf(rrp_instance->totemrrp_log_level_error,
-					"Membership left list contains incorrect address. "
-					"This is sign of misconfiguration between nodes!");
-			} else {
-				totemnet_member_set_active(rrp_instance->net_handles[interface],
-				    &left_list[i].addr[interface], 0);
-			}
-		}
-
-		for (i = 0; i < joined_list_entries; i++) {
-			if (joined_list->no_addrs < interface + 1 ||
-			    (joined_list[i].addr[interface].family != AF_INET &&
-			     joined_list[i].addr[interface].family != AF_INET6)) {
-				log_printf(rrp_instance->totemrrp_log_level_error,
-					"Membership join list contains incorrect address. "
-					"This is sign of misconfiguration between nodes!");
-			} else {
-				totemnet_member_set_active(rrp_instance->net_handles[interface],
-				    &joined_list[i].addr[interface], 1);
-			}
-		}
-	}
-}
-
-static void passive_ring_reenable (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no)
-{
-	struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
-	int i;
-
-	memset (rrp_algo_instance->mcast_recv_count, 0, sizeof (unsigned int) *
-		instance->interface_count);
-	memset (rrp_algo_instance->token_recv_count, 0, sizeof (unsigned int) *
-		instance->interface_count);
-
-	if (iface_no == instance->interface_count) {
-		memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
-			instance->interface_count);
-		for (i = 0; i < instance->interface_count; i++) {
-			stats_set_interface_faulty (instance, i, 0);
-		}
-	} else {
-		rrp_algo_instance->faulty[iface_no] = 0;
-		stats_set_interface_faulty (instance, iface_no, 0);
-	}
-}
-
-/*
- * Active Replication Implementation
- */
-void *active_instance_initialize (
-	struct totemrrp_instance *rrp_instance,
-	int interface_count)
-{
-	struct active_instance *instance;
-	int i;
-
-	instance = malloc (sizeof (struct active_instance));
-	if (instance == 0) {
-		goto error_exit;
-	}
-	memset (instance, 0, sizeof (struct active_instance));
-
-	instance->faulty = malloc (sizeof (int) * interface_count);
-	if (instance->faulty == 0) {
-		free (instance);
-		instance = 0;
-		goto error_exit;
-	}
-	memset (instance->faulty, 0, sizeof (unsigned int) * interface_count);
-
-	for (i = 0; i < interface_count; i++) {
-		stats_set_interface_faulty (rrp_instance, i, 0);
-	}
-
-	instance->last_token_recv = malloc (sizeof (int) * interface_count);
-	if (instance->last_token_recv == 0) {
-		free (instance->faulty);
-		free (instance);
-		instance = 0;
-		goto error_exit;
-	}
-	memset (instance->last_token_recv, 0, sizeof (unsigned int) * interface_count);
-
-	instance->counter_problems = malloc (sizeof (int) * interface_count);
-	if (instance->counter_problems == 0) {
-		free (instance->last_token_recv);
-		free (instance->faulty);
-		free (instance);
-		instance = 0;
-		goto error_exit;
-	}
-	memset (instance->counter_problems, 0, sizeof (unsigned int) * interface_count);
-
-	instance->timer_expired_token = 0;
-
-	instance->timer_problem_decrementer = 0;
-
-	instance->rrp_instance = rrp_instance;
-
-	instance->last_token_seq = ARR_SEQNO_START_TOKEN - 1;
-
-error_exit:
-	return ((void *)instance);
-}
-static void timer_function_active_problem_decrementer (void *context)
-{
-	struct active_instance *active_instance = (struct active_instance *)context;
-	struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
-	unsigned int problem_found = 0;
-	unsigned int i;
-
-	for (i = 0; i < rrp_instance->interface_count; i++) {
-		if (active_instance->counter_problems[i] > 0) {
-			problem_found = 1;
-			active_instance->counter_problems[i] -= 1;
-			if (active_instance->counter_problems[i] == 0) {
-				snprintf (rrp_instance->status[i], STATUS_STR_LEN,
-					"ring %d active with no faults", i);
-			} else {
-				snprintf (rrp_instance->status[i], STATUS_STR_LEN,
-					"Decrementing problem counter for iface %s to [%d of %d]",
-					totemnet_iface_print (rrp_instance->net_handles[i]),
-					active_instance->counter_problems[i],
-					rrp_instance->totem_config->rrp_problem_count_threshold);
-			}
-				log_printf (
-					rrp_instance->totemrrp_log_level_warning,
-					"%s",
-					rrp_instance->status[i]);
-		}
-	}
-	if (problem_found) {
-		active_timer_problem_decrementer_start (active_instance);
-	} else {
-		active_instance->timer_problem_decrementer = 0;
-	}
-}
-
-static void timer_function_active_token_expired (void *context)
-{
-	struct active_instance *active_instance = (struct active_instance *)context;
-	struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
-	unsigned int i;
-
-	for (i = 0; i < rrp_instance->interface_count; i++) {
-		if (active_instance->last_token_recv[i] == 0) {
-			active_instance->counter_problems[i] += 1;
-
-			if (active_instance->timer_problem_decrementer == 0) {
-				active_timer_problem_decrementer_start (active_instance);
-			}
-			snprintf (rrp_instance->status[i], STATUS_STR_LEN,
-				"Incrementing problem counter for seqid %d iface %s to [%d of %d]",
-				active_instance->last_token_seq,
-				totemnet_iface_print (rrp_instance->net_handles[i]),
-				active_instance->counter_problems[i],
-				rrp_instance->totem_config->rrp_problem_count_threshold);
-			log_printf (
-				rrp_instance->totemrrp_log_level_warning,
-				"%s",
-				rrp_instance->status[i]);
-		}
-	}
-	for (i = 0; i < rrp_instance->interface_count; i++) {
-		if (active_instance->counter_problems[i] >= rrp_instance->totem_config->rrp_problem_count_threshold &&
-		    active_instance->faulty[i] == 0) {
-			active_instance->faulty[i] = 1;
-
-			qb_loop_timer_add (rrp_instance->poll_handle,
-				QB_LOOP_MED,
-				rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
-				rrp_instance->deliver_fn_context[i],
-				timer_function_test_ring_timeout,
-				&rrp_instance->timer_active_test_ring_timeout[i]);
-
-			stats_set_interface_faulty (rrp_instance, i, active_instance->faulty[i]);
-
-			snprintf (rrp_instance->status[i], STATUS_STR_LEN,
-				"Marking seqid %d ringid %u interface %s FAULTY",
-				active_instance->last_token_seq,
-				i,
-				totemnet_iface_print (rrp_instance->net_handles[i]));
-			log_printf (
-				rrp_instance->totemrrp_log_level_error,
-				"%s",
-				rrp_instance->status[i]);
-			active_timer_problem_decrementer_cancel (active_instance);
-		}
-	}
-
-	rrp_instance->totemrrp_deliver_fn (
-		active_instance->totemrrp_context,
-		active_instance->token,
-		active_instance->token_len);
-}
-
-static void active_timer_expired_token_start (
-	struct active_instance *active_instance)
-{
-        qb_loop_timer_add (
-		active_instance->rrp_instance->poll_handle,
-		QB_LOOP_MED,
-		active_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
-		(void *)active_instance,
-		timer_function_active_token_expired,
-		&active_instance->timer_expired_token);
-}
-
-static void active_timer_expired_token_cancel (
-	struct active_instance *active_instance)
-{
-        qb_loop_timer_del (
-		active_instance->rrp_instance->poll_handle,
-		active_instance->timer_expired_token);
-}
-
-static void active_timer_problem_decrementer_start (
-	struct active_instance *active_instance)
-{
-        qb_loop_timer_add (
-		active_instance->rrp_instance->poll_handle,
-		QB_LOOP_MED,
-		active_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
-		(void *)active_instance,
-		timer_function_active_problem_decrementer,
-		&active_instance->timer_problem_decrementer);
-}
-
-static void active_timer_problem_decrementer_cancel (
-	struct active_instance *active_instance)
-{
-        qb_loop_timer_del (
-		active_instance->rrp_instance->poll_handle,
-		active_instance->timer_problem_decrementer);
-	active_instance->timer_problem_decrementer = 0;
-}
-
-
-/*
- * active replication
- */
-static void active_mcast_recv (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	instance->totemrrp_deliver_fn (
-		context,
-		msg,
-		msg_len);
-}
-
-static void active_mcast_flush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	int i;
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-			totemnet_mcast_flush_send (instance->net_handles[i], msg, msg_len);
-		}
-	}
-}
-
-static void active_mcast_noflush_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	int i;
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-			totemnet_mcast_noflush_send (instance->net_handles[i], msg, msg_len);
-		}
-	}
-}
-
-static void active_token_recv (
-	struct totemrrp_instance *rrp_instance,
-	unsigned int iface_no,
-	void *context,
-	const void *msg,
-	unsigned int msg_len,
-	unsigned int token_seq)
-{
-	int i;
-	struct active_instance *active_instance = (struct active_instance *)rrp_instance->rrp_algo_instance;
-
-	active_instance->totemrrp_context = context;
-	if (sq_lt_compare (active_instance->last_token_seq, token_seq)) {
-		memcpy (active_instance->token, msg, msg_len);
-		active_instance->token_len = msg_len;
-		for (i = 0; i < rrp_instance->interface_count; i++) {
-			active_instance->last_token_recv[i] = 0;
-		}
-
-		active_instance->last_token_recv[iface_no] = 1;
-		active_timer_expired_token_start (active_instance);
-	}
-
-	/*
-	 * This doesn't follow spec because the spec assumes we will know
-	 * when token resets occur.
-	 */
-	active_instance->last_token_seq = token_seq;
-
-	if (token_seq == active_instance->last_token_seq) {
-		active_instance->last_token_recv[iface_no] = 1;
-		for (i = 0; i < rrp_instance->interface_count; i++) {
-			if ((active_instance->last_token_recv[i] == 0) &&
-				active_instance->faulty[i] == 0) {
-				return; /* don't deliver token */
-			}
-		}
-		active_timer_expired_token_cancel (active_instance);
-
-		rrp_instance->totemrrp_deliver_fn (
-			context,
-			msg,
-			msg_len);
-	}
-}
-
-static void active_token_send (
-	struct totemrrp_instance *instance,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-	int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-			totemnet_token_send (
-				instance->net_handles[i],
-				msg, msg_len);
-
-		}
-	}
-}
-
-static void active_recv_flush (struct totemrrp_instance *instance)
-{
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_recv_flush (instance->net_handles[i]);
-		}
-	}
-}
-
-static void active_send_flush (struct totemrrp_instance *instance)
-{
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_send_flush (instance->net_handles[i]);
-		}
-	}
-}
-
-static int active_member_add (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no)
-{
-	int res;
-	res = totemnet_member_add (instance->net_handles[iface_no], member);
-	return (res);
-}
-
-static int active_member_remove (
-	struct totemrrp_instance *instance,
-	const struct totem_ip_address *member,
-	unsigned int iface_no)
-{
-	int res;
-	res = totemnet_member_remove (instance->net_handles[iface_no], member);
-	return (res);
-}
-
-static void active_membership_changed (
-	struct totemrrp_instance *rrp_instance,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id)
-{
-	int i;
-	int interface;
-
-	for (interface = 0; interface < rrp_instance->interface_count; interface++) {
-		for (i = 0; i < left_list_entries; i++) {
-			if (left_list->no_addrs < interface + 1 ||
-			    (left_list[i].addr[interface].family != AF_INET &&
-			     left_list[i].addr[interface].family != AF_INET6)) {
-				log_printf(rrp_instance->totemrrp_log_level_error,
-					"Membership left list contains incorrect address. "
-					"This is sign of misconfiguration between nodes!");
-			} else {
-				totemnet_member_set_active(rrp_instance->net_handles[interface],
-				    &left_list[i].addr[interface], 0);
-			}
-		}
-
-		for (i = 0; i < joined_list_entries; i++) {
-			if (joined_list->no_addrs < interface + 1 ||
-			    (joined_list[i].addr[interface].family != AF_INET &&
-			     joined_list[i].addr[interface].family != AF_INET6)) {
-				log_printf(rrp_instance->totemrrp_log_level_error,
-					"Membership join list contains incorrect address. "
-					"This is sign of misconfiguration between nodes!");
-			} else {
-				totemnet_member_set_active(rrp_instance->net_handles[interface],
-				    &joined_list[i].addr[interface], 1);
-			}
-		}
-	}
-}
-
-static void active_iface_check (struct totemrrp_instance *instance)
-{
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_iface_check (instance->net_handles[i]);
-		}
-	}
-}
-
-static void active_processor_count_set (
-	struct totemrrp_instance *instance,
-	unsigned int processor_count)
-{
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-	unsigned int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		if (rrp_algo_instance->faulty[i] == 0) {
-
-			totemnet_processor_count_set (instance->net_handles[i],
-				processor_count);
-		}
-	}
-}
-
-static void active_token_target_set (
-	struct totemrrp_instance *instance,
-	struct totem_ip_address *token_target,
-	unsigned int iface_no)
-{
-	totemnet_token_target_set (instance->net_handles[iface_no], token_target);
-}
-
-static int active_mcast_recv_empty (
-	struct totemrrp_instance *instance)
-{
-	int res;
-	int msgs_emptied = 0;
-	int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		res = totemnet_recv_mcast_empty (instance->net_handles[i]);
-		if (res == -1) {
-			return (-1);
-		}
-		if (res == 1) {
-			msgs_emptied = 1;
-		}
-	}
-
-	return (msgs_emptied);
-}
-
-static void active_ring_reenable (
-	struct totemrrp_instance *instance,
-	unsigned int iface_no)
-{
-	struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
-	int i;
-
-	if (iface_no == instance->interface_count) {
-		memset (rrp_algo_instance->last_token_recv, 0, sizeof (unsigned int) *
-			instance->interface_count);
-		memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
-			instance->interface_count);
-		memset (rrp_algo_instance->counter_problems, 0, sizeof (unsigned int) *
-			instance->interface_count);
-
-		for (i = 0; i < instance->interface_count; i++) {
-			stats_set_interface_faulty (instance, i, 0);
-		}
-	} else {
-		rrp_algo_instance->last_token_recv[iface_no] = 0;
-		rrp_algo_instance->faulty[iface_no] = 0;
-		rrp_algo_instance->counter_problems[iface_no] = 0;
-
-		stats_set_interface_faulty (instance, iface_no, 0);
-	}
-}
-
-static void totemrrp_instance_initialize (struct totemrrp_instance *instance)
-{
-	memset (instance, 0, sizeof (struct totemrrp_instance));
-}
-
-static int totemrrp_algorithm_set (
-	struct totem_config *totem_config,
-	struct totemrrp_instance *instance)
-{
-	unsigned int res = -1;
-	unsigned int i;
-
-	for (i = 0; i < RRP_ALGOS_COUNT; i++) {
-		if (strcmp (totem_config->rrp_mode, rrp_algos[i]->name) == 0) {
-			instance->rrp_algo = rrp_algos[i];
-			if (rrp_algos[i]->initialize) {
-				instance->rrp_algo_instance = rrp_algos[i]->initialize (
-					instance,
-					totem_config->interface_count);
-			}
-			res = 0;
-			break;
-		}
-	}
-	for (i = 0; i < totem_config->interface_count; i++) {
-		instance->status[i] = malloc (STATUS_STR_LEN+1);
-		snprintf (instance->status[i], STATUS_STR_LEN,
-			  "ring %d active with no faults", i);
-	}
-	return (res);
-}
-
-void rrp_deliver_fn (
-	void *context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	unsigned int token_seqid;
-	unsigned int token_is;
-
-	struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
-	struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
-	const struct message_header *hdr = msg;
-	struct message_header tmp_msg, activate_msg;
-
-	memset(&tmp_msg, 0, sizeof(struct message_header));
-	memset(&activate_msg, 0, sizeof(struct message_header));
-
-	rrp_instance->totemrrp_token_seqid_get (
-		msg,
-		&token_seqid,
-		&token_is);
-
-	if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVE) {
-		log_printf (
-			rrp_instance->totemrrp_log_level_debug,
-			"received message requesting test of ring now active");
-
-		if (hdr->endian_detector != ENDIAN_LOCAL) {
-			test_active_msg_endian_convert(hdr, &tmp_msg);
-			hdr = &tmp_msg;
-		}
-
-		if (hdr->nodeid_activator == rrp_instance->my_nodeid) {
-			/*
-			 * Send an activate message
-			 */
-			activate_msg.type = MESSAGE_TYPE_RING_TEST_ACTIVATE;
-			activate_msg.endian_detector = ENDIAN_LOCAL;
-			activate_msg.ring_number = hdr->ring_number;
-			activate_msg.nodeid_activator = rrp_instance->my_nodeid;
-			totemnet_token_send (
-				rrp_instance->net_handles[deliver_fn_context->iface_no],
-				&activate_msg, sizeof (struct message_header));
-		} else {
-			/*
-			 * Send a ring test message
-			 */
-			totemnet_token_send (
-				rrp_instance->net_handles[deliver_fn_context->iface_no],
-				msg, msg_len);
-		}
-	} else 
-	if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVATE) {
-
-		if (hdr->endian_detector != ENDIAN_LOCAL) {
-			test_active_msg_endian_convert(hdr, &tmp_msg);
-			hdr = &tmp_msg;
-		}
-
-		log_printf (
-			rrp_instance->totemrrp_log_level_debug,
-			"Received ring test activate message for ring %d sent by node %u",
-			hdr->ring_number,
-			hdr->nodeid_activator);
-
-		if (rrp_instance->stats.faulty[deliver_fn_context->iface_no]) {
-			log_printf (rrp_instance->totemrrp_log_level_notice,
-			    "Automatically recovered ring %d", hdr->ring_number);
-		}
-
-		totemrrp_ring_reenable (rrp_instance, deliver_fn_context->iface_no);
-		if (hdr->nodeid_activator != rrp_instance->my_nodeid) {
-			totemnet_token_send (
-				rrp_instance->net_handles[deliver_fn_context->iface_no],
-				msg, msg_len);
-		}
-	} else 
-	if (token_is) {
-		/*
-		 * Deliver to the token receiver for this rrp algorithm
-		 */
-		rrp_instance->rrp_algo->token_recv (
-			rrp_instance,
-			deliver_fn_context->iface_no,
-			deliver_fn_context->context,
-			msg,
-			msg_len,
-			token_seqid);
-	} else {
-		/*
-		 * Deliver to the mcast receiver for this rrp algorithm
-		 */
-		rrp_instance->rrp_algo->mcast_recv (
-			rrp_instance,
-			deliver_fn_context->iface_no,
-			deliver_fn_context->context,
-			msg,
-			msg_len);
-	}
-}
-
-void rrp_iface_change_fn (
-	void *context,
-	const struct totem_ip_address *iface_addr)
-{
-	struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
-
-	deliver_fn_context->instance->my_nodeid = iface_addr->nodeid;
-	deliver_fn_context->instance->totemrrp_iface_change_fn (
-		deliver_fn_context->context,
-		iface_addr,
-		deliver_fn_context->iface_no);
-}
-
-int totemrrp_finalize (
-	void *rrp_context)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int i;
-
-	for (i = 0; i < instance->interface_count; i++) {
-		totemnet_finalize (instance->net_handles[i]);
-	}
-	free (instance->net_handles);
-	free (instance);
-	return (0);
-}
-
-static void rrp_target_set_completed (void *context)
-{
-	struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
-
-	deliver_fn_context->instance->totemrrp_target_set_completed (deliver_fn_context->context);
-}
-
-/*
- * Totem Redundant Ring interface
- * depends on poll abstraction, POSIX, IPV4
- */
-
-/*
- * Create an instance
- */
-int totemrrp_initialize (
-	qb_loop_t *poll_handle,
-	void **rrp_context,
-	struct totem_config *totem_config,
-	totemsrp_stats_t *stats,
-	void *context,
-
-	void (*deliver_fn) (
-		void *context,
-		const void *msg,
-		unsigned int msg_len),
-
-	void (*iface_change_fn) (
-		void *context,
-		const struct totem_ip_address *iface_addr,
-		unsigned int iface_no),
-
-	void (*token_seqid_get) (
-		const void *msg,
-		unsigned int *seqid,
-		unsigned int *token_is),
-
-	unsigned int (*msgs_missing) (void),
-
-	void (*target_set_completed) (void *context))
-{
-	struct totemrrp_instance *instance;
-	unsigned int res;
-	int i;
-
-	instance = malloc (sizeof (struct totemrrp_instance));
-	if (instance == 0) {
-		return (-1);
-	}
-
-	totemrrp_instance_initialize (instance);
-
-	instance->totem_config = totem_config;
-	stats->rrp = &instance->stats;
-	instance->stats.interface_count = totem_config->interface_count;
-	instance->stats.faulty = calloc(instance->stats.interface_count, sizeof(uint8_t));
-
-	res = totemrrp_algorithm_set (
-		instance->totem_config,
-		instance);
-	if (res == -1) {
-		goto error_destroy;
-	}
-
-	/*
-	* Configure logging
-	*/
-	instance->totemrrp_log_level_security = totem_config->totem_logging_configuration.log_level_security;
-	instance->totemrrp_log_level_error = totem_config->totem_logging_configuration.log_level_error;
-	instance->totemrrp_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
-	instance->totemrrp_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
-	instance->totemrrp_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
-	instance->totemrrp_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
-	instance->totemrrp_log_printf = totem_config->totem_logging_configuration.log_printf;
-
-	instance->interfaces = totem_config->interfaces;
-
-	instance->poll_handle = poll_handle;
-
-	instance->totemrrp_deliver_fn = deliver_fn;
-
-	instance->totemrrp_iface_change_fn = iface_change_fn;
-
-	instance->totemrrp_token_seqid_get = token_seqid_get;
-
-	instance->totemrrp_target_set_completed = target_set_completed;
-
-	instance->totemrrp_msgs_missing = msgs_missing;
-
-	instance->interface_count = totem_config->interface_count;
-
-	instance->net_handles = malloc (sizeof (void *) * totem_config->interface_count);
-
-	instance->context = context;
-
-	instance->poll_handle = poll_handle;
-
-
-	for (i = 0; i < totem_config->interface_count; i++) {
-		struct deliver_fn_context *deliver_fn_context;
-
-		deliver_fn_context = malloc (sizeof (struct deliver_fn_context));
-		assert (deliver_fn_context);
-		deliver_fn_context->instance = instance;
-		deliver_fn_context->context = context;
-		deliver_fn_context->iface_no = i;
-		instance->deliver_fn_context[i] = (void *)deliver_fn_context;
-
-		totemnet_initialize (
-			poll_handle,
-			&instance->net_handles[i],
-			totem_config,
-			stats,
-			i,
-			(void *)deliver_fn_context,
-			rrp_deliver_fn,
-			rrp_iface_change_fn,
-			rrp_target_set_completed);
-
-		totemnet_net_mtu_adjust (instance->net_handles[i], totem_config);
-	}
-
-	*rrp_context = instance;
-	return (0);
-
-error_destroy:
-	free (instance);
-	return (res);
-}
-
-void *totemrrp_buffer_alloc (void *rrp_context)
-{
-	struct totemrrp_instance *instance = rrp_context;
-	assert (instance != NULL);
-	return totemnet_buffer_alloc (instance->net_handles[0]);
-}
-
-void totemrrp_buffer_release (void *rrp_context, void *ptr)
-{
-	struct totemrrp_instance *instance = rrp_context;
-	assert (instance != NULL);
-	totemnet_buffer_release (instance->net_handles[0], ptr);
-}
-
-int totemrrp_processor_count_set (
-	void *rrp_context,
-	unsigned int processor_count)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	instance->rrp_algo->processor_count_set (instance, processor_count);
-
-	instance->processor_count = processor_count;
-
-	return (0);
-}
-
-int totemrrp_token_target_set (
-	void *rrp_context,
-	struct totem_ip_address *addr,
-	unsigned int iface_no)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	instance->rrp_algo->token_target_set (instance, addr, iface_no);
-
-	return (0);
-}
-int totemrrp_recv_flush (void *rrp_context)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-
-	instance->rrp_algo->recv_flush (instance);
-
-	return (0);
-}
-
-int totemrrp_send_flush (void *rrp_context)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	instance->rrp_algo->send_flush (instance);
-
-	return (0);
-}
-
-int totemrrp_token_send (
-	void *rrp_context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	instance->rrp_algo->token_send (instance, msg, msg_len);
-
-	return (0);
-}
-
-int totemrrp_mcast_flush_send (
-	void *rrp_context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int res = 0;
-
-// TODO this needs to return the result
-	instance->rrp_algo->mcast_flush_send (instance, msg, msg_len);
-
-	return (res);
-}
-
-int totemrrp_mcast_noflush_send (
-	void *rrp_context,
-	const void *msg,
-	unsigned int msg_len)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	/*
-	 * merge detects go out through mcast_flush_send so it is safe to
-	 * flush these messages if we are only one processor.  This avoids
-	 * an encryption/hmac and decryption/hmac
-	 */
-	if (instance->processor_count > 1) {
-
-// TODO this needs to return the result
-		instance->rrp_algo->mcast_noflush_send (instance, msg, msg_len);
-	}
-
-	return (0);
-}
-
-int totemrrp_iface_check (void *rrp_context)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-
-	instance->rrp_algo->iface_check (instance);
-
-	return (0);
-}
-
-int totemrrp_ifaces_get (
-	void *rrp_context,
-	char ***status,
-	unsigned int *iface_count)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	*status = instance->status;
-
-	if (iface_count) {
-		*iface_count = instance->interface_count;
-	}
-
-	return (0);
-}
-
-int totemrrp_crypto_set (
-	void *rrp_context,
-	const char *cipher_type,
-	const char *hash_type)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int res;
-
-	res = totemnet_crypto_set(instance->net_handles[0], cipher_type, hash_type);
-
-	return (res);
-}
-
-
-/*
- * iface_no indicates the interface number [0, ..., interface_count-1] of the
- * specific ring which will be reenabled. We specify iface_no == interface_count
- * means reenabling all the rings.
- */
-int totemrrp_ring_reenable (
-        void *rrp_context,
-	unsigned int iface_no)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int res = 0;
-	unsigned int i;
-
-	instance->rrp_algo->ring_reenable (instance, iface_no);
-
-	if (iface_no == instance->interface_count) {
-		for (i = 0; i < instance->interface_count; i++) {
-			snprintf (instance->status[i], STATUS_STR_LEN,
-				"ring %d active with no faults", i);
-		}
-	} else {
-		snprintf (instance->status[iface_no], STATUS_STR_LEN,
-			"ring %d active with no faults", iface_no);
-	}
-
-	return (res);
-}
-
-extern int totemrrp_mcast_recv_empty (
-	void *rrp_context)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int res;
-
-	res = instance->rrp_algo->mcast_recv_empty (instance);
-
-	return (res);
-}
-
-int totemrrp_member_add (
-        void *rrp_context,
-        const struct totem_ip_address *member,
-	int iface_no)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int res;
-
-	res = instance->rrp_algo->member_add (instance, member, iface_no);
-
-	return (res);
-}
-
-int totemrrp_member_remove (
-        void *rrp_context,
-        const struct totem_ip_address *member,
-	int iface_no)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-	int res;
-
-	res = instance->rrp_algo->member_remove (instance, member, iface_no);
-
-	return (res);
-}
-
-void totemrrp_membership_changed (
-        void *rrp_context,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id)
-{
-	struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
-
-	instance->rrp_algo->membership_changed (instance,
-	    configuration_type,
-	    member_list, member_list_entries,
-	    left_list, left_list_entries,
-	    joined_list, joined_list_entries,
-	    ring_id);
-}

exec/totemrrp.h

@@ -1,170 +0,0 @@
-/*
- * Copyright (c) 2005 MontaVista Software, Inc.
- * Copyright (c) 2006-2007, 2009 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Steven Dake (sdake@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/**
- * @file
- * Totem Network interface - also does encryption/decryption
- *
- * depends on poll abstraction, POSIX, IPV4
- */
-
-#ifndef TOTEMRRP_H_DEFINED
-#define TOTEMRRP_H_DEFINED
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <qb/qbloop.h>
-#include <corosync/totem/totem.h>
-
-#define TOTEMRRP_NOFLUSH	0
-#define TOTEMRRP_FLUSH		1
-
-/*
- * SRP address. Used mainly in totemsrp.c, but we need it here to inform RRP about
- * membership change.
- */
-struct srp_addr {
-	uint8_t no_addrs;
-	struct totem_ip_address addr[INTERFACE_MAX];
-};
-
-/**
- * Create an instance
- */
-extern int totemrrp_initialize (
-	qb_loop_t *poll_handle,
-	void **rrp_context,
-	struct totem_config *totem_config,
-	totemsrp_stats_t *stats,
-	void *context,
-
-	void (*deliver_fn) (
-		void *context,
-		const void *msg,
-		unsigned int msg_len),
-
-	void (*iface_change_fn) (
-		void *context,
-		const struct totem_ip_address *iface_addr,
-		unsigned int iface_no),
-
-	void (*token_seqid_get) (
-		const void *msg,
-		unsigned int *seqid,
-		unsigned int *token_is),
-
-	unsigned int (*msgs_missing) (void),
-
-	void (*target_set_completed) (
-		void *context)
-	);
-
-extern void *totemrrp_buffer_alloc (
-	void *rrp_context);
-
-extern void totemrrp_buffer_release (
-	void *rrp_context,
-	void *ptr);
-
-extern int totemrrp_processor_count_set (
-	void *rrp_context,
-	unsigned int processor_count);
-
-extern int totemrrp_token_send (
-	void *rrp_context,
-	const void *msg,
-	unsigned int msg_len);
-
-extern int totemrrp_mcast_noflush_send (
-	void *rrp_context,
-	const void *msg,
-	unsigned int msg_len);
-
-extern int totemrrp_mcast_flush_send (
-	void *rrp_context,
-	const void *msg,
-	unsigned int msg_len);
-
-extern int totemrrp_recv_flush (
-	void *rrp_context);
-
-extern int totemrrp_send_flush (
-	void *rrp_context);
-
-extern int totemrrp_token_target_set (
-	void *rrp_context,
-	struct totem_ip_address *target,
-	unsigned int iface_no);
-
-extern int totemrrp_iface_check (void *rrp_context);
-
-extern int totemrrp_finalize (void *rrp_context);
-
-extern int totemrrp_ifaces_get (
-	void *rrp_context,
-	char ***status,
-	unsigned int *iface_count);
-
-extern int totemrrp_crypto_set (
-	void *rrp_context,
-	const char *cipher_type,
-	const char *hash_type);
-
-extern int totemrrp_ring_reenable (
-	void *rrp_context,
-	unsigned int iface_no);
-
-extern int totemrrp_mcast_recv_empty (
-	void *rrp_context);
-
-extern int totemrrp_member_add (
-        void *net_context,
-        const struct totem_ip_address *member,
-	int iface_no);
-
-extern int totemrrp_member_remove (
-        void *net_context,
-        const struct totem_ip_address *member,
-	int iface_no);
-
-extern void totemrrp_membership_changed (
-	void *rrp_context,
-	enum totem_configuration_type configuration_type,
-	const struct srp_addr *member_list, size_t member_list_entries,
-	const struct srp_addr *left_list, size_t left_list_entries,
-	const struct srp_addr *joined_list, size_t joined_list_entries,
-	const struct memb_ring_id *ring_id);
-
-#endif /* TOTEMRRP_H_DEFINED */

exec/totemsrp.c

@@ -87,7 +87,6 @@
 #include <corosync/logsys.h>
 
 #include "totemsrp.h"
-#include "totemrrp.h"
 #include "totemnet.h"
 
 #include "cs_queue.h"
@@ -101,6 +100,15 @@
 #define TOKEN_SIZE_MAX				64000 /* bytes */
 #define LEAVE_DUMMY_NODEID                      0
 
+/*
+ * SRP address.
+ * CC: TODO: Can we remove IP address from this and just use nodeids?
+ */
+struct srp_addr {
+	uint8_t no_addrs;
+	struct totem_ip_address addr[INTERFACE_MAX];
+};
+
 /*
  * Rollover handling:
  * SEQNO_START_MSG is the starting sequence number after a new configuration
@@ -173,16 +181,8 @@ struct totemsrp_socket {
 	int token;
 };
 
-struct message_header {
-	char type;
-	char encapsulated;
-	unsigned short endian_detector;
-	unsigned int nodeid;
-} __attribute__((packed));
-
-
 struct mcast {
-	struct message_header header;
+	struct totem_message_header header;
 	struct srp_addr system_from;
 	unsigned int seq;
 	int this_seqno;
@@ -199,7 +199,7 @@ struct rtr_item  {
 
 
 struct orf_token {
-	struct message_header header;
+	struct totem_message_header header;
 	unsigned int seq;
 	unsigned int token_seq;
 	unsigned int aru;
@@ -214,7 +214,7 @@ struct orf_token {
 
 
 struct memb_join {
-	struct message_header header;
+	struct totem_message_header header;
 	struct srp_addr system_from;
 	unsigned int proc_list_entries;
 	unsigned int failed_list_entries;
@@ -229,14 +229,14 @@ struct memb_join {
 
 
 struct memb_merge_detect {
-	struct message_header header;
+	struct totem_message_header header;
 	struct srp_addr system_from;
 	struct memb_ring_id ring_id;
 } __attribute__((packed));
 
 
 struct token_hold_cancel {
-	struct message_header header;
+	struct totem_message_header header;
 	struct memb_ring_id ring_id;
 } __attribute__((packed));
 
@@ -250,7 +250,7 @@ struct memb_commit_token_memb_entry {
 
 
 struct memb_commit_token {
-	struct message_header header;
+	struct totem_message_header header;
 	unsigned int token_seq;
 	struct memb_ring_id ring_id;
 	unsigned int retrans_flg;
@@ -317,7 +317,7 @@ struct totemsrp_instance {
 	struct srp_addr my_left_memb_list[PROCESSOR_COUNT_MAX];
 
 	unsigned int my_leave_memb_list[PROCESSOR_COUNT_MAX];
-	
+
 	int my_proc_list_entries;
 
 	int my_failed_list_entries;
@@ -331,7 +331,7 @@ struct totemsrp_instance {
 	int my_deliver_memb_entries;
 
 	int my_left_memb_entries;
-	
+
 	int my_leave_memb_entries;
 
 	struct memb_ring_id my_ring_id;
@@ -492,7 +492,7 @@ struct totemsrp_instance {
 
 	struct timeval tv_old;
 
-	void *totemrrp_context;
+	void *totemnet_context;
 
 	struct totem_config *totem_config;
 
@@ -519,7 +519,7 @@ struct totemsrp_instance {
 	uint32_t waiting_trans_ack;
 
 	int 	flushing;
-	
+
 	void * token_recv_event_handle;
 	void * token_sent_event_handle;
 	char commit_token_storage[40000];
@@ -614,13 +614,6 @@ static int message_handler_token_hold_cancel (
 
 static void totemsrp_instance_initialize (struct totemsrp_instance *instance);
 
-static unsigned int main_msgs_missing (void);
-
-static void main_token_seqid_get (
-	const void *msg,
-	unsigned int *seqid,
-	unsigned int *token_is);
-
 static void srp_addr_copy (struct srp_addr *dest, const struct srp_addr *src);
 
 static void srp_addr_to_nodeid (
@@ -748,27 +741,6 @@ static void totemsrp_instance_initialize (struct totemsrp_instance *instance)
 	instance->waiting_trans_ack = 1;
 }
 
-static void main_token_seqid_get (
-	const void *msg,
-	unsigned int *seqid,
-	unsigned int *token_is)
-{
-	const struct orf_token *token = msg;
-
-	*seqid = 0;
-	*token_is = 0;
-	if (token->header.type == MESSAGE_TYPE_ORF_TOKEN) {
-		*seqid = token->token_seq;
-		*token_is = 1;
-	}
-}
-
-static unsigned int main_msgs_missing (void)
-{
-// TODO
-	return (0);
-}
-
 static int pause_flush (struct totemsrp_instance *instance)
 {
 	uint64_t now_msec;
@@ -785,7 +757,7 @@ static int pause_flush (struct totemsrp_instance *instance)
 		 * -1 indicates an error from recvmsg
 		 */
 		do {
-			res = totemrrp_mcast_recv_empty (instance->totemrrp_context);
+			res = totemnet_recv_mcast_empty (instance->totemnet_context);
 		} while (res == -1);
 	}
 	return (res);
@@ -826,6 +798,17 @@ static int token_event_stats_collector (enum totem_callback_token_type type, con
 	return 0;
 }
 
+static void totempg_mtu_changed(void *context, int net_mtu)
+{
+	struct totemsrp_instance *instance = context;
+
+	instance->totem_config->net_mtu = net_mtu - sizeof (struct mcast);
+
+	log_printf (instance->totemsrp_log_level_debug,
+		    "Net MTU changed to %d, new value is %d",
+		    net_mtu, instance->totem_config->net_mtu);
+}
+
 /*
  * Exported interfaces
  */
@@ -833,7 +816,7 @@ int totemsrp_initialize (
 	qb_loop_t *poll_handle,
 	void **srp_context,
 	struct totem_config *totem_config,
-	totemmrp_stats_t *stats,
+	totempg_stats_t *stats,
 
 	void (*deliver_fn) (
 		unsigned int nodeid,
@@ -923,23 +906,6 @@ int totemsrp_initialize (
 
 	log_printf (instance->totemsrp_log_level_debug,
 		"send threads (%d threads)", totem_config->threads);
-	log_printf (instance->totemsrp_log_level_debug,
-		"RRP token expired timeout (%d ms)",
-		totem_config->rrp_token_expired_timeout);
-	log_printf (instance->totemsrp_log_level_debug,
-		"RRP token problem counter (%d ms)",
-		totem_config->rrp_problem_count_timeout);
-	log_printf (instance->totemsrp_log_level_debug,
-		"RRP threshold (%d problem count)",
-		totem_config->rrp_problem_count_threshold);
-	log_printf (instance->totemsrp_log_level_debug,
-		"RRP multicast threshold (%d problem count)",
-		totem_config->rrp_problem_count_mcast_threshold);
-	log_printf (instance->totemsrp_log_level_debug,
-		"RRP automatic recovery check timeout (%d ms)",
-		totem_config->rrp_autorecovery_check_timeout);
-	log_printf (instance->totemsrp_log_level_debug,
-		"RRP mode set to %s.", instance->totem_config->rrp_mode);
 
 	log_printf (instance->totemsrp_log_level_debug,
 		"heartbeat_failures_allowed (%d)", totem_config->heartbeat_failures_allowed);
@@ -993,20 +959,19 @@ int totemsrp_initialize (
 		}
 	}
 
-	totemrrp_initialize (
+	totemnet_initialize (
 		poll_handle,
-		&instance->totemrrp_context,
+		&instance->totemnet_context,
 		totem_config,
 		stats->srp,
 		instance,
 		main_deliver_fn,
 		main_iface_change_fn,
-		main_token_seqid_get,
-		main_msgs_missing,
+		totempg_mtu_changed,
 		target_set_completed);
 
 	/*
-	 * Must have net_mtu adjusted by totemrrp_initialize first
+	 * Must have net_mtu adjusted by totemnet_initialize first
 	 */
 	cs_queue_init (&instance->new_message_queue,
 		MESSAGE_QUEUE_MAX,
@@ -1040,9 +1005,8 @@ void totemsrp_finalize (
 {
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context;
 
-
 	memb_leave_message_send (instance);
-	totemrrp_finalize (instance->totemrrp_context);
+	totemnet_finalize (instance->totemnet_context);
 	cs_queue_free (&instance->new_message_queue);
 	cs_queue_free (&instance->new_message_queue_trans);
 	cs_queue_free (&instance->retrans_message_queue);
@@ -1113,7 +1077,7 @@ int totemsrp_ifaces_get (
 	}
 
 finish:
-	totemrrp_ifaces_get (instance->totemrrp_context, status, NULL);
+	totemnet_ifaces_get(instance->totemnet_context, status, iface_count);
 	return (res);
 }
 
@@ -1125,7 +1089,7 @@ int totemsrp_crypto_set (
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context;
 	int res;
 
-	res = totemrrp_crypto_set(instance->totemrrp_context, cipher_type, hash_type);
+	res = totemnet_crypto_set(instance->totemnet_context, cipher_type, hash_type);
 
 	return (res);
 }
@@ -1153,16 +1117,10 @@ int totemsrp_my_family_get (
 	return (res);
 }
 
-
 int totemsrp_ring_reenable (
         void *srp_context)
 {
-	struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context;
-
-	totemrrp_ring_reenable (instance->totemrrp_context,
-		instance->totem_config->interface_count);
-
-	return (0);
+       return (0);
 }
 
 
@@ -1532,13 +1490,13 @@ static void my_leave_memb_set(
 static void *totemsrp_buffer_alloc (struct totemsrp_instance *instance)
 {
 	assert (instance != NULL);
-	return totemrrp_buffer_alloc (instance->totemrrp_context);
+	return totemnet_buffer_alloc (instance->totemnet_context);
 }
 
 static void totemsrp_buffer_release (struct totemsrp_instance *instance, void *ptr)
 {
 	assert (instance != NULL);
-	totemrrp_buffer_release (instance->totemrrp_context, ptr);
+	totemnet_buffer_release (instance->totemnet_context, ptr);
 }
 
 static void reset_token_retransmit_timeout (struct totemsrp_instance *instance)
@@ -1758,7 +1716,7 @@ static void timer_function_orf_token_timeout (void *data)
 				"The token was lost in the OPERATIONAL state.");
 			log_printf (instance->totemsrp_log_level_notice,
 				"A processor failed, forming new configuration.");
-			totemrrp_iface_check (instance->totemrrp_context);
+			totemnet_iface_check (instance->totemnet_context);
 			memb_state_gather_enter (instance, TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_OPERATIONAL_STATE);
 			instance->stats.operational_token_lost++;
 			break;
@@ -1964,16 +1922,6 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 	instance->last_released = 0;
 	instance->my_set_retrans_flg = 0;
 
-	/*
-	 * Inform RRP about transitional change
-	 */
-	totemrrp_membership_changed (
-		instance->totemrrp_context,
-		TOTEM_CONFIGURATION_TRANSITIONAL,
-		instance->my_trans_memb_list, instance->my_trans_memb_entries,
-		instance->my_left_memb_list, instance->my_left_memb_entries,
-		NULL, 0,
-		&instance->my_ring_id);
 	/*
 	 * Deliver transitional configuration to application
 	 */
@@ -1994,16 +1942,6 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 
 	instance->my_aru = aru_save;
 
-	/*
-	 * Inform RRP about regular membership change
-	 */
-	totemrrp_membership_changed (
-		instance->totemrrp_context,
-		TOTEM_CONFIGURATION_REGULAR,
-		instance->my_new_memb_list, instance->my_new_memb_entries,
-		NULL, 0,
-		joined_list, joined_list_entries,
-		&instance->my_ring_id);
 	/*
 	 * Deliver regular configuration to application
 	 */
@@ -2107,7 +2045,7 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 					sptr2 += snprintf(failed_node_msg, sizeof(failed_node_msg)-sptr2, " failed:");
 				}
 				sptr2 += snprintf(failed_node_msg+sptr2, sizeof(left_node_msg)-sptr2, " %u", left_list[i]);
-			}		
+			}
 		}
 		if (sptr2 == 0) {
 			failed_node_msg[0] = '\0';
@@ -2319,8 +2257,8 @@ static void memb_state_recovery_enter (
 	/*
 	 * Build regular configuration
 	 */
-	totemrrp_processor_count_set (
-		instance->totemrrp_context,
+	totemnet_processor_count_set (
+		instance->totemnet_context,
 		commit_token->addr_entries);
 
 	/*
@@ -2607,8 +2545,8 @@ static int orf_token_remcast (
 
 	sort_queue_item = ptr;
 
-	totemrrp_mcast_noflush_send (
-		instance->totemrrp_context,
+	totemnet_mcast_noflush_send (
+		instance->totemnet_context,
 		sort_queue_item->mcast,
 		sort_queue_item->msg_len);
 
@@ -2760,8 +2698,8 @@ static int orf_token_mcast (
 		 */
 		sq_item_add (sort_queue, &sort_queue_item, message_item->mcast->seq);
 
-		totemrrp_mcast_noflush_send (
-			instance->totemrrp_context,
+		totemnet_mcast_noflush_send (
+			instance->totemnet_context,
 			message_item->mcast,
 			message_item->msg_len);
 
@@ -2921,7 +2859,7 @@ static int orf_token_rtr (
 
 static void token_retransmit (struct totemsrp_instance *instance)
 {
-	totemrrp_token_send (instance->totemrrp_context,
+	totemnet_token_send (instance->totemnet_context,
 		instance->orf_token_retransmit,
 		instance->orf_token_retransmit_size);
 }
@@ -3005,7 +2943,7 @@ static int token_send (
 		return (0);
 	}
 
-	totemrrp_token_send (instance->totemrrp_context,
+	totemnet_token_send (instance->totemnet_context,
 		orf_token,
 		orf_token_size);
 
@@ -3037,7 +2975,7 @@ static int token_hold_cancel_send (struct totemsrp_instance *instance)
 
 	instance->stats.token_hold_cancel_tx++;
 
-	totemrrp_mcast_flush_send (instance->totemrrp_context, &token_hold_cancel,
+	totemnet_mcast_flush_send (instance->totemnet_context, &token_hold_cancel,
 		sizeof (struct token_hold_cancel));
 
 	return (0);
@@ -3153,17 +3091,14 @@ static void memb_state_commit_token_target_set (
 	struct totemsrp_instance *instance)
 {
 	struct srp_addr *addr;
-	unsigned int i;
 
 	addr = (struct srp_addr *)instance->commit_token->end_of_commit_token;
 
-	for (i = 0; i < instance->totem_config->interface_count; i++) {
-		totemrrp_token_target_set (
-			instance->totemrrp_context,
-			&addr[instance->commit_token->memb_index %
-				instance->commit_token->addr_entries].addr[i],
-			i);
-	}
+	/* Totemnet just looks at the node id */
+	totemnet_token_target_set (
+		instance->totemnet_context,
+		&addr[instance->commit_token->memb_index %
+		      instance->commit_token->addr_entries].addr[0]);
 }
 
 static int memb_state_commit_token_send_recovery (
@@ -3185,7 +3120,7 @@ static int memb_state_commit_token_send_recovery (
 
 	instance->stats.memb_commit_token_tx++;
 
-	totemrrp_token_send (instance->totemrrp_context,
+	totemnet_token_send (instance->totemnet_context,
 		commit_token,
 		commit_token_size);
 
@@ -3214,7 +3149,7 @@ static int memb_state_commit_token_send (
 
 	instance->stats.memb_commit_token_tx++;
 
-	totemrrp_token_send (instance->totemrrp_context,
+	totemnet_token_send (instance->totemnet_context,
 		instance->commit_token,
 		commit_token_size);
 
@@ -3349,8 +3284,8 @@ static void memb_join_message_send (struct totemsrp_instance *instance)
 
 	instance->stats.memb_join_tx++;
 
-	totemrrp_mcast_flush_send (
-		instance->totemrrp_context,
+	totemnet_mcast_flush_send (
+		instance->totemnet_context,
 		memb_join,
 		addr_idx);
 }
@@ -3419,8 +3354,8 @@ static void memb_leave_message_send (struct totemsrp_instance *instance)
 	}
 	instance->stats.memb_join_tx++;
 
-	totemrrp_mcast_flush_send (
-		instance->totemrrp_context,
+	totemnet_mcast_flush_send (
+		instance->totemnet_context,
 		memb_join,
 		addr_idx);
 }
@@ -3439,7 +3374,7 @@ static void memb_merge_detect_transmit (struct totemsrp_instance *instance)
 	assert (memb_merge_detect.header.nodeid);
 
 	instance->stats.memb_merge_detect_tx++;
-	totemrrp_mcast_flush_send (instance->totemrrp_context,
+	totemnet_mcast_flush_send (instance->totemnet_context,
 		&memb_merge_detect,
 		sizeof (struct memb_merge_detect));
 }
@@ -3716,7 +3651,7 @@ static int message_handler_orf_token (
 	}
 #endif
 	instance->flushing = 1;
-	totemrrp_recv_flush (instance->totemrrp_context);
+	totemnet_recv_flush (instance->totemnet_context);
 	instance->flushing = 0;
 
 	/*
@@ -3906,7 +3841,7 @@ printf ("token seq %d\n", token->seq);
 				}
 			}
 
-			totemrrp_send_flush (instance->totemrrp_context);
+			totemnet_send_flush (instance->totemnet_context);
 			token_send (instance, token, forward_token);
 
 #ifdef GIVEINFO
@@ -4253,7 +4188,7 @@ static void memb_join_process (
 		if (instance->flushing) {
 			if (memb_join->header.nodeid == LEAVE_DUMMY_NODEID) {
 				log_printf (instance->totemsrp_log_level_warning,
-			    		"Discarding LEAVE message during flush, nodeid=%u", 
+			    		"Discarding LEAVE message during flush, nodeid=%u",
 						memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid : LEAVE_DUMMY_NODEID);
 				if (memb_join->failed_list_entries > 0) {
 					my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid);
@@ -4272,7 +4207,7 @@ static void memb_join_process (
 				}
 			}
 		}
-		
+
 	}
 
 	if (memb_set_equal (proc_list,
@@ -4709,9 +4644,9 @@ void main_deliver_fn (
 	unsigned int msg_len)
 {
 	struct totemsrp_instance *instance = context;
-	const struct message_header *message_header = msg;
+	const struct totem_message_header *message_header = msg;
 
-	if (msg_len < sizeof (struct message_header)) {
+	if (msg_len < sizeof (struct totem_message_header)) {
 		log_printf (instance->totemsrp_log_level_security,
 			    "Received message is too short...  ignoring %u.",
 			    (unsigned int)msg_len);
@@ -4810,12 +4745,12 @@ void totemsrp_service_ready_register (
 int totemsrp_member_add (
         void *context,
         const struct totem_ip_address *member,
-        int ring_no)
+        int link_no)
 {
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)context;
 	int res;
 
-	res = totemrrp_member_add (instance->totemrrp_context, member, ring_no);
+	res = totemnet_member_add (instance->totemnet_context, &instance->my_id.addr[link_no], member, link_no);
 
 	return (res);
 }
@@ -4823,12 +4758,12 @@ int totemsrp_member_add (
 int totemsrp_member_remove (
         void *context,
         const struct totem_ip_address *member,
-        int ring_no)
+        int link_no)
 {
 	struct totemsrp_instance *instance = (struct totemsrp_instance *)context;
 	int res;
 
-	res = totemrrp_member_remove (instance->totemrrp_context, member, ring_no);
+	res = totemnet_member_remove (instance->totemnet_context, member, link_no);
 
 	return (res);
 }

exec/totemsrp.h

@@ -53,7 +53,7 @@ int totemsrp_initialize (
 	qb_loop_t *poll_handle,
 	void **srp_context,
 	struct totem_config *totem_config,
-	totemmrp_stats_t *stats,
+	totempg_stats_t *stats,
 
 	void (*deliver_fn) (
 		unsigned int nodeid,

exec/totemudp.c

@@ -69,7 +69,6 @@
 #include "totemudp.h"
 
 #include "util.h"
-#include "totemcrypto.h"
 
 #include <nss.h>
 #include <pk11pub.h>
@@ -104,8 +103,6 @@ struct totemudp_socket {
 };
 
 struct totemudp_instance {
-	struct crypto_instance *crypto_inst;
-
 	qb_loop_t *totemudp_poll_handle;
 
 	struct totem_interface *totem_interface;
@@ -123,7 +120,8 @@ struct totemudp_instance {
 
 	void (*totemudp_iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address);
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no);
 
 	void (*totemudp_target_set_completed) (void *context);
 
@@ -264,27 +262,12 @@ static inline void ucast_sendmsg (
 {
 	struct msghdr msg_ucast;
 	int res = 0;
-	size_t buf_out_len;
-	unsigned char buf_out[FRAME_SIZE_MAX];
 	struct sockaddr_storage sockaddr;
 	struct iovec iovec;
 	int addrlen;
 
-	/*
-	 * Encrypt and digest the message
-	 */
-	if (crypto_encrypt_and_sign (
-		instance->crypto_inst,
-		(const unsigned char *)msg,
-		msg_len,
-		buf_out,
-		&buf_out_len) != 0) {
-		log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
-		return;
-	}
-
-	iovec.iov_base = (void *)buf_out;
-	iovec.iov_len = buf_out_len;
+	iovec.iov_base = (void*)msg;
+	iovec.iov_len = msg_len;
 
 	/*
 	 * Build unicast message
@@ -332,27 +315,12 @@ static inline void mcast_sendmsg (
 {
 	struct msghdr msg_mcast;
 	int res = 0;
-	size_t buf_out_len;
-	unsigned char buf_out[FRAME_SIZE_MAX];
 	struct iovec iovec;
 	struct sockaddr_storage sockaddr;
 	int addrlen;
 
-	/*
-	 * Encrypt and digest the message
-	 */
-	if (crypto_encrypt_and_sign (
-		instance->crypto_inst,
-		(const unsigned char *)msg,
-		msg_len,
-		buf_out,
-		&buf_out_len) != 0) {
-		log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
-		return;
-	}
-
-	iovec.iov_base = (void *)&buf_out;
-	iovec.iov_len = buf_out_len;
+	iovec.iov_base = (void *)msg;
+	iovec.iov_len = msg_len;
 
 	/*
 	 * Build multicast message
@@ -453,7 +421,6 @@ static int net_deliver_fn (
 	struct iovec *iovec;
 	struct sockaddr_storage system_from;
 	int bytes_received;
-	int res = 0;
 	char *message_type;
 
 	if (instance->flushing == 1) {
@@ -492,17 +459,6 @@ static int net_deliver_fn (
 		instance->stats_recv += bytes_received;
 	}
 
-	/*
-	 * Authenticate and if authenticated, decrypt datagram
-	 */
-	res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
-	if (res == -1) {
-		log_printf (instance->totemudp_log_level_security, "Received message has invalid digest... ignoring.");
-		log_printf (instance->totemudp_log_level_security,
-			"Invalid packet data");
-		iovec->iov_len = FRAME_SIZE_MAX;
-		return 0;
-	}
 	iovec->iov_len = bytes_received;
 
 	/*
@@ -670,7 +626,7 @@ static void timer_function_netif_check_timeout (
 				"The network interface [%s] is now up.",
 				totemip_print (&instance->totem_interface->boundto));
 			instance->netif_state_report = NETIF_STATE_REPORT_DOWN;
-			instance->totemudp_iface_change_fn (instance->context, &instance->my_id);
+			instance->totemudp_iface_change_fn (instance->context, &instance->my_id, 0);
 		}
 		/*
 		 * Add a timer to check for interface going down in single membership
@@ -688,7 +644,7 @@ static void timer_function_netif_check_timeout (
 		if (instance->netif_state_report & NETIF_STATE_REPORT_DOWN) {
 			log_printf (instance->totemudp_log_level_notice,
 				"The network interface is down.");
-			instance->totemudp_iface_change_fn (instance->context, &instance->my_id);
+			instance->totemudp_iface_change_fn (instance->context, &instance->my_id, 0);
 		}
 		instance->netif_state_report = NETIF_STATE_REPORT_UP;
 
@@ -1094,7 +1050,7 @@ static int totemudp_build_sockets (
 }
 
 /*
- * Totem Network interface - also does encryption/decryption
+ * Totem Network interface
  * depends on poll abstraction, POSIX, IPV4
  */
 
@@ -1106,7 +1062,7 @@ int totemudp_initialize (
 	void **udp_context,
 	struct totem_config *totem_config,
 	totemsrp_stats_t *stats,
-	int interface_no,
+
 	void *context,
 
 	void (*deliver_fn) (
@@ -1116,7 +1072,12 @@ int totemudp_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address),
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no),
+
+	void (*mtu_changed) (
+		void *context,
+		int net_mtu),
 
 	void (*target_set_completed) (
 		void *context))
@@ -1144,26 +1105,10 @@ int totemudp_initialize (
 	instance->totemudp_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
 	instance->totemudp_log_printf = totem_config->totem_logging_configuration.log_printf;
 
-	/*
-	* Initialize random number generator for later use to generate salt
-	*/
-	instance->crypto_inst = crypto_init (totem_config->private_key,
-			totem_config->private_key_len,
-			totem_config->crypto_cipher_type,
-			totem_config->crypto_hash_type,
-			instance->totemudp_log_printf,
-			instance->totemudp_log_level_security,
-			instance->totemudp_log_level_notice,
-			instance->totemudp_log_level_error,
-			instance->totemudp_subsys_id);
-	if (instance->crypto_inst == NULL) {
-		free(instance);
-		return (-1);
-	}
 	/*
 	 * Initialize local variables for totemudp
 	 */
-	instance->totem_interface = &totem_config->interfaces[interface_no];
+	instance->totem_interface = &totem_config->interfaces[0];
 	totemip_copy (&instance->mcast_address, &instance->totem_interface->mcast_addr);
 	memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
 
@@ -1317,35 +1262,27 @@ extern int totemudp_iface_check (void *udp_context)
 	return (res);
 }
 
-extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *totem_config)
+int totemudp_ifaces_get (
+	void *net_context,
+	char ***status,
+	unsigned int *iface_count)
 {
+	static char *statuses[INTERFACE_MAX] = {(char*)"OK"};
 
-	assert(totem_config->interface_count > 0);
-
-	totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
-							totem_config->crypto_hash_type) +
-				 totemip_udpip_header_size(totem_config->interfaces[0].bindnet.family);
-}
-
-const char *totemudp_iface_print (void *udp_context)  {
-	struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
-	const char *ret_char;
-
-	ret_char = totemip_print (&instance->my_id);
+	if (status) {
+		*status = statuses;
+	}
+	*iface_count = 1;
 
-	return (ret_char);
+	return (0);
 }
 
-int totemudp_iface_get (
-	void *udp_context,
-	struct totem_ip_address *addr)
+extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *totem_config)
 {
-	struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
-	int res = 0;
 
-	memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
+	assert(totem_config->interface_count > 0);
 
-	return (res);
+	totem_config->net_mtu -= totemip_udpip_header_size(totem_config->interfaces[0].bindnet.family);
 }
 
 int totemudp_token_target_set (

exec/totemudp.h

@@ -49,7 +49,6 @@ extern int totemudp_initialize (
 	void **udp_context,
 	struct totem_config *totem_config,
 	totemsrp_stats_t *stats,
-	int interface_no,
 	void *context,
 
 	void (*deliver_fn) (
@@ -59,7 +58,12 @@ extern int totemudp_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address),
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no),
+
+	void (*mtu_changed) (
+		void *context,
+		int net_mtu),
 
 	void (*target_set_completed) (
 		void *context));
@@ -87,6 +91,10 @@ extern int totemudp_mcast_noflush_send (
 	const void *msg,
 	unsigned int msg_len);
 
+extern int totemudp_ifaces_get (void *net_context,
+	char ***status,
+	unsigned int *iface_count);
+
 extern int totemudp_recv_flush (void *udp_context);
 
 extern int totemudp_send_flush (void *udp_context);
@@ -97,12 +105,6 @@ extern int totemudp_finalize (void *udp_context);
 
 extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *totem_config);
 
-extern const char *totemudp_iface_print (void *udp_context);
-
-extern int totemudp_iface_get (
-	void *udp_context,
-	struct totem_ip_address *addr);
-
 extern int totemudp_token_target_set (
 	void *udp_context,
 	const struct totem_ip_address *token_target);

exec/totemudpu.c

@@ -69,7 +69,6 @@
 #include "totemudpu.h"
 
 #include "util.h"
-#include "totemcrypto.h"
 
 #include <nss.h>
 #include <pk11pub.h>
@@ -96,8 +95,6 @@ struct totemudpu_member {
 };
 
 struct totemudpu_instance {
-	struct crypto_instance *crypto_inst;
-
 	qb_loop_t *totemudpu_poll_handle;
 
 	struct totem_interface *totem_interface;
@@ -115,7 +112,8 @@ struct totemudpu_instance {
 
 	void (*totemudpu_iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address);
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no);
 
 	void (*totemudpu_target_set_completed) (void *context);
 
@@ -265,27 +263,12 @@ static inline void ucast_sendmsg (
 {
 	struct msghdr msg_ucast;
 	int res = 0;
-	size_t buf_out_len;
-	unsigned char buf_out[FRAME_SIZE_MAX];
 	struct sockaddr_storage sockaddr;
 	struct iovec iovec;
 	int addrlen;
 
-	/*
-	 * Encrypt and digest the message
-	 */
-	if (crypto_encrypt_and_sign (
-		instance->crypto_inst,
-		(const unsigned char *)msg,
-		msg_len,
-		buf_out,
-		&buf_out_len) != 0) {
-		log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
-		return;
-	}
-
-	iovec.iov_base = (void *)buf_out;
-	iovec.iov_len = buf_out_len;
+	iovec.iov_base = (void *)msg;
+	iovec.iov_len = msg_len;
 
 	/*
 	 * Build unicast message
@@ -333,29 +316,14 @@ static inline void mcast_sendmsg (
 {
 	struct msghdr msg_mcast;
 	int res = 0;
-	size_t buf_out_len;
-	unsigned char buf_out[FRAME_SIZE_MAX];
 	struct iovec iovec;
 	struct sockaddr_storage sockaddr;
 	int addrlen;
         struct list_head *list;
 	struct totemudpu_member *member;
 
-	/*
-	 * Encrypt and digest the message
-	 */
-	if (crypto_encrypt_and_sign (
-		instance->crypto_inst,
-		(const unsigned char *)msg,
-		msg_len,
-		buf_out,
-		&buf_out_len) != 0) {
-		log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
-		return;
-	}
-
-	iovec.iov_base = (void *)buf_out;
-	iovec.iov_len = buf_out_len;
+	iovec.iov_base = (void *)msg;
+	iovec.iov_len = msg_len;
 
 	memset(&msg_mcast, 0, sizeof(msg_mcast));
 	/*
@@ -445,7 +413,6 @@ static int net_deliver_fn (
 	struct iovec *iovec;
 	struct sockaddr_storage system_from;
 	int bytes_received;
-	int res = 0;
 
 	iovec = &instance->totemudpu_iov_recv;
 
@@ -479,18 +446,6 @@ static int net_deliver_fn (
 		instance->stats_recv += bytes_received;
 	}
 
-	/*
-	 * Authenticate and if authenticated, decrypt datagram
-	 */
-
-	res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
-	if (res == -1) {
-		log_printf (instance->totemudpu_log_level_security, "Received message has invalid digest... ignoring.");
-		log_printf (instance->totemudpu_log_level_security,
-			"Invalid packet data");
-		iovec->iov_len = FRAME_SIZE_MAX;
-		return 0;
-	}
 	iovec->iov_len = bytes_received;
 
 	/*
@@ -618,7 +573,7 @@ static void timer_function_netif_check_timeout (
 				"The network interface [%s] is now up.",
 				totemip_print (&instance->totem_interface->boundto));
 			instance->netif_state_report = NETIF_STATE_REPORT_DOWN;
-			instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
+			instance->totemudpu_iface_change_fn (instance->context, &instance->my_id, 0);
 		}
 		/*
 		 * Add a timer to check for interface going down in single membership
@@ -636,7 +591,7 @@ static void timer_function_netif_check_timeout (
 		if (instance->netif_state_report & NETIF_STATE_REPORT_DOWN) {
 			log_printf (instance->totemudpu_log_level_notice,
 				"The network interface is down.");
-			instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
+			instance->totemudpu_iface_change_fn (instance->context, &instance->my_id, 0);
 		}
 		instance->netif_state_report = NETIF_STATE_REPORT_UP;
 
@@ -714,6 +669,22 @@ static int totemudpu_build_sockets_ip (
 	return 0;
 }
 
+int totemudpu_ifaces_get (
+	void *net_context,
+	char ***status,
+	unsigned int *iface_count)
+{
+	static char *statuses[INTERFACE_MAX] = {(char*)"OK"};
+
+	if (status) {
+		*status = statuses;
+	}
+	*iface_count = 1;
+
+	return (0);
+}
+
+
 static int totemudpu_build_sockets (
 	struct totemudpu_instance *instance,
 	struct totem_ip_address *bindnet_address,
@@ -753,7 +724,7 @@ static int totemudpu_build_sockets (
 }
 
 /*
- * Totem Network interface - also does encryption/decryption
+ * Totem Network interface
  * depends on poll abstraction, POSIX, IPV4
  */
 
@@ -765,7 +736,6 @@ int totemudpu_initialize (
 	void **udpu_context,
 	struct totem_config *totem_config,
 	totemsrp_stats_t *stats,
-	int interface_no,
 	void *context,
 
 	void (*deliver_fn) (
@@ -775,7 +745,12 @@ int totemudpu_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address),
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no),
+
+	void (*mtu_changed) (
+		void *context,
+		int net_mtu),
 
 	void (*target_set_completed) (
 		void *context))
@@ -803,26 +778,10 @@ int totemudpu_initialize (
 	instance->totemudpu_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
 	instance->totemudpu_log_printf = totem_config->totem_logging_configuration.log_printf;
 
-	/*
-	* Initialize random number generator for later use to generate salt
-	*/
-	instance->crypto_inst = crypto_init (totem_config->private_key,
-		totem_config->private_key_len,
-		totem_config->crypto_cipher_type,
-		totem_config->crypto_hash_type,
-		instance->totemudpu_log_printf,
-		instance->totemudpu_log_level_security,
-		instance->totemudpu_log_level_notice,
-		instance->totemudpu_log_level_error,
-		instance->totemudpu_subsys_id);
-	if (instance->crypto_inst == NULL) {
-		free(instance);
-		return (-1);
-	}
 	/*
 	 * Initialize local variables for totemudpu
 	 */
-	instance->totem_interface = &totem_config->interfaces[interface_no];
+	instance->totem_interface = &totem_config->interfaces[0];
 	memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
 
 	instance->totemudpu_poll_handle = poll_handle;
@@ -955,31 +914,9 @@ extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *t
 
 	assert(totem_config->interface_count > 0);
 
-	totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
-							totem_config->crypto_hash_type) +
-				 totemip_udpip_header_size(totem_config->interfaces[0].bindnet.family);
-}
-
-const char *totemudpu_iface_print (void *udpu_context)  {
-	struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
-	const char *ret_char;
-
-	ret_char = totemip_print (&instance->my_id);
-
-	return (ret_char);
+	totem_config->net_mtu -= totemip_udpip_header_size(totem_config->interfaces[0].bindnet.family);
 }
 
-int totemudpu_iface_get (
-	void *udpu_context,
-	struct totem_ip_address *addr)
-{
-	struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
-	int res = 0;
-
-	memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
-
-	return (res);
-}
 
 int totemudpu_token_target_set (
 	void *udpu_context,
@@ -1108,7 +1045,9 @@ error_close_fd:
 
 int totemudpu_member_add (
 	void *udpu_context,
-	const struct totem_ip_address *member)
+	const struct totem_ip_address *local,
+	const struct totem_ip_address *member,
+	int ring_no)
 {
 	struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
 
@@ -1127,14 +1066,15 @@ int totemudpu_member_add (
 	list_add_tail (&new_member->list, &instance->member_list);
 	memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
 	new_member->fd = totemudpu_create_sending_socket(udpu_context, member);
-	new_member->active = 0;
+	new_member->active = 1;
 
 	return (0);
 }
 
 int totemudpu_member_remove (
 	void *udpu_context,
-	const struct totem_ip_address *token_target)
+	const struct totem_ip_address *token_target,
+	int ring_no)
 {
 	int found = 0;
 	struct list_head *list;
@@ -1208,45 +1148,6 @@ int totemudpu_member_list_rebind_ip (
 	return (0);
 }
 
-int totemudpu_member_set_active (
-	void *udpu_context,
-	const struct totem_ip_address *member_ip,
-	int active)
-{
-	struct list_head *list;
-	struct totemudpu_member *member;
-	int addr_found = 0;
-
-	struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
-
-	/*
-	 * Find the member to set active flag
-	 */
-	for (list = instance->member_list.next; list != &instance->member_list;	list = list->next) {
-		member = list_entry (list, struct totemudpu_member, list);
-
-		if (totemip_compare (member_ip, &member->member) == 0) {
-			log_printf(LOGSYS_LEVEL_DEBUG,
-			    "Marking UDPU member %s %s",
-			    totemip_print(&member->member),
-			    (active ? "active" : "inactive"));
-
-			member->active = active;
-			addr_found = 1;
-
-			break;
-		}
-	}
-
-	if (!addr_found) {
-		log_printf(LOGSYS_LEVEL_DEBUG,
-		    "Can't find UDPU member %s (should be marked as %s)",
-			    totemip_print(member_ip),
-			    (active ? "active" : "inactive"));
-	}
-
-	return (0);
-}
 
 static void timer_function_merge_detect_timeout (
 	void *data)

exec/totemudpu.h

@@ -49,7 +49,6 @@ extern int totemudpu_initialize (
 	void **udpu_context,
 	struct totem_config *totem_config,
 	totemsrp_stats_t *stats,
-	int interface_no,
 	void *context,
 
 	void (*deliver_fn) (
@@ -59,7 +58,12 @@ extern int totemudpu_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address),
+		const struct totem_ip_address *iface_address,
+		unsigned int ring_no),
+
+	void (*mtu_changed) (
+		void *context,
+		int net_mtu),
 
 	void (*target_set_completed) (
 		void *context));
@@ -87,6 +91,10 @@ extern int totemudpu_mcast_noflush_send (
 	const void *msg,
 	unsigned int msg_len);
 
+extern int totemudpu_ifaces_get (void *net_context,
+	char ***status,
+	unsigned int *iface_count);
+
 extern int totemudpu_recv_flush (void *udpu_context);
 
 extern int totemudpu_send_flush (void *udpu_context);
@@ -97,12 +105,6 @@ extern int totemudpu_finalize (void *udpu_context);
 
 extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *totem_config);
 
-extern const char *totemudpu_iface_print (void *udpu_context);
-
-extern int totemudpu_iface_get (
-	void *udpu_context,
-	struct totem_ip_address *addr);
-
 extern int totemudpu_token_target_set (
 	void *udpu_context,
 	const struct totem_ip_address *token_target);
@@ -117,15 +119,13 @@ extern int totemudpu_recv_mcast_empty (
 
 extern int totemudpu_member_add (
 	void *udpu_context,
-	const struct totem_ip_address *member);
+	const struct totem_ip_address *local,
+	const struct totem_ip_address *member,
+	int ring_no);
 
 extern int totemudpu_member_remove (
 	void *udpu_context,
-	const struct totem_ip_address *member);
-
-extern int totemudpu_member_set_active (
-	void *udpu_context,
-	const struct totem_ip_address *member_ip,
-	int active);
+	const struct totem_ip_address *member,
+	int ring_no);
 
 #endif /* TOTEMUDPU_H_DEFINED */

include/corosync/coroapi.h

@@ -85,7 +85,7 @@ struct corosync_tpg_group {
 
 #define TOTEMIP_ADDRLEN (sizeof(struct in6_addr))
 
-#define INTERFACE_MAX 2
+#define INTERFACE_MAX 8
 
 #ifndef MESSAGE_QUEUE_MAX
 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT

include/corosync/totem/totem.h

@@ -50,8 +50,9 @@
 #define FRAME_SIZE_MAX		10000
 #define TRANSMITS_ALLOWED	16
 #define SEND_THREADS_MAX	16
-#define INTERFACE_MAX		2
 
+/* This must be <= KNET_MAX_LINK */
+#define INTERFACE_MAX		8
 /**
  * Maximum number of continuous gather states
  */
@@ -68,6 +69,10 @@ struct totem_interface {
 	uint16_t ip_port;
 	uint16_t ttl;
 	int member_count;
+	int knet_link_priority;
+	int knet_ping_interval;
+	int knet_ping_timeout;
+	int knet_ping_precision;
 	struct totem_ip_address member_list[PROCESSOR_COUNT_MAX];
 };
 
@@ -90,13 +95,21 @@ struct totem_logging_configuration {
 	int log_subsys_id;
 };
 
-enum { TOTEM_PRIVATE_KEY_LEN = 128 };
-enum { TOTEM_RRP_MODE_BYTES = 64 };
+struct totem_message_header {
+	char type;
+	char encapsulated;
+	unsigned short endian_detector;
+	unsigned int nodeid;
+	unsigned int target_nodeid;
+} __attribute__((packed));
+
+enum { TOTEM_PRIVATE_KEY_LEN = 4096 };
+enum { TOTEM_LINK_MODE_BYTES = 64 };
 
 typedef enum {
 	TOTEM_TRANSPORT_UDP = 0,
 	TOTEM_TRANSPORT_UDPU = 1,
-	TOTEM_TRANSPORT_RDMA = 2
+	TOTEM_TRANSPORT_KNET = 2
 } totem_transport_t;
 
 #define MEMB_RING_ID
@@ -148,17 +161,7 @@ struct totem_config {
 
 	unsigned int seqno_unchanged_const;
 
-	unsigned int rrp_token_expired_timeout;
-
-	unsigned int rrp_problem_count_timeout;
-
-	unsigned int rrp_problem_count_threshold;
-
-	unsigned int rrp_problem_count_mcast_threshold;
-
-	unsigned int rrp_autorecovery_check_timeout;
-
-	char rrp_mode[TOTEM_RRP_MODE_BYTES];
+	char link_mode[TOTEM_LINK_MODE_BYTES];
 
 	struct totem_logging_configuration totem_logging_configuration;
 
@@ -224,15 +227,6 @@ typedef struct {
 	uint32_t iface_changes;
 } totemnet_stats_t;
 
-typedef struct {
-	totem_stats_header_t hdr;
-	totemnet_stats_t *net;
-	char *algo_name;
-	uint8_t *faulty;
-	uint32_t interface_count;
-} totemrrp_stats_t;
-
-
 typedef struct {
 	uint32_t rx;
 	uint32_t tx;
@@ -241,7 +235,6 @@ typedef struct {
 
 typedef struct {
 	totem_stats_header_t hdr;
-	totemrrp_stats_t *rrp;
 	uint64_t orf_token_tx;
 	uint64_t orf_token_rx;
 	uint64_t memb_merge_detect_tx;
@@ -281,11 +274,6 @@ typedef struct {
 typedef struct {
 	totem_stats_header_t hdr;
 	totemsrp_stats_t *srp;
-} totemmrp_stats_t;
-
-typedef struct {
-	totem_stats_header_t hdr;
-	totemmrp_stats_t *mrp;
 	uint32_t msg_reserved;
 	uint32_t msg_queue_avail;
 } totempg_stats_t;

man/corosync.conf.5

@@ -62,31 +62,52 @@ qb { }
 This top level directive contains configuration options related to libqb.
 
 .PP
-.PP
-Within the
-.B totem
-directive, an interface directive is required.  There is also one configuration
-option which is required:
-.PP
-.PP
-Within the
-.B interface
-sub-directive of totem there are four parameters which are required.  There is
-one parameter which is optional.
+The 
+.B interface sub-directive of totem is optional for UDP and knet transports.
+
+For knet, multiple interface subsections define parameters for each knet link on the
+system.
+
+For UDP, there should be just one interface section that defines the multicast or
+broadcast options for the link.
+
+For UDPU an interface section is not needed and it is recommended that the nodelist
+is used to define cluster nodes.
+
+.TP
+linknumber
+This specifies the link number for the interface.  When using the knet
+protocol, each interface should specify separate link numbers to uniquely
+identify to the membership protocol which interface to use for which link. 
+The linknumber must start at 0. For UDP the only supported linknumber is 0.
 
 .TP
-ringnumber
-This specifies the ring number for the interface.  When using the redundant
-ring protocol, each interface should specify separate ring numbers to uniquely
-identify to the membership protocol which interface to use for which redundant
-ring. The ringnumber must start at 0.
+knet_link_priority
+This specifies the priority for the link when knet is used in 'passive'
+mode. (see link_mode below)
 
 .TP
-bindnetaddr
+knet_ping_interval
+This specifies the interval between knet link pings. 
+(default 1000 ms)
+
+.TP
+knet_ping_timeout
+If no ping is received within this time, the knet link is declared dead. 
+(default 2000 ms)
+
+.TP
+knet_ping_precision
+How many values of latency are used to calculate
+the average link latency. (default 2048 samples)
+
+.TP
+bindnetaddr (udp only)
 This specifies the network address the corosync executive should bind
-to.
+to when using udp.
 
-bindnetaddr should be an IP address configured on the system, or a network
+bindnetaddr (udp only) 
+should be an IP address configured on the system, or a network
 address.
 
 For example, if the local interface is 192.168.5.92 with netmask
@@ -101,13 +122,13 @@ selection of the network interface within a specific subnet as with IPv4.
 If IPv6 networking is used, the nodeid field in nodelist must be specified.
 
 .TP
-broadcast
+broadcast (udp only)
 This is optional and can be set to yes.  If it is set to yes, the broadcast
 address will be used for communication.  If this option is set, mcastaddr
 should not be set.
 
 .TP
-mcastaddr
+mcastaddr (udp only)
 This is the multicast address used by corosync executive.  The default
 should work for most networks, but the network administrator should be queried
 about a multicast address to use.  Avoid 224.x.x.x because this is a "config"
@@ -117,11 +138,11 @@ This may also be an IPV6 multicast address, in which case IPV6 networking
 will be used.  If IPv6 networking is used, the nodeid field in nodelist must
 be specified.
 
-It's not needed to use this option if cluster_name option is used. If both options
+It's not necessary to use this option if cluster_name option is used. If both options
 are used, mcastaddr has higher priority.
 
 .TP
-mcastport
+mcastport (udp only)
 This specifies the UDP port number.  It is possible to use the same multicast
 address on a network with the corosync services configured for different
 UDP ports.
@@ -131,11 +152,10 @@ If you have multiple clusters on the same network using the same mcastaddr
 please configure the mcastports with a gap.
 
 .TP
-ttl
+ttl (udp only)
 This specifies the Time To Live (TTL). If you run your cluster on a routed
 network then the default of "1" will be too small. This option provides
 a way to increase this up to 255. The valid range is 0..255.
-Note that this is only valid on multicast transport types.
 
 .PP
 .PP
@@ -146,7 +166,7 @@ five are optional, and one is required when IPV6 is configured in the interface
 subdirective.  The required directive controls the version of the totem
 configuration.  The optional option unless using IPV6 directive controls
 identification of the processor.  The optional options control secrecy and
-authentication, the redundant ring mode of operation and maximum network MTU
+authentication, the network mode of operation and maximum network MTU
 field.
 
 .TP
@@ -169,7 +189,8 @@ a subset of the cluster (for example during a rolling upgrade).
 crypto_hash
 This specifies which HMAC authentication should be used to authenticate all
 messages. Valid values are none (no authentication), md5, sha1, sha256,
-sha384 and sha512.
+sha384 and sha512. Encrypted transmission is only supported for
+the knet transport.
 
 The default is sha1.
 
@@ -177,47 +198,34 @@ The default is sha1.
 crypto_cipher
 This specifies which cipher should be used to encrypt all messages.
 Valid values are none (no encryption), aes256, aes192, aes128 and 3des.
-Enabling crypto_cipher, requires also enabling of crypto_hash.
+Enabling crypto_cipher, requires also enabling of crypto_hash. Encrypted
+transmission is only supported for the knet transport.
 
 The default is aes256.
 
 .TP
-secauth
-This specifies that HMAC/SHA1 authentication should be used to authenticate
-all messages.  It further specifies that all data should be encrypted with the
-nss library and aes256 encryption algorithm to protect data from eavesdropping.
-
-Enabling this option adds a encryption header to every message sent by totem which
-reduces total throughput. Also encryption and authentication consume extra CPU
-cycles in corosync.
-
-The default is on.
+link_mode
+This specifies the Kronosnet mode, which may be passive, active, or
+rr (round-robin). 
+.B passive:
+the active link with the lowest priority will be used. If one or more 
+links share the same priority the one with the lowest link ID will
+be used.
+.B active: 
+All active links will be used simultaneously to send traffic.
+link priority is ignored.
+.B rr:
+Round-Robin policy. Each packet will be sent to the next active link in 
+order.
 
-WARNING: This parameter is deprecated. It's recomended to use combination of
-crypto_cipher and crypto_hash.
-
-.TP
-rrp_mode
-This specifies the mode of redundant ring, which may be none, active, or
-passive.  Currently only 'passive' is supported or tested
-(using  'active'  is  not recommended). Active replication offers
-slightly lower latency from transmit to delivery in faulty network
-environments but with less performance.
-Passive replication may nearly double the speed of the totem protocol
-if the protocol doesn't become cpu bound.  The final option is none, in
-which case only one network interface will be used to operate the totem
-protocol.
+If only one interface directive is specified, passive is automatically chosen.
 
-If only one interface directive is specified, none is automatically chosen.
-If multiple interface directives are specified, only active or passive may
-be chosen.
-
-The maximum number of interface directives that is allowed for either 
-modes (active or passive) is 2.
+The maximum number of interface directives that is allowed with Kronosnet
+is 8. For other transports it is 1.
 
 When using multiple interfaces, make sure to use different multicast
 address/port (port for same address must differ by at least two) pair
-for each interface (this is checked by parser) to make rrp works.
+for each interface (this is checked by parser).
 
 .TP
 netmtu
@@ -245,14 +253,9 @@ The default is 1500.
 
 .TP
 transport
-This directive controls the transport mechanism used.  If the interface to
-which corosync is binding is an RDMA interface such as RoCEE or Infiniband, the
-"iba" parameter may be specified.  To avoid the use of multicast entirely, a
-unicast transport parameter "udpu" can be specified.  This requires specifying
-the list of members in nodelist directive, that could potentially make up
-the membership before deployment.
-
-The default is udp.  The transport type can also be set to udpu or iba.
+This directive controls the transport mechanism used.  
+The default is knet.  The transport type can also be set to udpu or udp.
+Only knet allows crypto or multiple interfaces per node.
 
 .TP
 cluster_name
@@ -460,56 +463,6 @@ modern switches.
 
 The default is 5 messages.
 
-.TP
-rrp_problem_count_timeout
-This specifies the time in milliseconds to wait before decrementing the
-problem count by 1 for a particular ring to ensure a link is not marked
-faulty for transient network failures.
-
-The default is 2000 milliseconds.
-
-.TP
-rrp_problem_count_threshold
-This specifies the number of times a problem is detected with a link before
-setting the link faulty.  Once a link is set faulty, no more data is
-transmitted upon it.  Also, the problem counter is no longer decremented when
-the problem count timeout expires.
-
-A problem is detected whenever all tokens from the proceeding processor have
-not been received within the rrp_token_expired_timeout.  The
-rrp_problem_count_threshold * rrp_token_expired_timeout should be atleast 50
-milliseconds less then the token timeout, or a complete reconfiguration
-may occur.
-
-The default is 10 problem counts.
-
-.TP
-rrp_problem_count_mcast_threshold
-This specifies the number of times a problem is detected with multicast before
-setting the link faulty for passive rrp mode. This variable is unused in active
-rrp mode.
-
-The default is 10 times rrp_problem_count_threshold.
-
-.TP
-rrp_token_expired_timeout
-This specifies the time in milliseconds to increment the problem counter for
-the redundant ring protocol after not having received a token from all rings
-for a particular processor.
-
-This value will automatically be calculated from the token timeout and
-problem_count_threshold but may be overridden.  It is not recommended to
-override this value without guidance from the corosync community.
-
-The default is 47 milliseconds.
-
-.TP
-rrp_autorecovery_check_timeout
-This specifies the time in milliseconds to check if the failed ring can be
-auto-recovered.
-
-The default is 1000 milliseconds.
-
 .PP
 Within the
 .B logging
@@ -651,20 +604,18 @@ can contain only
 sub-directive, which specifies every node that should be a member of the membership, and where
 non-default options are needed. Every node must have at least ring0_addr field filled.
 
-For UDPU, every node that should be a member of the membership must be specified.
+Every node that should be a member of the membership must be specified.
 
 Possible options are:
 .TP
 ringX_addr
-This specifies ip address of one of the nodes. X is ring number.
+This specifies ip address of one of the nodes. X is link number. 
 
 .TP
 nodeid
-This configuration option is optional when using IPv4 and required when using
-IPv6.  This is a 32 bit value specifying the node identifier delivered to the
-cluster membership service.  If this is not specified with IPv4, the node id
-will be determined from the 32 bit IP address the system to which the system
-is bound with ring identifier of 0.  The node identifier value of zero is
+This configuration option is required for each node. It is a 32 bit value 
+specifying the node identifier delivered to the
+cluster membership service. The node identifier value of zero is
 reserved and should not be used.
 
 .PP