Totem IBA implementation.

git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@2481 fd59a12c-fef9-0310-b244-a6a79926bd2f

INSTALL

@@ -67,12 +67,13 @@ balance:~/corosync/trunk# make install
 * A notice about dependencies *
 -------------------------------
 We have strived very hard to avoid dependencies as much as possible, and in
-fact corosync can be compiled without any dependencies.  The one optional
-dependency corosync has today is the nss package when built with support for
-nss encryption.  NSS encryption performs more poorly then the default
-encryption, but may be a good choice if your organization has policies
-regarding encryption technology requiring the use of a common cryptographic
-library such as Netscape Security Services.
+fact corosync can be compiled without any dependencies.
+
+Optional dependencies are support for NSS and support for Infiniband RDMA.
+
+NSS encryption performs more poorly then the default encryption, but may be
+ a good choice if your organization has policies regarding encryption technology
+requiring the use of a common cryptographic library such as Netscape Security Services.
 
 To enable nss during building:
 balance:~/corosync/trunk% ./configure --enable-nss
@@ -86,6 +87,42 @@ distribution.  Note nss is enabled by default, so if you don't want nss support
 or don't want to install nss devel packages, make sure to set the appropriate
 configure option as described above.
 
+-----------------------------------------
+* Building with Infiniband RDMA support *
+-----------------------------------------
+Infiniband RDMA build support is disabled by default.  To enable rdma during
+building:
+balance:~/corosync/trunk% ./configure --enable-rdma
+
+Infiniband rdma support requires the libibverbs and librdmacm packages (and
+their -devel counterparts).  Corosync requires the use of pkgconfig in these
+packages.  The Corosync maintainers have sent patches for this feature upstream
+to the Infiniband maintainers.
+
+If your version of these packages don't have pkgconfig support, you will see an
+error during configure such as:
+
+checking for ibverbs... configure: error: Package requirements (ibverbs) were not met:
+
+No package 'ibverbs' found
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables ibverbs_CFLAGS
+and ibverbs_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+** Follow the directions:
+balance:~/corosync/trunk% export ibverbs_CFLAGS=-I/usr/include/infiniband
+balance:~/corosync/trunk% export ibverbs_LIBS=-libverbs
+balance:~/corosync/trunk% export rdmacm_CFLAGS=-I/usr/include/rdma
+balance:~/corosync/trunk% export rdmacm_LIBS=-lrdmacm
+balance:~/corosync/trunk% ./configure --enable-rdma
+
+Hopefully pkgconfig support is added in your distribution's version of
+libibverbs and librdmacm soon if you run into this problem.
+
 ------------------------
 * Configuring Corosync *
 ------------------------

configure.ac

@@ -203,6 +203,7 @@ AC_ARG_ENABLE([nss],
 AC_ARG_ENABLE([rdma],
 	[  --enable-rdma           : Infiniband RDMA transport support ],,
 	[ enable_rdma="no" ])
+AM_CONDITIONAL(BUILD_RDMA, test x$enable_rdma = xyes)
 
 AC_ARG_WITH([lcrso-dir],
 	[  --with-lcrso-dir=DIR    : corosync lcrso files. ],
@@ -310,6 +311,7 @@ fi
 
 if test "x${enable_rdma}" = xyes; then
 	PKG_CHECK_MODULES([rdmacm],[rdmacm])
+	PKG_CHECK_MODULES([ibverbs],[ibverbs])
 	AC_DEFINE_UNQUOTED([HAVE_RDMA], 1, [have rdmacm])
 	PACKAGE_FEATURES="$PACKAGE_FEATURES rdma"
 fi

exec/Makefile.am

@@ -33,11 +33,15 @@ MAINTAINERCLEANFILES	= Makefile.in
 
 AM_CFLAGS		= -fPIC
 
-INCLUDES		= -I$(top_builddir)/include -I$(top_srcdir)/include $(nss_CFLAGS)
+INCLUDES		= -I$(top_builddir)/include -I$(top_srcdir)/include $(nss_CFLAGS) $(rdmacm_CFLAGS) $(ibverbs_CFLAGS)
 
-TOTEM_SRC		= coropoll.c totemip.c totemnet.c totemudp.c totemiba.c \
+TOTEM_SRC		= coropoll.c totemip.c totemnet.c totemudp.c \
 			  totemrrp.c totemsrp.c totemmrp.c totempg.c \
 			  crypto.c wthread.c
+if BUILD_RDMA
+TOTEM_SRC		+= totemiba.c
+endif
+
 LOGSYS_SRC		= wthread.c logsys.c
 COROIPCS_SRC		= coroipcs.c
 
@@ -123,7 +127,7 @@ else
 libtotem_pg.so.$(SONAME): $(TOTEM_OBJS)
 	$(CC) -shared -o $@ \
 		-Wl,-soname=libtotem_pg.so.$(SOMAJOR) \
-		$^ $(LDFLAGS) $(nss_LIBS) -lpthread
+		$^ $(LDFLAGS) $(nss_LIBS) $(rdmacm_LIBS) $(ibverbs_LIBS) -lpthread
 	ln -sf libtotem_pg.so.$(SONAME) libtotem_pg.so
 	ln -sf libtotem_pg.so.$(SONAME) libtotem_pg.so.$(SOMAJOR)
 

exec/totemconfig.c

@@ -220,7 +220,6 @@ static void totem_volatile_config_read (
 	objdb_get_string (objdb, object_totem_handle, "vsftype", &totem_config->vsf_type);
 
 	objdb_get_int (objdb,object_totem_handle, "max_messages", &totem_config->max_messages);
-
 }
 
 
@@ -275,6 +274,7 @@ extern int totem_config_read (
 	const char *str;
 	unsigned int ringnumber = 0;
 	hdb_handle_t object_find_interface_handle;
+	const char *transport_type;
 
 	res = totem_handle_find (objdb, &object_totem_handle);
 	if (res == -1) {
@@ -392,6 +392,14 @@ printf ("couldn't find totem handle\n");
 
 	add_totem_config_notification(objdb, totem_config, object_totem_handle);
 
+	totem_config->transport_number = 0;
+	objdb_get_string (objdb, object_totem_handle, "transport", &transport_type);
+	if (transport_type) {
+		if (strcmp (transport_type, "iba") == 0) {
+			totem_config->transport_number = 1;
+		}
+	}
+
 	return 0;
 }
 

exec/totemiba.h

@@ -57,7 +57,10 @@ extern int totemiba_initialize (
 
 	void (*iface_change_fn) (
 		void *context,
-		const struct totem_ip_address *iface_address));
+		const struct totem_ip_address *iface_address),
+
+	void (*target_set_completed) (
+		void *context));
 
 extern int totemiba_processor_count_set (
 	void *iba_context,

exec/totemnet.c

@@ -35,10 +35,14 @@
 
 #include <config.h>
 
+#ifdef HAVE_RDMA
 #include <totemiba.h>
+#endif
 #include <totemudp.h>
 #include <totemnet.h>
 
+#define LOGSYS_UTILS_ONLY 1
+#include <corosync/engine/logsys.h>
 
 struct transport {
 	const char *name;
@@ -129,6 +133,7 @@ struct transport transport_entries[] = {
 		.crypto_set = totemudp_crypto_set,
 		.recv_mcast_empty = totemudp_recv_mcast_empty
 	},
+#ifdef HAVE_RDMA
 	{
 		.name = "Infiniband/IP",
 		.initialize = totemiba_initialize,
@@ -148,17 +153,57 @@ struct transport transport_entries[] = {
 		.recv_mcast_empty = totemiba_recv_mcast_empty
 
 	}
+#endif
 };
 	
 struct totemnet_instance {
 	void *transport_context;
 
 	struct transport *transport;
+
+        void (*totemnet_log_printf) (
+                unsigned int rec_ident,
+                const char *function,
+                const char *file,
+                int line,
+                const char *format,
+                ...)__attribute__((format(printf, 5, 6)));
+
+        int totemnet_subsys_id;
 };
 
-static void totemnet_instance_initialize (struct totemnet_instance *instance)
+#define log_printf(level, format, args...)				\
+do {									\
+	instance->totemnet_log_printf (					\
+		LOGSYS_ENCODE_RECID(level,				\
+		instance->totemnet_subsys_id,				\
+		LOGSYS_RECID_LOG),					\
+		__FUNCTION__, __FILE__, __LINE__,			\
+		(const char *)format, ##args);				\
+} while (0);
+
+static void totemnet_instance_initialize (
+	struct totemnet_instance *instance,
+	struct totem_config *config)
 {
-	instance->transport = &transport_entries[0];
+	int transport;
+
+	instance->totemnet_log_printf = config->totem_logging_configuration.log_printf;
+	instance->totemnet_subsys_id = config->totem_logging_configuration.log_subsys_id;
+
+
+	transport = 0;
+
+#ifdef HAVE_RDMA
+	if (config->transport_number == 1) {
+		transport = 1;
+	}
+#endif
+
+	log_printf (LOGSYS_LEVEL_NOTICE,
+		"Initializing transport (%s).\n", transport_entries[transport].name);
+
+	instance->transport = &transport_entries[transport];
 }
 
 int totemnet_crypto_set (
@@ -210,7 +255,7 @@ int totemnet_initialize (
 	if (instance == NULL) {
 		return (-1);
 	}
-	totemnet_instance_initialize (instance);
+	totemnet_instance_initialize (instance, totem_config);
 
 	res = instance->transport->initialize (poll_handle,
 		&instance->transport_context, totem_config,

exec/totemudp.h

@@ -37,6 +37,7 @@
 
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <corosync/hdb.h>
 
 #include <corosync/totem/totem.h>
 

include/corosync/totem/totem.h

@@ -162,6 +162,8 @@ struct totem_config {
 
 	int crypto_crypt_type;
 	int crypto_sign_type;
+
+	int transport_number;
 };
 
 #define TOTEM_CONFIGURATION_TYPE

man/corosync.conf.5

@@ -251,6 +251,15 @@ are not safe for use.
 
 The default is ykd.  The vsftype can also be set to none.
 
+.TP
+transport
+This directive controls the transport mechanism used.  If the interface to
+which corosync is binding is Infiniband, you can specify the "iba" option.  Any
+other option is ignored.  Note Infiniband interfaces will use RDMA transport
+techniques and perform at higher bandwidths and lower latency than gige networks.
+
+The default is udp.  The transport type can also be set to iba.
+
 Within the
 .B totem
 directive, there are several configuration options which are used to control