Add libnss security support to corosync.

git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@2145 fd59a12c-fef9-0310-b244-a6a79926bd2f

SECURITY

@@ -61,11 +61,23 @@ encrypt_and_sign to prepare the message to be sent.  When the executive
 wants to receive a message from the network, it uses
 authenticate_and_decrypt to verify the message is valid and decrypt it.
 
-These two functions utilize the following algorithms:
+There are currently two encryption methods available in corosync.
+sha1/hmac/sober which are coded internally, and AES/SHA256 which
+are in the Mozilla Network Security Services (libnss) library.
+
+The internal functions utilize the following algorithms:
 sha1 - hash algorithm secure for using with hmac
 hmac - produces a 16 byte digest from any length input
 sober - pseudo random number generator and stream cipher
 
+Every message starts with a
+struct security {
+	unsigned char digest[20]; A one way hash digest
+	unsigned char salt[16]; A securely generated random number
+}
+
+INTERNAL SECURITY CODE:
+-----------------------
 The hmac algorithm requires a 16 byte key.
 The sober algorithm requires a 16 byte private key.
 The sober algorithm requires a 16 byte public initial vector.
@@ -73,12 +85,6 @@ The sober algorithm requires a 16 byte public initial vector.
 The private key is read from disk and stored in memory for use with the
 sober algorithm to generate the three required keys.
 
-Every message starts with a
-struct security {
-	unsigned char digest[20]; A one way hash digest
-	unsigned char salt[16]; A securely generated random number
-}
-
 When a message is sent (encrypt_and_sign):
 ------------------------------------------
 1. sober is used to create a 16 byte random number (salt) using the md4
@@ -123,4 +129,59 @@ is randomly unique (within the 2^128 search space of the input to sober) to
 ensure that keys are never reused, nonce's are never reused, and hmac's are
 never reused.
 
+
+USING LIBNSS
+------------
+
+The process is similar in concept to the above, but most of the details are 
+hidden inside the NSS library. When corosync is started up libnss is initialised,
+the private key is read into memory and stored for later use by the code.
+
+When a message is sent (encrypt_and_sign):
+------------------------------------------
+- The message is encrypted using AES.
+- A digest of that message is then created using SHA256 and based on the
+  private key.
+- the message is then transmitted.
+
+When a message is received (decrypt_and_authenticate):
+- A Digest of the encrypted message is created using the private key
+- That digest is compared to the one in the message security_header
+- If they do not match the packet is rejected
+- If they do match then the message is decrypted using the private key.
+- The message is processed.
+
+
+Compatibility
+-------------
+
+The default mode of operation is to allow for wire-compatibility with existing
+openais systems. That means that the internal encryption system is used 
+and all received packets are expected to use that system. This allows a rolling
+upgrade from openais to corosync. 
+
+Once all nodes in the cluster are running corosync they can be changed to allow
+the newer libnss-based encryption by setting the 
+totem {
+    crypto_accept: new
+}
+option in corosync.conf.
+
+This enables the new encryption system but does not switch it on. It simply
+adds a byte to the end of the packets to indicate the encryption type.
+
+Once all nodes have been upgraded and 'crypto_accept: new' has been set, 
+the encryption type can be set using a single command:
+
+# corosync-cfgtool -c1
+
+This will tell all cluster nodes to start using libnss encryption. Note that
+it is possible to upgrade node individially by seetting the encryption type in
+corosync.conf. The last byte of the packet indicates the decryption algorithm 
+that the receiver should use.
+
+Once all nodes are using libnss encryption, the option should be set in
+in corosync.conf so that it takes effect at the next system reboot.
+
+
 Comments welcome mailto:corosync@lists.osdl.org

configure.ac

@@ -51,6 +51,7 @@ AC_PROG_LN_S
 AC_PROG_MAKE_SET
 AC_PROG_RANLIB
 AC_CHECK_PROGS([GROFF], [groff])
+AC_CHECK_PROGS([PKGCONFIG], [pkg-config])
 
 # Checks for libraries.
 AC_CHECK_LIB([dl], [dlopen])
@@ -176,6 +177,10 @@ AC_ARG_ENABLE([coverage],
 	[  --enable-coverage       : coverage analysis of the codebase. ],
 	[ default="no" ])
 
+AC_ARG_ENABLE([nss],
+	[  --enable-nss            : Network Security Services encryption. ],
+	[ default="yes" ])
+
 AC_ARG_WITH([lcrso-dir],
 	[  --with-lcrso-dir=DIR    : corosync lcrso files. ],
 	[ LCRSODIR="$withval" ],
@@ -257,6 +262,16 @@ else
 	GDB_FLAGS="-g"
 fi
 
+# Look for libnss
+if test "x${enable_nss}" = xyes; then
+    if $PKGCONFIG --exists nss; then
+	NSS_CFLAGS="`$PKGCONFIG --cflags nss`"
+	NSS_LDFLAGS="`$PKGCONFIG --libs nss`"
+	AC_DEFINE_UNQUOTED([HAVE_LIBNSS], 1, [have libnss])	
+	PKG_FEATURES="$PKG_FEATURES nss"
+    fi
+fi
+
 # extra warnings
 EXTRA_WARNINGS=""
 
@@ -320,7 +335,7 @@ fi
 
 # final build of *FLAGS
 CFLAGS="$ENV_CFLAGS $OPT_CFLAGS $GDB_FLAGS $OS_CFLAGS \
-	$COVERAGE_CFLAGS $EXTRA_WARNINGS $WERROR_CFLAGS"
+	$COVERAGE_CFLAGS $EXTRA_WARNINGS $WERROR_CFLAGS $NSS_CFLAGS"
 CPPFLAGS="$ENV_CPPFLAGS $ANSI_CPPFLAGS $OS_CPPFLAGS"
 LDFLAGS="$ENV_LDFLAGS $COVERAGE_LDFLAGS $OS_LDFLAGS"
 
@@ -335,6 +350,8 @@ AC_SUBST([SONAME])
 
 AC_SUBST([OS_DYFLAGS])
 
+AC_SUBST([NSS_LDFLAGS])
+
 AM_CONDITIONAL(BUILD_DARWIN, test -n "${DARWIN_OPTS}")
 AC_SUBST([DARWIN_OPTS])
 

exec/Makefile.am

@@ -94,7 +94,7 @@ else
 libtotem_pg.so.$(SONAME): $(TOTEM_OBJS)
 	$(CC) -shared -o $@ \
 		-Wl,-soname=libtotem_pg.so.$(SOMAJOR) \
-		$^ $(LDFLAGS) -lpthread
+		$^ $(LDFLAGS) $(NSS_LDFLAGS) -lpthread
 	ln -sf libtotem_pg.so.$(SONAME) libtotem_pg.so
 	ln -sf libtotem_pg.so.$(SONAME) libtotem_pg.so.$(SOMAJOR)
 

exec/apidef.c

@@ -116,6 +116,7 @@ static struct corosync_api_v1 apidef_corosync_api_v1 = {
 	.totem_ifaces_get = totempg_ifaces_get,
 	.totem_ifaces_print = totempg_ifaces_print,
 	.totem_ip_print = totemip_print,
+	.totem_crypto_set = totempg_crypto_set,
 	.totem_callback_token_create = totempg_callback_token_create,
 	.tpg_init = totempg_groups_initialize,
 	.tpg_exit = NULL, /* missing from totempg api */

exec/totemconfig.c

@@ -56,6 +56,13 @@
 #include <corosync/engine/config.h>
 #include <corosync/engine/logsys.h>
 
+#ifdef HAVE_LIBNSS
+#include <nss.h>
+#include <pk11pub.h>
+#include <pkcs11.h>
+#include <prerror.h>
+#endif
+
 #include "util.h"
 #include "totemconfig.h"
 #include "tlist.h" /* for HZ */
@@ -218,6 +225,46 @@ static void totem_volatile_config_read (
 }
 
 
+static void totem_get_crypto_type(
+	const struct objdb_iface_ver0 *objdb,
+	hdb_handle_t object_totem_handle,
+	struct totem_config *totem_config)
+{
+	const char *str;
+
+	totem_config->crypto_accept = TOTEM_CRYPTO_ACCEPT_OLD;
+	if (!objdb_get_string (objdb, object_totem_handle, "crypto_accept", &str)) {
+		if (strcmp(str, "new") == 0) {
+			totem_config->crypto_accept = TOTEM_CRYPTO_ACCEPT_NEW;
+		}
+	}
+
+	totem_config->crypto_type = TOTEM_CRYPTO_SOBER;
+
+#ifdef HAVE_LIBNSS
+	/*
+	 * We must set these even if the key does not exist.
+	 * Encryption type can be set on-the-fly using CFG
+	 */
+	totem_config->crypto_crypt_type = CKM_AES_CBC_PAD;
+	totem_config->crypto_sign_type = CKM_SHA256_RSA_PKCS;
+#endif
+
+	if (!objdb_get_string (objdb, object_totem_handle, "crypto_type", &str)) {
+		if (strcmp(str, "sober") == 0) {
+			return;
+		}
+#ifdef HAVE_LIBNSS
+		if (strcmp(str, "nss") == 0) {
+			totem_config->crypto_type = TOTEM_CRYPTO_NSS;
+
+		}
+#endif
+	}
+}
+
+
+
 extern int totem_config_read (
 	struct objdb_iface_ver0 *objdb,
 	struct totem_config *totem_config,
@@ -263,6 +310,11 @@ printf ("couldn't find totem handle\n");
 			totem_config->secauth = 0;
 		}
 	}
+
+	if (totem_config->secauth == 1) {
+		totem_get_crypto_type(objdb, object_totem_handle, totem_config);
+	}
+
 	if (!objdb_get_string (objdb, object_totem_handle, "rrp_mode", &str)) {
 		strcpy (totem_config->rrp_mode, str);
 	}

exec/totemmrp.c

@@ -210,6 +210,13 @@ int totemmrp_ifaces_get (
 	return (res);
 }
 
+int totemmrp_crypto_set (
+	unsigned int type)
+{
+	return totemsrp_crypto_set (totemsrp_handle_in,
+				    type);
+}
+
 unsigned int totemmrp_my_nodeid_get (void)
 {
 	return (totemsrp_my_nodeid_get (totemsrp_handle_in));

exec/totemmrp.h

@@ -108,6 +108,8 @@ extern unsigned int totemmrp_my_nodeid_get (void);
 
 extern int totemmrp_my_family_get (void);
 
+extern int totemmrp_crypto_set (unsigned int);
+
 extern int totemmrp_ring_reenable (void);
 
 #endif /* TOTEMMRP_H_DEFINED */

exec/totemnet.c

@@ -69,8 +69,14 @@
 
 #include "crypto.h"
 
-#define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
+#ifdef HAVE_LIBNSS
+#include <nss.h>
+#include <pk11pub.h>
+#include <pkcs11.h>
+#include <prerror.h>
+#endif
 
+#define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
 #define NETIF_STATE_REPORT_UP		1
 #define NETIF_STATE_REPORT_DOWN		2
 
@@ -101,6 +107,12 @@ struct totemnet_instance {
 
 	prng_state totemnet_prng_state;
 
+#ifdef HAVE_LIBNSS
+	SECItem      *nss_sec_param;
+	PK11SymKey   *nss_sym_key;
+	unsigned char nss_iv_data[16];
+#endif
+
 	unsigned char totemnet_private_key[1024];
 
 	unsigned int totemnet_private_key_len;
@@ -234,9 +246,11 @@ do {									\
 		level, (const char *)format, ##args);			\
 } while (0);
 
-static int authenticate_and_decrypt (
+
+static int authenticate_and_decrypt_sober (
 	struct totemnet_instance *instance,
-	struct iovec *iov)
+	struct iovec *iov,
+	unsigned int iov_len)
 {
 	unsigned char keys[48];
 	struct security_header *header = iov[0].iov_base;
@@ -280,7 +294,6 @@ static int authenticate_and_decrypt (
 	hmac_done (&instance->totemnet_hmac_state, digest_comparison, &len);
 
 	if (memcmp (digest_comparison, header->hash_digest, len) != 0) {
-		log_printf (instance->totemnet_log_level_security, "Received message has invalid digest... ignoring.\n");
 		return (-1);
 	}
 
@@ -294,13 +307,333 @@ static int authenticate_and_decrypt (
 
 	return (0);
 }
-static void encrypt_and_sign_worker (
+
+static void init_sober_crypto(
+	struct totemnet_instance *instance)
+{
+	log_printf(instance->totemnet_log_level_notice, "Initialising SOBER128 crypto\n");
+	rng_make_prng (128, PRNG_SOBER, &instance->totemnet_prng_state, NULL);
+}
+
+#ifdef HAVE_LIBNSS
+
+static unsigned char *copy_from_iovec(
+	const struct iovec *iov,
+	unsigned int iov_len,
+	size_t *buf_size)
+{
+	int i;
+	size_t bufptr;
+	size_t buflen = 0;
+	unsigned char *newbuf;
+
+	for (i=0; i<iov_len; i++)
+		buflen += iov[i].iov_len;
+
+	newbuf = malloc(buflen);
+	if (!newbuf)
+		return NULL;
+
+	bufptr=0;
+	for (i=0; i<iov_len; i++) {
+		memcpy(newbuf+bufptr, iov[i].iov_base, iov[i].iov_len);
+		bufptr += iov[i].iov_len;
+	}
+	*buf_size = buflen;
+	return newbuf;
+}
+
+static void copy_to_iovec(
+	struct iovec *iov,
+	unsigned int iov_len,
+	const unsigned char *buf,
+	size_t buf_size)
+{
+	int i;
+	size_t copylen;
+	size_t bufptr = 0;
+
+	bufptr=0;
+	for (i=0; i<iov_len; i++) {
+		copylen = iov[i].iov_len;
+		if (bufptr + copylen > buf_size) {
+			copylen = buf_size - bufptr;
+		}
+		memcpy(iov[i].iov_base, buf+bufptr, copylen);
+		bufptr += copylen;
+		if (iov[i].iov_len != copylen) {
+			iov[i].iov_len = copylen;
+			return;
+		}
+	}
+}
+
+
+static void init_nss_crypto(
+	struct totemnet_instance *instance)
+{
+	PK11SlotInfo*      slot = NULL;
+	SECItem            key_item, iv_item;
+	SECStatus          rv;
+
+	log_printf(instance->totemnet_log_level_notice, "Initialising NSS crypto\n");
+	rv = NSS_NoDB_Init(".");
+	if (rv != SECSuccess)
+	{
+		log_printf(instance->totemnet_log_level_security, "NSS initialization failed (err %d)\n",
+			PR_GetError());
+		goto out;
+	}
+
+	slot = PK11_GetBestSlot(instance->totem_config->crypto_crypt_type, NULL);
+	if (slot == NULL)
+	{
+		log_printf(instance->totemnet_log_level_security, "Unable to find security slot (err %d)\n",
+			PR_GetError());
+		goto out;
+	}
+
+	/*
+	 * Make the private key into a SymKey that we can use
+	 */
+	key_item.type = siBuffer;
+	key_item.data = instance->totem_config->private_key;
+	key_item.len = 32; /* Use 128 bits */
+
+	instance->nss_sym_key = PK11_ImportSymKey(slot, instance->totem_config->crypto_crypt_type,
+						  PK11_OriginUnwrap, CKA_ENCRYPT|CKA_DECRYPT|CKA_SIGN,
+						  &key_item, NULL);
+	if (instance->nss_sym_key == NULL)
+	{
+		log_printf(instance->totemnet_log_level_security, "Failure to import key into NSS (err %d)\n",
+			PR_GetError());
+		goto out;
+	}
+
+	/* set up the PKCS11 encryption paramters.
+	 * when not using CBC mode, iv_item.data and iv_item.len can be 0, or you
+	 * can simply pass NULL for the iv parameter in PK11_ParamFromIV func
+	 */
+	rng_get_bytes(instance->nss_iv_data, sizeof(instance->nss_iv_data), NULL);
+	iv_item.type = siBuffer;
+	iv_item.data = instance->nss_iv_data;
+	iv_item.len = sizeof(instance->nss_iv_data);
+	instance->nss_sec_param = PK11_ParamFromIV(instance->totem_config->crypto_crypt_type, &iv_item);
+	if (instance->nss_sec_param == NULL)
+	{
+		log_printf(instance->totemnet_log_level_security, "Failure to set up PKCS11 param (err %d)\n",
+			PR_GetError());
+		goto out;
+	}
+
+out:
+	return;
+}
+
+static int encrypt_and_sign_nss (
+	struct totemnet_instance *instance,
+	unsigned char *buf,
+	size_t *buf_len,
+	const struct iovec *iovec,
+	unsigned int iov_len)
+{
+	PK11Context*       enc_context = NULL;
+	SECStatus          rv1, rv2;
+	int                tmp1_outlen;
+	unsigned int       tmp2_outlen;
+	unsigned char      *inbuf;
+	unsigned char      *data;
+	unsigned char      *outdata;
+	size_t             datalen;
+	SECItem            no_params;
+	struct security_header *header;
+
+	no_params.type = siBuffer;
+	no_params.data = 0;
+	no_params.len = 0;
+
+	tmp1_outlen = tmp2_outlen = 0;
+	inbuf = copy_from_iovec(iovec, iov_len, &datalen);
+	if (!inbuf) {
+		log_printf(instance->totemnet_log_level_security, "malloc error copying buffer from iovec\n");
+		return -1;
+	}
+
+	data = inbuf + sizeof (struct security_header);
+	datalen -= sizeof (struct security_header);
+
+	outdata = buf + sizeof (struct security_header);
+	header = (struct security_header *)buf;
+
+	/*
+	 * Create cipher context for encryption
+	 */
+	enc_context = PK11_CreateContextBySymKey(instance->totem_config->crypto_crypt_type, CKA_ENCRYPT,
+						instance->nss_sym_key, instance->nss_sec_param);
+	if (!enc_context) {
+		char err[1024];
+		PR_GetErrorText(err);
+		err[PR_GetErrorTextLength()] = 0;
+		log_printf(instance->totemnet_log_level_security, "PK11_CreateContext failed (encrypt) crypt_type=%d (err %d): %s\n",
+			instance->totem_config->crypto_crypt_type,
+			PR_GetError(), err);
+		return -1;
+	}
+	rv1 = PK11_CipherOp(enc_context, outdata,
+			    &tmp1_outlen, FRAME_SIZE_MAX - sizeof(struct security_header),
+			    data, datalen);
+	rv2 = PK11_DigestFinal(enc_context, outdata + tmp1_outlen, &tmp2_outlen,
+			       FRAME_SIZE_MAX - tmp1_outlen);
+	PK11_DestroyContext(enc_context, PR_TRUE);
+
+	*buf_len = tmp1_outlen + tmp2_outlen;
+	free(inbuf);
+
+	if (rv1 != SECSuccess || rv2 != SECSuccess)
+		goto out;
+
+	/* Now do the digest */
+	enc_context = PK11_CreateContextBySymKey(CKM_MD5, CKA_DIGEST, instance->nss_sym_key, &no_params);
+	if (!enc_context) {
+		char err[1024];
+		PR_GetErrorText(err);
+		err[PR_GetErrorTextLength()] = 0;
+		log_printf(instance->totemnet_log_level_security, "encrypt: PK11_CreateContext failed (digest) err %d: %s\n",
+			PR_GetError(), err);
+		return -1;
+	}
+
+	PK11_DigestBegin(enc_context);
+
+	rv1 = PK11_DigestOp(enc_context, outdata, *buf_len);
+	rv2 = PK11_DigestFinal(enc_context, header->hash_digest, &tmp2_outlen, sizeof(header->hash_digest));
+
+	PK11_DestroyContext(enc_context, PR_TRUE);
+
+	if (rv1 != SECSuccess || rv2 != SECSuccess)
+		goto out;
+
+	memcpy(header->salt, instance->nss_iv_data, sizeof(instance->nss_iv_data));
+
+	*buf_len += sizeof(struct security_header);
+	return 0;
+
+out:
+	return -1;
+}
+
+
+static int authenticate_and_decrypt_nss (
+	struct totemnet_instance *instance,
+	struct iovec *iov,
+	unsigned int iov_len)
+{
+	PK11Context*  enc_context = NULL;
+	SECStatus     rv1, rv2;
+	int           tmp1_outlen;
+	unsigned int  tmp2_outlen;
+	unsigned char outbuf[FRAME_SIZE_MAX];
+	unsigned char digest[HMAC_HASH_SIZE];
+	unsigned char *outdata;
+	int           result_len;
+	unsigned char *data;
+	unsigned char *inbuf;
+	size_t        datalen;
+	struct security_header *header = iov[0].iov_base;
+	SECItem no_params;
+	SECItem ivdata;
+
+	no_params.type = siBuffer;
+	no_params.data = 0;
+	no_params.len = 0;
+
+	tmp1_outlen = tmp2_outlen = 0;
+	if (iov_len > 1) {
+		inbuf = copy_from_iovec(iov, iov_len, &datalen);
+		if (!inbuf) {
+			log_printf(instance->totemnet_log_level_security, "malloc error copying buffer from iovec\n");
+			return -1;
+		}
+	}
+	else {
+		inbuf = iov[0].iov_base;
+		datalen = iov[0].iov_len;
+	}
+	data = inbuf + sizeof (struct security_header);
+	datalen -= sizeof (struct security_header);
+
+	outdata = outbuf + sizeof (struct security_header);
+
+	/* Check the digest */
+	enc_context = PK11_CreateContextBySymKey(CKM_MD5, CKA_DIGEST, instance->nss_sym_key, &no_params);
+	if (!enc_context) {
+		char err[1024];
+		PR_GetErrorText(err);
+		err[PR_GetErrorTextLength()] = 0;
+		log_printf(instance->totemnet_log_level_security, "PK11_CreateContext failed (check digest) err %d: %s\n",
+			PR_GetError(), err);
+		return -1;
+	}
+
+	PK11_DigestBegin(enc_context);
+
+	rv1 = PK11_DigestOp(enc_context, data, datalen);
+	rv2 = PK11_DigestFinal(enc_context, digest, &tmp2_outlen, sizeof(digest));
+
+	PK11_DestroyContext(enc_context, PR_TRUE);
+
+	if (rv1 != SECSuccess || rv2 != SECSuccess) {
+		log_printf(instance->totemnet_log_level_security, "Digest check failed\n");
+		return -1;
+	}
+
+	if (memcmp(digest, header->hash_digest, tmp2_outlen) != 0) {
+		log_printf(instance->totemnet_log_level_error, "Digest does not match\n");
+		return -1;
+	}
+
+	/* Create cipher context for decryption */
+	ivdata.type = siBuffer;
+	ivdata.data = header->salt;
+	ivdata.len = sizeof(header->salt);
+	enc_context = PK11_CreateContextBySymKey(instance->totem_config->crypto_crypt_type, CKA_DECRYPT,
+						instance->nss_sym_key, &ivdata);
+	if (!enc_context) {
+		log_printf(instance->totemnet_log_level_security, "PK11_CreateContext (decrypt) failed (err %d)\n",
+			PR_GetError());
+		return -1;
+	}
+
+	rv1 = PK11_CipherOp(enc_context, outdata, &tmp1_outlen,
+			    sizeof(outbuf) - sizeof (struct security_header),
+			    data, datalen);
+	if (rv1 != SECSuccess) {
+		log_printf(instance->totemnet_log_level_security, "PK11_CipherOp (decrypt) failed (err %d)\n",
+			PR_GetError());
+	}
+	rv2 = PK11_DigestFinal(enc_context, outdata + tmp1_outlen, &tmp2_outlen,
+			       sizeof(outbuf) - tmp1_outlen);
+	PK11_DestroyContext(enc_context, PR_TRUE);
+	result_len = tmp1_outlen + tmp2_outlen + sizeof (struct security_header);
+
+	/* Copy it back to the buffer */
+	copy_to_iovec(iov, iov_len, outbuf, result_len);
+	if (iov_len > 1)
+		free(inbuf);
+
+	if (rv1 != SECSuccess || rv2 != SECSuccess)
+		return -1;
+
+	return 0;
+}
+#endif
+
+static int encrypt_and_sign_sober (
 	struct totemnet_instance *instance,
 	unsigned char *buf,
 	size_t *buf_len,
 	const struct iovec *iovec,
-	size_t iov_len,
-	prng_state *prng_state_in)
+	unsigned int iov_len)
 {
 	int i;
 	unsigned char *addr;
@@ -314,6 +647,7 @@ static void encrypt_and_sign_worker (
 	hmac_state hmac_st;
 	prng_state keygen_prng_state;
 	prng_state stream_prng_state;
+	prng_state *prng_state_in = &instance->totemnet_prng_state;
 
 	header = (struct security_header *)buf;
 	addr = buf + sizeof (struct security_header);
@@ -374,8 +708,124 @@ static void encrypt_and_sign_worker (
 	hmac_done (&hmac_st, header->hash_digest, &len);
 
 	*buf_len = outlen;
+
+	return 0;
+}
+
+static int encrypt_and_sign_worker (
+	struct totemnet_instance *instance,
+	unsigned char *buf,
+	size_t *buf_len,
+	const struct iovec *iovec,
+	unsigned int iov_len)
+{
+	if (instance->totem_config->crypto_type == TOTEM_CRYPTO_SOBER ||
+	    instance->totem_config->crypto_accept == TOTEM_CRYPTO_ACCEPT_OLD)
+		return encrypt_and_sign_sober(instance, buf, buf_len, iovec, iov_len);
+#ifdef HAVE_LIBNSS
+	if (instance->totem_config->crypto_type == TOTEM_CRYPTO_NSS)
+		return encrypt_and_sign_nss(instance, buf, buf_len, iovec, iov_len);
+#endif
+	return -1;
 }
 
+static int authenticate_and_decrypt (
+	struct totemnet_instance *instance,
+	struct iovec *iov,
+	unsigned int iov_len)
+{
+	unsigned char type;
+	unsigned char *endbuf = iov[iov_len-1].iov_base;
+	int res = -1;
+
+	/*
+	 * Get the encryption type and remove it from the buffer
+	 */
+	type = endbuf[iov[iov_len-1].iov_len-1];
+	iov[iov_len-1].iov_len -= 1;
+
+	if (type == TOTEM_CRYPTO_SOBER)
+		res = authenticate_and_decrypt_sober(instance, iov, iov_len);
+
+	/*
+	 * Only try higher crypto options if NEW has been requested
+	 */
+	if (instance->totem_config->crypto_accept == TOTEM_CRYPTO_ACCEPT_NEW) {
+#ifdef HAVE_LIBNSS
+		if (type == TOTEM_CRYPTO_NSS)
+		    res = authenticate_and_decrypt_nss(instance, iov, iov_len);
+#endif
+	}
+
+	/*
+	 * If it failed, then try decrypting the whole packet as it might be
+	 * from aisexec
+	 */
+	if (res == -1) {
+		iov[iov_len-1].iov_len += 1;
+		res = authenticate_and_decrypt_sober(instance, iov, iov_len);
+	}
+
+	return res;
+}
+
+static void init_crypto(
+	struct totemnet_instance *instance)
+{
+	/*
+	 * If we are expecting NEW crypto type then initialise all available
+	 * crypto options. For OLD then we only need SOBER128.
+	 */
+
+	init_sober_crypto(instance);
+
+	if (instance->totem_config->crypto_accept == TOTEM_CRYPTO_ACCEPT_OLD)
+		return;
+
+#ifdef HAVE_LIBNSS
+	init_nss_crypto(instance);
+#endif
+}
+
+int totemnet_crypto_set (hdb_handle_t handle,
+			 unsigned int type)
+{
+	struct totemnet_instance *instance;
+	int res = 0;
+
+	res = hdb_handle_get (&totemnet_instance_database, handle,
+		(void *)&instance);
+	if (res != 0) {
+		res = ENOENT;
+		goto error_exit;
+	}
+
+	/*
+	 * Can't set crypto type if OLD is selected
+	 */
+	if (instance->totem_config->crypto_accept == TOTEM_CRYPTO_ACCEPT_OLD) {
+		res = -1;
+	}
+	else {
+		/*
+		 * Validate number
+		 */
+		if (type == TOTEM_CRYPTO_SOBER ||
+		    type == TOTEM_CRYPTO_NSS) {
+			instance->totem_config->crypto_type = type;
+			log_printf(instance->totemnet_log_level_security, "Encryption type set to %d\n", type);
+		}
+		else {
+			res = -1;
+		}
+	}
+	hdb_handle_put (&totemnet_instance_database, handle);
+
+error_exit:
+	return res;
+}
+
+
 static inline void ucast_sendmsg (
 	struct totemnet_instance *instance,
 	struct totem_ip_address *system_to,
@@ -408,8 +858,14 @@ static inline void ucast_sendmsg (
 			encrypt_data,
 			&buf_len,
 			iovec_encrypt,
-			iov_len_in + 1,
-			&instance->totemnet_prng_state);
+			iov_len_in + 1);
+
+		if (instance->totem_config->crypto_accept == TOTEM_CRYPTO_ACCEPT_NEW) {
+			encrypt_data[buf_len++] = instance->totem_config->crypto_type;
+		}
+		else {
+			encrypt_data[buf_len++] = 0;
+		}
 
 		iovec_encrypt[0].iov_base = encrypt_data;
 		iovec_encrypt[0].iov_len = buf_len;
@@ -472,8 +928,14 @@ static inline void mcast_sendmsg (
 			encrypt_data,
 			&buf_len,
 			iovec_encrypt,
-			iov_len_in + 1,
-			&instance->totemnet_prng_state);
+			iov_len_in + 1);
+
+		if (instance->totem_config->crypto_accept == TOTEM_CRYPTO_ACCEPT_NEW) {
+			encrypt_data[buf_len++] = instance->totem_config->crypto_type;
+		}
+		else {
+			encrypt_data[buf_len++] = 0;
+		}
 
 		iovec_encrypt[0].iov_base = encrypt_data;
 		iovec_encrypt[0].iov_len = buf_len;
@@ -546,8 +1008,7 @@ static void totemnet_mcast_worker_fn (void *thread_state, void *work_item_in)
 		encrypt_and_sign_worker (
 			instance,
 			totemnet_mcast_thread_state->iobuf, &buf_len,
-			work_item->iovec, work_item->iov_len + 1,
-			&totemnet_mcast_thread_state->prng_state);
+			work_item->iovec, work_item->iov_len + 1);
 
 			iovec_sendmsg = &iovec_encrypted;
 			iovec_sendmsg->iov_base = totemnet_mcast_thread_state->iobuf;
@@ -660,8 +1121,9 @@ static int net_deliver_fn (
 		 * Authenticate and if authenticated, decrypt datagram
 		 */
 
-		res = authenticate_and_decrypt (instance, iovec);
+		res = authenticate_and_decrypt (instance, iovec, 1);
 		if (res == -1) {
+			log_printf (instance->totemnet_log_level_security, "Received message has invalid digest... ignoring.\n");
 			log_printf (instance->totemnet_log_level_security,
 				"Invalid packet data\n");
 			iovec->iov_len = FRAME_SIZE_MAX;
@@ -698,7 +1160,7 @@ static int netif_determine (
 
 	res = totemip_iface_check (bindnet, bound_to,
 		interface_up, interface_num,
-+               0); // TODO andrew can address this instance->totem_config->clear_node_high_bit);
+                0); // TODO andrew can address this instance->totem_config->clear_node_high_bit);
 
 
 	return (res);
@@ -1200,7 +1662,7 @@ int totemnet_initialize (
 
 	instance->totemnet_private_key_len = totem_config->private_key_len;
 
-        rng_make_prng (128, PRNG_SOBER, &instance->totemnet_prng_state, NULL);
+	init_crypto(instance);
 
 	/*
 	 * Initialize local variables for totemnet
@@ -1233,8 +1695,6 @@ int totemnet_initialize (
 
 	instance->handle = *handle;
 
-	rng_make_prng (128, PRNG_SOBER, &instance->totemnet_prng_state, NULL);
-
 	totemip_localhost (instance->mcast_address.family, &localhost);
 
 	netif_down_check (instance);

exec/totemnet.h

@@ -105,4 +105,8 @@ extern int totemnet_token_target_set (
 	hdb_handle_t handle,
 	const struct totem_ip_address *token_target);
 
+extern int totemnet_crypto_set (
+	hdb_handle_t handle,
+	unsigned int type);
+
 #endif /* TOTEMNET_H_DEFINED */

exec/totempg.c

@@ -1293,6 +1293,17 @@ int totempg_ifaces_get (
 	return (res);
 }
 
+int totempg_crypto_set (
+	unsigned int type)
+{
+	int res;
+
+	res = totemmrp_crypto_set (
+		type);
+
+	return (res);
+}
+
 int totempg_ring_reenable (void)
 {
 	int res;

exec/totemrrp.c

@@ -1710,6 +1710,26 @@ error_exit:
 	return (res);
 }
 
+int totemrrp_crypto_set (
+	hdb_handle_t handle,
+	unsigned int type)
+{
+	int res;
+	struct totemrrp_instance *instance;
+
+	res = hdb_handle_get (&totemrrp_instance_database, handle,
+		(void *)&instance);
+	if (res != 0) {
+		return (0);
+	}
+
+	res = totemnet_crypto_set(instance->net_handles[0], type);
+
+	hdb_handle_put (&totemrrp_instance_database, handle);
+	return (res);
+}
+
+
 int totemrrp_ring_reenable (
         hdb_handle_t handle)
 {

exec/totemrrp.h

@@ -112,6 +112,10 @@ extern int totemrrp_ifaces_get (
 	char ***status,
 	unsigned int *iface_count);
 
+extern int totemrrp_crypto_set (
+	hdb_handle_t handle,
+	unsigned int type);
+
 extern int totemrrp_ring_reenable (
 	hdb_handle_t handle);
 

exec/totemsrp.c

@@ -79,6 +79,7 @@
 #include <corosync/totem/coropoll.h>
 #include "totemsrp.h"
 #include "totemrrp.h"
+#include "totemnet.h"
 #include "wthread.h"
 
 #include "crypto.h"
@@ -942,6 +943,26 @@ error_exit:
 	return (res);
 }
 
+int totemsrp_crypto_set (
+	hdb_handle_t handle,
+	unsigned int type)
+{
+	int res;
+	struct totemsrp_instance *instance;
+
+	res = hdb_handle_get (&totemsrp_instance_database, handle,
+		(void *)&instance);
+	if (res != 0) {
+		return (0);
+	}
+
+	res = totemrrp_crypto_set(instance->totemrrp_handle, type);
+
+	hdb_handle_put (&totemsrp_instance_database, handle);
+	return (res);
+}
+
+
 unsigned int totemsrp_my_nodeid_get (
 	hdb_handle_t handle)
 {

exec/totemsrp.h

@@ -108,6 +108,10 @@ extern unsigned int totemsrp_my_nodeid_get (
 extern int totemsrp_my_family_get (
 	hdb_handle_t handle);
 
+extern int totemsrp_crypto_set (
+	hdb_handle_t handle,
+	unsigned int type);
+
 extern int totemsrp_ring_reenable (
 	hdb_handle_t handle);
 

include/corosync/cfg.h

@@ -228,6 +228,11 @@ corosync_cfg_local_get (
 	corosync_cfg_handle_t handle,
 	unsigned int *local_nodeid);
 
+cs_error_t
+corosync_cfg_crypto_set (
+	corosync_cfg_handle_t handle,
+	unsigned int type);
+
 #ifdef __cplusplus
 }
 #endif

include/corosync/engine/coroapi.h

@@ -443,6 +443,7 @@ struct corosync_api_v1 {
 
 	const char *(*totem_ip_print) (const struct totem_ip_address *addr);
 
+	int (*totem_crypto_set) (unsigned int type);
 
 	int (*totem_callback_token_create) (
 		void **handle_out,

include/corosync/ipc_cfg.h

@@ -52,7 +52,8 @@ enum req_lib_cfg_types {
 	MESSAGE_REQ_CFG_TRYSHUTDOWN = 9,
 	MESSAGE_REQ_CFG_REPLYTOSHUTDOWN = 10,
 	MESSAGE_REQ_CFG_GET_NODE_ADDRS = 11,
-	MESSAGE_REQ_CFG_LOCAL_GET = 12
+	MESSAGE_REQ_CFG_LOCAL_GET = 12,
+	MESSAGE_REQ_CFG_CRYPTO_SET = 13
 };
 
 enum res_lib_cfg_types {
@@ -69,7 +70,8 @@ enum res_lib_cfg_types {
 	MESSAGE_RES_CFG_TESTSHUTDOWN = 10,
 	MESSAGE_RES_CFG_GET_NODE_ADDRS = 11,
 	MESSAGE_RES_CFG_LOCAL_GET = 12,
-	MESSAGE_RES_CFG_REPLYTOSHUTDOWN = 13
+	MESSAGE_RES_CFG_REPLYTOSHUTDOWN = 13,
+	MESSAGE_RES_CFG_CRYPTO_SET = 14,
 };
 
 struct req_lib_cfg_statetrack {
@@ -205,6 +207,15 @@ struct res_lib_cfg_local_get {
 	mar_uint32_t local_nodeid __attribute__((aligned(8)));
 };
 
+struct req_lib_cfg_crypto_set {
+	coroipc_response_header_t header __attribute__((aligned(8)));
+	mar_uint32_t type __attribute__((aligned(8)));
+};
+
+struct res_lib_cfg_crypto_set {
+	coroipc_response_header_t header __attribute__((aligned(8)));
+};
+
 typedef enum {
 	AIS_AMF_ADMINISTRATIVETARGET_SERVICEUNIT = 0,
 	AIS_AMF_ADMINISTRATIVETARGET_SERVICEGROUP = 1,

include/corosync/totem/totem.h

@@ -145,6 +145,12 @@ struct totem_config {
 	unsigned int max_messages;
 
 	const char *vsf_type;
+
+	enum { TOTEM_CRYPTO_SOBER=0, TOTEM_CRYPTO_NSS } crypto_type;
+	enum { TOTEM_CRYPTO_ACCEPT_OLD=0, TOTEM_CRYPTO_ACCEPT_NEW } crypto_accept;
+
+	int crypto_crypt_type;
+	int crypto_sign_type;
 };
 
 #define TOTEM_CONFIGURATION_TYPE

include/corosync/totem/totempg.h

@@ -145,6 +145,8 @@ extern unsigned int totempg_my_nodeid_get (void);
 
 extern int totempg_my_family_get (void);
 
+extern int totempg_crypto_set (unsigned int type);
+
 extern int totempg_ring_reenable (void);
 
 #endif /* TOTEMPG_H_DEFINED */

lib/cfg.c

@@ -822,3 +822,44 @@ error_exit:
 
 	return (error);
 }
+
+cs_error_t
+corosync_cfg_crypto_set (
+	corosync_cfg_handle_t handle,
+	unsigned int type)
+{
+	struct cfg_instance *cfg_instance;
+	struct req_lib_cfg_crypto_set req_lib_cfg_crypto_set;
+	struct res_lib_cfg_crypto_set res_lib_cfg_crypto_set;
+	struct iovec iov;
+	cs_error_t error;
+
+
+	error = hdb_error_to_cs(hdb_handle_get (&cfg_hdb, handle, (void *)&cfg_instance));
+	if (error != CS_OK) {
+		return (error);
+	}
+
+	req_lib_cfg_crypto_set.header.id = MESSAGE_REQ_CFG_CRYPTO_SET;
+	req_lib_cfg_crypto_set.header.size = sizeof (struct req_lib_cfg_crypto_set);
+	req_lib_cfg_crypto_set.type = type;
+
+	iov.iov_base = &req_lib_cfg_crypto_set;
+	iov.iov_len = sizeof (struct req_lib_cfg_crypto_set);
+
+	pthread_mutex_lock (&cfg_instance->response_mutex);
+
+	error = coroipcc_msg_send_reply_receive (cfg_instance->ipc_ctx,
+		&iov,
+		1,
+		&res_lib_cfg_crypto_set,
+		sizeof (struct res_lib_cfg_crypto_set));
+
+	pthread_mutex_unlock (&cfg_instance->response_mutex);
+
+	if (error == CS_OK)
+		error = res_lib_cfg_crypto_set.header.error;
+
+	(void)hdb_handle_put (&cfg_hdb, handle);
+	return (error);
+}

services/cfg.c

@@ -57,6 +57,7 @@
 #include <corosync/queue.h>
 #include <corosync/mar_gen.h>
 #include <corosync/totem/totemip.h>
+#include <corosync/totem/totem.h>
 #include <corosync/ipc_cfg.h>
 #include <corosync/lcr/lcr_comp.h>
 #include <corosync/engine/logsys.h>
@@ -68,7 +69,8 @@ LOGSYS_DECLARE_SUBSYS ("CFG");
 enum cfg_message_req_types {
         MESSAGE_REQ_EXEC_CFG_RINGREENABLE = 0,
 	MESSAGE_REQ_EXEC_CFG_KILLNODE = 1,
-	MESSAGE_REQ_EXEC_CFG_SHUTDOWN = 2
+	MESSAGE_REQ_EXEC_CFG_SHUTDOWN = 2,
+	MESSAGE_REQ_EXEC_CFG_CRYPTO_SET = 3
 };
 
 #define DEFAULT_SHUTDOWN_TIMEOUT 5
@@ -120,6 +122,10 @@ static void message_handler_req_exec_cfg_shutdown (
         const void *message,
         unsigned int nodeid);
 
+static void message_handler_req_exec_cfg_crypto_set (
+        const void *message,
+        unsigned int nodeid);
+
 static void exec_cfg_killnode_endian_convert (void *msg);
 
 static void message_handler_req_lib_cfg_ringstatusget (
@@ -174,6 +180,10 @@ static void message_handler_req_lib_cfg_local_get (
 	void *conn,
 	const void *msg);
 
+static void message_handler_req_lib_cfg_crypto_set (
+	void *conn,
+	const void *msg);
+
 /*
  * Service Handler Definition
  */
@@ -256,6 +266,12 @@ static struct corosync_lib_handler cfg_lib_engine[] =
 		.response_size		= sizeof (struct res_lib_cfg_local_get),
 		.response_id		= MESSAGE_RES_CFG_LOCAL_GET,
 		.flow_control		= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
+	},
+	{ /* 13 */
+		.lib_handler_fn		= message_handler_req_lib_cfg_crypto_set,
+		.response_size		= sizeof (struct res_lib_cfg_crypto_set),
+		.response_id		= MESSAGE_RES_CFG_CRYPTO_SET,
+		.flow_control		= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
 	}
 };
 
@@ -270,6 +286,9 @@ static struct corosync_exec_handler cfg_exec_engine[] =
 	},
 	{ /* 2 */
 		.exec_handler_fn = message_handler_req_exec_cfg_shutdown,
+	},
+	{ /* 3 */
+		.exec_handler_fn = message_handler_req_exec_cfg_crypto_set,
 	}
 };
 
@@ -342,6 +361,11 @@ struct req_exec_cfg_killnode {
 	mar_name_t reason __attribute__((aligned(8)));
 };
 
+struct req_exec_cfg_crypto_set {
+	coroipc_request_header_t header __attribute__((aligned(8)));
+	mar_uint32_t type __attribute__((aligned(8)));
+};
+
 struct req_exec_cfg_shutdown {
 	coroipc_request_header_t header __attribute__((aligned(8)));
 };
@@ -630,6 +654,19 @@ static void message_handler_req_exec_cfg_shutdown (
 	LEAVE();
 }
 
+static void message_handler_req_exec_cfg_crypto_set (
+        const void *message,
+        unsigned int nodeid)
+{
+	const struct req_exec_cfg_crypto_set *req_exec_cfg_crypto_set = message;
+	ENTER();
+
+	log_printf(LOGSYS_LEVEL_NOTICE, "Node %d requested set crypto to %d\n", nodeid, req_exec_cfg_crypto_set->type);
+
+	api->totem_crypto_set(req_exec_cfg_crypto_set->type);
+	LEAVE();
+}
+
 
 /*
  * Library Interface Implementation
@@ -1058,3 +1095,40 @@ static void message_handler_req_lib_cfg_local_get (void *conn, const void *msg)
 	api->ipc_response_send(conn, &res_lib_cfg_local_get,
 		sizeof(res_lib_cfg_local_get));
 }
+
+
+static void message_handler_req_lib_cfg_crypto_set (
+	void *conn,
+	const void *msg)
+{
+	const struct req_lib_cfg_crypto_set *req_lib_cfg_crypto_set = msg;
+	struct res_lib_cfg_crypto_set res_lib_cfg_crypto_set;
+	struct req_exec_cfg_crypto_set req_exec_cfg_crypto_set;
+	struct iovec iovec;
+	int ret = CS_ERR_INVALID_PARAM;
+
+	req_exec_cfg_crypto_set.header.size =
+		sizeof (struct req_exec_cfg_crypto_set);
+	req_exec_cfg_crypto_set.header.id = SERVICE_ID_MAKE (CFG_SERVICE,
+		MESSAGE_REQ_EXEC_CFG_CRYPTO_SET);
+
+	/*
+	 * Set it locally first so we can tell if it is allowed
+	 */
+	if (api->totem_crypto_set(req_lib_cfg_crypto_set->type) == 0) {
+
+		req_exec_cfg_crypto_set.type = req_lib_cfg_crypto_set->type;
+
+		iovec.iov_base = (char *)&req_exec_cfg_crypto_set;
+		iovec.iov_len = sizeof (struct req_exec_cfg_crypto_set);
+		assert (api->totem_mcast (&iovec, 1, TOTEM_SAFE) == 0);
+		ret = CS_OK;
+	}
+
+	res_lib_cfg_crypto_set.header.size = sizeof(res_lib_cfg_crypto_set);
+	res_lib_cfg_crypto_set.header.id = MESSAGE_RES_CFG_CRYPTO_SET;
+	res_lib_cfg_crypto_set.header.error = ret;
+
+	api->ipc_response_send(conn, &res_lib_cfg_crypto_set,
+		sizeof(res_lib_cfg_crypto_set));
+}

tools/corosync-cfgtool.c

@@ -211,6 +211,26 @@ static void showaddrs_do(int nodeid)
 	(void)corosync_cfg_finalize (handle);
 }
 
+
+static void crypto_do(unsigned int type)
+{
+	cs_error_t result;
+	corosync_cfg_handle_t handle;
+
+	printf ("Setting crypto to mode %d\n", type);
+	result = corosync_cfg_initialize (&handle, NULL);
+	if (result != CS_OK) {
+		printf ("Could not initialize corosync configuration API error %d\n", result);
+		exit (1);
+	}
+	result = corosync_cfg_crypto_set (handle, type);
+	if (result != CS_OK) {
+		printf ("Could not set crypto mode (error = %d)\n", result);
+	}
+	(void)corosync_cfg_finalize (handle);
+
+}
+
 static void killnode_do(unsigned int nodeid)
 {
 	cs_error_t result;
@@ -241,6 +261,7 @@ static void usage_do (void)
 	printf ("\t-l\tLoad a service identified by name.\n");
 	printf ("\t-u\tUnload a service identified by name.\n");
 	printf ("\t-a\tDisplay the IP address(es) of a node\n");
+	printf ("\t-c\tSet the cryptography mode of cluster communications\n");
 	printf ("\t-k\tKill a node identified by node id.\n");
 	printf ("\t-H\tShutdown corosync cleanly on this node.\n");
 }
@@ -257,7 +278,7 @@ xstrdup (char const *s)
 }
 
 int main (int argc, char *argv[]) {
-	const char *options = "srl:u:v:k:a:hH";
+	const char *options = "srl:u:v:k:a:c:hH";
 	int opt;
 	int service_load = 0;
 	unsigned int nodeid;
@@ -294,6 +315,9 @@ int main (int argc, char *argv[]) {
 		case 'a':
 			showaddrs_do( atoi(optarg) );
 			break;
+		case 'c':
+			crypto_do( atoi(optarg) );
+			break;
 		case 'v':
 			version = atoi (optarg);
 			break;