totemcrypto: implement crypto packet format 2.2 and crypto_compat: config opt

Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

conf/corosync.conf.example

@@ -8,6 +8,13 @@ totem {
 	crypto_cipher: none
 	crypto_hash: none
 
+	# crypto_compat: 2.0|2.2 (default higher) can be used to change
+	# on-wire crypto packet format. Unless performing some special
+	# rolling upgrades from corosync < 2.2 to 2.2, to keep the cluster
+	# running, do not touch this option. This option cannot be changed
+	# at runtime.
+	#crypto_compat: 2.2
+
 	# interface: define at least one interface to communicate
 	# over. If you define more than one interface stanza, you must
 	# also set rrp_mode.

conf/lenses/corosync.aug

@@ -53,6 +53,7 @@ let totem =
     |kv "crypto_type" /nss|aes256|aes192|aes128|3des/
     |kv "crypto_cipher" /none|nss|aes256|aes192|aes128|3des/
     |kv "crypto_hash" /none|md5|sha1|sha256|sha384|sha512/
+    |kv "crypto_compat" /2.0|2.2/
     |kv "transport" /udp|iba/
     |kv "version" Rx.integer
     |kv "nodeid" Rx.integer

conf/lenses/tests/test_corosync.aug

@@ -7,6 +7,7 @@ totem {
 	secauth: off
 	crypto_cipher: none
 	crypto_hash: none
+	crypto_compat: 2.2
 	threads: 0
 	clear_node_high_bit: no
 	rrp_mode: none
@@ -96,6 +97,7 @@ test Corosync.lns get conf =
 	{ "secauth" = "off" }
 	{ "crypto_cipher" = "none" }
 	{ "crypto_hash" = "none" }
+	{ "crypto_compat" = "2.2" }
 	{ "threads" = "0" }
     { "clear_node_high_bit" = "no" }
     { "rrp_mode" = "none" }

exec/coroparse.c

@@ -535,6 +535,14 @@ static int main_config_parser_cb(const char *path,
 					return (0);
 				}
 			}
+			if (strcmp(path, "totem.crypto_compat") == 0) {
+				if ((strcmp(value, "2.0") != 0) &&
+				    (strcmp(value, "2.2") != 0)) {
+					*error_string = "Invalid crypto compat type";
+
+					return (0);
+				}
+			}
 			break;
 
 		case MAIN_CP_CB_DATA_STATE_QB:

exec/main.c

@@ -907,6 +907,7 @@ static void set_icmap_ro_keys_flag (void)
 	 */
 	icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE);
 	icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE);
+	icmap_set_ro_access("totem.crypto_compat", CS_FALSE, CS_TRUE);
 	icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE);
 	icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE);
 	icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE);

exec/totemconfig.c

@@ -119,9 +119,11 @@ static void totem_get_crypto(struct totem_config *totem_config)
 	char *str;
 	const char *tmp_cipher;
 	const char *tmp_hash;
+	const char *tmp_compat;
 
 	tmp_hash = "sha1";
 	tmp_cipher = "aes256";
+	tmp_compat = "2.2";
 
 	if (icmap_get_string("totem.secauth", &str) == CS_OK) {
 		if (strcmp (str, "off") == 0) {
@@ -172,11 +174,23 @@ static void totem_get_crypto(struct totem_config *totem_config)
 		free(str);
 	}
 
+	if (icmap_get_string("totem.crypto_compat", &str) == CS_OK) {
+		if (strcmp(str, "2.0") == 0) {
+			tmp_compat = "2.0";
+		}
+		if (strcmp(str, "2.2") == 0) {
+			tmp_compat = "2.2";
+		}
+		free(str);
+	}
+
 	free(totem_config->crypto_cipher_type);
 	free(totem_config->crypto_hash_type);
+	free(totem_config->crypto_compat_type);
 
 	totem_config->crypto_cipher_type = strdup(tmp_cipher);
 	totem_config->crypto_hash_type = strdup(tmp_hash);
+	totem_config->crypto_compat_type = strdup(tmp_compat);
 }
 
 static uint16_t generate_cluster_id (const char *cluster_name)

exec/totemcrypto.c

@@ -81,12 +81,19 @@ struct crypto_config_header {
 #define AES_128_KEY_LENGTH 16
 #endif
 
+/*
+ * while CRYPTO_CIPHER_TYPE_2_2 is not a real cipher at all,
+ * we still allocate a value for it because we use crypto_crypt_t
+ * internally and we don't want overlaps
+ */
+
 enum crypto_crypt_t {
 	CRYPTO_CIPHER_TYPE_NONE = 0,
 	CRYPTO_CIPHER_TYPE_AES256 = 1,
 	CRYPTO_CIPHER_TYPE_AES192 = 2,
 	CRYPTO_CIPHER_TYPE_AES128 = 3,
-	CRYPTO_CIPHER_TYPE_3DES = 4
+	CRYPTO_CIPHER_TYPE_3DES = 4,
+	CRYPTO_CIPHER_TYPE_2_2 = UINT8_MAX
 };
 
 CK_MECHANISM_TYPE cipher_to_nss[] = {
@@ -117,13 +124,20 @@ size_t cypher_block_len[] = {
  * hash definitions and conversion tables
  */
 
+/*
+ * while CRYPTO_HASH_TYPE_2_2 is not a real hash mechanism at all,
+ * we still allocate a value for it because we use crypto_hash_t
+ * internally and we don't want overlaps
+ */
+
 enum crypto_hash_t {
 	CRYPTO_HASH_TYPE_NONE	= 0,
 	CRYPTO_HASH_TYPE_MD5	= 1,
 	CRYPTO_HASH_TYPE_SHA1	= 2,
 	CRYPTO_HASH_TYPE_SHA256	= 3,
 	CRYPTO_HASH_TYPE_SHA384	= 4,
-	CRYPTO_HASH_TYPE_SHA512	= 5
+	CRYPTO_HASH_TYPE_SHA512	= 5,
+	CRYPTO_HASH_TYPE_2_2	= UINT8_MAX
 };
 
 CK_MECHANISM_TYPE hash_to_nss[] = {
@@ -153,6 +167,15 @@ size_t hash_block_len[] = {
 	SHA512_BLOCK_LENGTH		/* CRYPTO_HASH_TYPE_SHA512 */
 };
 
+/*
+ * crypto on-wire compat
+ */
+
+enum crypto_compat_t {
+	CRYPTO_COMPAT_2_0	= 0,
+	CRYPTO_COMPAT_2_2	= 1
+};
+
 struct crypto_instance {
 	PK11SymKey   *nss_sym_key;
 	PK11SymKey   *nss_sym_key_sign;
@@ -165,6 +188,8 @@ struct crypto_instance {
 
 	enum crypto_hash_t crypto_hash_type;
 
+	enum crypto_compat_t crypto_compat_type;
+
 	unsigned int crypto_header_size;
 
 	void (*log_printf_func) (
@@ -190,6 +215,20 @@ do {									\
 		(const char *)format, ##args);				\
 } while (0);
 
+/*
+ * compat functions
+ */
+
+static int string_to_crypto_compat_type(const char* crypto_compat_type)
+{
+	if (strcmp(crypto_compat_type, "2.0") == 0) {
+		return CRYPTO_COMPAT_2_0;
+	} else if (strcmp(crypto_compat_type, "2.1") == 0) {
+		return CRYPTO_COMPAT_2_2;
+	}
+	return CRYPTO_COMPAT_2_2;
+}
+
 /*
  * crypt/decrypt functions
  */
@@ -556,11 +595,12 @@ static int init_nss_db(struct crypto_instance *instance)
 
 static int init_nss(struct crypto_instance *instance,
 		    const char *crypto_cipher_type,
-		    const char *crypto_hash_type)
+		    const char *crypto_hash_type,
+		    const char *crypto_compat_type)
 {
 	log_printf(instance->log_level_notice,
-		   "Initializing transmit/receive security (NSS) crypto: %s hash: %s",
-		   crypto_cipher_type, crypto_hash_type);
+		   "Initializing transmit/receive security (NSS) crypto: %s hash: %s compat: %s",
+		   crypto_cipher_type, crypto_hash_type, crypto_compat_type);
 
 	if (init_nss_db(instance) < 0) {
 		return -1;
@@ -577,7 +617,7 @@ static int init_nss(struct crypto_instance *instance,
 	return 0;
 }
 
-static int encrypt_and_sign_nss (
+static int encrypt_and_sign_nss_2_0 (
 	struct crypto_instance *instance,
 	const unsigned char *buf_in,
 	const size_t buf_in_len,
@@ -601,7 +641,32 @@ static int encrypt_and_sign_nss (
 	return 0;
 }
 
-static int authenticate_and_decrypt_nss (
+static int encrypt_and_sign_nss_2_2 (
+	struct crypto_instance *instance,
+	const unsigned char *buf_in,
+	const size_t buf_in_len,
+	unsigned char *buf_out,
+	size_t *buf_out_len)
+{
+	if (encrypt_nss(instance,
+			buf_in, buf_in_len,
+			buf_out + sizeof(struct crypto_config_header), buf_out_len) < 0) {
+		return -1;
+	}
+
+	*buf_out_len += sizeof(struct crypto_config_header);
+
+	if (hash_to_nss[instance->crypto_hash_type]) {
+		if (calculate_nss_hash(instance, buf_out, *buf_out_len, buf_out + *buf_out_len) < 0) {
+			return -1;
+		}
+		*buf_out_len += hash_len[instance->crypto_hash_type];
+	}
+
+	return 0;
+}
+
+static int authenticate_and_decrypt_nss_2_0 (
 	struct crypto_instance *instance,
 	unsigned char *buf,
 	int *buf_len)
@@ -632,6 +697,43 @@ static int authenticate_and_decrypt_nss (
 	return 0;
 }
 
+static int authenticate_nss_2_2 (
+	struct crypto_instance *instance,
+	unsigned char *buf,
+	int *buf_len)
+{
+	if (hash_to_nss[instance->crypto_hash_type]) {
+		unsigned char	tmp_hash[hash_len[instance->crypto_hash_type]];
+		int             datalen = *buf_len - hash_len[instance->crypto_hash_type];
+
+		if (calculate_nss_hash(instance, buf, datalen, tmp_hash) < 0) {
+			return -1;
+		}
+
+		if (memcmp(tmp_hash, buf + datalen, hash_len[instance->crypto_hash_type]) != 0) {
+			log_printf(instance->log_level_error, "Digest does not match");
+			return -1;
+		}
+		*buf_len = datalen;
+	}
+
+	return 0;
+}
+
+static int decrypt_nss_2_2 (
+	struct crypto_instance *instance,
+	unsigned char *buf,
+	int *buf_len)
+{
+	*buf_len -= sizeof(struct crypto_config_header);
+
+	if (decrypt_nss(instance, buf + sizeof(struct crypto_config_header), buf_len) < 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
 /*
  * exported API
  */
@@ -658,6 +760,20 @@ size_t crypto_sec_header_size(
 	return hdr_size;
 }
 
+/*
+ * 2.0 packet format:
+ *   crypto_cipher_type | crypto_hash_type | __pad0 | __pad1 | hash | salt | data
+ *   only data is encrypted, hash only covers salt + data
+ *
+ * 2.2 packet format
+ *   fake_crypto_cipher_type | fake_crypto_hash_type | __pad0 | __pad1 | salt | data | hash
+ *   only data is encrypted, hash covers the whole packet
+ *
+ *  we need to leave fake_* unencrypted for older versions of corosync to reject the packets,
+ *  we need to leave __pad0|1 unencrypted for performance reasons (saves at least 2 memcpy and
+ *  and extra buffer but values are hashed and verified.
+ */
+
 int crypto_encrypt_and_sign (
 	struct crypto_instance *instance,
 	const unsigned char *buf_in,
@@ -668,18 +784,35 @@ int crypto_encrypt_and_sign (
 	struct crypto_config_header *cch = (struct crypto_config_header *)buf_out;
 	int err;
 
-	cch->crypto_cipher_type = instance->crypto_cipher_type;
-	cch->crypto_hash_type = instance->crypto_hash_type;
-	cch->__pad0 = 0;
-	cch->__pad1 = 0;
+	switch (instance->crypto_compat_type) {
+		case CRYPTO_COMPAT_2_0:
+			cch->crypto_cipher_type = instance->crypto_cipher_type;
+			cch->crypto_hash_type = instance->crypto_hash_type;
+			cch->__pad0 = 0;
+			cch->__pad1 = 0;
+
+			buf_out += sizeof(struct crypto_config_header);
 
-	buf_out += sizeof(struct crypto_config_header);
+			err = encrypt_and_sign_nss_2_0(instance,
+						       buf_in, buf_in_len,
+						       buf_out, buf_out_len);
 
-	err = encrypt_and_sign_nss(instance,
-				   buf_in, buf_in_len,
-				   buf_out, buf_out_len);
+			*buf_out_len = *buf_out_len + sizeof(struct crypto_config_header);
+			break;
+		case CRYPTO_COMPAT_2_2:
+			cch->crypto_cipher_type = CRYPTO_CIPHER_TYPE_2_2;
+			cch->crypto_hash_type = CRYPTO_HASH_TYPE_2_2;
+			cch->__pad0 = 0;
+			cch->__pad1 = 0;
 
-	*buf_out_len = *buf_out_len + sizeof(struct crypto_config_header);
+			err = encrypt_and_sign_nss_2_2(instance,
+						   buf_in, buf_in_len,
+						   buf_out, buf_out_len);
+			break;
+		default:
+			err = -1;
+			break;
+	}
 
 	return err;
 }
@@ -690,36 +823,92 @@ int crypto_authenticate_and_decrypt (struct crypto_instance *instance,
 {
 	struct crypto_config_header *cch = (struct crypto_config_header *)buf;
 
-	/*
-	 * decode crypto config of incoming packets
-	 */
-
-	if (cch->crypto_cipher_type != instance->crypto_cipher_type) {
-		log_printf(instance->log_level_security,
-			   "Incoming packet has different crypto type. Rejecting");
-		return -1;
-	}
-
-	if (cch->crypto_hash_type != instance->crypto_hash_type) {
-		log_printf(instance->log_level_security,
-			   "Incoming packet has different hash type. Rejecting");
-		return -1;
-	}
-
-	if ((cch->__pad0 != 0) || (cch->__pad1 != 0)) {
-		log_printf(instance->log_level_security,
-			   "Incoming packet appears to have features not supported by this version of corosync. Rejecting");
-		return -1;
+	switch (instance->crypto_compat_type) {
+		case CRYPTO_COMPAT_2_0:
+
+			/*
+			 * decode crypto config of incoming packets
+			 */
+
+			if (cch->crypto_cipher_type != instance->crypto_cipher_type) {
+				log_printf(instance->log_level_security,
+					   "Incoming packet has different crypto type. Rejecting");
+				return -1;
+			}
+
+			if (cch->crypto_hash_type != instance->crypto_hash_type) {
+				log_printf(instance->log_level_security,
+					   "Incoming packet has different hash type. Rejecting");
+				return -1;
+			}
+
+			if ((cch->__pad0 != 0) || (cch->__pad1 != 0)) {
+				log_printf(instance->log_level_security,
+					   "Incoming packet appears to have features not supported by this version of corosync. Rejecting");
+				return -1;
+			}
+
+			/*
+			 * invalidate config header and kill it
+			 */
+
+			cch = NULL;
+			*buf_len -= sizeof(struct crypto_config_header);
+			memmove(buf, buf + sizeof(struct crypto_config_header), *buf_len);
+
+			return authenticate_and_decrypt_nss_2_0(instance, buf, buf_len);
+			break;
+		case CRYPTO_COMPAT_2_2:
+			if (cch->crypto_cipher_type != CRYPTO_CIPHER_TYPE_2_2) {
+				log_printf(instance->log_level_security,
+					   "Incoming packet has different crypto type. Rejecting");
+				return -1;
+			}
+
+			if (cch->crypto_hash_type != CRYPTO_HASH_TYPE_2_2) {
+				log_printf(instance->log_level_security,
+					   "Incoming packet has different hash type. Rejecting");
+				return -1;
+			}
+
+			/*
+			 * authenticate packet first
+			 */
+
+			if (authenticate_nss_2_2(instance, buf, buf_len) != 0) {
+				return -1;
+			}
+
+			/*
+			 * now we can "trust" the padding bytes/future features
+			 */
+
+			if ((cch->__pad0 != 0) || (cch->__pad1 != 0)) {
+				log_printf(instance->log_level_security,
+					   "Incoming packet appears to have features not supported by this version of corosync. Rejecting");
+				return -1;
+			}
+
+			/*
+			 * decrypt
+			 */
+
+			if (decrypt_nss_2_2(instance, buf, buf_len) != 0) {
+				return -1;
+			}
+
+			/*
+			 * invalidate config header and kill it
+			 */
+			cch = NULL;
+			memmove(buf, buf + sizeof(struct crypto_config_header), *buf_len);
+
+			return 0;
+			break;
+		default:
+			return -1;
+			break;
 	}
-
-	/*
-	 * invalidate config header and kill it
-	 */
-	cch = NULL;
-	*buf_len -= sizeof(struct crypto_config_header);
-	memmove(buf, buf + sizeof(struct crypto_config_header), *buf_len);
-
-	return authenticate_and_decrypt_nss(instance, buf, buf_len);
 }
 
 struct crypto_instance *crypto_init(
@@ -727,6 +916,7 @@ struct crypto_instance *crypto_init(
 	unsigned int private_key_len,
 	const char *crypto_cipher_type,
 	const char *crypto_hash_type,
+	const char *crypto_compat_type,
 	void (*log_printf_func) (
 		int level,
 		int subsys,
@@ -752,6 +942,7 @@ struct crypto_instance *crypto_init(
 
 	instance->crypto_cipher_type = string_to_crypto_cipher_type(crypto_cipher_type);
 	instance->crypto_hash_type = string_to_crypto_hash_type(crypto_hash_type);
+	instance->crypto_compat_type = string_to_crypto_compat_type(crypto_compat_type);
 
 	instance->crypto_header_size = crypto_sec_header_size(crypto_cipher_type, crypto_hash_type);
 
@@ -761,7 +952,7 @@ struct crypto_instance *crypto_init(
 	instance->log_level_error = log_level_error;
 	instance->log_subsys_id = log_subsys_id;
 
-	if (init_nss(instance, crypto_cipher_type, crypto_hash_type) < 0) {
+	if (init_nss(instance, crypto_cipher_type, crypto_hash_type, crypto_compat_type) < 0) {
 		free(instance);
 		return(NULL);
 	}

exec/totemcrypto.h

@@ -61,6 +61,7 @@ extern struct crypto_instance *crypto_init(
 	unsigned int private_key_len,
 	const char *crypto_cipher_type,
 	const char *crypto_hash_type,
+	const char *crypto_compat_type,
 	void (*log_printf_func) (
 		int level,
 		int subsys,

exec/totemudp.c

@@ -1148,6 +1148,7 @@ int totemudp_initialize (
 			totem_config->private_key_len,
 			totem_config->crypto_cipher_type,
 			totem_config->crypto_hash_type,
+			totem_config->crypto_compat_type,
 			instance->totemudp_log_printf,
 			instance->totemudp_log_level_security,
 			instance->totemudp_log_level_notice,

exec/totemudpu.c

@@ -779,6 +779,7 @@ int totemudpu_initialize (
 		totem_config->private_key_len,
 		totem_config->crypto_cipher_type,
 		totem_config->crypto_hash_type,
+		totem_config->crypto_compat_type,
 		instance->totemudpu_log_printf,
 		instance->totemudpu_log_level_security,
 		instance->totemudpu_log_level_notice,

include/corosync/totem/totem.h

@@ -176,6 +176,8 @@ struct totem_config {
 
 	char *crypto_hash_type;
 
+	char *crypto_compat_type;
+
 	totem_transport_t transport_number;
 
 	unsigned int miss_count_const;

man/corosync.conf.5

@@ -180,6 +180,19 @@ Valid values are none (no encryption), aes256, aes192, aes128 and 3des.
 
 The default is aes256.
 
+.TP
+crypto_compat
+This specifies which crypto protocol version should be used to encrypt all messages.
+Valid values are 2.0 and 2.2.
+
+The default is always is the higher supported version.
+
+This value should only be used when performing rolling upgrades from older
+versions of corosync to newer ones. It cannot be changed at runtime.
+
+Once the upgrade is completed, the cluster should be temporary halted to
+switch to the latest version of the protocol.
+
 .TP
 secauth
 This specifies that HMAC/SHA1 authentication should be used to authenticate