Add totem stats to objdb.

git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@2517 fd59a12c-fef9-0310-b244-a6a79926bd2f

exec/apidef.c

@@ -117,6 +117,7 @@ static struct corosync_api_v1 apidef_corosync_api_v1 = {
 	.totem_ip_print = totemip_print,
 	.totem_crypto_set = totempg_crypto_set,
 	.totem_callback_token_create = totempg_callback_token_create,
+	.totem_get_stats = totempg_get_stats,
 	.tpg_init = totempg_groups_initialize,
 	.tpg_exit = NULL, /* missing from totempg api */
 	.tpg_join = (typedef_tpg_join)totempg_groups_join,

exec/main.c

@@ -131,6 +131,8 @@ struct sched_param global_sched_param;
 
 static hdb_handle_t object_connection_handle;
 
+static corosync_timer_handle_t corosync_stats_timer_handle;
+
 hdb_handle_t corosync_poll_handle_get (void)
 {
 	return (corosync_poll_handle);
@@ -418,6 +420,254 @@ static void corosync_mlockall (void)
 #endif
 }
 
+
+static void corosync_totem_stats_updater (void *data)
+{
+	totempg_stats_t * stats;
+	uint32_t mtt_rx_token;
+	uint32_t total_mtt_rx_token;
+	uint32_t avg_backlog_calc;
+	uint32_t total_backlog_calc;
+	uint32_t avg_token_holdtime;
+	uint32_t total_token_holdtime;
+	int t, prev;
+	int32_t token_count;
+
+	stats = api->totem_get_stats();
+
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"orf_token_tx", strlen("orf_token_tx"),
+		&stats->mrp->srp->orf_token_tx, sizeof (stats->mrp->srp->orf_token_tx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"orf_token_rx", strlen("orf_token_rx"),
+		&stats->mrp->srp->orf_token_rx, sizeof (stats->mrp->srp->orf_token_rx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"memb_merge_detect_tx", strlen("memb_merge_detect_tx"),
+		&stats->mrp->srp->memb_merge_detect_tx, sizeof (stats->mrp->srp->memb_merge_detect_tx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"memb_merge_detect_rx", strlen("memb_merge_detect_rx"),
+		&stats->mrp->srp->memb_merge_detect_rx, sizeof (stats->mrp->srp->memb_merge_detect_rx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"memb_join_tx", strlen("memb_join_tx"),
+		&stats->mrp->srp->memb_join_tx, sizeof (stats->mrp->srp->memb_join_tx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"memb_join_rx", strlen("memb_join_rx"),
+		&stats->mrp->srp->memb_join_rx, sizeof (stats->mrp->srp->memb_join_rx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"mcast_tx", strlen("mcast_tx"),
+		&stats->mrp->srp->mcast_tx,	sizeof (stats->mrp->srp->mcast_tx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"mcast_retx", strlen("mcast_retx"),
+		&stats->mrp->srp->mcast_retx, sizeof (stats->mrp->srp->mcast_retx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"mcast_rx", strlen("mcast_rx"),
+		&stats->mrp->srp->mcast_rx, sizeof (stats->mrp->srp->mcast_rx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"memb_commit_token_tx", strlen("memb_commit_token_tx"),
+		&stats->mrp->srp->memb_commit_token_tx, sizeof (stats->mrp->srp->memb_commit_token_tx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"memb_commit_token_rx", strlen("memb_commit_token_rx"),
+		&stats->mrp->srp->memb_commit_token_rx, sizeof (stats->mrp->srp->memb_commit_token_rx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"token_hold_cancel_tx", strlen("token_hold_cancel_tx"),
+		&stats->mrp->srp->token_hold_cancel_tx, sizeof (stats->mrp->srp->token_hold_cancel_tx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"token_hold_cancel_rx", strlen("token_hold_cancel_rx"),
+		&stats->mrp->srp->token_hold_cancel_rx, sizeof (stats->mrp->srp->token_hold_cancel_rx));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"operational_entered", strlen("operational_entered"),
+		&stats->mrp->srp->operational_entered, sizeof (stats->mrp->srp->operational_entered));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"operational_token_lost", strlen("operational_token_lost"),
+		&stats->mrp->srp->operational_token_lost, sizeof (stats->mrp->srp->operational_token_lost));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"gather_entered", strlen("gather_entered"),
+		&stats->mrp->srp->gather_entered, sizeof (stats->mrp->srp->gather_entered));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"gather_token_lost", strlen("gather_token_lost"),
+		&stats->mrp->srp->gather_token_lost, sizeof (stats->mrp->srp->gather_token_lost));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"commit_entered", strlen("commit_entered"),
+		&stats->mrp->srp->commit_entered, sizeof (stats->mrp->srp->commit_entered));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"commit_token_lost", strlen("commit_token_lost"),
+		&stats->mrp->srp->commit_token_lost, sizeof (stats->mrp->srp->commit_token_lost));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"recovery_entered", strlen("recovery_entered"),
+		&stats->mrp->srp->recovery_entered, sizeof (stats->mrp->srp->recovery_entered));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"recovery_token_lost", strlen("recovery_token_lost"),
+		&stats->mrp->srp->recovery_token_lost, sizeof (stats->mrp->srp->recovery_token_lost));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"consensus_timeouts", strlen("consensus_timeouts"),
+		&stats->mrp->srp->consensus_timeouts, sizeof (stats->mrp->srp->consensus_timeouts));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+		"rx_msg_dropped", strlen("rx_msg_dropped"),
+		&stats->mrp->srp->rx_msg_dropped, sizeof (stats->mrp->srp->rx_msg_dropped));
+
+	total_mtt_rx_token = 0;
+	total_token_holdtime = 0;
+	total_backlog_calc = 0;
+	token_count = 0;
+	t = stats->mrp->srp->latest_token;
+	while (1) {
+		if (t == 0)
+			prev = TOTEM_TOKEN_STATS_MAX - 1;
+		else
+			prev = t - 1;
+		if (prev == stats->mrp->srp->earliest_token)
+			break;
+		/* if tx == 0, then dropped token (not ours) */
+		if (stats->mrp->srp->token[t].tx != 0 ||
+			(stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx) > 0 ) {
+			total_mtt_rx_token += (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx);
+			total_token_holdtime += (stats->mrp->srp->token[t].tx - stats->mrp->srp->token[t].rx);
+			total_backlog_calc += stats->mrp->srp->token[t].backlog_calc;
+			token_count++;
+		}
+		t = prev;
+	}
+	mtt_rx_token = (total_mtt_rx_token / token_count);
+	avg_backlog_calc = (total_backlog_calc / token_count);
+	avg_token_holdtime = (total_token_holdtime / token_count);
+
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+			"mtt_rx_token", strlen("mtt_rx_token"),
+			&mtt_rx_token, sizeof (mtt_rx_token));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+			"avg_token_workload", strlen("avg_token_workload"),
+			&avg_token_holdtime, sizeof (avg_backlog_calc));
+	objdb->object_key_replace (stats->mrp->srp->hdr.handle,
+			"avg_backlog_calc", strlen("avg_backlog_calc"),
+			&avg_backlog_calc, sizeof (avg_backlog_calc));
+
+	api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
+					corosync_totem_stats_updater,
+					&corosync_stats_timer_handle);
+}
+
+static void corosync_totem_stats_init (void)
+{
+	totempg_stats_t * stats;
+	hdb_handle_t object_find_handle;
+	hdb_handle_t object_runtime_handle;
+	hdb_handle_t object_totem_handle;
+	uint32_t zero_32 = 0;
+	uint64_t zero_64 = 0;
+
+	stats = api->totem_get_stats();
+
+	objdb->object_find_create (
+		OBJECT_PARENT_HANDLE,
+		"runtime",
+		strlen ("runtime"),
+		&object_find_handle);
+
+	if (objdb->object_find_next (
+			object_find_handle,
+			&object_runtime_handle) == 0) {
+
+		objdb->object_create (object_runtime_handle,
+							  &object_totem_handle,
+							  "totem", strlen ("totem"));
+		objdb->object_create (object_totem_handle,
+							  &stats->hdr.handle,
+							  "pg", strlen ("pg"));
+		objdb->object_create (stats->hdr.handle,
+							  &stats->mrp->hdr.handle,
+							  "mrp", strlen ("mrp"));
+		objdb->object_create (stats->mrp->hdr.handle,
+							  &stats->mrp->srp->hdr.handle,
+							  "srp", strlen ("srp"));
+
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"orf_token_tx",	&stats->mrp->srp->orf_token_tx,
+			sizeof (stats->mrp->srp->orf_token_tx),	OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"orf_token_rx", &stats->mrp->srp->orf_token_rx,
+			sizeof (stats->mrp->srp->orf_token_rx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"memb_merge_detect_tx", &stats->mrp->srp->memb_merge_detect_tx,
+			sizeof (stats->mrp->srp->memb_merge_detect_tx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"memb_merge_detect_rx", &stats->mrp->srp->memb_merge_detect_rx,
+			sizeof (stats->mrp->srp->memb_merge_detect_rx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"memb_join_tx", &stats->mrp->srp->memb_join_tx,
+			sizeof (stats->mrp->srp->memb_join_tx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"memb_join_rx", &stats->mrp->srp->memb_join_rx,
+			 sizeof (stats->mrp->srp->memb_join_rx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"mcast_tx", &stats->mrp->srp->mcast_tx,
+			sizeof (stats->mrp->srp->mcast_tx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"mcast_retx", &stats->mrp->srp->mcast_retx,
+			sizeof (stats->mrp->srp->mcast_retx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"mcast_rx", &stats->mrp->srp->mcast_rx,
+			sizeof (stats->mrp->srp->mcast_rx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"memb_commit_token_tx", &stats->mrp->srp->memb_commit_token_tx,
+			sizeof (stats->mrp->srp->memb_commit_token_tx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"memb_commit_token_rx", &stats->mrp->srp->memb_commit_token_rx,
+			sizeof (stats->mrp->srp->memb_commit_token_rx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"token_hold_cancel_tx", &stats->mrp->srp->token_hold_cancel_tx,
+			sizeof (stats->mrp->srp->token_hold_cancel_tx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"token_hold_cancel_rx", &stats->mrp->srp->token_hold_cancel_rx,
+			sizeof (stats->mrp->srp->token_hold_cancel_rx), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"operational_entered", &stats->mrp->srp->operational_entered,
+			sizeof (stats->mrp->srp->operational_entered), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"operational_token_lost", &stats->mrp->srp->operational_token_lost,
+			sizeof (stats->mrp->srp->operational_token_lost), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"gather_entered", &stats->mrp->srp->gather_entered,
+			sizeof (stats->mrp->srp->gather_entered), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"gather_token_lost", &stats->mrp->srp->gather_token_lost,
+			sizeof (stats->mrp->srp->gather_token_lost), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"commit_entered", &stats->mrp->srp->commit_entered,
+			sizeof (stats->mrp->srp->commit_entered), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"commit_token_lost", &stats->mrp->srp->commit_token_lost,
+			sizeof (stats->mrp->srp->commit_token_lost), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"recovery_entered", &stats->mrp->srp->recovery_entered,
+			sizeof (stats->mrp->srp->recovery_entered), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"recovery_token_lost", &stats->mrp->srp->recovery_token_lost,
+			sizeof (stats->mrp->srp->recovery_token_lost), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"consensus_timeouts", &stats->mrp->srp->consensus_timeouts,
+			sizeof (stats->mrp->srp->consensus_timeouts), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"mtt_rx_token", &zero_32,
+			sizeof (zero_32), OBJDB_VALUETYPE_UINT32);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"avg_token_workload", &zero_32,
+			sizeof (zero_32), OBJDB_VALUETYPE_UINT32);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"avg_backlog_calc", &zero_64,
+			sizeof (zero_64), OBJDB_VALUETYPE_UINT64);
+		objdb->object_key_create_typed (stats->mrp->srp->hdr.handle,
+			"rx_msg_dropped", &zero_64,
+			sizeof (zero_64), OBJDB_VALUETYPE_UINT64);
+
+	}
+	/* start stats timer */
+	api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
+					corosync_totem_stats_updater,
+					&corosync_stats_timer_handle);
+
+}
+
+
 static void deliver_fn (
 	unsigned int nodeid,
 	const void *msg,
@@ -898,6 +1148,7 @@ static void main_service_ready (void)
 	}
 	evil_init (api);
 	corosync_stats_init ();
+	corosync_totem_stats_init ();
 }
 
 int main (int argc, char **argv)

exec/service.c

@@ -345,7 +345,6 @@ extern unsigned int corosync_service_unlink_and_exit (
 	unsigned short *service_id;
 	unsigned int *found_service_ver;
 	hdb_handle_t object_find_handle;
-	char object_name[32];
 	char *name_sufix;
 
 	name_sufix = strrchr (service_name, '_');

exec/totemmrp.c

@@ -57,7 +57,6 @@
 
 #include <corosync/totem/totem.h>
 #include <corosync/totem/coropoll.h>
-#include <corosync/hdb.h>
 
 #include "totemmrp.h"
 #include "totemsrp.h"
@@ -119,6 +118,7 @@ void totemmrp_confchg_fn (
 int totemmrp_initialize (
 	hdb_handle_t poll_handle,
 	struct totem_config *totem_config,
+	totempg_stats_t *stats,
 
 	void (*deliver_fn) (
 		unsigned int nodeid,
@@ -136,10 +136,12 @@ int totemmrp_initialize (
 	pg_deliver_fn = deliver_fn;
 	pg_confchg_fn = confchg_fn;
 
+	stats->mrp = calloc (sizeof(totemmrp_stats_t), 1);
 	result = totemsrp_initialize (
 		poll_handle,
 		&totemsrp_context,
 		totem_config,
+		stats->mrp,
 		totemmrp_deliver_fn,
 		totemmrp_confchg_fn);
 

exec/totemmrp.h

@@ -58,6 +58,7 @@ extern void totemmrp_log_printf_init (
 extern int totemmrp_initialize (
 	hdb_handle_t poll_handle,
 	struct totem_config *totem_config,
+	totempg_stats_t *stats,
 
 	void (*deliver_fn) (
 		unsigned int nodeid,

exec/totempg.c

@@ -175,6 +175,8 @@ static void (*totempg_log_printf) (
 
 struct totem_config *totempg_totem_config;
 
+static totempg_stats_t totempg_stats;
+
 enum throw_away_mode {
 	THROW_AWAY_INACTIVE,
 	THROW_AWAY_ACTIVE
@@ -726,6 +728,7 @@ int totempg_initialize (
 	res = totemmrp_initialize (
 		poll_handle,
 		totem_config,
+		&totempg_stats,
 		totempg_deliver_fn,
 		totempg_confchg_fn);
 
@@ -1304,6 +1307,11 @@ int totempg_ifaces_get (
 	return (res);
 }
 
+void* totempg_get_stats (void)
+{
+	return &totempg_stats;
+}
+
 int totempg_crypto_set (
 	unsigned int type)
 {

exec/totemsrp.c

@@ -501,6 +501,9 @@ struct totemsrp_instance {
 
 	struct memb_commit_token *commit_token;
 
+	totemsrp_stats_t *stats;
+	void * token_recv_event_handle;
+	void * token_sent_event_handle;
 	char commit_token_storage[9000];
 };
 
@@ -707,6 +710,41 @@ static int pause_flush (struct totemsrp_instance *instance)
 	return (res);
 }
 
+static int token_event_stats_collector (enum totem_callback_token_type type, const void *void_instance)
+{
+	struct totemsrp_instance *instance = (struct totemsrp_instance *)void_instance;
+	uint32_t time_now;
+	unsigned long long nano_secs = timerlist_nano_current_get ();
+
+	time_now = (nano_secs / TIMERLIST_NS_IN_MSEC);
+
+	if (type == TOTEM_CALLBACK_TOKEN_RECEIVED) {
+		/* incr latest token the index */
+		if (instance->stats->latest_token == (TOTEM_TOKEN_STATS_MAX - 1))
+			instance->stats->latest_token = 0;
+		else
+			instance->stats->latest_token++;
+
+		if (instance->stats->earliest_token == instance->stats->latest_token) {
+			/* we have filled up the array, start overwriting */
+			if (instance->stats->earliest_token == (TOTEM_TOKEN_STATS_MAX - 1))
+				instance->stats->earliest_token = 0;
+			else
+				instance->stats->earliest_token++;
+
+			instance->stats->token[instance->stats->earliest_token].rx = 0;
+			instance->stats->token[instance->stats->earliest_token].tx = 0;
+			instance->stats->token[instance->stats->earliest_token].backlog_calc = 0;
+		}
+
+		instance->stats->token[instance->stats->latest_token].rx = time_now;
+		instance->stats->token[instance->stats->latest_token].tx = 0; /* in case we drop the token */
+	} else {
+		instance->stats->token[instance->stats->latest_token].tx = time_now;
+	}
+	return 0;
+}
+
 /*
  * Exported interfaces
  */
@@ -714,6 +752,7 @@ int totemsrp_initialize (
 	hdb_handle_t poll_handle,
 	void **srp_context,
 	struct totem_config *totem_config,
+	totemmrp_stats_t *stats,
 
 	void (*deliver_fn) (
 		unsigned int nodeid,
@@ -753,6 +792,11 @@ int totemsrp_initialize (
 
 	totemsrp_instance_initialize (instance);
 
+	instance->stats = calloc (sizeof(totemsrp_stats_t), 1);
+	stats->srp = instance->stats;
+	instance->stats->latest_token = 0;
+	instance->stats->earliest_token = 0;
+
 	instance->totem_config = totem_config;
 
 	/*
@@ -881,6 +925,18 @@ int totemsrp_initialize (
 		MESSAGE_QUEUE_MAX,
 		sizeof (struct message_item));
 
+	totemsrp_callback_token_create (instance,
+									&instance->token_recv_event_handle,
+									TOTEM_CALLBACK_TOKEN_RECEIVED,
+									0,
+									token_event_stats_collector,
+									instance);
+	totemsrp_callback_token_create (instance,
+									&instance->token_sent_event_handle,
+									TOTEM_CALLBACK_TOKEN_SENT,
+									0,
+									token_event_stats_collector,
+									instance);
 	*srp_context = instance;
 	return (0);
 
@@ -1432,6 +1488,7 @@ static void memb_state_consensus_timeout_expired (
         struct srp_addr no_consensus_list[PROCESSOR_COUNT_MAX];
 	int no_consensus_list_entries;
 
+	instance->stats->consensus_timeouts++;
 	if (memb_consensus_agreed (instance)) {
 		memb_consensus_reset (instance);
 
@@ -1479,6 +1536,7 @@ static void timer_function_orf_token_timeout (void *data)
 				"A processor failed, forming new configuration.\n");
 			totemrrp_iface_check (instance->totemrrp_context);
 			memb_state_gather_enter (instance, 2);
+			instance->stats->operational_token_lost++;
 			break;
 
 		case MEMB_STATE_GATHER:
@@ -1486,12 +1544,14 @@ static void timer_function_orf_token_timeout (void *data)
 				"The consensus timeout expired.\n");
 			memb_state_consensus_timeout_expired (instance);
 			memb_state_gather_enter (instance, 3);
+			instance->stats->gather_token_lost++;
 			break;
 
 		case MEMB_STATE_COMMIT:
 			log_printf (instance->totemsrp_log_level_debug,
 				"The token was lost in the COMMIT state.\n");
 			memb_state_gather_enter (instance, 4);
+			instance->stats->commit_token_lost++;
 			break;
 
 		case MEMB_STATE_RECOVERY:
@@ -1499,6 +1559,7 @@ static void timer_function_orf_token_timeout (void *data)
 				"The token was lost in the RECOVERY state.\n");
 			ring_state_restore (instance);
 			memb_state_gather_enter (instance, 5);
+			instance->stats->recovery_token_lost++;
 			break;
 	}
 }
@@ -1730,6 +1791,7 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
 		"A processor joined or left the membership and a new membership was formed.\n");
 	instance->memb_state = MEMB_STATE_OPERATIONAL;
 
+	instance->stats->operational_entered++;
 	instance->my_received_flg = 1;
 
 	reset_pause_timeout (instance);
@@ -1786,6 +1848,7 @@ static void memb_state_gather_enter (
 		"entering GATHER state from %d.\n", gather_from);
 
 	instance->memb_state = MEMB_STATE_GATHER;
+	instance->stats->gather_entered++;
 
 	return;
 }
@@ -1831,6 +1894,7 @@ static void memb_state_commit_enter (
 	reset_token_retransmit_timeout (instance); // REVIEWED
 	reset_token_timeout (instance); // REVIEWED
 
+	instance->stats->commit_entered++;
 
 	/*
 	 * reset all flow control variables since we are starting a new ring
@@ -2022,6 +2086,7 @@ originated:
 	reset_token_retransmit_timeout (instance); // REVIEWED
 
 	instance->memb_state = MEMB_STATE_RECOVERY;
+	instance->stats->recovery_entered++;
 	return;
 }
 
@@ -2083,6 +2148,7 @@ int totemsrp_mcast (
 	message_item.msg_len = addr_idx;
 
 	log_printf (instance->totemsrp_log_level_debug, "mcasted message added to pending queue\n");
+	instance->stats->mcast_tx++;
 	cs_queue_item_add (&instance->new_message_queue, &message_item);
 
 	return (0);
@@ -2398,6 +2464,7 @@ static int orf_token_rtr (
 			memmove (&rtr_list[i], &rtr_list[i + 1],
 				sizeof (struct rtr_item) * (orf_token->rtr_list_entries));
 
+			instance->stats->mcast_retx++;
 			instance->fcc_remcast_current++;
 		} else {
 			i += 1;
@@ -2567,6 +2634,8 @@ static int token_hold_cancel_send (struct totemsrp_instance *instance)
 		sizeof (struct memb_ring_id));
 	assert (token_hold_cancel.header.nodeid);
 
+	instance->stats->token_hold_cancel_tx++;
+
 	totemrrp_mcast_flush_send (instance->totemrrp_context, &token_hold_cancel,
 		sizeof (struct token_hold_cancel));
 
@@ -2587,6 +2656,7 @@ static int orf_token_send_initial (struct totemsrp_instance *instance)
 	orf_token.token_seq = SEQNO_START_TOKEN;
 	orf_token.retrans_flg = 1;
 	instance->my_set_retrans_flg = 1;
+	instance->stats->orf_token_tx++;
 
 	if (cs_queue_is_empty (&instance->retrans_message_queue) == 1) {
 		orf_token.retrans_flg = 0;
@@ -2748,6 +2818,8 @@ static int memb_state_commit_token_send (
 	memcpy (instance->orf_token_retransmit, instance->commit_token, commit_token_size);
 	instance->orf_token_retransmit_size = commit_token_size;
 
+	instance->stats->memb_commit_token_tx++;
+
 	totemrrp_token_send (instance->totemrrp_context,
 		instance->commit_token,
 		commit_token_size);
@@ -2882,6 +2954,8 @@ static void memb_join_message_send (struct totemsrp_instance *instance)
 		usleep (random() % (instance->totem_config->send_join_timeout * 1000));
 	}
 
+	instance->stats->memb_join_tx++;
+
 	totemrrp_mcast_flush_send (
 		instance->totemrrp_context,
 		memb_join,
@@ -2950,6 +3024,7 @@ static void memb_leave_message_send (struct totemsrp_instance *instance)
 	if (instance->totem_config->send_join_timeout) {
 		usleep (random() % (instance->totem_config->send_join_timeout * 1000));
 	}
+	instance->stats->memb_join_tx++;
 
 	totemrrp_mcast_flush_send (
 		instance->totemrrp_context,
@@ -2970,6 +3045,7 @@ static void memb_merge_detect_transmit (struct totemsrp_instance *instance)
 		sizeof (struct memb_ring_id));
 	assert (memb_merge_detect.header.nodeid);
 
+	instance->stats->memb_merge_detect_tx++;
 	totemrrp_mcast_flush_send (instance->totemrrp_context,
 		&memb_merge_detect,
 		sizeof (struct memb_merge_detect));
@@ -3152,6 +3228,7 @@ static unsigned int backlog_get (struct totemsrp_instance *instance)
 	if (instance->memb_state == MEMB_STATE_RECOVERY) {
 		backlog = cs_queue_used (&instance->retrans_message_queue);
 	}
+	instance->stats->token[instance->stats->latest_token].backlog_calc = backlog;
 	return (backlog);
 }
 
@@ -3704,10 +3781,12 @@ static int message_handler_mcast (
 
 		case MEMB_STATE_COMMIT:
 			/* discard message */
+			instance->stats->rx_msg_dropped++;
 			break;
 
 		case MEMB_STATE_RECOVERY:
 			/* discard message */
+			instance->stats->rx_msg_dropped++;
 			break;
 		}
 		return (0);
@@ -4203,9 +4282,32 @@ void main_deliver_fn (
 	if ((int)message_header->type >= totemsrp_message_handlers.count) {
 		log_printf (instance->totemsrp_log_level_security, "Type of received message is wrong...  ignoring %d.\n", (int)message_header->type);
 printf ("wrong message type\n");
+		instance->stats->rx_msg_dropped++;
 		return;
 	}
 
+	switch (message_header->type) {
+	case MESSAGE_TYPE_ORF_TOKEN:
+		instance->stats->orf_token_rx++;
+		break;
+	case MESSAGE_TYPE_MCAST:
+		instance->stats->mcast_rx++;
+		break;
+	case MESSAGE_TYPE_MEMB_MERGE_DETECT:
+		instance->stats->memb_merge_detect_rx++;
+		break;
+	case MESSAGE_TYPE_MEMB_JOIN:
+		instance->stats->memb_join_rx++;
+		break;
+	case MESSAGE_TYPE_MEMB_COMMIT_TOKEN:
+		instance->stats->memb_commit_token_rx++;
+		break;
+	case MESSAGE_TYPE_TOKEN_HOLD_CANCEL:
+		instance->stats->token_hold_cancel_rx++;
+		break;
+	default:
+		break;
+	}
 	/*
 	 * Handle incoming message
 	 */

exec/totemsrp.h

@@ -50,6 +50,7 @@ int totemsrp_initialize (
 	hdb_handle_t poll_handle,
 	void **srp_context,
 	struct totem_config *totem_config,
+	totemmrp_stats_t *stats,
 
 	void (*deliver_fn) (
 		unsigned int nodeid,

include/corosync/coroipcs.h

@@ -36,6 +36,7 @@
 #define COROIPCS_H_DEFINED
 
 #include <stdlib.h>
+#include <corosync/engine/objdb.h>
 
 #ifdef __cplusplus
 extern "C" {

include/corosync/engine/coroapi.h

@@ -625,6 +625,8 @@ struct corosync_api_v1 {
 		void **value,
 		size_t *value_len,
 		objdb_value_types_t *type);
+
+	void *(*totem_get_stats)(void);
 };
 
 #define SERVICE_ID_MAKE(a,b) ( ((a)<<16) | (b) )

include/corosync/totem/totem.h

@@ -35,6 +35,7 @@
 #ifndef TOTEM_H_DEFINED
 #define TOTEM_H_DEFINED
 #include "totemip.h"
+#include <corosync/hdb.h>
 
 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
 #define PROCESSOR_COUNT_MAX	16
@@ -184,5 +185,72 @@ struct memb_ring_id {
 	unsigned long long seq;
 } __attribute__((packed));
 
+typedef struct {
+	hdb_handle_t handle;
+	int is_dirty;
+	time_t last_updated;
+} totem_stats_header_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	uint32_t iface_changes;
+} totemnet_stats_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	totemnet_stats_t *net;
+	char *algo_name;
+} totemrrp_stats_t;
+
+
+typedef struct {
+	uint32_t rx;
+	uint32_t tx;
+	int backlog_calc;
+} totemsrp_token_stats_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	totemrrp_stats_t *rrp;
+	uint64_t orf_token_tx;
+	uint64_t orf_token_rx;
+	uint64_t memb_merge_detect_tx;
+	uint64_t memb_merge_detect_rx;
+	uint64_t memb_join_tx;
+	uint64_t memb_join_rx;
+	uint64_t mcast_tx;
+	uint64_t mcast_retx;
+	uint64_t mcast_rx;
+	uint64_t memb_commit_token_tx;
+	uint64_t memb_commit_token_rx;
+	uint64_t token_hold_cancel_tx;
+	uint64_t token_hold_cancel_rx;
+	uint64_t operational_entered;
+	uint64_t operational_token_lost;
+	uint64_t gather_entered;
+	uint64_t gather_token_lost;
+	uint64_t commit_entered;
+	uint64_t commit_token_lost;
+	uint64_t recovery_entered;
+	uint64_t recovery_token_lost;
+	uint64_t consensus_timeouts;
+	uint64_t rx_msg_dropped;
+
+	int earliest_token;
+	int latest_token;
+#define TOTEM_TOKEN_STATS_MAX 100
+	totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX];
+
+} totemsrp_stats_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	totemsrp_stats_t *srp;
+} totemmrp_stats_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	totemmrp_stats_t *mrp;
+} totempg_stats_t;
 
 #endif /* TOTEM_H_DEFINED */

include/corosync/totem/totempg.h

@@ -143,6 +143,8 @@ extern int totempg_ifaces_get (
 	char ***status,
         unsigned int *iface_count);
 
+extern void* totempg_get_stats (void);
+
 extern const char *totempg_ifaces_print (unsigned int nodeid);
 
 extern unsigned int totempg_my_nodeid_get (void);