stats: Add map with on-demand statistics

Icmap is factored out so it's possible to add other
maps for cmap. API call to switch maps from application
end is added.

Corosync-cmapctl is enhanced with -m option.

Stats contains all statistics previously found in runtime.connections,
runtime.services and runtime.totem prefixes together with new knet
related. All stats are read only.

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

exec/Makefile.am

@@ -36,7 +36,7 @@ noinst_HEADERS		= apidef.h cs_queue.h logconfig.h main.h \
 			  totemnet.h totemudp.h \
 			  totemudpu.h totemsrp.h util.h vsf.h \
 			  schedwrk.h sync.h fsm.h votequorum.h vsf_ykd.h \
-			  totemknet.h
+			  totemknet.h stats.h
 
 TOTEM_SRC		= totemip.c totemnet.c totemudp.c \
 			  totemudpu.c totemsrp.c \
@@ -55,7 +55,7 @@ sbin_PROGRAMS		= corosync
 corosync_SOURCES	= vsf_ykd.c coroparse.c vsf_quorum.c sync.c \
 			  logsys.c cfg.c cmap.c cpg.c pload.c \
 			  votequorum.c util.c schedwrk.c main.c \
-			  apidef.c quorum.c icmap.c timer.c \
+			  apidef.c quorum.c icmap.c timer.c stats.c \
 			  ipc_glue.c service.c logconfig.c totemconfig.c
 
 if BUILD_MONITORING

exec/cmap.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2012 Red Hat, Inc.
+ * Copyright (c) 2011-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -43,6 +43,8 @@
 #include <assert.h>
 
 #include <qb/qbloop.h>
+#include <qb/qblist.h>
+#include <qb/qbipcs.h>
 #include <qb/qbipc_common.h>
 
 #include <corosync/corotypes.h>
@@ -54,15 +56,77 @@
 #include <corosync/icmap.h>
 
 #include "service.h"
+#include "ipcs_stats.h"
+#include "stats.h"
 
 LOGSYS_DECLARE_SUBSYS ("CMAP");
 
 #define MAX_REQ_EXEC_CMAP_MCAST_ITEMS		32
 #define ICMAP_VALUETYPE_NOT_EXIST		0
 
+struct cmap_map {
+	cs_error_t (*map_get)(const char *key_name,
+			      void *value,
+			      size_t *value_len,
+			      icmap_value_types_t *type);
+
+	cs_error_t (*map_set)(const char *key_name,
+			      const void *value,
+			      size_t value_len,
+			      icmap_value_types_t type);
+
+	cs_error_t (*map_adjust_int)(const char *key_name, int32_t step);
+
+	cs_error_t (*map_delete)(const char *key_name);
+
+	int (*map_is_key_ro)(const char *key_name);
+
+	icmap_iter_t (*map_iter_init)(const char *prefix);
+	const char * (*map_iter_next)(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type);
+	void (*map_iter_finalize)(icmap_iter_t iter);
+
+	cs_error_t (*map_track_add)(const char *key_name,
+				    int32_t track_type,
+				    icmap_notify_fn_t notify_fn,
+				    void *user_data,
+				    icmap_track_t *icmap_track);
+
+	cs_error_t (*map_track_delete)(icmap_track_t icmap_track);
+	void * (*map_track_get_user_data)(icmap_track_t icmap_track);
+};
+
+struct cmap_map icmap_map = {
+	.map_get = icmap_get,
+	.map_set = icmap_set,
+	.map_adjust_int = icmap_adjust_int,
+	.map_delete = icmap_delete,
+	.map_is_key_ro = icmap_is_key_ro,
+	.map_iter_init = icmap_iter_init,
+	.map_iter_next = icmap_iter_next,
+	.map_iter_finalize = icmap_iter_finalize,
+	.map_track_add = icmap_track_add,
+	.map_track_delete = icmap_track_delete,
+	.map_track_get_user_data = icmap_track_get_user_data,
+};
+
+struct cmap_map stats_map = {
+	.map_get = stats_map_get,
+	.map_set = stats_map_set,
+	.map_adjust_int = stats_map_adjust_int,
+	.map_delete = stats_map_delete,
+	.map_is_key_ro = stats_map_is_key_ro,
+	.map_iter_init = stats_map_iter_init,
+	.map_iter_next = stats_map_iter_next,
+	.map_iter_finalize = stats_map_iter_finalize,
+	.map_track_add = stats_map_track_add,
+	.map_track_delete = stats_map_track_delete,
+	.map_track_get_user_data = stats_map_track_get_user_data,
+};
+
 struct cmap_conn_info {
 	struct hdb_handle_database iter_db;
 	struct hdb_handle_database track_db;
+	struct cmap_map map_fns;
 };
 
 typedef uint64_t cmap_iter_handle_t;
@@ -100,6 +164,7 @@ static void message_handler_req_lib_cmap_iter_next(void *conn, const void *messa
 static void message_handler_req_lib_cmap_iter_finalize(void *conn, const void *message);
 static void message_handler_req_lib_cmap_track_add(void *conn, const void *message);
 static void message_handler_req_lib_cmap_track_delete(void *conn, const void *message);
+static void message_handler_req_lib_cmap_set_current_map(void *conn, const void *message);
 
 static void cmap_notify_fn(int32_t event,
 		const char *key_name,
@@ -181,6 +246,10 @@ static struct corosync_lib_handler cmap_lib_engine[] =
 		.lib_handler_fn				= message_handler_req_lib_cmap_track_delete,
 		.flow_control				= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
 	},
+	{ /* 9 */
+		.lib_handler_fn				= message_handler_req_lib_cmap_set_current_map,
+		.flow_control				= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
+	},
 };
 
 static struct corosync_exec_handler cmap_exec_engine[] =
@@ -306,6 +375,7 @@ static int cmap_lib_init_fn (void *conn)
 	api->ipc_refcnt_inc(conn);
 
 	memset(conn_info, 0, sizeof(*conn_info));
+	conn_info->map_fns = icmap_map;
 	hdb_create(&conn_info->iter_db);
 	hdb_create(&conn_info->track_db);
 
@@ -326,7 +396,7 @@ static int cmap_lib_exit_fn (void *conn)
         while (hdb_iterator_next(&conn_info->iter_db,
                 (void*)&iter, &iter_handle) == 0) {
 
-		icmap_iter_finalize(*iter);
+		conn_info->map_fns.map_iter_finalize(*iter);
 
 		(void)hdb_handle_put (&conn_info->iter_db, iter_handle);
         }
@@ -337,9 +407,9 @@ static int cmap_lib_exit_fn (void *conn)
         while (hdb_iterator_next(&conn_info->track_db,
                 (void*)&track, &track_handle) == 0) {
 
-		free(icmap_track_get_user_data(*track));
+		free(conn_info->map_fns.map_track_get_user_data(*track));
 
-		icmap_track_delete(*track);
+		conn_info->map_fns.map_track_delete(*track);
 
 		(void)hdb_handle_put (&conn_info->track_db, track_handle);
         }
@@ -425,14 +495,15 @@ static void cmap_sync_abort (void)
 static void message_handler_req_lib_cmap_set(void *conn, const void *message)
 {
 	const struct req_lib_cmap_set *req_lib_cmap_set = message;
+	struct cmap_conn_info *conn_info = (struct cmap_conn_info *)api->ipc_private_data_get (conn);
 	struct res_lib_cmap_set res_lib_cmap_set;
 	cs_error_t ret;
 
-	if (icmap_is_key_ro((char *)req_lib_cmap_set->key_name.value)) {
+	if (conn_info->map_fns.map_is_key_ro((char *)req_lib_cmap_set->key_name.value)) {
 		ret = CS_ERR_ACCESS;
 	} else {
-		ret = icmap_set((char *)req_lib_cmap_set->key_name.value, &req_lib_cmap_set->value,
-				req_lib_cmap_set->value_len, req_lib_cmap_set->type);
+		ret = conn_info->map_fns.map_set((char *)req_lib_cmap_set->key_name.value, &req_lib_cmap_set->value,
+						 req_lib_cmap_set->value_len, req_lib_cmap_set->type);
 	}
 
 	memset(&res_lib_cmap_set, 0, sizeof(res_lib_cmap_set));
@@ -446,13 +517,14 @@ static void message_handler_req_lib_cmap_set(void *conn, const void *message)
 static void message_handler_req_lib_cmap_delete(void *conn, const void *message)
 {
 	const struct req_lib_cmap_set *req_lib_cmap_set = message;
+	struct cmap_conn_info *conn_info = (struct cmap_conn_info *)api->ipc_private_data_get (conn);
 	struct res_lib_cmap_delete res_lib_cmap_delete;
 	cs_error_t ret;
 
-	if (icmap_is_key_ro((char *)req_lib_cmap_set->key_name.value)) {
+	if (conn_info->map_fns.map_is_key_ro((char *)req_lib_cmap_set->key_name.value)) {
 		ret = CS_ERR_ACCESS;
 	} else {
-		ret = icmap_delete((char *)req_lib_cmap_set->key_name.value);
+		ret = conn_info->map_fns.map_delete((char *)req_lib_cmap_set->key_name.value);
 	}
 
 	memset(&res_lib_cmap_delete, 0, sizeof(res_lib_cmap_delete));
@@ -466,6 +538,7 @@ static void message_handler_req_lib_cmap_delete(void *conn, const void *message)
 static void message_handler_req_lib_cmap_get(void *conn, const void *message)
 {
 	const struct req_lib_cmap_get *req_lib_cmap_get = message;
+	struct cmap_conn_info *conn_info = (struct cmap_conn_info *)api->ipc_private_data_get (conn);
 	struct res_lib_cmap_get *res_lib_cmap_get;
 	struct res_lib_cmap_get error_res_lib_cmap_get;
 	cs_error_t ret;
@@ -491,10 +564,10 @@ static void message_handler_req_lib_cmap_get(void *conn, const void *message)
 		value = NULL;
 	}
 
-	ret = icmap_get((char *)req_lib_cmap_get->key_name.value,
-			value,
-			&value_len,
-			&type);
+	ret = conn_info->map_fns.map_get((char *)req_lib_cmap_get->key_name.value,
+					  value,
+					  &value_len,
+					  &type);
 
 	if (ret != CS_OK) {
 		free(res_lib_cmap_get);
@@ -524,14 +597,16 @@ error_exit:
 static void message_handler_req_lib_cmap_adjust_int(void *conn, const void *message)
 {
 	const struct req_lib_cmap_adjust_int *req_lib_cmap_adjust_int = message;
+	struct cmap_conn_info *conn_info = (struct cmap_conn_info *)api->ipc_private_data_get (conn);
 	struct res_lib_cmap_adjust_int res_lib_cmap_adjust_int;
 	cs_error_t ret;
 
-	if (icmap_is_key_ro((char *)req_lib_cmap_adjust_int->key_name.value)) {
+	if (conn_info->map_fns.map_is_key_ro((char *)req_lib_cmap_adjust_int->key_name.value)) {
 		ret = CS_ERR_ACCESS;
 	} else {
-		ret = icmap_adjust_int((char *)req_lib_cmap_adjust_int->key_name.value,
-		    req_lib_cmap_adjust_int->step);
+		ret = conn_info->map_fns.map_adjust_int((char *)req_lib_cmap_adjust_int->key_name.value,
+								req_lib_cmap_adjust_int->step);
+
 	}
 
 	memset(&res_lib_cmap_adjust_int, 0, sizeof(res_lib_cmap_adjust_int));
@@ -559,7 +634,7 @@ static void message_handler_req_lib_cmap_iter_init(void *conn, const void *messa
 		prefix = NULL;
 	}
 
-	iter = icmap_iter_init(prefix);
+	iter = conn_info->map_fns.map_iter_init(prefix);
 	if (iter == NULL) {
 		ret = CS_ERR_NO_SECTIONS;
 		goto reply_send;
@@ -606,7 +681,7 @@ static void message_handler_req_lib_cmap_iter_next(void *conn, const void *messa
 		goto reply_send;
 	}
 
-	res = icmap_iter_next(*iter, &value_len, &type);
+	res = conn_info->map_fns.map_iter_next(*iter, &value_len, &type);
 	if (res == NULL) {
 		ret = CS_ERR_NO_SECTIONS;
 	}
@@ -644,7 +719,7 @@ static void message_handler_req_lib_cmap_iter_finalize(void *conn, const void *m
 		goto reply_send;
 	}
 
-	icmap_iter_finalize(*iter);
+	conn_info->map_fns.map_iter_finalize(*iter);
 
 	(void)hdb_handle_destroy(&conn_info->iter_db, req_lib_cmap_iter_finalize->iter_handle);
 
@@ -722,11 +797,11 @@ static void message_handler_req_lib_cmap_track_add(void *conn, const void *messa
 		key_name = NULL;
 	}
 
-	ret = icmap_track_add(key_name,
-			req_lib_cmap_track_add->track_type,
-			cmap_notify_fn,
-			cmap_track_user_data,
-			&track);
+	ret = conn_info->map_fns.map_track_add(key_name,
+					       req_lib_cmap_track_add->track_type,
+					       cmap_notify_fn,
+					       cmap_track_user_data,
+					       &track);
 	if (ret != CS_OK) {
 		free(cmap_track_user_data);
 
@@ -781,9 +856,9 @@ static void message_handler_req_lib_cmap_track_delete(void *conn, const void *me
 
 	track_inst_handle = ((struct cmap_track_user_data *)icmap_track_get_user_data(*track))->track_inst_handle;
 
-	free(icmap_track_get_user_data(*track));
+	free(conn_info->map_fns.map_track_get_user_data(*track));
 
-	ret = icmap_track_delete(*track);
+	ret = conn_info->map_fns.map_track_delete(*track);
 
 	(void)hdb_handle_put (&conn_info->track_db, req_lib_cmap_track_delete->track_handle);
 	(void)hdb_handle_destroy(&conn_info->track_db, req_lib_cmap_track_delete->track_handle);
@@ -798,6 +873,57 @@ reply_send:
 	api->ipc_response_send(conn, &res_lib_cmap_track_delete, sizeof(res_lib_cmap_track_delete));
 }
 
+
+static void message_handler_req_lib_cmap_set_current_map(void *conn, const void *message)
+{
+	const struct req_lib_cmap_set_current_map *req_lib_cmap_set_current_map = message;
+	struct qb_ipc_response_header res;
+	cs_error_t ret = CS_OK;
+	struct cmap_conn_info *conn_info = (struct cmap_conn_info *)api->ipc_private_data_get (conn);
+	int handles_open = 0;
+	hdb_handle_t iter_handle = 0;
+	icmap_iter_t *iter;
+	hdb_handle_t track_handle = 0;
+	icmap_track_t *track;
+
+	/* Cannot switch maps while there are tracks or iterators active */
+	hdb_iterator_reset(&conn_info->iter_db);
+        while (hdb_iterator_next(&conn_info->iter_db,
+                (void*)&iter, &iter_handle) == 0) {
+		handles_open++;
+        }
+
+	hdb_iterator_reset(&conn_info->track_db);
+        while (hdb_iterator_next(&conn_info->track_db,
+                (void*)&track, &track_handle) == 0) {
+		handles_open++;
+        }
+
+	if (handles_open) {
+		ret = CS_ERR_BUSY;
+		goto reply_send;
+	}
+
+	switch (req_lib_cmap_set_current_map->map) {
+		case CMAP_SETMAP_DEFAULT:
+			conn_info->map_fns = icmap_map;
+			break;
+		case CMAP_SETMAP_STATS:
+			conn_info->map_fns = stats_map;
+			break;
+		default:
+			ret = CS_ERR_NOT_EXIST;
+			break;
+	}
+
+reply_send:
+	res.size = sizeof(res);
+	res.id = MESSAGE_RES_CMAP_SET_CURRENT_MAP;
+	res.error = ret;
+
+	api->ipc_response_send(conn, &res, sizeof(res));
+}
+
 static cs_error_t cmap_mcast_send(enum cmap_mcast_reason reason, int argc, char *argv[])
 {
 	int i;
@@ -937,7 +1063,7 @@ static void message_handler_req_exec_cmap_mcast_reason_sync_nv(
 	}
 
 	snprintf(member_config_version, ICMAP_KEYNAME_MAXLEN,
-		"runtime.totem.pg.mrp.srp.members.%u.config_version", nodeid);
+		"runtime.members.%u.config_version", nodeid);
 	icmap_set_uint64(member_config_version, config_version);
 
 	LEAVE();

exec/icmap.c

@@ -90,21 +90,6 @@ static int icmap_check_key_name(const char *key_name);
  */
 static int icmap_check_value_len(const void *value, size_t value_len, icmap_value_types_t type);
 
-/*
- * Returns length of value of given type, or 0 for string and binary data type
- */
-static size_t icmap_get_valuetype_len(icmap_value_types_t type);
-
-/*
- * Converts track type of icmap to qb
- */
-static int32_t icmap_tt_to_qbtt(int32_t track_type);
-
-/*
- * Convert track type of qb to icmap
- */
-static int32_t icmap_qbtt_to_tt(int32_t track_type);
-
 /*
  * Checks if item has same value as value with value_len and given type. Returns 0 if not, otherwise !0.
  */
@@ -139,7 +124,7 @@ static cs_error_t icmap_get_ref_r(
 /*
  * Function implementation
  */
-static int32_t icmap_tt_to_qbtt(int32_t track_type)
+int32_t icmap_tt_to_qbtt(int32_t track_type)
 {
 	int32_t res = 0;
 
@@ -162,7 +147,7 @@ static int32_t icmap_tt_to_qbtt(int32_t track_type)
 	return (res);
 }
 
-static int32_t icmap_qbtt_to_tt(int32_t track_type)
+int32_t icmap_qbtt_to_tt(int32_t track_type)
 {
 	int32_t res = 0;
 
@@ -316,7 +301,7 @@ static int icmap_check_key_name(const char *key_name)
 	return (0);
 }
 
-static size_t icmap_get_valuetype_len(icmap_value_types_t type)
+size_t icmap_get_valuetype_len(icmap_value_types_t type)
 {
 	size_t res = 0;
 

exec/ipc_glue.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010-2012 Red Hat, Inc.
+ * Copyright (c) 2010-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -60,6 +60,8 @@
 #include "util.h"
 #include "apidef.h"
 #include "service.h"
+#include "ipcs_stats.h"
+#include "stats.h"
 
 LOGSYS_DECLARE_SUBSYS ("MAIN");
 
@@ -117,6 +119,8 @@ static struct qb_ipcs_service_handlers corosync_service_funcs = {
 	.connection_destroyed	= cs_ipcs_connection_destroyed,
 };
 
+static struct ipcs_global_stats global_stats;
+
 static const char* cs_ipcs_serv_short_name(int32_t service_id)
 {
 	const char *name;
@@ -255,27 +259,12 @@ static char * pid_to_name (pid_t pid, char *out_name, size_t name_len)
 	return out_name;
 }
 
-struct cs_ipcs_conn_context {
-	char *icmap_path;
-	struct qb_list_head outq_head;
-	int32_t queuing;
-	uint32_t queued;
-	uint64_t invalid_request;
-	uint64_t overload;
-	uint32_t sent;
-	char data[1];
-};
-
 static void cs_ipcs_connection_created(qb_ipcs_connection_t *c)
 {
 	int32_t service = 0;
 	struct cs_ipcs_conn_context *context;
-	char proc_name[32];
 	struct qb_ipcs_connection_stats stats;
 	size_t size = sizeof(struct cs_ipcs_conn_context);
-	char key_name[ICMAP_KEYNAME_MAXLEN];
-	int set_client_pid = 0;
-	int set_proc_name = 0;
 
 	log_printf(LOG_DEBUG, "connection created");
 
@@ -300,76 +289,14 @@ static void cs_ipcs_connection_created(qb_ipcs_connection_t *c)
 		qb_ipcs_disconnect(c);
 		return;
 	}
-	icmap_inc("runtime.connections.active");
 
 	qb_ipcs_connection_stats_get(c, &stats, QB_FALSE);
 
-	if (stats.client_pid > 0) {
-		if (pid_to_name (stats.client_pid, proc_name, sizeof(proc_name))) {
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.connections.%s:%u:%p",
-					proc_name, stats.client_pid, c);
-			set_proc_name = 1;
-		} else {
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.connections.%u:%p",
-					stats.client_pid, c);
-		}
-		set_client_pid = 1;
-	} else {
-		snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.connections.%p", c);
-	}
-
-	icmap_convert_name_to_valid_name(key_name);
-
-	context->icmap_path = strdup(key_name);
-	if (context->icmap_path == NULL) {
-		qb_ipcs_disconnect(c);
-		return;
-	}
-
-	if (set_proc_name) {
-		snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.name", context->icmap_path);
-		icmap_set_string(key_name, proc_name);
-	}
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.client_pid", context->icmap_path);
-	if (set_client_pid) {
-		icmap_set_uint32(key_name, stats.client_pid);
-	} else {
-		icmap_set_uint32(key_name, 0);
+	if (!pid_to_name (stats.client_pid, context->proc_name, sizeof(context->proc_name))) {
+		context->proc_name[0] = '\0';
 	}
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.service_id", context->icmap_path);
-	icmap_set_uint32(key_name, service);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.responses", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.dispatched", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.requests", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.send_retries", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.recv_retries", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.flow_control", context->icmap_path);
-	icmap_set_uint32(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.flow_control_count", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.queue_size", context->icmap_path);
-	icmap_set_uint32(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.invalid_request", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
-
-	snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.overload", context->icmap_path);
-	icmap_set_uint64(key_name, 0);
+	stats_ipcs_add_connection(service, stats.client_pid, c);
+	global_stats.active++;
 }
 
 void cs_ipc_refcnt_inc(void *conn)
@@ -414,10 +341,7 @@ static int32_t cs_ipcs_connection_closed (qb_ipcs_connection_t *c)
 {
 	int32_t res = 0;
 	int32_t service = qb_ipcs_service_id_get(c);
-	icmap_iter_t iter;
-	char prefix[ICMAP_KEYNAME_MAXLEN];
-	const char *key_name;
-	struct cs_ipcs_conn_context *cnx;
+	struct qb_ipcs_connection_stats stats;
 
 	log_printf(LOG_DEBUG, "%s() ", __func__);
 	res = corosync_service[service]->lib_exit_fn(c);
@@ -427,19 +351,12 @@ static int32_t cs_ipcs_connection_closed (qb_ipcs_connection_t *c)
 
 	qb_loop_job_del(cs_poll_handle_get(), QB_LOOP_HIGH, c, outq_flush);
 
-	cnx = qb_ipcs_context_get(c);
-
-	snprintf(prefix, ICMAP_KEYNAME_MAXLEN, "%s.", cnx->icmap_path);
-	iter = icmap_iter_init(prefix);
-	while ((key_name = icmap_iter_next(iter, NULL, NULL)) != NULL) {
-		icmap_delete(key_name);
-	}
-	icmap_iter_finalize(iter);
-	free(cnx->icmap_path);
+	qb_ipcs_connection_stats_get(c, &stats, QB_FALSE);
 
-	icmap_inc("runtime.connections.closed");
-	icmap_dec("runtime.connections.active");
+	stats_ipcs_del_connection(service, stats.client_pid, c);
 
+	global_stats.active--;
+	global_stats.closed++;
 	return 0;
 }
 
@@ -750,64 +667,43 @@ void cs_ipcs_sync_state_changed(int32_t sync_in_process)
 	cs_ipcs_check_for_flow_control();
 }
 
-void cs_ipcs_stats_update(void)
+void cs_ipcs_get_global_stats(struct ipcs_global_stats *ipcs_stats)
 {
-	int32_t i;
-	struct qb_ipcs_stats srv_stats;
-	struct qb_ipcs_connection_stats stats;
-	qb_ipcs_connection_t *c, *prev;
-	struct cs_ipcs_conn_context *cnx;
-	char key_name[ICMAP_KEYNAME_MAXLEN];
-
-	for (i = 0; i < SERVICES_COUNT_MAX; i++) {
-		if (corosync_service[i] == NULL || ipcs_mapper[i].inst == NULL) {
-			continue;
-		}
-		qb_ipcs_stats_get(ipcs_mapper[i].inst, &srv_stats, QB_FALSE);
-
-		for (c = qb_ipcs_connection_first_get(ipcs_mapper[i].inst);
-			 c;
-			 prev = c, c = qb_ipcs_connection_next_get(ipcs_mapper[i].inst, prev), qb_ipcs_connection_unref(prev)) {
-
-			cnx = qb_ipcs_context_get(c);
-			if (cnx == NULL) continue;
-
-			qb_ipcs_connection_stats_get(c, &stats, QB_FALSE);
-
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.client_pid", cnx->icmap_path);
-			icmap_set_uint32(key_name, stats.client_pid);
-
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.requests", cnx->icmap_path);
-			icmap_set_uint64(key_name, stats.requests);
-
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.responses", cnx->icmap_path);
-			icmap_set_uint64(key_name, stats.responses);
-
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.dispatched", cnx->icmap_path);
-			icmap_set_uint64(key_name, stats.events);
-
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.send_retries", cnx->icmap_path);
-			icmap_set_uint64(key_name, stats.send_retries);
+	memcpy(ipcs_stats, &global_stats, sizeof(global_stats));
+}
 
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.recv_retries", cnx->icmap_path);
-			icmap_set_uint64(key_name, stats.recv_retries);
+cs_error_t cs_ipcs_get_conn_stats(int service_id, uint32_t pid, void *conn_ptr, struct ipcs_conn_stats *ipcs_stats)
+{
+	struct cs_ipcs_conn_context *cnx;
+	qb_ipcs_connection_t *c, *prev;
+	int found = 0;
 
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.flow_control", cnx->icmap_path);
-			icmap_set_uint32(key_name, stats.flow_control_state);
+	if (corosync_service[service_id] == NULL || ipcs_mapper[service_id].inst == NULL) {
+		return CS_ERR_NOT_EXIST;
+	}
 
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.flow_control_count", cnx->icmap_path);
-			icmap_set_uint64(key_name, stats.flow_control_count);
+	qb_ipcs_stats_get(ipcs_mapper[service_id].inst, &ipcs_stats->srv, QB_FALSE);
 
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.queue_size", cnx->icmap_path);
-			icmap_set_uint32(key_name, cnx->queued);
+	for (c = qb_ipcs_connection_first_get(ipcs_mapper[service_id].inst);
+	     c;
+	     prev = c, c = qb_ipcs_connection_next_get(ipcs_mapper[service_id].inst, prev), qb_ipcs_connection_unref(prev)) {
 
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.invalid_request", cnx->icmap_path);
-			icmap_set_uint64(key_name, cnx->invalid_request);
+		cnx = qb_ipcs_context_get(c);
+		if (cnx == NULL) continue;
+		if (c != conn_ptr) continue;
 
-			snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "%s.overload", cnx->icmap_path);
-			icmap_set_uint64(key_name, cnx->overload);
+		qb_ipcs_connection_stats_get(c, &ipcs_stats->conn, QB_FALSE);
+		if (ipcs_stats->conn.client_pid != pid) {
+			continue;
 		}
+		found = 1;
+		memcpy(&ipcs_stats->cnx, cnx, sizeof(struct cs_ipcs_conn_context));
 	}
+	if (!found) {
+		return CS_ERR_NOT_EXIST;
+	}
+
+	return CS_OK;
 }
 
 static enum qb_ipc_type cs_get_ipc_type (void)
@@ -896,7 +792,6 @@ void cs_ipcs_init(void)
 	api->quorum_register_callback (cs_ipcs_fc_quorum_changed, NULL);
 	totempg_queue_level_register_callback (cs_ipcs_totem_queue_level_changed);
 
-	icmap_set_uint64("runtime.connections.active", 0);
-	icmap_set_uint64("runtime.connections.closed", 0);
+	global_stats.active = 0;
+	global_stats.closed = 0;
 }
-

exec/ipcs_stats.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2017 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Authors: Christine Caulfield (ccaulfie@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+struct cs_ipcs_conn_context {
+	struct qb_list_head outq_head;
+	int32_t queuing;
+	uint32_t queued;
+	uint64_t invalid_request;
+	uint64_t overload;
+	uint32_t sent;
+	char proc_name[32];
+	char data[1];
+};
+
+struct ipcs_global_stats
+{
+	uint64_t active;
+	uint64_t closed;
+};
+
+struct ipcs_conn_stats
+{
+	struct qb_ipcs_stats srv;
+	struct qb_ipcs_connection_stats conn;
+	struct cs_ipcs_conn_context cnx;
+};
+
+cs_error_t cs_ipcs_get_conn_stats(int service_id, uint32_t pid, void *conn_ptr, struct ipcs_conn_stats *ipcs_stats);
+void cs_ipcs_get_global_stats(struct ipcs_global_stats *ipcs_stats);

exec/main.c

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2002-2006 MontaVista Software, Inc.
- * Copyright (c) 2006-2012 Red Hat, Inc.
+ * Copyright (c) 2006-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -120,6 +120,8 @@
 #include "apidef.h"
 #include "service.h"
 #include "schedwrk.h"
+#include "ipcs_stats.h"
+#include "stats.h"
 
 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
 #define IPC_LOGSYS_SIZE			1024*64
@@ -318,11 +320,11 @@ static void member_object_joined (unsigned int nodeid)
 	char member_status[ICMAP_KEYNAME_MAXLEN];
 
 	snprintf(member_ip, ICMAP_KEYNAME_MAXLEN,
-		"runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
+		"runtime.members.%u.ip", nodeid);
 	snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN,
-		"runtime.totem.pg.mrp.srp.members.%u.join_count", nodeid);
+		"runtime.members.%u.join_count", nodeid);
 	snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
-		"runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
+		"runtime.members.%u.status", nodeid);
 
 	if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) {
 		icmap_inc(member_join_count);
@@ -342,7 +344,7 @@ static void member_object_left (unsigned int nodeid)
 	char member_status[ICMAP_KEYNAME_MAXLEN];
 
 	snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
-		"runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
+		"runtime.members.%u.status", nodeid);
 	icmap_set_string(member_status, "left");
 
 	log_printf (LOGSYS_LEVEL_DEBUG,
@@ -475,37 +477,8 @@ static void corosync_totem_stats_updater (void *data)
 
 	stats = api->totem_get_stats();
 
-	icmap_set_uint32("runtime.totem.pg.msg_reserved", stats->msg_reserved);
-	icmap_set_uint32("runtime.totem.pg.msg_queue_avail", stats->msg_queue_avail);
-	icmap_set_uint64("runtime.totem.pg.srp.orf_token_tx", stats->srp->orf_token_tx);
-	icmap_set_uint64("runtime.totem.pg.srp.orf_token_rx", stats->srp->orf_token_rx);
-	icmap_set_uint64("runtime.totem.pg.srp.memb_merge_detect_tx", stats->srp->memb_merge_detect_tx);
-	icmap_set_uint64("runtime.totem.pg.srp.memb_merge_detect_rx", stats->srp->memb_merge_detect_rx);
-	icmap_set_uint64("runtime.totem.pg.srp.memb_join_tx", stats->srp->memb_join_tx);
-	icmap_set_uint64("runtime.totem.pg.srp.memb_join_rx", stats->srp->memb_join_rx);
-	icmap_set_uint64("runtime.totem.pg.srp.mcast_tx", stats->srp->mcast_tx);
-	icmap_set_uint64("runtime.totem.pg.srp.mcast_retx", stats->srp->mcast_retx);
-	icmap_set_uint64("runtime.totem.pg.srp.mcast_rx", stats->srp->mcast_rx);
-	icmap_set_uint64("runtime.totem.pg.srp.memb_commit_token_tx", stats->srp->memb_commit_token_tx);
-	icmap_set_uint64("runtime.totem.pg.srp.memb_commit_token_rx", stats->srp->memb_commit_token_rx);
-	icmap_set_uint64("runtime.totem.pg.srp.token_hold_cancel_tx", stats->srp->token_hold_cancel_tx);
-	icmap_set_uint64("runtime.totem.pg.srp.token_hold_cancel_rx", stats->srp->token_hold_cancel_rx);
-	icmap_set_uint64("runtime.totem.pg.srp.operational_entered", stats->srp->operational_entered);
-	icmap_set_uint64("runtime.totem.pg.srp.operational_token_lost", stats->srp->operational_token_lost);
-	icmap_set_uint64("runtime.totem.pg.srp.gather_entered", stats->srp->gather_entered);
-	icmap_set_uint64("runtime.totem.pg.srp.gather_token_lost", stats->srp->gather_token_lost);
-	icmap_set_uint64("runtime.totem.pg.srp.commit_entered", stats->srp->commit_entered);
-	icmap_set_uint64("runtime.totem.pg.srp.commit_token_lost", stats->srp->commit_token_lost);
-	icmap_set_uint64("runtime.totem.pg.srp.recovery_entered", stats->srp->recovery_entered);
-	icmap_set_uint64("runtime.totem.pg.srp.recovery_token_lost", stats->srp->recovery_token_lost);
-	icmap_set_uint64("runtime.totem.pg.srp.consensus_timeouts", stats->srp->consensus_timeouts);
-	icmap_set_uint64("runtime.totem.pg.srp.rx_msg_dropped", stats->srp->rx_msg_dropped);
-	icmap_set_uint32("runtime.totem.pg.srp.continuous_gather", stats->srp->continuous_gather);
-	icmap_set_uint32("runtime.totem.pg.srp.continuous_sendmsg_failures",
-	    stats->srp->continuous_sendmsg_failures);
-
-	icmap_set_uint8("runtime.totem.pg.srp.firewall_enabled_or_nic_failure",
-		stats->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0);
+
+	stats->srp->firewall_enabled_or_nic_failure = stats->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0;
 
 	if (stats->srp->continuous_gather > MAX_NO_CONT_GATHER ||
 	    stats->srp->continuous_sendmsg_failures > MAX_NO_CONT_SENDMSG_FAILURES) {
@@ -524,9 +497,9 @@ static void corosync_totem_stats_updater (void *data)
 			"operating system or network fault (reason: %s). The most common "
 			"cause of this message is that the local firewall is "
 			"configured improperly.", cstr);
-		icmap_set_uint8("runtime.totem.pg.srp.firewall_enabled_or_nic_failure", 1);
+		stats->srp->firewall_enabled_or_nic_failure = 1;
 	} else {
-		icmap_set_uint8("runtime.totem.pg.srp.firewall_enabled_or_nic_failure", 0);
+		stats->srp->firewall_enabled_or_nic_failure = 0;
 	}
 
 	total_mtt_rx_token = 0;
@@ -552,12 +525,12 @@ static void corosync_totem_stats_updater (void *data)
 		t = prev;
 	}
 	if (token_count) {
-		icmap_set_uint32("runtime.totem.pg.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
-		icmap_set_uint32("runtime.totem.pg.srp.avg_token_workload", (total_token_holdtime / token_count));
-		icmap_set_uint32("runtime.totem.pg.srp.avg_backlog_calc", (total_backlog_calc / token_count));
+		stats->srp->mtt_rx_token = (total_mtt_rx_token / token_count);
+		stats->srp->avg_token_workload = (total_token_holdtime / token_count);
+		stats->srp->avg_backlog_calc = (total_backlog_calc / token_count);
 	}
 
-	cs_ipcs_stats_update();
+	stats_trigger_trackers();
 
 	api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
 		corosync_totem_stats_updater,
@@ -566,10 +539,6 @@ static void corosync_totem_stats_updater (void *data)
 
 static void corosync_totem_stats_init (void)
 {
-	icmap_set_uint32("runtime.totem.pg.srp.mtt_rx_token", 0);
-	icmap_set_uint32("runtime.totem.pg.srp.avg_token_workload", 0);
-	icmap_set_uint32("runtime.totem.pg.srp.avg_backlog_calc", 0);
-
 	/* start stats timer */
 	api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
 		corosync_totem_stats_updater,
@@ -999,10 +968,9 @@ static void set_icmap_ro_keys_flag (void)
 	 * Set RO flag for all keys of internal configuration and runtime statistics
 	 */
 	icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE);
-	icmap_set_ro_access("runtime.connections.", CS_TRUE, CS_TRUE);
-	icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
 	icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE);
 	icmap_set_ro_access("runtime.config.", CS_TRUE, CS_TRUE);
+	icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
 	icmap_set_ro_access("uidgid.config.", CS_TRUE, CS_TRUE);
 
 	/*
@@ -1232,6 +1200,11 @@ int main (int argc, char **argv, char **envp)
 		corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD);
 	}
 
+	if (stats_map_init(api) != CS_OK) {
+		log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize statistics component.");
+		corosync_exit_error (COROSYNC_DONE_STATS);
+	}
+
 	res = corosync_log_config_read (&error_string);
 	if (res == -1) {
 		/*
@@ -1336,8 +1309,8 @@ int main (int argc, char **argv, char **envp)
 
 	totem_config.totem_logging_configuration = totem_logging_configuration;
 	totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
-			"totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
-			"totempg.c,totemiba.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c");
+			"totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
+			"totempg.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c");
 
 	totem_config.totem_logging_configuration.log_level_security = LOGSYS_LEVEL_WARNING;
 	totem_config.totem_logging_configuration.log_level_error = LOGSYS_LEVEL_ERROR;

exec/stats.c

@@ -0,0 +1,589 @@
+/*
+ * Copyright (c) 2017 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Authors: Christine Caulfield (ccaulfie@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <libknet.h>
+
+#include <qb/qblist.h>
+#include <qb/qbipcs.h>
+#include <qb/qbipc_common.h>
+
+#include <corosync/corodefs.h>
+#include <corosync/coroapi.h>
+#include <corosync/logsys.h>
+#include <corosync/icmap.h>
+#include <corosync/totem/totemstats.h>
+
+#include "util.h"
+#include "ipcs_stats.h"
+#include "stats.h"
+
+LOGSYS_DECLARE_SUBSYS ("STATS");
+
+static qb_map_t *stats_map;
+
+/* Convert iterator number to text and a stats pointer */
+struct cs_stats_conv {
+	enum {STAT_PG, STAT_SRP, STAT_KNET, STAT_IPCSC, STAT_IPCSG} type;
+	const char *name;
+	const size_t offset;
+	const icmap_value_types_t value_type;
+};
+
+struct cs_stats_conv cs_pg_stats[] = {
+	{ STAT_PG, "msg_queue_avail",         offsetof(totempg_stats_t, msg_queue_avail),         ICMAP_VALUETYPE_UINT32},
+	{ STAT_PG, "msg_reserved",            offsetof(totempg_stats_t, msg_reserved),            ICMAP_VALUETYPE_UINT32},
+};
+struct cs_stats_conv cs_srp_stats[] = {
+	{ STAT_SRP, "orf_token_tx",           offsetof(totemsrp_stats_t, orf_token_tx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "orf_token_rx",           offsetof(totemsrp_stats_t, orf_token_rx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_merge_detect_tx",   offsetof(totemsrp_stats_t, memb_merge_detect_tx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_merge_detect_rx",   offsetof(totemsrp_stats_t, memb_merge_detect_rx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_join_tx",           offsetof(totemsrp_stats_t, memb_join_tx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_join_rx",           offsetof(totemsrp_stats_t, memb_join_rx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "mcast_tx",               offsetof(totemsrp_stats_t, mcast_tx),               ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "mcast_retx",             offsetof(totemsrp_stats_t, mcast_retx),             ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "mcast_rx",               offsetof(totemsrp_stats_t, mcast_rx),               ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_commit_token_tx",   offsetof(totemsrp_stats_t, memb_commit_token_tx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_commit_token_rx",   offsetof(totemsrp_stats_t, memb_commit_token_rx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "token_hold_cancel_tx",   offsetof(totemsrp_stats_t, token_hold_cancel_tx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "token_hold_cancel_rx",   offsetof(totemsrp_stats_t, token_hold_cancel_rx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "operational_entered",    offsetof(totemsrp_stats_t, operational_entered),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "operational_token_lost", offsetof(totemsrp_stats_t, operational_token_lost), ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "gather_entered",         offsetof(totemsrp_stats_t, gather_entered),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "gather_token_lost",      offsetof(totemsrp_stats_t, gather_token_lost),      ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "commit_entered",         offsetof(totemsrp_stats_t, commit_entered),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "commit_token_lost",      offsetof(totemsrp_stats_t, commit_token_lost),      ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "recovery_entered",       offsetof(totemsrp_stats_t, recovery_entered),       ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "recovery_token_lost",    offsetof(totemsrp_stats_t, recovery_token_lost),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "consensus_timeouts",     offsetof(totemsrp_stats_t, consensus_timeouts),     ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "rx_msg_dropped",         offsetof(totemsrp_stats_t, rx_msg_dropped),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "continuous_gather",      offsetof(totemsrp_stats_t, continuous_gather),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "continuous_sendmsg_failures", offsetof(totemsrp_stats_t, continuous_sendmsg_failures), ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "firewall_enabled_or_nic_failure", offsetof(totemsrp_stats_t, firewall_enabled_or_nic_failure), ICMAP_VALUETYPE_UINT8},
+	{ STAT_SRP, "mtt_rx_token",           offsetof(totemsrp_stats_t, mtt_rx_token),           ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "avg_token_workload",     offsetof(totemsrp_stats_t, avg_token_workload),     ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "avg_backlog_calc",       offsetof(totemsrp_stats_t, avg_backlog_calc),       ICMAP_VALUETYPE_UINT32},
+};
+
+struct cs_stats_conv cs_knet_stats[] = {
+	{ STAT_KNET, "enabled",          offsetof(struct knet_link_status, enabled),                ICMAP_VALUETYPE_UINT8},
+	{ STAT_KNET, "connected",        offsetof(struct knet_link_status, connected),              ICMAP_VALUETYPE_UINT8},
+	{ STAT_KNET, "mtu",              offsetof(struct knet_link_status, mtu),                    ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_data_packets",  offsetof(struct knet_link_status, stats.tx_data_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_data_packets",  offsetof(struct knet_link_status, stats.rx_data_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_data_bytes",    offsetof(struct knet_link_status, stats.tx_data_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_data_bytes",    offsetof(struct knet_link_status, stats.rx_data_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_ping_packets",  offsetof(struct knet_link_status, stats.tx_ping_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_ping_packets",  offsetof(struct knet_link_status, stats.rx_ping_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_ping_bytes",    offsetof(struct knet_link_status, stats.tx_ping_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_ping_bytes",    offsetof(struct knet_link_status, stats.rx_ping_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pong_packets",  offsetof(struct knet_link_status, stats.tx_pong_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pong_packets",  offsetof(struct knet_link_status, stats.rx_pong_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pong_bytes",    offsetof(struct knet_link_status, stats.tx_pong_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pong_bytes",    offsetof(struct knet_link_status, stats.rx_pong_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pmtu_packets",  offsetof(struct knet_link_status, stats.tx_pmtu_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pmtu_packets",  offsetof(struct knet_link_status, stats.rx_pmtu_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pmtu_bytes",    offsetof(struct knet_link_status, stats.tx_pmtu_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pmtu_bytes",    offsetof(struct knet_link_status, stats.rx_pmtu_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_total_packets", offsetof(struct knet_link_status, stats.tx_total_packets), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_total_packets", offsetof(struct knet_link_status, stats.rx_total_packets), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_total_bytes",   offsetof(struct knet_link_status, stats.tx_total_bytes),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_total_bytes",   offsetof(struct knet_link_status, stats.rx_total_bytes),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_total_errors",  offsetof(struct knet_link_status, stats.tx_total_errors),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_total_retries", offsetof(struct knet_link_status, stats.tx_total_retries), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pmtu_errors",   offsetof(struct knet_link_status, stats.tx_pmtu_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_pmtu_retries",  offsetof(struct knet_link_status, stats.tx_pmtu_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_ping_errors",   offsetof(struct knet_link_status, stats.tx_ping_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_ping_retries",  offsetof(struct knet_link_status, stats.tx_ping_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_pong_errors",   offsetof(struct knet_link_status, stats.tx_pong_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_pong_retries",  offsetof(struct knet_link_status, stats.tx_pong_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_data_errors",   offsetof(struct knet_link_status, stats.tx_data_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_data_retries",  offsetof(struct knet_link_status, stats.tx_data_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_min",      offsetof(struct knet_link_status, stats.latency_min),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_max",      offsetof(struct knet_link_status, stats.latency_max),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_ave",      offsetof(struct knet_link_status, stats.latency_ave),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_samples",  offsetof(struct knet_link_status, stats.latency_samples),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "down_count",       offsetof(struct knet_link_status, stats.down_count),       ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "up_count",         offsetof(struct knet_link_status, stats.up_count),         ICMAP_VALUETYPE_UINT32},
+};
+struct cs_stats_conv cs_ipcs_conn_stats[] = {
+	{ STAT_IPCSC, "queueing",        offsetof(struct ipcs_conn_stats, cnx.queuing),          ICMAP_VALUETYPE_INT32},
+	{ STAT_IPCSC, "queued",          offsetof(struct ipcs_conn_stats, cnx.queued),           ICMAP_VALUETYPE_UINT32},
+	{ STAT_IPCSC, "invalid_request", offsetof(struct ipcs_conn_stats, cnx.invalid_request),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "overload",        offsetof(struct ipcs_conn_stats, cnx.overload),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "sent",            offsetof(struct ipcs_conn_stats, cnx.sent),             ICMAP_VALUETYPE_UINT32},
+	{ STAT_IPCSC, "procname",        offsetof(struct ipcs_conn_stats, cnx.proc_name),        ICMAP_VALUETYPE_STRING},
+	{ STAT_IPCSC, "requests",        offsetof(struct ipcs_conn_stats, conn.requests),        ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "responses",       offsetof(struct ipcs_conn_stats, conn.responses),       ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "dispatched",      offsetof(struct ipcs_conn_stats, conn.events),          ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "send_retries",    offsetof(struct ipcs_conn_stats, conn.send_retries),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "recv_retries",    offsetof(struct ipcs_conn_stats, conn.recv_retries),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "flow_control",    offsetof(struct ipcs_conn_stats, conn.flow_control_state),    ICMAP_VALUETYPE_UINT32},
+	{ STAT_IPCSC, "flow_control_count",   offsetof(struct ipcs_conn_stats, conn.flow_control_count),    ICMAP_VALUETYPE_UINT64},
+};
+struct cs_stats_conv cs_ipcs_global_stats[] = {
+	{ STAT_IPCSG, "global.active",        offsetof(struct ipcs_global_stats, active),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSG, "global.closed",        offsetof(struct ipcs_global_stats, closed),           ICMAP_VALUETYPE_UINT64},
+};
+
+#define NUM_PG_STATS (sizeof(cs_pg_stats) / sizeof(struct cs_stats_conv))
+#define NUM_SRP_STATS (sizeof(cs_srp_stats) / sizeof(struct cs_stats_conv))
+#define NUM_KNET_STATS (sizeof(cs_knet_stats) / sizeof(struct cs_stats_conv))
+#define NUM_IPCSC_STATS (sizeof(cs_ipcs_conn_stats) / sizeof(struct cs_stats_conv))
+#define NUM_IPCSG_STATS (sizeof(cs_ipcs_global_stats) / sizeof(struct cs_stats_conv))
+
+/* What goes in the trie */
+struct stats_item {
+	char *key_name;
+	struct cs_stats_conv * cs_conv;
+};
+
+/* One of these per tracker */
+struct cs_stats_tracker
+{
+	char *key_name;
+	void *user_data;
+	int32_t events;
+	icmap_notify_fn_t notify_fn;
+	uint64_t old_value;
+	struct qb_list_head list;
+};
+QB_LIST_DECLARE (stats_tracker_list_head);
+static const struct corosync_api_v1 *api;
+
+static void stats_map_set_value(struct cs_stats_conv *conv,
+				void *stat_array,
+				void *value,
+				size_t *value_len,
+				icmap_value_types_t *type)
+{
+	if (value_len) {
+		*value_len = icmap_get_valuetype_len(conv->value_type);
+	}
+	if (type) {
+		*type = conv->value_type;
+		if ((*type == ICMAP_VALUETYPE_STRING) && value_len && stat_array) {
+			*value_len = strlen((char *)(stat_array) + conv->offset)+1;
+		}
+	}
+	if (value) {
+		memcpy(value, (char *)(stat_array) + conv->offset, *value_len);
+	}
+}
+
+static void stats_add_entry(const char *key, struct cs_stats_conv *cs_conv)
+{
+	struct stats_item *item = malloc(sizeof(struct stats_item));
+
+	if (item) {
+		item->cs_conv = cs_conv;
+		item->key_name = strdup(key);
+		qb_map_put(stats_map, item->key_name, item);
+	}
+}
+static void stats_rm_entry(const char *key)
+{
+	struct stats_item *item = qb_map_get(stats_map, key);
+
+	if (item) {
+		qb_map_rm(stats_map, item->key_name);
+		free(item->key_name);
+		free(item);
+	}
+}
+
+cs_error_t stats_map_init(const struct corosync_api_v1 *corosync_api)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	api = corosync_api;
+
+	stats_map = qb_trie_create();
+	if (!stats_map) {
+		return CS_ERR_INIT;
+	}
+
+	/* Populate the static portions of the trie */
+	for (i = 0; i<NUM_PG_STATS; i++) {
+		sprintf(param, "stats.pg.%s", cs_pg_stats[i].name);
+		stats_add_entry(param, &cs_pg_stats[i]);
+	}
+	for (i = 0; i<NUM_SRP_STATS; i++) {
+		sprintf(param, "stats.srp.%s", cs_srp_stats[i].name);
+		stats_add_entry(param, &cs_srp_stats[i]);
+	}
+	for (i = 0; i<NUM_IPCSG_STATS; i++) {
+		sprintf(param, "stats.ipcs.%s", cs_ipcs_global_stats[i].name);
+		stats_add_entry(param, &cs_ipcs_global_stats[i]);
+	}
+
+	/* KNET and IPCS stats are added when appropriate */
+	return CS_OK;
+}
+
+cs_error_t stats_map_get(const char *key_name,
+			 void *value,
+			 size_t *value_len,
+			 icmap_value_types_t *type)
+{
+	struct cs_stats_conv *statinfo;
+	struct stats_item *item;
+	totempg_stats_t *pg_stats;
+	struct knet_link_status link_status;
+	struct ipcs_conn_stats ipcs_conn_stats;
+	struct ipcs_global_stats ipcs_global_stats;
+	int res;
+	int nodeid;
+	int link_no;
+	int service_id;
+	uint32_t pid;
+	void *conn_ptr;
+
+	item = qb_map_get(stats_map, key_name);
+	if (!item) {
+		return CS_ERR_NOT_EXIST;
+	}
+
+	statinfo = item->cs_conv;
+	switch (statinfo->type) {
+		case STAT_PG:
+			pg_stats = api->totem_get_stats();
+			stats_map_set_value(statinfo, pg_stats, value, value_len, type);
+			break;
+		case STAT_SRP:
+			pg_stats = api->totem_get_stats();
+			stats_map_set_value(statinfo, pg_stats->srp, value, value_len, type);
+			break;
+		case STAT_KNET:
+			if (sscanf(key_name, "stats.knet.node%d.link%d", &nodeid, &link_no) != 2) {
+				return CS_ERR_NOT_EXIST;
+			}
+
+			/* Validate node & link IDs */
+			if (nodeid <= 0 || nodeid > KNET_MAX_HOST ||
+			    link_no < 0 || link_no > KNET_MAX_LINK) {
+				return CS_ERR_NOT_EXIST;
+			}
+
+			/* Always get the latest stats */
+			res = totemknet_link_get_status((knet_node_id_t)nodeid, (uint8_t)link_no, &link_status);
+			if (res != CS_OK) {
+				return CS_ERR_LIBRARY;
+			}
+			stats_map_set_value(statinfo, &link_status, value, value_len, type);
+			break;
+		case STAT_IPCSC:
+			if (sscanf(key_name, "stats.ipcs.service%d.%d.%p", &service_id, &pid, &conn_ptr) != 3) {
+				return CS_ERR_NOT_EXIST;
+			}
+			res = cs_ipcs_get_conn_stats(service_id, pid, conn_ptr, &ipcs_conn_stats);
+			if (res != CS_OK) {
+				return res;
+			}
+			stats_map_set_value(statinfo, &ipcs_conn_stats, value, value_len, type);
+			break;
+		case STAT_IPCSG:
+			cs_ipcs_get_global_stats(&ipcs_global_stats);
+			stats_map_set_value(statinfo, &ipcs_global_stats, value, value_len, type);
+			break;
+		default:
+			return CS_ERR_LIBRARY;
+	}
+	return CS_OK;
+}
+
+cs_error_t stats_map_set(const char *key_name,
+			 const void *value,
+			 size_t value_len,
+			 icmap_value_types_t type)
+{
+	return CS_ERR_NOT_SUPPORTED;
+}
+
+cs_error_t stats_map_adjust_int(const char *key_name, int32_t step)
+{
+	return CS_ERR_NOT_SUPPORTED;
+}
+
+cs_error_t stats_map_delete(const char *key_name)
+{
+	return CS_ERR_NOT_SUPPORTED;
+}
+
+int stats_map_is_key_ro(const char *key_name)
+{
+	/* It's all read-only */
+	return 1;
+}
+
+icmap_iter_t stats_map_iter_init(const char *prefix)
+{
+	return (qb_map_pref_iter_create(stats_map, prefix));
+}
+
+
+const char *stats_map_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
+{
+	const char *res;
+	struct stats_item *item;
+
+	res = qb_map_iter_next(iter, (void **)&item);
+	if (res == NULL) {
+		return (res);
+	}
+	stats_map_set_value(item->cs_conv, NULL, NULL, value_len, type);
+
+	return res;
+}
+
+void stats_map_iter_finalize(icmap_iter_t iter)
+{
+	qb_map_iter_free(iter);
+}
+
+
+void stats_trigger_trackers()
+{
+	struct cs_stats_tracker *tracker;
+	struct qb_list_head *iter;
+	cs_error_t res;
+	size_t value_len;
+	icmap_value_types_t type;
+	uint64_t value;
+	struct icmap_notify_value new_val;
+	struct icmap_notify_value old_val;
+
+	qb_list_for_each(iter, &stats_tracker_list_head) {
+
+		tracker = qb_list_entry(iter, struct cs_stats_tracker, list);
+		if (tracker->events & ICMAP_TRACK_PREFIX || !tracker->key_name ) {
+			continue;
+		}
+
+		res = stats_map_get(tracker->key_name,
+				    &value, &value_len, &type);
+
+		/* Check if it has changed */
+		if ((res == CS_OK) && (memcmp(&value, &tracker->old_value, value_len) != 0)) {
+
+			old_val.type = new_val.type = type;
+			old_val.len = new_val.len = value_len;
+			old_val.data = new_val.data = &value;
+
+			tracker->notify_fn(ICMAP_TRACK_MODIFY, tracker->key_name,
+					   old_val, new_val, tracker->user_data);
+
+			memcpy(&tracker->old_value, &value, value_len);
+		}
+	}
+}
+
+
+/* Callback from libqb when a key is added/removed */
+static void stats_map_notify_fn(uint32_t event, char *key, void *old_value, void *value, void *user_data)
+{
+	struct cs_stats_tracker *tracker = user_data;
+	struct icmap_notify_value new_val;
+	struct icmap_notify_value old_val;
+	char new_value[64];
+
+	if (value == NULL && old_value == NULL) {
+		return ;
+	}
+
+	new_val.data = new_value;
+	if (stats_map_get(key,
+			  &new_value,
+			  &new_val.len,
+			  &new_val.type) != CS_OK) {
+	}
+
+	/* We don't know what the old value was
+	   but as this only tracks ADD & DELETE I'm not worried
+	   about it */
+	memcpy(&old_val, &new_val, sizeof(new_val));
+
+	tracker->notify_fn(icmap_qbtt_to_tt(event),
+			   key,
+			   new_val,
+			   old_val,
+			   tracker->user_data);
+
+}
+
+cs_error_t stats_map_track_add(const char *key_name,
+			       int32_t track_type,
+			       icmap_notify_fn_t notify_fn,
+			       void *user_data,
+			       icmap_track_t *icmap_track)
+{
+	struct cs_stats_tracker *tracker;
+	size_t value_len;
+	icmap_value_types_t type;
+	cs_error_t err;
+
+	/* We can track adding or deleting a key under a prefix */
+	if ((track_type & ICMAP_TRACK_PREFIX) &&
+	    (!(track_type & ICMAP_TRACK_DELETE) ||
+	     !(track_type & ICMAP_TRACK_ADD))) {
+		return CS_ERR_NOT_SUPPORTED;
+	}
+
+	tracker = malloc(sizeof(struct cs_stats_tracker));
+	if (!tracker) {
+		return CS_ERR_NO_MEMORY;
+	}
+
+	tracker->notify_fn = notify_fn;
+	tracker->user_data = user_data;
+	if (key_name) {
+		tracker->key_name = strdup(key_name);
+		if (!tracker->key_name) {
+			free(tracker);
+			return CS_ERR_NO_MEMORY;
+		}
+		/* Get initial value */
+		if (stats_map_get(tracker->key_name,
+				  &tracker->old_value, &value_len, &type) == CS_OK) {
+			tracker->old_value = 0ULL;
+		}
+	} else {
+		tracker->key_name = NULL;
+		tracker->old_value = 0ULL;
+	}
+
+	/* Add/delete trackers can use the qb_map tracking */
+	if ((track_type & ICMAP_TRACK_ADD) ||
+	    (track_type & ICMAP_TRACK_DELETE)) {
+		err = qb_map_notify_add(stats_map, tracker->key_name,
+					stats_map_notify_fn,
+					icmap_tt_to_qbtt(track_type),
+					tracker);
+		if (err != 0) {
+			log_printf(LOGSYS_LEVEL_ERROR, "creating stats tracker %s failed. %d\n", tracker->key_name, err);
+			free(tracker->key_name);
+			free(tracker);
+			return (qb_to_cs_error(err));
+		}
+		tracker->events = track_type;
+	}
+
+	qb_list_add (&tracker->list, &stats_tracker_list_head);
+
+	*icmap_track = (icmap_track_t)tracker;
+	return CS_OK;
+}
+
+cs_error_t stats_map_track_delete(icmap_track_t icmap_track)
+{
+	struct cs_stats_tracker *tracker = (struct cs_stats_tracker *)icmap_track;
+	int err;
+
+	if ((tracker->events & ICMAP_TRACK_ADD) ||
+	    (tracker->events & ICMAP_TRACK_DELETE)) {
+		err = qb_map_notify_del_2(stats_map,
+					  tracker->key_name, stats_map_notify_fn,
+					  icmap_tt_to_qbtt(tracker->events), tracker);
+		if (err) {
+			log_printf(LOGSYS_LEVEL_ERROR, "deleting tracker %s failed. %d\n", tracker->key_name, err);
+		}
+	}
+
+	qb_list_del(&tracker->list);
+	free(tracker->key_name);
+	free(tracker);
+
+	return CS_OK;
+}
+
+void *stats_map_track_get_user_data(icmap_track_t icmap_track)
+{
+	struct cs_stats_tracker *tracker = (struct cs_stats_tracker *)icmap_track;
+
+	return tracker->user_data;
+}
+
+/* Called from totemknet to add/remove keys from our map */
+void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link_no)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_KNET_STATS; i++) {
+		sprintf(param, "stats.knet.node%d.link%d.%s", nodeid, link_no, cs_knet_stats[i].name);
+		stats_add_entry(param, &cs_knet_stats[i]);
+	}
+}
+void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link_no)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_KNET_STATS; i++) {
+		sprintf(param, "stats.knet.node%d.link%d.%s", nodeid, link_no, cs_knet_stats[i].name);
+		stats_rm_entry(param);
+	}
+}
+
+
+/* Called from ipc_glue to add/remove keys from our map */
+void stats_ipcs_add_connection(int service_id, uint32_t pid, void *ptr)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_IPCSC_STATS; i++) {
+		sprintf(param, "stats.ipcs.service%d.%d.%p.%s", service_id, pid, ptr, cs_ipcs_conn_stats[i].name);
+		stats_add_entry(param, &cs_ipcs_conn_stats[i]);
+	}
+}
+void stats_ipcs_del_connection(int service_id, uint32_t pid, void *ptr)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_IPCSC_STATS; i++) {
+		sprintf(param, "stats.ipcs.service%d.%d.%p.%s", service_id, pid, ptr, cs_ipcs_conn_stats[i].name);
+		stats_rm_entry(param);
+	}
+}

exec/stats.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2017 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Authors: Christine Caulfield (ccaulfie@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+cs_error_t stats_map_init(const struct corosync_api_v1 *api);
+
+cs_error_t stats_map_get(const char *key_name,
+		   void *value,
+		   size_t *value_len,
+		   icmap_value_types_t *type);
+
+cs_error_t stats_map_set(const char *key_name,
+		   const void *value,
+		   size_t value_len,
+		   icmap_value_types_t type);
+
+cs_error_t stats_map_adjust_int(const char *key_name, int32_t step);
+
+cs_error_t stats_map_delete(const char *key_name);
+
+int stats_map_is_key_ro(const char *key_name);
+
+icmap_iter_t stats_map_iter_init(const char *prefix);
+const char *stats_map_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type);
+void stats_map_iter_finalize(icmap_iter_t iter);
+
+cs_error_t stats_map_track_add(const char *key_name,
+			 int32_t track_type,
+			 icmap_notify_fn_t notify_fn,
+			 void *user_data,
+			 icmap_track_t *icmap_track);
+
+cs_error_t stats_map_track_delete(icmap_track_t icmap_track);
+void *stats_map_track_get_user_data(icmap_track_t icmap_track);
+
+void stats_trigger_trackers(void);
+
+
+void stats_ipcs_add_connection(int service_id, uint32_t pid, void *ptr);
+void stats_ipcs_del_connection(int service_id, uint32_t pid, void *ptr);
+cs_error_t cs_ipcs_get_conn_stats(int service_id, uint32_t pid, void *conn_ptr, struct ipcs_conn_stats *ipcs_stats);

exec/totemknet.c

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2016 Red Hat, Inc.
+ * Copyright (c) 2016-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -76,6 +76,7 @@
 #include <prerror.h>
 
 #include <libknet.h>
+#include <corosync/totem/totemstats.h>
 
 #ifndef MSG_NOSIGNAL
 #define MSG_NOSIGNAL 0
@@ -141,18 +142,6 @@ struct totemknet_instance {
 
 	char iov_buffer[KNET_MAX_PACKET_SIZE];
 
-	int stats_sent;
-
-	int stats_recv;
-
-	int stats_delv;
-
-	int stats_remcasts;
-
-	int stats_orf_token;
-
-	struct timeval stats_tv_start;
-
 	char *link_status[INTERFACE_MAX];
 
 	int num_links;
@@ -165,8 +154,6 @@ struct totemknet_instance {
 
 	struct totem_config *totem_config;
 
-	totemsrp_stats_t *stats;
-
 	struct totem_ip_address token_target;
 
 	qb_loop_timer_handle timer_netif_check_timeout;
@@ -181,6 +168,9 @@ struct totemknet_instance {
 	int knet_fd;
 };
 
+/* Awkward. But needed to get stats from knet */
+struct totemknet_instance *global_instance;
+
 struct work_item {
 	const void *msg;
 	unsigned int msg_len;
@@ -632,8 +622,6 @@ static int data_deliver_fn (
 		return (0);
 	}
 
-	instance->stats_recv += msg_len;
-
 	/*
 	 * Handle incoming message
 	 */
@@ -820,7 +808,6 @@ int totemknet_initialize (
 	totemknet_instance_initialize (instance);
 
 	instance->totem_config = totem_config;
-	instance->stats = stats;
 
 	/*
 	* Configure logging
@@ -895,6 +882,7 @@ int totemknet_initialize (
 	if (res) {
 		KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
 	}
+	global_instance = instance;
 
 	/* Get an fd into knet */
 	instance->knet_fd = 0;
@@ -1204,6 +1192,8 @@ int totemknet_member_add (
 		return -1;
 	}
 
+	/* register stats */
+	stats_knet_add_member(member->nodeid, link_no);
 	return (0);
 }
 
@@ -1220,6 +1210,8 @@ int totemknet_member_remove (
 	if (token_target->nodeid == instance->our_nodeid) {
 		return 0; /* Don't remove ourself */
 	}
+	/* Tidy stats */
+	stats_knet_del_member(token_target->nodeid, link_no);
 
 	/* Remove the link first */
 	res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
@@ -1243,6 +1235,44 @@ int totemknet_member_list_rebind_ip (
 	return (0);
 }
 
+/* For the stats module */
+int totemknet_link_get_status (
+	knet_node_id_t node, uint8_t link_no,
+	struct knet_link_status *status)
+{
+	int res;
+	int ret = CS_OK;
+
+	/* We are probably not using knet */
+	if (!global_instance) {
+		return CS_ERR_NOT_EXIST;
+	}
+
+	if (link_no > global_instance->num_links) {
+		return -1; /* no more links */
+	}
+
+	res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
+	if (res) {
+		switch (errno) {
+			case EINVAL:
+				ret = CS_ERR_INVALID_PARAM;
+				break;
+			case EBUSY:
+				ret = CS_ERR_BUSY;
+				break;
+			case EDEADLK:
+				ret = CS_ERR_TRY_AGAIN;
+				break;
+			default:
+				ret = CS_ERR_LIBRARY;
+				break;
+		}
+	}
+
+	return (ret);
+}
+
 static void timer_function_merge_detect_timeout (
 	void *data)
 {

exec/util.h

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2002-2004 MontaVista Software, Inc.
  * Copyright (c) 2004 Open Source Development Lab
- * Copyright (c) 2006-2011 Red Hat, Inc.
+ * Copyright (c) 2006-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -59,6 +59,7 @@ enum e_corosync_done {
 	COROSYNC_DONE_STD_TO_NULL_REDIR = 19,
 	COROSYNC_DONE_SERVICE_ENGINE_INIT = 20,
 	COROSYNC_DONE_STORE_RINGID = 21,
+	COROSYNC_DONE_STATS = 22,
 	COROSYNC_DONE_PLOAD = 99
 };
 

include/corosync/cmap.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011 Red Hat, Inc.
+ * Copyright (c) 2011-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -104,6 +104,12 @@ typedef enum {
     CMAP_VALUETYPE_BINARY	= 12,
 } cmap_value_types_t;
 
+typedef enum {
+	CMAP_MAP_DEFAULT        = 0,
+	CMAP_MAP_ICMAP          = 0,
+	CMAP_MAP_STATS          = 1,
+} cmap_map_t;
+
 /**
  * Structure passed as new_value and old_value in change callback. It contains type of
  * key, length of key and pointer to value of key
@@ -138,6 +144,17 @@ typedef void (*cmap_notify_fn_t) (
 extern cs_error_t cmap_initialize (
 	cmap_handle_t *handle);
 
+/**
+ * Create a new cmap connection on a specified map
+ *
+ * @param handle will be filled with handle to be used for all following
+ * operations with cht.
+ * @param map is the 'map' to use for this connection
+ */
+extern cs_error_t cmap_initialize_map (
+	cmap_handle_t *handle,
+	cmap_map_t map);
+
 /**
  * Close the cmap handle
  * @param handle cmap handle

include/corosync/icmap.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2012 Red Hat, Inc.
+ * Copyright (c) 2011-2017 Red Hat, Inc.
  *
  * Author: Jan Friesse (jfriesse@redhat.com)
  *
@@ -542,6 +542,21 @@ extern void icmap_convert_name_to_valid_name(char *key_name);
  */
 extern cs_error_t icmap_copy_map(icmap_map_t dst_map, const icmap_map_t src_map);
 
+/*
+ * Returns length of value of given type, or 0 for string and binary data type
+ */
+size_t icmap_get_valuetype_len(icmap_value_types_t type);
+
+/*
+ * Converts track type of icmap to qb
+ */
+int32_t icmap_tt_to_qbtt(int32_t track_type);
+
+/*
+ * Convert track type of qb to icmap
+ */
+int32_t icmap_qbtt_to_tt(int32_t track_type);
+
 #ifdef __cplusplus
 }
 #endif

include/corosync/ipc_cmap.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011 Red Hat, Inc.
+ * Copyright (c) 2011-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -52,6 +52,7 @@ enum req_cmap_types {
 	MESSAGE_REQ_CMAP_ITER_FINALIZE = 6,
 	MESSAGE_REQ_CMAP_TRACK_ADD = 7,
 	MESSAGE_REQ_CMAP_TRACK_DELETE = 8,
+	MESSAGE_REQ_CMAP_SET_CURRENT_MAP = 9,
 };
 
 /**
@@ -68,6 +69,12 @@ enum res_cmap_types {
 	MESSAGE_RES_CMAP_TRACK_ADD = 7,
 	MESSAGE_RES_CMAP_TRACK_DELETE = 8,
 	MESSAGE_RES_CMAP_NOTIFY_CALLBACK = 9,
+	MESSAGE_RES_CMAP_SET_CURRENT_MAP = 10,
+};
+
+enum {
+	CMAP_SETMAP_DEFAULT        = 0,
+	CMAP_SETMAP_STATS          = 1,
 };
 
 /**
@@ -243,4 +250,14 @@ struct res_lib_cmap_notify_callback {
 	mar_uint8_t new_value[];
 };
 
+/**
+ * @brief The req_lib_cmap_set_current_map struct
+ * used by cmap_initialize_map()
+ */
+struct req_lib_cmap_set_current_map {
+	struct qb_ipc_request_header header __attribute__((aligned(8)));
+	mar_int32_t map __attribute__((aligned(8)));
+};
+
+
 #endif /* IPC_CMAP_H_DEFINED */

include/corosync/totem/totem.h

@@ -37,6 +37,7 @@
 #include "totemip.h"
 #include "libknet.h"
 #include <corosync/hdb.h>
+#include <corosync/totem/totemstats.h>
 
 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
 #define PROCESSOR_COUNT_MAX	16
@@ -225,65 +226,4 @@ enum totem_event_type {
 	TOTEM_EVENT_NEW_MSG,
 };
 
-typedef struct {
-	int is_dirty;
-	time_t last_updated;
-} totem_stats_header_t;
-
-typedef struct {
-	totem_stats_header_t hdr;
-	uint32_t iface_changes;
-} totemnet_stats_t;
-
-typedef struct {
-	uint32_t rx;
-	uint32_t tx;
-	int backlog_calc;
-} totemsrp_token_stats_t;
-
-typedef struct {
-	totem_stats_header_t hdr;
-	uint64_t orf_token_tx;
-	uint64_t orf_token_rx;
-	uint64_t memb_merge_detect_tx;
-	uint64_t memb_merge_detect_rx;
-	uint64_t memb_join_tx;
-	uint64_t memb_join_rx;
-	uint64_t mcast_tx;
-	uint64_t mcast_retx;
-	uint64_t mcast_rx;
-	uint64_t memb_commit_token_tx;
-	uint64_t memb_commit_token_rx;
-	uint64_t token_hold_cancel_tx;
-	uint64_t token_hold_cancel_rx;
-	uint64_t operational_entered;
-	uint64_t operational_token_lost;
-	uint64_t gather_entered;
-	uint64_t gather_token_lost;
-	uint64_t commit_entered;
-	uint64_t commit_token_lost;
-	uint64_t recovery_entered;
-	uint64_t recovery_token_lost;
-	uint64_t consensus_timeouts;
-	uint64_t rx_msg_dropped;
-	uint32_t continuous_gather;
-	uint32_t continuous_sendmsg_failures;
-
-	int earliest_token;
-	int latest_token;
-#define TOTEM_TOKEN_STATS_MAX 100
-	totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX];
-
-} totemsrp_stats_t;
-
- 
- #define TOTEM_CONFIGURATION_TYPE
-
-typedef struct {
-	totem_stats_header_t hdr;
-	totemsrp_stats_t *srp;
-	uint32_t msg_reserved;
-	uint32_t msg_queue_avail;
-} totempg_stats_t;
-
 #endif /* TOTEM_H_DEFINED */

include/corosync/totem/totemstats.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2017 Red Hat, Inc.
+ *
+ * Author: Christine Caulfield (ccaulfie@redhat.com)
+ *
+ * All rights reserved.
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef TOTEMSTATS_H_DEFINED
+#define TOTEMSTATS_H_DEFINED
+
+typedef struct {
+	int is_dirty;
+	time_t last_updated;
+} totem_stats_header_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	uint32_t iface_changes;
+} totemnet_stats_t;
+
+typedef struct {
+	uint32_t rx;
+	uint32_t tx;
+	int backlog_calc;
+} totemsrp_token_stats_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	uint64_t orf_token_tx;
+	uint64_t orf_token_rx;
+	uint64_t memb_merge_detect_tx;
+	uint64_t memb_merge_detect_rx;
+	uint64_t memb_join_tx;
+	uint64_t memb_join_rx;
+	uint64_t mcast_tx;
+	uint64_t mcast_retx;
+	uint64_t mcast_rx;
+	uint64_t memb_commit_token_tx;
+	uint64_t memb_commit_token_rx;
+	uint64_t token_hold_cancel_tx;
+	uint64_t token_hold_cancel_rx;
+	uint64_t operational_entered;
+	uint64_t operational_token_lost;
+	uint64_t gather_entered;
+	uint64_t gather_token_lost;
+	uint64_t commit_entered;
+	uint64_t commit_token_lost;
+	uint64_t recovery_entered;
+	uint64_t recovery_token_lost;
+	uint64_t consensus_timeouts;
+	uint64_t rx_msg_dropped;
+	uint32_t continuous_gather;
+	uint32_t continuous_sendmsg_failures;
+
+	uint8_t  firewall_enabled_or_nic_failure;
+	uint32_t mtt_rx_token;
+	uint32_t avg_token_workload;
+	uint32_t avg_backlog_calc;
+
+	int earliest_token;
+	int latest_token;
+#define TOTEM_TOKEN_STATS_MAX 100
+	totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX];
+
+} totemsrp_stats_t;
+
+typedef struct {
+	totem_stats_header_t hdr;
+	totemsrp_stats_t *srp;
+	uint32_t msg_reserved;
+	uint32_t msg_queue_avail;
+} totempg_stats_t;
+
+
+extern int totemknet_link_get_status (
+	knet_node_id_t node, uint8_t link,
+	struct knet_link_status *status);
+
+void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link);
+
+void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link);
+
+
+#endif /* TOTEMSTATS_H_DEFINED */

lib/cmap.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2012 Red Hat, Inc.
+ * Copyright (c) 2011-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -121,6 +121,49 @@ error_no_destroy:
 	return (error);
 }
 
+cs_error_t cmap_initialize_map (cmap_handle_t *handle,
+				cmap_map_t map)
+{
+	cs_error_t error;
+	struct iovec iov[1];
+	struct cmap_inst *cmap_inst;
+	struct req_lib_cmap_set_current_map req_lib_cmap_set_current_map;
+	struct qb_ipc_response_header res_lib_cmap_set_current_map;
+
+	error = cmap_initialize(handle);
+
+	if (error == CS_OK) {
+		error = hdb_error_to_cs(hdb_handle_get (&cmap_handle_t_db, *handle, (void *)&cmap_inst));
+		if (error != CS_OK) {
+			return (error);
+		}
+
+		memset(&req_lib_cmap_set_current_map, 0, sizeof(req_lib_cmap_set_current_map));
+		req_lib_cmap_set_current_map.header.size = sizeof(req_lib_cmap_set_current_map);
+		req_lib_cmap_set_current_map.header.id = MESSAGE_REQ_CMAP_SET_CURRENT_MAP;
+		req_lib_cmap_set_current_map.map = map;
+
+		iov[0].iov_base = (char *)&req_lib_cmap_set_current_map;
+		iov[0].iov_len = sizeof(req_lib_cmap_set_current_map);
+
+		error = qb_to_cs_error(qb_ipcc_sendv_recv(
+					       cmap_inst->c,
+					       iov,
+					       1,
+					       &res_lib_cmap_set_current_map,
+					       sizeof (res_lib_cmap_set_current_map), CS_IPC_TIMEOUT_MS));
+
+		if (error == CS_OK) {
+			error = res_lib_cmap_set_current_map.error;
+		}
+
+		(void)hdb_handle_put (&cmap_handle_t_db, *handle);
+
+		return (error);
+	}
+	return (error);
+}
+
 static void cmap_inst_free (void *inst)
 {
 	struct cmap_inst *cmap_inst = (struct cmap_inst *)inst;

man/Makefile.am

@@ -114,6 +114,7 @@ autogen_man		= cpg_context_get.3 \
 			  cmap_finalize.3 \
 			  cmap_dispatch.3  \
 			  cmap_initialize.3 \
+			  cmap_initialize_map.3 \
 			  cmap_track_add.3 \
 			  cmap_context_set.3 \
 			  cmap_fd_get.3 \

man/cmap_initialize.3.in

@@ -60,4 +60,5 @@ This call returns the CS_OK value if successful, otherwise an error is returned.
 
 .SH "SEE ALSO"
 .BR cmap_finalize (3),
+.BR cmap_initialize_map (3),
 .BR cmap_overview (8)

man/cmap_initialize_map.3.in

@@ -0,0 +1,73 @@
+.\"/*
+.\" * Copyright (c) 2017 Red Hat, Inc.
+.\" *
+.\" * All rights reserved.
+.\" *
+.\" * Author: Jan Friesse (jfriesse@redhat.com)
+.\" *
+.\" * This software licensed under BSD license, the text of which follows:
+.\" *
+.\" * Redistribution and use in source and binary forms, with or without
+.\" * modification, are permitted provided that the following conditions are met:
+.\" *
+.\" * - Redistributions of source code must retain the above copyright notice,
+.\" *   this list of conditions and the following disclaimer.
+.\" * - Redistributions in binary form must reproduce the above copyright notice,
+.\" *   this list of conditions and the following disclaimer in the documentation
+.\" *   and/or other materials provided with the distribution.
+.\" * - Neither the name of the Red Hat, Inc. nor the names of its
+.\" *   contributors may be used to endorse or promote products derived from this
+.\" *   software without specific prior written permission.
+.\" *
+.\" * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+.\" * THE POSSIBILITY OF SUCH DAMAGE.
+.\" */
+.TH "CMAP_INITIALIZE_MAP" 3 "20/07/2017" "corosync Man Page" "Corosync Cluster Engine Programmer's Manual"
+
+.SH NAME
+.P
+cmap_initialize_map \- Initialize CMAP API with a specific map
+
+.SH SYNOPSIS
+.P
+\fB#include <corosync/cmap.h>\fR
+
+.P
+\fBcs_error_t cmap_initialize_map (cmap_handle_t \fI*handle\fB, cmap_map_t \fImap\fB);\fR
+
+.SH DESCRIPTION
+.P
+The \fBcmap_initialize_map\fR function is used to initialize a connection to
+the Configuration Map API and specify a particular map to use. Each application may have
+several connections to the CMAP API and to different maps.
+Each application uses the
+.I handle
+argument to uniquely identify the connection.  The
+.I handle
+argument is then used in other function calls to identify the connection to be used
+for communication with the CMAP service.
+.P
+The
+.I map
+argument specifies which map to connect to. \fICMAP_MAP_ICMAP\fB is the configuration map
+that contains the current corosyn cconfiguraton and some runtime variables that maybe useful
+to external agents. \fICMAP_MAP_STATS\fB is the statistics map that conains detailed information
+about network and inter-process communications.
+
+
+.SH RETURN VALUE
+This call returns the CS_OK value if successful, otherwise an error is returned.
+
+.SH "SEE ALSO"
+.BR cmap_initialize (3),
+.BR cmap_finalize (3),
+.BR cmap_overview (8)

man/cmap_keys.8

@@ -1,5 +1,5 @@
 .\"/*
-.\" * Copyright (c) 2012-2014 Red Hat, Inc.
+.\" * Copyright (c) 2012-2017 Red Hat, Inc.
 .\" *
 .\" * All rights reserved.
 .\" *
@@ -49,7 +49,8 @@ There are 3 main types of keys stored in CMAP:
 
 In this man page, wild-cards have the usual meaning.
 
-.SH KEYS
+.SH ICMAP KEYS
+These keys are in the icmap (default) map
 .TP
 internal_configuration.*
 Internal configuration data. All keys in this prefix are read only.
@@ -83,54 +84,6 @@ runtime.blackbox.*
 Trigger keys for storing fplay data. It's recommended that you the corosync-blackbox command
 to change keys in this prefix.
 
-.TP
-runtime.connections.*
-There is information about total number of active connections in given moment in the
-.B active
-key, number of closed connections during whole runtime of corosync in the
-.B closed
-key and information about each active IPC connection. All keys in this prefix are read-only.
-
-.TP
-runtime.connections.ID.*
-Each IPC connection has a unique ID. This is in the form [[short_name:][PID:]internal_id. On some
-platforms, short_name and PID are not filled and only internal_id is used.
-
-Typical keys in this prefix are:
-
-.B client_pid
-containing PID of IPC connection (unavailable on some platforms).
-
-.B dispatched
-number of dispatched messages.
-
-.B invalid_request
-number of requests made by IPC which are invalid (calling non-existing call, ...).
-
-.B name
-contains short name of the IPC connection (unavailable on some platforms).
-
-.B overload
-is number of requests which were not processed because of overload.
-
-.B queue_size
-contains the number of messages in the queue waiting for send.
-
-.B recv_retries
-is the total number of interrupted receives.
-
-.B requests
-contains the number of requests made by IPC.
-
-.B responses
-is the number of responses sent to the IPC client.
-
-.B send_retries
-contains the total number of interrupted sends.
-
-.B service_id
-contains the ID of service which the IPC is connected to.
-
 .TP
 runtime.config.*
 Contains the values actually in use by the totem membership protocol.
@@ -151,8 +104,67 @@ call (so for example 3 in cpg service is receive of multicast message from other
 nodes).
 
 .TP
-runtime.totem.pg.mrp.srp.*
-Prefix containing statistics about totem. All keys here are read only.
+runtime.totem.members.*
+Prefix containing members of the totem single ring protocol. Each member
+keys has format runtime.totem.members.NODEID.KEY, where key is
+one of:
+
+.B config_version
+Config version of the member node.
+
+.TP
+resources.process.PID.*
+Prefix created by applications using SAM with CMAP integration.
+It contains the following keys:
+
+.B recovery
+Recovery policy of the process. Can be one of quit or restart.
+
+.B poll_period
+Value passed in sam_initialize as a time_interval.
+
+.B last_updated
+Last time SAM received a heartbeat from the client.
+
+.B state
+State of the client. Can be one of failed, stopped, running and waiting for quorum.
+
+.TP
+uidgid.*
+Information about users/groups which are allowed to make IPC connections to
+corosync. Entries loaded from configuration file are stored with
+uidgid.config.* prefix and are pruned on configuration file reload. Dynamic
+entries has uidgid.* prefix and a configuration file reload doesn't affect them.
+
+.TP
+quorum.cancel_wait_for_all
+Tells votequorum to cancel waiting for all nodes at cluster startup. Can be used
+to unblock quorum if notes are known to be down. For pcs use only.
+
+.TP
+config.reload_in_progress
+This value will be set to 1 (or created) when a corosync.conf reload is started,
+and set to 0 when the reload is completed. This allows interested subsystems
+to do atomic reconfiguration rather than changing each key. Note that
+individual add/change/delete notifications will still be sent during a reload.
+
+.TP
+config.totemconfig_reload_in_progress
+This key is similar to
+.B config.totemconfig_reload_in_progress
+but changed after the totem config trigger is processed. It is useful (mainly)
+for situations when
+.B nodelist.local_node_pos
+must be correctly reinstated before anything else.
+
+.SH STATS KEYS
+These keys are in the stats map. All keys in this map are read-only.
+Modification tracking of individual keys is supported in the stats map, but not
+prefixes. Add/Delete operations are supported on prefixes though so you can track
+for new ipc connections or knet interfaces.
+.TP
+stats.srp.*
+Prefix containing statistics about totem.
 Typical key prefixes:
 
 .B commit_entered
@@ -237,68 +249,108 @@ Average time in milliseconds of holding time of token on the current processor.
 Average number of not yet sent messages on the current processor.
 
 .TP
-runtime.totem.pg.mrp.srp.members.*
-Prefix containing members of the totem single ring protocol. Each member
-keys has format runtime.totem.pg.mrp.srp.members.NODEID.KEY, where key is
-one of:
+stats.knet.nodeX.linkY.*
+Statistics about the network traffic to and from each node and link when using
+tke kronosnet transport
 
-.B ip
-IP address of member. It's stored in format r(RING_ID) ip(IP_ADDRESS).
+.B connected
+Whether the link is connected or not
 
-.B join_count
-Number of times the processor joined membership with local cluster. When
-processor fails and rejoins again, this value is incremented.
+.B up_count
+Number of times this link has changed state to UP
 
-.B status
-Status of the processor. Can be one of joined and left.
+.B down_count
+Number of times this link has changed state to DOWN
 
-.B config_version
-Config version of the member node.
+.B latency_ave / latency_max / latency_max
+Calculated latencies of this link. Note that if there has been no traffic
+on the link then latency_min will show a very large number.
 
-.TP
-resources.process.PID.*
-Prefix created by applications using SAM with CMAP integration.
-It contains the following keys:
+.B latency_samples
+The number of samples used to calculate the latency figures, so you have
+some idea of their precision.
 
-.B recovery
-Recovery policy of the process. Can be one of quit or restart.
+.B rx_data_packets / tx_data_packets
+The number of packets sent/received on this link
 
-.B poll_period
-Value passed in sam_initialize as a time_interval.
+.B rx_data_bytes / tx_data_bytes
+The number of bytes sent/received on this link
 
-.B last_updated
-Last time SAM received a heartbeat from the client.
+.B rx_pmtu_packets / tx_pmtu_packets
+The number of packets sent/received by the PMTUd subsystem
 
-.B state
-State of the client. Can be one of failed, stopped, running and waiting for quorum.
+.B rx_pmtu_bytes / tx_pmtu_bytes
+The number of bytes sent/received by the PMTUd subsystem
 
-.TP
-uidgid.*
-Information about users/groups which are allowed to make IPC connections to
-corosync. Entries loaded from configuration file are stored with
-uidgid.config.* prefix and are pruned on configuration file reload. Dynamic
-entries has uidgid.* prefix and a configuration file reload doesn't affect them.
+.B rx_ping_packets / tx_ping_packets
+The number of packets sent/received as pings
 
-.TP
-quorum.cancel_wait_for_all
-Tells votequorum to cancel waiting for all nodes at cluster startup. Can be used
-to unblock quorum if notes are known to be down. For pcs use only.
+.B rx_ping_bytes / tx_ping_bytes
+The number of bytes sent/received as pings
+
+.B rx_pong_packets / tx_pong_packets
+The number of packets sent/received as pongs
+
+.B rx_pong_bytes / tx_pong_bytes
+The number of bytes sent/received as pongs
+
+.B rx_total_packets / tx_total_packets
+The total number of packets sent/received. The aggregate of all of the above packet stats
+
+.B rx_total_bytes / tx_total_bytes
+The total number of bytes sent/received. The aggregate of all of the above bytes stats
+
+.B tx_data_retries / tx_pmtu_retries / tx_ping_retries / tx_pong_retries / tx_total_retries
+Number of times a transmit operation had to be retried due to the socket returning EAGAIN
 
 .TP
-config.reload_in_progress
-This value will be set to 1 (or created) when a corosync.conf reload is started,
-and set to 0 when the reload is completed. This allows interested subsystems
-to do atomic reconfiguration rather than changing each key. Note that 
-individual add/change/delete notifications will still be sent during a reload.
+stats.ipcs.*
+There is information about total number of active connections from client programs
+at the time the request was made.
+.B active
+number of closed connections during whole runtime of corosync
+.B closed
+Total number of connnections that have been made since corosync was started
 
 .TP
-config.totemconfig_reload_in_progress
-This key is similar to
-.B config.totemconfig_reload_in_progress
-but changed after the totem config trigger is processed. It is useful (mainly)
-for situations when
-.B nodelist.local_node_pos
-must be correctly reinstated before anything else.
+stats.ipcs.ID.*
+Each IPC connection has a unique ID. This is in the form [[serviceX:][PID:]internal_id.
+
+Typical keys in this prefix are:
+
+.B proc_name
+process name of connected process (unavailable on some platforms)
+
+.B dispatched
+number of dispatched messages.
+
+.B invalid_request
+number of requests made by IPC which are invalid (calling non-existing call, ...).
+
+.B name
+contains short name of the IPC connection (unavailable on some platforms).
+
+.B overload
+is number of requests which were not processed because of overload.
+
+.B queue_size
+contains the number of messages in the queue waiting for send.
+
+.B recv_retries
+is the total number of interrupted receives.
+
+.B requests
+contains the number of requests made by IPC.
+
+.B responses
+is the number of responses sent to the IPC client.
+
+.B send_retries
+contains the total number of interrupted sends.
+
+.B service_id
+contains the ID of service which the IPC is connected to.
+
 
 .SH DYNAMIC CHANGE USER/GROUP PERMISSION TO USE COROSYNC IPC
 Is the same as in the configuration file. eg: to add UID 500 use

man/cmap_overview.8

@@ -39,7 +39,7 @@ cmap_overview \- Overview of the Configuration Map
 
 .SH OVERVIEW
 .P
-The CMAP library is used to interact with the configuration database used by corosync.
+The CMAP library is used to interact with the configuration & statistics databases used by corosync.
 
 .PP
 The library provides a mechanism to:
@@ -59,6 +59,7 @@ Description of most keys created by corosync itself can be found in cmap_keys (8
 .SH BUGS
 .SH "SEE ALSO"
 .BR cmap_initialize (3),
+.BR cmap_initialize_map (3),
 .BR cmap_finalize (3),
 .BR cmap_get (3),
 .BR cmap_set (3),

man/cmap_track_add.3.in

@@ -156,6 +156,12 @@ free it.
 This call returns the CS_OK value if successful. It can return CS_ERR_INVALID_PARAM if
 notify_fn is NULL or track_type is invalid value.
 
+.SH NOTES
+Modification tracking of individual keys is supported in the stats map, but not
+prefixes. Add/Delete operations are supported on prefixes though so you can track
+for new ipc connections or knet interfaces.
+In the icmap map, prefix tracking is fully supported.
+
 .SH "SEE ALSO"
 .BR cmap_track_delete (3),
 .BR cmap_initialize (3),

man/corosync-cmapctl.8

@@ -35,9 +35,17 @@
 .SH NAME
 corosync-cmapctl: \- A tool for accessing the object database.
 .SH DESCRIPTION
-usage:  corosync\-cmapctl [\-b] [\-DdghsTt] [\-p filename] [params...]
+usage:  corosync\-cmapctl [\-b] [\-DdghsTt] [\-m map] [\-p filename] [params...]
 .HP
 \fB\-b\fR show binary values
+.HP
+\fB\-m\fR select map to use
+.IP
+The default map is 'icmap' which contains configuration information and some runtime variables
+used by corosync. A 'stats' map is also available which displays network statistics - in
+great detail when knet is used as the transport. Tracking of individual keys (but not prefixes)
+works on the stats map but notifications are sent on a timer, and not every time a value changes.
+
 .SS "Set key:"
 .IP
 corosync\-cmapctl \fB\-s\fR key_name type value

tools/corosync-cmapctl.c

@@ -53,7 +53,6 @@ enum user_action {
 	ACTION_SET,
 	ACTION_DELETE,
 	ACTION_DELETE_PREFIX,
-	ACTION_PRINT_ALL,
 	ACTION_PRINT_PREFIX,
 	ACTION_TRACK,
 	ACTION_LOAD,
@@ -96,7 +95,7 @@ static int convert_name_to_type(const char *name)
 static int print_help(void)
 {
 	printf("\n");
-	printf("usage:  corosync-cmapctl [-b] [-DdghsTt] [-p filename] [params...]\n");
+	printf("usage:  corosync-cmapctl [-b] [-DdghsTt] [-p filename] [-m map] [params...]\n");
 	printf("\n");
 	printf("    -b show binary values\n");
 	printf("\n");
@@ -106,6 +105,10 @@ static int print_help(void)
 	printf("    where type is one of ([i|u][8|16|32|64] | flt | dbl | str | bin)\n");
 	printf("    for bin, value is file name (or - for stdin)\n");
 	printf("\n");
+	printf("    map can be either 'icmap' (the default) which contains corosync\n");
+	printf("    configuration information, or 'stats' which contains statistics\n");
+	printf("    about the networking and IPC traffic in some detail.\n");
+	printf("\n");
 	printf("Load settings from a file:\n");
 	printf("    corosync-cmapctl -p filename\n");
 	printf("\n");
@@ -745,14 +748,16 @@ int main(int argc, char *argv[])
 	int i;
 	size_t value_len;
 	cmap_value_types_t type;
+	cmap_map_t map = CMAP_MAP_DEFAULT;
 	int track_prefix;
+	int map_set = 0;
 	int no_retries;
 	char * settings_file = NULL;
 
 	action = ACTION_PRINT_PREFIX;
 	track_prefix = 1;
 
-	while ((c = getopt(argc, argv, "hgsdDtTbp:")) != -1) {
+	while ((c = getopt(argc, argv, "m:hgsdDtTbp:")) != -1) {
 		switch (c) {
 		case 'h':
 			return print_help();
@@ -783,6 +788,21 @@ int main(int argc, char *argv[])
 		case 'T':
 			action = ACTION_TRACK;
 			break;
+		case 'm':
+			if (strcmp(optarg, "icmap") == 0 ||
+			    strcmp(optarg, "default") == 0) {
+				map = CMAP_MAP_ICMAP;
+				map_set = 1;
+			}
+			if (strcmp(optarg, "stats") == 0) {
+				map = CMAP_MAP_STATS;
+				map_set = 1;
+			}
+			if (!map_set) {
+				fprintf(stderr, "invalid map name, must be 'default', 'icmap' or 'stats'\n");
+				return (EXIT_FAILURE);
+			}
+			break;
 		case '?':
 			return (EXIT_FAILURE);
 			break;
@@ -792,22 +812,19 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	if (argc == 1 || (argc == 2 && show_binary)) {
-		action = ACTION_PRINT_ALL;
-	}
-
 	argc -= optind;
 	argv += optind;
 
 	if (argc == 0 &&
 	    action != ACTION_LOAD &&
-	    action != ACTION_PRINT_ALL) {
+	    action != ACTION_PRINT_PREFIX) {
 		fprintf(stderr, "Expected key after options\n");
 		return (EXIT_FAILURE);
 	}
 
 	no_retries = 0;
-	while ((err = cmap_initialize(&handle)) == CS_ERR_TRY_AGAIN && no_retries++ < MAX_TRY_AGAIN) {
+
+	while ((err = cmap_initialize_map(&handle, map)) == CS_ERR_TRY_AGAIN && no_retries++ < MAX_TRY_AGAIN) {
 		sleep(1);
 	}
 
@@ -817,12 +834,13 @@ int main(int argc, char *argv[])
 	}
 
 	switch (action) {
-	case ACTION_PRINT_ALL:
-		print_iter(handle, NULL);
-		break;
 	case ACTION_PRINT_PREFIX:
-		for (i = 0; i < argc; i++) {
-			print_iter(handle, argv[i]);
+		if (argc == 0) {
+			print_iter(handle, NULL);
+		} else {
+			for (i = 0; i < argc; i++) {
+				print_iter(handle, argv[i]);
+			}
 		}
 		break;
 	case ACTION_GET:

tools/corosync-notifyd.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2012 Red Hat, Inc.
+ * Copyright (c) 2011-2017 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -52,6 +52,7 @@
 
 #include <qb/qbdefs.h>
 #include <qb/qbloop.h>
+#include <qb/qbmap.h>
 #include <qb/qblog.h>
 
 #include <corosync/corotypes.h>
@@ -77,13 +78,18 @@ static int32_t _cs_is_quorate = 0;
 typedef void (*node_membership_fn_t)(char *nodename, uint32_t nodeid, char *state, char* ip);
 typedef void (*node_quorum_fn_t)(char *nodename, uint32_t nodeid, const char *state);
 typedef void (*application_connection_fn_t)(char *nodename, uint32_t nodeid, char *app_name, const char *state);
-typedef void (*rrp_faulty_fn_t)(char *nodename, uint32_t nodeid, uint32_t iface_no, const char *state);
+typedef void (*link_faulty_fn_t)(char *nodename, uint32_t local_nodeid, uint32_t nodeid, uint32_t iface_no, const char *state);
 
 struct notify_callbacks {
 	node_membership_fn_t node_membership_fn;
 	node_quorum_fn_t node_quorum_fn;
 	application_connection_fn_t application_connection_fn;
-	rrp_faulty_fn_t rrp_faulty_fn;
+	link_faulty_fn_t link_faulty_fn;
+};
+
+struct track_item {
+	char key_name[CMAP_KEYNAME_MAXLEN + 1];
+	cmap_track_handle_t track_handle;
 };
 
 #define MAX_NOTIFIERS 5
@@ -93,11 +99,12 @@ static uint32_t local_nodeid = 0;
 static char local_nodename[CS_MAX_NAME_LENGTH];
 static qb_loop_t *main_loop;
 static quorum_handle_t quorum_handle;
+static qb_map_t *tracker_map;
 
 static void _cs_node_membership_event(char *nodename, uint32_t nodeid, char *state, char* ip);
 static void _cs_node_quorum_event(const char *state);
 static void _cs_application_connection_event(char *app_name, const char *state);
-static void _cs_rrp_faulty_event(uint32_t iface_no, const char *state);
+static void _cs_link_faulty_event(uint32_t nodeid, uint32_t iface_no, const char *state);
 
 #ifdef HAVE_DBUS
 #include <dbus/dbus.h>
@@ -165,6 +172,7 @@ static char *snmp_community = NULL;
  * cmap
  */
 static cmap_handle_t cmap_handle;
+static cmap_handle_t stats_handle;
 
 static int32_t _cs_ip_to_hostname(char* ip, char* name_out)
 {
@@ -214,7 +222,7 @@ static void _cs_cmap_members_key_changed (
 		return ;
 	}
 
-	res = sscanf(key_name, "runtime.totem.pg.mrp.srp.members.%u.%s", &nodeid, tmp_key);
+	res = sscanf(key_name, "runtime.members.%u.%s", &nodeid, tmp_key);
 	if (res != 2)
 		return ;
 
@@ -222,7 +230,7 @@ static void _cs_cmap_members_key_changed (
 		return ;
 	}
 
-	snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
+	snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "runtime.members.%u.ip", nodeid);
 	no_retries = 0;
 	while ((err = cmap_get_string(cmap_handle, tmp_key, &ip_str)) == CS_ERR_TRY_AGAIN &&
 			no_retries++ < CMAP_MAX_RETRIES) {
@@ -257,18 +265,19 @@ static void _cs_cmap_connections_key_changed (
 	char obj_name[CS_MAX_NAME_LENGTH];
 	char conn_str[CMAP_KEYNAME_MAXLEN];
 	char tmp_key[CMAP_KEYNAME_MAXLEN];
+	int service, pid;
 	int res;
 
-	res = sscanf(key_name, "runtime.connections.%[^.].%s", conn_str, tmp_key);
-	if (res != 2) {
+	res = sscanf(key_name, "stats.ipcs.service%d.%d.%[^.].%s", &service,&pid, conn_str, tmp_key);
+	if (res != 4) {
 		return ;
 	}
 
-	if (strcmp(tmp_key, "service_id") != 0) {
+	if (strcmp(tmp_key, "procname") != 0) {
 		return ;
 	}
 
-	snprintf(obj_name, CS_MAX_NAME_LENGTH, "%s", conn_str);
+	snprintf(obj_name, CS_MAX_NAME_LENGTH, "%s.%d.%s", conn_str, pid, (char*)new_value.data);
 
 	if (event == CMAP_TRACK_ADD) {
 		_cs_application_connection_event(obj_name, "connected");
@@ -279,7 +288,7 @@ static void _cs_cmap_connections_key_changed (
 	}
 }
 
-static void _cs_cmap_rrp_faulty_key_changed (
+static void _cs_cmap_link_faulty_key_changed (
 	cmap_handle_t cmap_handle_c,
 	cmap_track_handle_t cmap_track_handle,
 	int32_t event,
@@ -289,23 +298,19 @@ static void _cs_cmap_rrp_faulty_key_changed (
 	void *user_data)
 {
 	uint32_t iface_no;
-	char tmp_key[CMAP_KEYNAME_MAXLEN];
+	uint32_t nodeid;
 	int res;
 	int no_retries;
-	uint8_t faulty;
+	uint8_t connected;
 	cs_error_t err;
 
-	res = sscanf(key_name, "runtime.totem.pg.mrp.rrp.%u.%s", &iface_no, tmp_key);
+	res = sscanf(key_name, "stats.knet.node%u.link%u.connected", &nodeid, &iface_no);
 	if (res != 2) {
 		return ;
 	}
 
-	if (strcmp(tmp_key, "faulty") != 0) {
-		return ;
-	}
-
 	no_retries = 0;
-	while ((err = cmap_get_uint8(cmap_handle, key_name, &faulty)) == CS_ERR_TRY_AGAIN &&
+	while ((err = cmap_get_uint8(stats_handle, key_name, &connected)) == CS_ERR_TRY_AGAIN &&
 			no_retries++ < CMAP_MAX_RETRIES) {
 		sleep(1);
 	}
@@ -314,19 +319,55 @@ static void _cs_cmap_rrp_faulty_key_changed (
 		return ;
 	}
 
-	if (faulty) {
-		_cs_rrp_faulty_event(iface_no, "faulty");
+	if (connected) {
+		_cs_link_faulty_event(nodeid, iface_no, "operational");
 	} else {
-		_cs_rrp_faulty_event(iface_no, "operational");
+		_cs_link_faulty_event(nodeid, iface_no, "disconnected");
+	}
+}
+
+static void _cs_cmap_link_added_removed (
+	cmap_handle_t cmap_handle_c,
+	cmap_track_handle_t track_handle,
+	int32_t event,
+	const char *key_name,
+	struct cmap_notify_value new_value,
+	struct cmap_notify_value old_value,
+	void *user_data)
+{
+	struct track_item *track_item;
+
+	/* Add/remove a tracker for a new/removed knet link */
+	if (strstr(key_name, ".connected")) {
+		if (event == CMAP_TRACK_ADD) {
+
+			track_item = malloc(sizeof(struct track_item));
+			if (!track_item) {
+				return;
+			}
+			cmap_track_add(stats_handle, key_name, CMAP_TRACK_MODIFY, _cs_cmap_link_faulty_key_changed, NULL, &track_handle);
+
+			strcpy(track_item->key_name, key_name);
+			track_item->track_handle = track_handle;
+			qb_map_put(tracker_map, track_item->key_name, track_item);
+		} else {
+			track_item = qb_map_get(tracker_map, key_name);
+			if (track_item) {
+				cmap_track_delete(stats_handle, track_item->track_handle);
+				qb_map_rm(tracker_map, track_item->key_name);
+				free(track_item);
+			}
+		}
 	}
 }
 
+
 static int
 _cs_cmap_dispatch(int fd, int revents, void *data)
 {
 	cs_error_t err;
 
-	err = cmap_dispatch(cmap_handle, CS_DISPATCH_ONE);
+	err = cmap_dispatch(*(cmap_handle_t *)data, CS_DISPATCH_ONE);
 
 	if (err != CS_OK && err != CS_ERR_TRY_AGAIN && err != CS_ERR_TIMEOUT &&
 		err != CS_ERR_QUEUE_FULL) {
@@ -589,7 +630,7 @@ out_free:
 }
 
 static void
-_cs_dbus_rrp_faulty_event(char *nodename, uint32_t nodeid, uint32_t iface_no, const char *state)
+_cs_dbus_link_faulty_event(char *nodename, uint32_t nodeid, uint32_t iface_no, const char *state)
 {
 	DBusMessage *msg = NULL;
 
@@ -901,9 +942,9 @@ _cs_syslog_application_connection_event(char *nodename, uint32_t nodeid, char* a
 }
 
 static void
-_cs_syslog_rrp_faulty_event(char *nodename, uint32_t nodeid, uint32_t iface_no, const char *state)
+_cs_syslog_link_faulty_event(char *nodename, uint32_t our_nodeid, uint32_t nodeid, uint32_t iface_no, const char *state)
 {
-	qb_log(LOG_NOTICE, "%s[%d] interface %u is now %s", nodename, nodeid, iface_no, state);
+	qb_log(LOG_NOTICE, "%s[%d] link %u to node %u is now %s", nodename, our_nodeid, iface_no, nodeid, state);
 }
 
 static void
@@ -978,17 +1019,17 @@ _cs_application_connection_event(char *app_name, const char *state)
 }
 
 static void
-_cs_rrp_faulty_event(uint32_t iface_no, const char *state)
+_cs_link_faulty_event(uint32_t nodeid, uint32_t iface_no, const char *state)
 {
 	int i;
 	char *nodename;
-	uint32_t nodeid;
+	uint32_t our_nodeid;
 
-	_cs_local_node_info_get(&nodename, &nodeid);
+	_cs_local_node_info_get(&nodename, &our_nodeid);
 
 	for (i = 0; i < num_notifiers; i++) {
-		if (notifiers[i].rrp_faulty_fn) {
-			notifiers[i].rrp_faulty_fn(nodename, nodeid, iface_no, state);
+		if (notifiers[i].link_faulty_fn) {
+			notifiers[i].link_faulty_fn(nodename, our_nodeid, nodeid, iface_no, state);
 		}
 	}
 }
@@ -1000,35 +1041,81 @@ sig_exit_handler(int32_t num, void *data)
 	return 0;
 }
 
+static void track_link_updown_events(void)
+{
+	cmap_iter_handle_t iter_handle;
+	cmap_track_handle_t track_handle;
+
+	char key_name[CMAP_KEYNAME_MAXLEN + 1];
+	size_t value_len;
+	cmap_value_types_t type;
+	cs_error_t err;
+	struct track_item *track_item;
+
+	err = cmap_iter_init(stats_handle, "stats.knet.", &iter_handle);
+	if (err != CS_OK) {
+		fprintf (stderr, "Failed to initialize knet stats iterator. Error %s\n", cs_strerror(err));
+		exit (EXIT_FAILURE);
+	}
+
+	while ((err = cmap_iter_next(stats_handle, iter_handle, key_name, &value_len, &type)) == CS_OK) {
+		if (strstr(key_name, ".connected")) {
+
+			track_item = malloc(sizeof(struct track_item));
+			if (!track_item) {
+				return;
+			}
+
+			if ((err = cmap_track_add(stats_handle, key_name, CMAP_TRACK_MODIFY, _cs_cmap_link_faulty_key_changed, NULL, &track_handle)) != CS_OK) {
+				fprintf (stderr, "Failed to add tracker for %s. Error %s\n", key_name, cs_strerror(err));
+				exit (EXIT_FAILURE);
+			}
+			strcpy(track_item->key_name, key_name);
+			track_item->track_handle = track_handle;
+			qb_map_put(tracker_map, track_item->key_name, track_item);
+
+		}
+	}
+	cmap_iter_finalize(stats_handle, iter_handle);
+}
+
 static void
 _cs_cmap_init(void)
 {
-	cs_error_t rc;
+	cs_error_t rc = CS_OK;
 	int cmap_fd = 0;
+	int stats_fd = 0;
 	cmap_track_handle_t track_handle;
 
-	rc = cmap_initialize (&cmap_handle);
+	tracker_map = qb_trie_create();
+	if (!tracker_map) {
+		qb_log(LOG_ERR, "Failed to initialize the track map. Error %d", rc);
+		exit (EXIT_FAILURE);
+	}
+
+	rc = cmap_initialize_map (&cmap_handle, CMAP_MAP_ICMAP);
 	if (rc != CS_OK) {
 		qb_log(LOG_ERR, "Failed to initialize the cmap API. Error %d", rc);
 		exit (EXIT_FAILURE);
 	}
 	cmap_fd_get(cmap_handle, &cmap_fd);
 
-	qb_loop_poll_add(main_loop, QB_LOOP_MED, cmap_fd, POLLIN|POLLNVAL, NULL,
+	qb_loop_poll_add(main_loop, QB_LOOP_MED, cmap_fd, POLLIN|POLLNVAL, (void*)&cmap_handle,
 		_cs_cmap_dispatch);
 
-	rc = cmap_track_add(cmap_handle, "runtime.connections.",
-			CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_PREFIX,
-			_cs_cmap_connections_key_changed,
-			NULL,
-			&track_handle);
+
+	rc = cmap_initialize_map (&stats_handle, CMAP_MAP_STATS);
 	if (rc != CS_OK) {
-		qb_log(LOG_ERR,
-			"Failed to track the connections key. Error %d", rc);
+		qb_log(LOG_ERR, "Failed to initialize the cmap stats API. Error %d", rc);
 		exit (EXIT_FAILURE);
 	}
+	cmap_fd_get(stats_handle, &stats_fd);
 
-	rc = cmap_track_add(cmap_handle, "runtime.totem.pg.mrp.srp.members.",
+	qb_loop_poll_add(main_loop, QB_LOOP_MED, stats_fd, POLLIN|POLLNVAL, (void*)&stats_handle,
+		_cs_cmap_dispatch);
+
+
+	rc = cmap_track_add(cmap_handle, "runtime.members.",
 			CMAP_TRACK_ADD | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX,
 			_cs_cmap_members_key_changed,
 			NULL,
@@ -1038,22 +1125,37 @@ _cs_cmap_init(void)
 			"Failed to track the members key. Error %d", rc);
 		exit (EXIT_FAILURE);
 	}
-	rc = cmap_track_add(cmap_handle, "runtime.totem.pg.mrp.rrp.",
-			CMAP_TRACK_ADD | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX,
-			_cs_cmap_rrp_faulty_key_changed,
+
+	rc = cmap_track_add(stats_handle, "stats.ipcs.",
+			CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_PREFIX,
+			_cs_cmap_connections_key_changed,
 			NULL,
 			&track_handle);
 	if (rc != CS_OK) {
 		qb_log(LOG_ERR,
-			"Failed to track the rrp key. Error %d", rc);
+			"Failed to track the connections key. Error %d", rc);
 		exit (EXIT_FAILURE);
 	}
+
+	rc = cmap_track_add(stats_handle, "stats.knet.",
+			CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_PREFIX,
+			_cs_cmap_link_added_removed,
+			NULL,
+			&track_handle);
+	if (rc != CS_OK) {
+		qb_log(LOG_ERR,
+			"Failed to track the knet link status key. Error %d", rc);
+		exit (EXIT_FAILURE);
+	}
+	track_link_updown_events();
+
 }
 
 static void
 _cs_cmap_finalize(void)
 {
 	cmap_finalize (cmap_handle);
+	cmap_finalize (stats_handle);
 }
 
 static void
@@ -1175,8 +1277,8 @@ main(int argc, char *argv[])
 			_cs_syslog_node_quorum_event;
 		notifiers[num_notifiers].application_connection_fn =
 			_cs_syslog_application_connection_event;
-		notifiers[num_notifiers].rrp_faulty_fn =
-			_cs_syslog_rrp_faulty_event;
+		notifiers[num_notifiers].link_faulty_fn =
+			_cs_syslog_link_faulty_event;
 		num_notifiers++;
 	}