corosync/exec/stats.c

/*
 * Copyright (c) 2017-2020 Red Hat, Inc.
 *
 * All rights reserved.
 *
 * Authors: Christine Caulfield (ccaulfie@redhat.com)
 *
 * This software licensed under BSD license, the text of which follows:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * - Neither the name of the MontaVista Software, Inc. nor the names of its
 *   contributors may be used to endorse or promote products derived from this
 *   software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <config.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdint.h>
#include <stddef.h>
#include <unistd.h>
#include <libknet.h>

#include <qb/qblist.h>
#include <qb/qbipcs.h>
#include <qb/qbipc_common.h>

#include <corosync/corodefs.h>
#include <corosync/coroapi.h>
#include <corosync/logsys.h>
#include <corosync/icmap.h>
#include <corosync/totem/totemstats.h>

#include "util.h"
#include "ipcs_stats.h"
#include "stats.h"

LOGSYS_DECLARE_SUBSYS ("STATS");

static qb_map_t *stats_map;

/* Structure of an element in the schedmiss array */
struct schedmiss_entry {
	uint64_t timestamp;
	float delay;
};
#define MAX_SCHEDMISS_EVENTS 10
static struct schedmiss_entry schedmiss_event[MAX_SCHEDMISS_EVENTS];
static unsigned int highest_schedmiss_event;

#define SCHEDMISS_PREFIX "stats.schedmiss"

/* Convert iterator number to text and a stats pointer */
struct cs_stats_conv {
	enum {STAT_PG, STAT_SRP, STAT_KNET, STAT_KNET_HANDLE, STAT_IPCSC, STAT_IPCSG, STAT_SCHEDMISS} type;
	const char *name;
	const size_t offset;
	const icmap_value_types_t value_type;
};

struct cs_stats_conv cs_pg_stats[] = {
	{ STAT_PG, "msg_queue_avail",         offsetof(totempg_stats_t, msg_queue_avail),         ICMAP_VALUETYPE_UINT32},
	{ STAT_PG, "msg_reserved",            offsetof(totempg_stats_t, msg_reserved),            ICMAP_VALUETYPE_UINT32},
};
struct cs_stats_conv cs_srp_stats[] = {
	{ STAT_SRP, "orf_token_tx",           offsetof(totemsrp_stats_t, orf_token_tx),           ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "orf_token_rx",           offsetof(totemsrp_stats_t, orf_token_rx),           ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "memb_merge_detect_tx",   offsetof(totemsrp_stats_t, memb_merge_detect_tx),   ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "memb_merge_detect_rx",   offsetof(totemsrp_stats_t, memb_merge_detect_rx),   ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "memb_join_tx",           offsetof(totemsrp_stats_t, memb_join_tx),           ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "memb_join_rx",           offsetof(totemsrp_stats_t, memb_join_rx),           ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "mcast_tx",               offsetof(totemsrp_stats_t, mcast_tx),               ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "mcast_retx",             offsetof(totemsrp_stats_t, mcast_retx),             ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "mcast_rx",               offsetof(totemsrp_stats_t, mcast_rx),               ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "memb_commit_token_tx",   offsetof(totemsrp_stats_t, memb_commit_token_tx),   ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "memb_commit_token_rx",   offsetof(totemsrp_stats_t, memb_commit_token_rx),   ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "token_hold_cancel_tx",   offsetof(totemsrp_stats_t, token_hold_cancel_tx),   ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "token_hold_cancel_rx",   offsetof(totemsrp_stats_t, token_hold_cancel_rx),   ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "operational_entered",    offsetof(totemsrp_stats_t, operational_entered),    ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "operational_token_lost", offsetof(totemsrp_stats_t, operational_token_lost), ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "gather_entered",         offsetof(totemsrp_stats_t, gather_entered),         ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "gather_token_lost",      offsetof(totemsrp_stats_t, gather_token_lost),      ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "commit_entered",         offsetof(totemsrp_stats_t, commit_entered),         ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "commit_token_lost",      offsetof(totemsrp_stats_t, commit_token_lost),      ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "recovery_entered",       offsetof(totemsrp_stats_t, recovery_entered),       ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "recovery_token_lost",    offsetof(totemsrp_stats_t, recovery_token_lost),    ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "consensus_timeouts",     offsetof(totemsrp_stats_t, consensus_timeouts),     ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "rx_msg_dropped",         offsetof(totemsrp_stats_t, rx_msg_dropped),         ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "time_since_token_last_received", offsetof(totemsrp_stats_t, time_since_token_last_received), ICMAP_VALUETYPE_UINT64},
	{ STAT_SRP, "continuous_gather",      offsetof(totemsrp_stats_t, continuous_gather),      ICMAP_VALUETYPE_UINT32},
	{ STAT_SRP, "continuous_sendmsg_failures", offsetof(totemsrp_stats_t, continuous_sendmsg_failures), ICMAP_VALUETYPE_UINT32},
	{ STAT_SRP, "firewall_enabled_or_nic_failure", offsetof(totemsrp_stats_t, firewall_enabled_or_nic_failure), ICMAP_VALUETYPE_UINT8},
	{ STAT_SRP, "mtt_rx_token",           offsetof(totemsrp_stats_t, mtt_rx_token),           ICMAP_VALUETYPE_UINT32},
	{ STAT_SRP, "avg_token_workload",     offsetof(totemsrp_stats_t, avg_token_workload),     ICMAP_VALUETYPE_UINT32},
	{ STAT_SRP, "avg_backlog_calc",       offsetof(totemsrp_stats_t, avg_backlog_calc),       ICMAP_VALUETYPE_UINT32},
};

struct cs_stats_conv cs_knet_stats[] = {
	{ STAT_KNET, "enabled",          offsetof(struct knet_link_status, enabled),                ICMAP_VALUETYPE_UINT8},
	{ STAT_KNET, "connected",        offsetof(struct knet_link_status, connected),              ICMAP_VALUETYPE_UINT8},
	{ STAT_KNET, "mtu",              offsetof(struct knet_link_status, mtu),                    ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_data_packets",  offsetof(struct knet_link_status, stats.tx_data_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_data_packets",  offsetof(struct knet_link_status, stats.rx_data_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_data_bytes",    offsetof(struct knet_link_status, stats.tx_data_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_data_bytes",    offsetof(struct knet_link_status, stats.rx_data_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_ping_packets",  offsetof(struct knet_link_status, stats.tx_ping_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_ping_packets",  offsetof(struct knet_link_status, stats.rx_ping_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_ping_bytes",    offsetof(struct knet_link_status, stats.tx_ping_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_ping_bytes",    offsetof(struct knet_link_status, stats.rx_ping_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_pong_packets",  offsetof(struct knet_link_status, stats.tx_pong_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_pong_packets",  offsetof(struct knet_link_status, stats.rx_pong_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_pong_bytes",    offsetof(struct knet_link_status, stats.tx_pong_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_pong_bytes",    offsetof(struct knet_link_status, stats.rx_pong_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_pmtu_packets",  offsetof(struct knet_link_status, stats.tx_pmtu_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_pmtu_packets",  offsetof(struct knet_link_status, stats.rx_pmtu_packets),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_pmtu_bytes",    offsetof(struct knet_link_status, stats.tx_pmtu_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_pmtu_bytes",    offsetof(struct knet_link_status, stats.rx_pmtu_bytes),    ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_total_packets", offsetof(struct knet_link_status, stats.tx_total_packets), ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_total_packets", offsetof(struct knet_link_status, stats.rx_total_packets), ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_total_bytes",   offsetof(struct knet_link_status, stats.tx_total_bytes),   ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_total_bytes",   offsetof(struct knet_link_status, stats.rx_total_bytes),   ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_total_errors",  offsetof(struct knet_link_status, stats.tx_total_errors),  ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "rx_total_retries", offsetof(struct knet_link_status, stats.tx_total_retries), ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET, "tx_pmtu_errors",   offsetof(struct knet_link_status, stats.tx_pmtu_errors),   ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_pmtu_retries",  offsetof(struct knet_link_status, stats.tx_pmtu_retries),  ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_ping_errors",   offsetof(struct knet_link_status, stats.tx_ping_errors),   ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_ping_retries",  offsetof(struct knet_link_status, stats.tx_ping_retries),  ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_pong_errors",   offsetof(struct knet_link_status, stats.tx_pong_errors),   ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_pong_retries",  offsetof(struct knet_link_status, stats.tx_pong_retries),  ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_data_errors",   offsetof(struct knet_link_status, stats.tx_data_errors),   ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "tx_data_retries",  offsetof(struct knet_link_status, stats.tx_data_retries),  ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "latency_min",      offsetof(struct knet_link_status, stats.latency_min),      ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "latency_max",      offsetof(struct knet_link_status, stats.latency_max),      ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "latency_ave",      offsetof(struct knet_link_status, stats.latency_ave),      ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "latency_samples",  offsetof(struct knet_link_status, stats.latency_samples),  ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "down_count",       offsetof(struct knet_link_status, stats.down_count),       ICMAP_VALUETYPE_UINT32},
	{ STAT_KNET, "up_count",         offsetof(struct knet_link_status, stats.up_count),         ICMAP_VALUETYPE_UINT32},
};
struct cs_stats_conv cs_knet_handle_stats[] = {
	{ STAT_KNET_HANDLE, "tx_uncompressed_packets",      offsetof(struct knet_handle_stats, tx_uncompressed_packets),      ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_compressed_packets",        offsetof(struct knet_handle_stats, tx_compressed_packets),        ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_compressed_original_bytes", offsetof(struct knet_handle_stats, tx_compressed_original_bytes), ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_compressed_size_bytes",     offsetof(struct knet_handle_stats, tx_compressed_size_bytes),     ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_compress_time_min",         offsetof(struct knet_handle_stats, tx_compress_time_min),         ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_compress_time_max",         offsetof(struct knet_handle_stats, tx_compress_time_max),         ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_compress_time_ave",         offsetof(struct knet_handle_stats, tx_compress_time_ave),         ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_compressed_packets",        offsetof(struct knet_handle_stats, rx_compressed_packets),        ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_compressed_original_bytes", offsetof(struct knet_handle_stats, rx_compressed_original_bytes), ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_compressed_size_bytes",     offsetof(struct knet_handle_stats, rx_compressed_size_bytes),     ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_compress_time_min",         offsetof(struct knet_handle_stats, rx_compress_time_min),         ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_compress_time_max",         offsetof(struct knet_handle_stats, rx_compress_time_max),         ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_compress_time_ave",         offsetof(struct knet_handle_stats, rx_compress_time_ave),         ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_crypt_time_min",            offsetof(struct knet_handle_stats, tx_crypt_time_min),            ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_crypt_time_max",            offsetof(struct knet_handle_stats, tx_crypt_time_max),            ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_crypt_time_ave",            offsetof(struct knet_handle_stats, tx_crypt_time_ave),            ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_crypt_byte_overhead",       offsetof(struct knet_handle_stats, tx_crypt_byte_overhead),       ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "tx_crypt_packets",             offsetof(struct knet_handle_stats, tx_crypt_packets),             ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_crypt_time_min",            offsetof(struct knet_handle_stats, rx_crypt_time_min),            ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_crypt_time_max",            offsetof(struct knet_handle_stats, rx_crypt_time_max),            ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_crypt_time_ave",            offsetof(struct knet_handle_stats, rx_crypt_time_ave),            ICMAP_VALUETYPE_UINT64},
	{ STAT_KNET_HANDLE, "rx_crypt_packets",             offsetof(struct knet_handle_stats, rx_crypt_packets),             ICMAP_VALUETYPE_UINT64},
};

struct cs_stats_conv cs_ipcs_conn_stats[] = {
	{ STAT_IPCSC, "queueing",        offsetof(struct ipcs_conn_stats, cnx.queuing),          ICMAP_VALUETYPE_INT32},
	{ STAT_IPCSC, "queued",          offsetof(struct ipcs_conn_stats, cnx.queued),           ICMAP_VALUETYPE_UINT32},
	{ STAT_IPCSC, "invalid_request", offsetof(struct ipcs_conn_stats, cnx.invalid_request),  ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "overload",        offsetof(struct ipcs_conn_stats, cnx.overload),         ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "sent",            offsetof(struct ipcs_conn_stats, cnx.sent),             ICMAP_VALUETYPE_UINT32},
	{ STAT_IPCSC, "procname",        offsetof(struct ipcs_conn_stats, cnx.proc_name),        ICMAP_VALUETYPE_STRING},
	{ STAT_IPCSC, "requests",        offsetof(struct ipcs_conn_stats, conn.requests),        ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "responses",       offsetof(struct ipcs_conn_stats, conn.responses),       ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "dispatched",      offsetof(struct ipcs_conn_stats, conn.events),          ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "send_retries",    offsetof(struct ipcs_conn_stats, conn.send_retries),    ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "recv_retries",    offsetof(struct ipcs_conn_stats, conn.recv_retries),    ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSC, "flow_control",    offsetof(struct ipcs_conn_stats, conn.flow_control_state),    ICMAP_VALUETYPE_UINT32},
	{ STAT_IPCSC, "flow_control_count",   offsetof(struct ipcs_conn_stats, conn.flow_control_count),    ICMAP_VALUETYPE_UINT64},
};
struct cs_stats_conv cs_ipcs_global_stats[] = {
	{ STAT_IPCSG, "global.active",        offsetof(struct ipcs_global_stats, active),           ICMAP_VALUETYPE_UINT64},
	{ STAT_IPCSG, "global.closed",        offsetof(struct ipcs_global_stats, closed),           ICMAP_VALUETYPE_UINT64},
};
struct cs_stats_conv cs_schedmiss_stats[] = {
	{ STAT_SCHEDMISS, "timestamp",    offsetof(struct schedmiss_entry, timestamp), ICMAP_VALUETYPE_UINT64},
	{ STAT_SCHEDMISS, "delay",        offsetof(struct schedmiss_entry, delay),     ICMAP_VALUETYPE_FLOAT},
};

#define NUM_PG_STATS (sizeof(cs_pg_stats) / sizeof(struct cs_stats_conv))
#define NUM_SRP_STATS (sizeof(cs_srp_stats) / sizeof(struct cs_stats_conv))
#define NUM_KNET_STATS (sizeof(cs_knet_stats) / sizeof(struct cs_stats_conv))
#define NUM_KNET_HANDLE_STATS (sizeof(cs_knet_handle_stats) / sizeof(struct cs_stats_conv))
#define NUM_IPCSC_STATS (sizeof(cs_ipcs_conn_stats) / sizeof(struct cs_stats_conv))
#define NUM_IPCSG_STATS (sizeof(cs_ipcs_global_stats) / sizeof(struct cs_stats_conv))

/* What goes in the trie */
struct stats_item {
	char *key_name;
	struct cs_stats_conv * cs_conv;
};

/* One of these per tracker */
struct cs_stats_tracker
{
	char *key_name;
	void *user_data;
	int32_t events;
	icmap_notify_fn_t notify_fn;
	uint64_t old_value;
	struct qb_list_head list;
};
QB_LIST_DECLARE (stats_tracker_list_head);
static const struct corosync_api_v1 *api;

static void stats_map_set_value(struct cs_stats_conv *conv,
				void *stat_array,
				void *value,
				size_t *value_len,
				icmap_value_types_t *type)
{
	if (value_len) {
		*value_len = icmap_get_valuetype_len(conv->value_type);
	}
	if (type) {
		*type = conv->value_type;
		if ((*type == ICMAP_VALUETYPE_STRING) && value_len && stat_array) {
			*value_len = strlen((char *)(stat_array) + conv->offset)+1;
		}
	}
	if (value) {
		assert(value_len != NULL);

		memcpy(value, (char *)(stat_array) + conv->offset, *value_len);
	}
}

static void stats_add_entry(const char *key, struct cs_stats_conv *cs_conv)
{
	struct stats_item *item = malloc(sizeof(struct stats_item));

	if (item) {
		item->cs_conv = cs_conv;
		item->key_name = strdup(key);
		qb_map_put(stats_map, item->key_name, item);
	}
}
static void stats_rm_entry(const char *key)
{
	struct stats_item *item = qb_map_get(stats_map, key);

	if (item) {
		qb_map_rm(stats_map, item->key_name);
		/* Structures freed in callback below */
	}
}

static void stats_map_free_cb(uint32_t event,
			      char* key, void* old_value,
			      void* value, void* user_data)
{
	struct stats_item *item = (struct stats_item *)old_value;

	if (item) {
		free(item->key_name);
		free(item);
	}
}

cs_error_t stats_map_init(const struct corosync_api_v1 *corosync_api)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];
	int32_t err;

	api = corosync_api;

	stats_map = qb_trie_create();
	if (!stats_map) {
		return CS_ERR_INIT;
	}

	/* Populate the static portions of the trie */
	for (i = 0; i<NUM_PG_STATS; i++) {
		sprintf(param, "stats.pg.%s", cs_pg_stats[i].name);
		stats_add_entry(param, &cs_pg_stats[i]);
	}
	for (i = 0; i<NUM_SRP_STATS; i++) {
		sprintf(param, "stats.srp.%s", cs_srp_stats[i].name);
		stats_add_entry(param, &cs_srp_stats[i]);
	}
	for (i = 0; i<NUM_IPCSG_STATS; i++) {
		sprintf(param, "stats.ipcs.%s", cs_ipcs_global_stats[i].name);
		stats_add_entry(param, &cs_ipcs_global_stats[i]);
	}

	/* KNET, IPCS & SCHEDMISS stats are added when appropriate */


	/* Call us when we can free things */
	err = qb_map_notify_add(stats_map, NULL, stats_map_free_cb, QB_MAP_NOTIFY_FREE, NULL);

	return (qb_to_cs_error(err));
}

cs_error_t stats_map_get(const char *key_name,
			 void *value,
			 size_t *value_len,
			 icmap_value_types_t *type)
{
	struct cs_stats_conv *statinfo;
	struct stats_item *item;
	totempg_stats_t *pg_stats;
	struct knet_link_status link_status;
	struct ipcs_conn_stats ipcs_conn_stats;
	struct ipcs_global_stats ipcs_global_stats;
	struct knet_handle_stats knet_handle_stats;
	int res;
	int nodeid;
	int link_no;
	int service_id;
	uint32_t pid;
	unsigned int sm_event;
	char *sm_type;
	void *conn_ptr;

	item = qb_map_get(stats_map, key_name);
	if (!item) {
		return CS_ERR_NOT_EXIST;
	}

	statinfo = item->cs_conv;
	switch (statinfo->type) {
		case STAT_PG:
			pg_stats = api->totem_get_stats();
			stats_map_set_value(statinfo, pg_stats, value, value_len, type);
			break;
		case STAT_SRP:
			pg_stats = api->totem_get_stats();
			stats_map_set_value(statinfo, pg_stats->srp, value, value_len, type);
			break;
		case STAT_KNET_HANDLE:
			res = totemknet_handle_get_stats(&knet_handle_stats);
			if (res != CS_OK) {
				return res;
			}
			stats_map_set_value(statinfo, &knet_handle_stats, value, value_len, type);
			break;
		case STAT_KNET:
			if (sscanf(key_name, "stats.knet.node%d.link%d", &nodeid, &link_no) != 2) {
				return CS_ERR_NOT_EXIST;
			}

			/* Validate node & link IDs */
			if (nodeid <= 0 || nodeid > KNET_MAX_HOST ||
			    link_no < 0 || link_no > KNET_MAX_LINK) {
				return CS_ERR_NOT_EXIST;
			}

			/* Always get the latest stats */
			res = totemknet_link_get_status((knet_node_id_t)nodeid, (uint8_t)link_no, &link_status);
			if (res != CS_OK) {
				return CS_ERR_LIBRARY;
			}
			stats_map_set_value(statinfo, &link_status, value, value_len, type);
			break;
		case STAT_IPCSC:
			if (sscanf(key_name, "stats.ipcs.service%d.%d.%p", &service_id, &pid, &conn_ptr) != 3) {
				return CS_ERR_NOT_EXIST;
			}
			res = cs_ipcs_get_conn_stats(service_id, pid, conn_ptr, &ipcs_conn_stats);
			if (res != CS_OK) {
				return res;
			}
			stats_map_set_value(statinfo, &ipcs_conn_stats, value, value_len, type);
			break;
		case STAT_IPCSG:
			cs_ipcs_get_global_stats(&ipcs_global_stats);
			stats_map_set_value(statinfo, &ipcs_global_stats, value, value_len, type);
			break;
		case STAT_SCHEDMISS:
			if (sscanf(key_name, SCHEDMISS_PREFIX ".%d", &sm_event) != 1) {
				return CS_ERR_NOT_EXIST;
			}

			sm_type = strrchr(key_name, '.');
			if (sm_type == NULL) {
				return CS_ERR_NOT_EXIST;
			}
			sm_type++;

			if (strcmp(sm_type, "timestamp") == 0) {
				memcpy(value, &schedmiss_event[sm_event].timestamp, sizeof(uint64_t));
				*value_len = sizeof(uint64_t);
				*type = ICMAP_VALUETYPE_UINT64;
			}
			if (strcmp(sm_type, "delay") == 0) {
				memcpy(value, &schedmiss_event[sm_event].delay, sizeof(float));
				*value_len = sizeof(float);
				*type = ICMAP_VALUETYPE_FLOAT;
			}
			break;
		default:
			return CS_ERR_LIBRARY;
	}
	return CS_OK;
}

static void schedmiss_clear_stats(void)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];

	for (i=0; i<MAX_SCHEDMISS_EVENTS; i++) {
		if (i < highest_schedmiss_event) {
			sprintf(param, SCHEDMISS_PREFIX ".%i.timestamp", i);
			stats_rm_entry(param);
			sprintf(param, SCHEDMISS_PREFIX ".%i.delay", i);
			stats_rm_entry(param);
		}
		schedmiss_event[i].timestamp = (uint64_t)0LL;
		schedmiss_event[i].delay = 0.0f;
	}
	highest_schedmiss_event = 0;
}

/* Called from main.c */
void stats_add_schedmiss_event(uint64_t timestamp, float delay)
{
	char param[ICMAP_KEYNAME_MAXLEN];
	int i;

	/* Move 'em all up */
	for (i=MAX_SCHEDMISS_EVENTS-2; i>=0; i--) {
		schedmiss_event[i+1].timestamp = schedmiss_event[i].timestamp;
		schedmiss_event[i+1].delay = schedmiss_event[i].delay;
	}

	/* New entries are always at the front */
	schedmiss_event[0].timestamp = timestamp;
	schedmiss_event[0].delay = delay;

	/* If we've not run off the end then add an entry in the trie for the new 'end' one */
	if (highest_schedmiss_event < MAX_SCHEDMISS_EVENTS) {
		sprintf(param, SCHEDMISS_PREFIX ".%i.timestamp", highest_schedmiss_event);
		stats_add_entry(param, &cs_schedmiss_stats[0]);
		sprintf(param, SCHEDMISS_PREFIX ".%i.delay", highest_schedmiss_event);
		stats_add_entry(param, &cs_schedmiss_stats[1]);
		highest_schedmiss_event++;
	}
	/* Notifications get sent by the stats_updater */
}

#define STATS_CLEAR           "stats.clear."
#define STATS_CLEAR_KNET      "stats.clear.knet"
#define STATS_CLEAR_IPC       "stats.clear.ipc"
#define STATS_CLEAR_TOTEM     "stats.clear.totem"
#define STATS_CLEAR_ALL       "stats.clear.all"
#define STATS_CLEAR_SCHEDMISS "stats.clear.schedmiss"

cs_error_t stats_map_set(const char *key_name,
			 const void *value,
			 size_t value_len,
			 icmap_value_types_t type)
{
	int cleared = 0;

	if (strncmp(key_name, STATS_CLEAR_KNET, strlen(STATS_CLEAR_KNET)) == 0) {
		totempg_stats_clear(TOTEMPG_STATS_CLEAR_TRANSPORT);
		cleared = 1;
	}
	if (strncmp(key_name, STATS_CLEAR_IPC, strlen(STATS_CLEAR_IPC)) == 0) {
		cs_ipcs_clear_stats();
		cleared = 1;
	}
	if (strncmp(key_name, STATS_CLEAR_TOTEM, strlen(STATS_CLEAR_TOTEM)) == 0) {
		totempg_stats_clear(TOTEMPG_STATS_CLEAR_TOTEM);
		cleared = 1;
	}
	if (strncmp(key_name, STATS_CLEAR_SCHEDMISS, strlen(STATS_CLEAR_SCHEDMISS)) == 0) {
		schedmiss_clear_stats();
		cleared = 1;
	}
	if (strncmp(key_name, STATS_CLEAR_ALL, strlen(STATS_CLEAR_ALL)) == 0) {
		totempg_stats_clear(TOTEMPG_STATS_CLEAR_TRANSPORT | TOTEMPG_STATS_CLEAR_TOTEM);
		cs_ipcs_clear_stats();
		schedmiss_clear_stats();
		cleared = 1;
	}
	if (!cleared) {
		return CS_ERR_NOT_SUPPORTED;
	}
	return CS_OK;
}

cs_error_t stats_map_adjust_int(const char *key_name, int32_t step)
{
	return CS_ERR_NOT_SUPPORTED;
}

cs_error_t stats_map_delete(const char *key_name)
{
	return CS_ERR_NOT_SUPPORTED;
}

int stats_map_is_key_ro(const char *key_name)
{
	/* It's all read-only apart from the 'clear' destinations */
	if (strncmp(key_name, STATS_CLEAR, strlen(STATS_CLEAR)) == 0) {
		return 0;
	} else {
		return 1;
	}
}

icmap_iter_t stats_map_iter_init(const char *prefix)
{
	return (qb_map_pref_iter_create(stats_map, prefix));
}


const char *stats_map_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
{
	const char *res;
	struct stats_item *item;

	res = qb_map_iter_next(iter, (void **)&item);
	if (res == NULL) {
		return (res);
	}
	stats_map_set_value(item->cs_conv, NULL, NULL, value_len, type);

	return res;
}

void stats_map_iter_finalize(icmap_iter_t iter)
{
	qb_map_iter_free(iter);
}


void stats_trigger_trackers()
{
	struct cs_stats_tracker *tracker;
	struct qb_list_head *iter;
	cs_error_t res;
	size_t value_len;
	icmap_value_types_t type;
	uint64_t value;
	struct icmap_notify_value new_val;
	struct icmap_notify_value old_val;

	qb_list_for_each(iter, &stats_tracker_list_head) {

		tracker = qb_list_entry(iter, struct cs_stats_tracker, list);
		if (tracker->events & ICMAP_TRACK_PREFIX || !tracker->key_name ) {
			continue;
		}

		res = stats_map_get(tracker->key_name,
				    &value, &value_len, &type);

		/* Check if it has changed */
		if ((res == CS_OK) && (memcmp(&value, &tracker->old_value, value_len) != 0)) {

			old_val.type = new_val.type = type;
			old_val.len = new_val.len = value_len;
			old_val.data = new_val.data = &value;

			tracker->notify_fn(ICMAP_TRACK_MODIFY, tracker->key_name,
					   old_val, new_val, tracker->user_data);

			memcpy(&tracker->old_value, &value, value_len);
		}
	}
}


/* Callback from libqb when a key is added/removed */
static void stats_map_notify_fn(uint32_t event, char *key, void *old_value, void *value, void *user_data)
{
	struct cs_stats_tracker *tracker = user_data;
	struct icmap_notify_value new_val;
	struct icmap_notify_value old_val;
	char new_value[64];

	if (value == NULL && old_value == NULL) {
		return ;
	}

	/* Ignore schedmiss trackers as the values are read from the circular buffer */
	if (strncmp(key, SCHEDMISS_PREFIX, strlen(SCHEDMISS_PREFIX)) == 0 ) {
		return ;
	}

	new_val.data = new_value;
	if (stats_map_get(key,
			  &new_value,
			  &new_val.len,
			  &new_val.type) != CS_OK) {
		log_printf(LOGSYS_LEVEL_WARNING, "get value of notified key %s failed", key);
		return ;
	}

	/* We don't know what the old value was
	   but as this only tracks ADD & DELETE I'm not worried
	   about it */
	memcpy(&old_val, &new_val, sizeof(new_val));

	tracker->notify_fn(icmap_qbtt_to_tt(event),
			   key,
			   new_val,
			   old_val,
			   tracker->user_data);

}

cs_error_t stats_map_track_add(const char *key_name,
			       int32_t track_type,
			       icmap_notify_fn_t notify_fn,
			       void *user_data,
			       icmap_track_t *icmap_track)
{
	struct cs_stats_tracker *tracker;
	size_t value_len;
	icmap_value_types_t type;
	cs_error_t err;

	/* We can track adding or deleting a key under a prefix */
	if ((track_type & ICMAP_TRACK_PREFIX) &&
	    (!(track_type & ICMAP_TRACK_DELETE) ||
	     !(track_type & ICMAP_TRACK_ADD))) {
		return CS_ERR_NOT_SUPPORTED;
	}

	tracker = malloc(sizeof(struct cs_stats_tracker));
	if (!tracker) {
		return CS_ERR_NO_MEMORY;
	}

	tracker->notify_fn = notify_fn;
	tracker->user_data = user_data;
	tracker->events = track_type;
	if (key_name) {
		tracker->key_name = strdup(key_name);
		if (!tracker->key_name) {
			free(tracker);
			return CS_ERR_NO_MEMORY;
		}
		/* Get initial value */
		if (stats_map_get(tracker->key_name,
				  &tracker->old_value, &value_len, &type) != CS_OK) {
			tracker->old_value = 0ULL;
		}
	} else {
		tracker->key_name = NULL;
		tracker->old_value = 0ULL;
	}

	/* Add/delete trackers can use the qb_map tracking */
	if ((track_type & ICMAP_TRACK_ADD) ||
	    (track_type & ICMAP_TRACK_DELETE)) {
		err = qb_map_notify_add(stats_map, tracker->key_name,
					stats_map_notify_fn,
					icmap_tt_to_qbtt(track_type),
					tracker);
		if (err != 0) {
			log_printf(LOGSYS_LEVEL_ERROR, "creating stats tracker %s failed. %d\n", tracker->key_name, err);
			free(tracker->key_name);
			free(tracker);
			return (qb_to_cs_error(err));
		}
	}

	qb_list_add (&tracker->list, &stats_tracker_list_head);

	*icmap_track = (icmap_track_t)tracker;
	return CS_OK;
}

cs_error_t stats_map_track_delete(icmap_track_t icmap_track)
{
	struct cs_stats_tracker *tracker = (struct cs_stats_tracker *)icmap_track;
	int err;

	if ((tracker->events & ICMAP_TRACK_ADD) ||
	    (tracker->events & ICMAP_TRACK_DELETE)) {
		err = qb_map_notify_del_2(stats_map,
					  tracker->key_name, stats_map_notify_fn,
					  icmap_tt_to_qbtt(tracker->events), tracker);
		if (err) {
			log_printf(LOGSYS_LEVEL_ERROR, "deleting tracker %s failed. %d\n", tracker->key_name, err);
		}
	}

	qb_list_del(&tracker->list);
	free(tracker->key_name);
	free(tracker);

	return CS_OK;
}

void *stats_map_track_get_user_data(icmap_track_t icmap_track)
{
	struct cs_stats_tracker *tracker = (struct cs_stats_tracker *)icmap_track;

	return tracker->user_data;
}

/* Called from totemknet to add/remove keys from our map */
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link_no)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];

	for (i = 0; i<NUM_KNET_STATS; i++) {
		sprintf(param, "stats.knet.node%d.link%d.%s", nodeid, link_no, cs_knet_stats[i].name);
		stats_add_entry(param, &cs_knet_stats[i]);
	}
}
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link_no)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];

	for (i = 0; i<NUM_KNET_STATS; i++) {
		sprintf(param, "stats.knet.node%d.link%d.%s", nodeid, link_no, cs_knet_stats[i].name);
		stats_rm_entry(param);
	}
}

/* This is separated out from  stats_map_init() because we don't know whether
   knet is in use until much later in the startup */
void stats_knet_add_handle(void)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];

	for (i = 0; i<NUM_KNET_HANDLE_STATS; i++) {
		sprintf(param, "stats.knet.handle.%s", cs_knet_handle_stats[i].name);
		stats_add_entry(param, &cs_knet_handle_stats[i]);
	}
}

/* Called from ipc_glue to add/remove keys from our map */
void stats_ipcs_add_connection(int service_id, uint32_t pid, void *ptr)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];

	for (i = 0; i<NUM_IPCSC_STATS; i++) {
		sprintf(param, "stats.ipcs.service%d.%d.%p.%s", service_id, pid, ptr, cs_ipcs_conn_stats[i].name);
		stats_add_entry(param, &cs_ipcs_conn_stats[i]);
	}
}
void stats_ipcs_del_connection(int service_id, uint32_t pid, void *ptr)
{
	int i;
	char param[ICMAP_KEYNAME_MAXLEN];

	for (i = 0; i<NUM_IPCSC_STATS; i++) {
		sprintf(param, "stats.ipcs.service%d.%d.%p.%s", service_id, pid, ptr, cs_ipcs_conn_stats[i].name);
		stats_rm_entry(param);
	}
}