cts: Remove CTS

There are several reasons for removal of CTS:
1. It's not actively maintained
2. It's quite hard to setup
3. It has hard to fix bug in it's design (syslog messages are thrown by
   rsyslog (configurable), journald (configurable) or when rsyslog is
   used together with journald (non configurable)) so test
   can fail just because of lost message.
4. It depends on pacemaker CTS, which is changed quite often
5. CTS itself is great tool for Pacemaker
   (shutdown/startup of the node), but Corosync has a slightly
   different needs
6. Bin Liu <bliu@suse.com> made a heroic effort to port it to Python 3
   (huge thanks), but it's still not fully complete

All and all, if somebody is interested in maintaining CTS code, please
create repository similar to corosync flatiron cts and let us know.

Signed-off-by: Jan Friesse <jfriesse@redhat.com>
Reviewed-by: Fabio M. Di Nitto <fdinitto@redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>

Makefile.am

@@ -48,7 +48,7 @@ MAINTAINERCLEANFILES	= Makefile.in aclocal.m4 configure depcomp \
 
 dist_doc_DATA		= LICENSE INSTALL README.recovery AUTHORS
 
-SUBDIRS			= include common_lib lib exec tools test cts pkgconfig \
+SUBDIRS			= include common_lib lib exec tools test pkgconfig \
 			  man init conf vqsim
 
 coverity:

autobuild.sh

@@ -1,112 +0,0 @@
-#!/bin/sh
-#
-# This script is called by buildbot to test
-# corosync. It is run continously to help catch regressions.
-#
-# ENVIRONMENT variables that affect it's behaviour:
-#
-# TEST_NODES - the hostnames of the nodes to be tested
-# TARGET - this is used by mock so look in /etc/mock for
-#          possible options.
-#
-
-LOG="echo CTS: "
-
-# required packages
-which mock >/dev/null 2>&1
-if [ $? -ne 0 ]
-then
-	$LOG 'please install mock (yum install mock).'
-        exit 1
-fi
-
-MOCK=/usr/bin/mock
-
-git clean -xfd
-
-set -e
-
-$LOG 'running autogen ...'
-./autogen.sh
-
-$LOG 'running configure ...'
-./configure --enable-testagents --enable-watchdog --enable-monitoring
-
-$LOG 'building source rpm'
-rm -f *.src.rpm
-make srpm 
-SRPM=$(ls *src.rpm)
-
-if [ ! -f $SRPM ]
-then
-	$LOG no source rpm to build from!
-	exit 1
-fi
-
-if [ -z "$TARGET" ]
-then
-	TARGET=fedora-16-x86_64
-fi
-case $TARGET in
-	fedora-15-x86_64)
-	EXTRA_WITH=" --with systemd"
-	;;
-	fedora-16-x86_64)
-	EXTRA_WITH=" --with systemd"
-	;;
-	fedora-17-x86_64)
-	EXTRA_WITH=" --with systemd"
-	;;
-	*)
-esac
-
-RPM_DIR=/var/lib/mock/$TARGET/result
-rm -f $RPM_DIR/corosync*.rpm
-
-$LOG "running mock rebuild ($SRPM)"
-$MOCK -v -r $TARGET --no-clean --rebuild $SRPM --with testagents --with watchdog --with monitoring $EXTRA_WITH
-
-if [ -z "$TEST_NODES" ]
-then
-	$LOG no test nodes, exiting without running cts.
-	exit 0
-else
-	# start the VMs, or leave them running?
-	true
-fi
-
-RPM_LIST=
-for r in $RPM_DIR/corosync*.rpm
-do
-  case $r in
-    *src.rpm)
-    ;;
-    *-devel-*)
-    ;;
-    *)
-    RPM_LIST="$RPM_LIST $r"
-    ;;
-  esac
-done
-
-$LOG installing $RPM_LIST
-$LOG onto the test nodes $TEST_NODES
-
-# load and install rpm(s) onto the nodes
-for n in $TEST_NODES
-do
-	$LOG "Installing onto $n"
-	sudo ssh $n "rm -rf /tmp/corosync*.rpm"
-	sudo ssh $n "rm -f /etc/corosync/corosync.conf.*"
-	sudo scp $RPM_LIST $n:/tmp/
-        sudo ssh $n "rpm --nodeps --force -Uvf /tmp/corosync*.rpm"
-done
-
-$LOG 'running CTS ...'
-CTS_LOG=$(pwd)/cts.log
-rm -f $CTS_LOG
-pushd cts
-# needs sudo to read /var/log/messages
-	sudo -n ./corolab.py --nodes "$TEST_NODES" --outputfile $CTS_LOG $CTS_ARGS
-popd
-

configure.ac

@@ -202,9 +202,6 @@ AC_CONFIG_FILES([Makefile
 		 man/Makefile
 		 pkgconfig/Makefile
 		 test/Makefile
-		 cts/Makefile
-		 cts/agents/Makefile
-		 cts/CTSvars.py
 		 tools/Makefile
 		 conf/Makefile
 		 vqsim/Makefile
@@ -334,10 +331,6 @@ AC_ARG_ENABLE([dbus],
 	[  --enable-dbus                   : dbus events. ],,
 	[ enable_dbus="no" ])
 
-AC_ARG_ENABLE([testagents],
-	[  --enable-testagents             : Install Test Agents. ],,
-	[ default="no" ])
-
 AC_ARG_ENABLE([monitoring],
 	[  --enable-monitoring             : resource monitoring ],,
 	[ default="no" ])
@@ -429,12 +422,6 @@ if test "x${enable_dbus}" = xyes; then
 	WITH_LIST="$WITH_LIST --with dbus"
 fi
 
-if test "x${enable_testagents}" = xyes; then
-	AC_DEFINE_UNQUOTED([HAVE_TESTAGENTS], 1, [have testagents])
-	PACKAGE_FEATURES="$PACKAGE_FEATURES testagents"
-	WITH_LIST="$WITH_LIST --with testagents"
-fi
-
 if test "x${enable_monitoring}" = xyes; then
 	PKG_CHECK_MODULES([statgrab], [libstatgrab])
 	PKG_CHECK_MODULES([statgrabge090], [libstatgrab >= 0.90],
@@ -688,7 +675,6 @@ AC_SUBST([SOMINOR])
 AC_SUBST([SOMICRO])
 AC_SUBST([SONAME])
 
-AM_CONDITIONAL(INSTALL_TESTAGENTS, test "${enable_testagents}" = "yes")
 AM_CONDITIONAL(INSTALL_MIB, test "${do_snmp}" = "1")
 AM_CONDITIONAL(INSTALL_DBUSCONF, test "${enable_dbus}" = "yes")
 AM_CONDITIONAL(AUGTOOL, test -n "${AUGTOOL}")

corosync.spec.in

@@ -5,7 +5,6 @@
 # Conditionals
 # Invoke "rpmbuild --without <feature>" or "rpmbuild --with <feature>"
 # to disable or enable specific features
-%bcond_with testagents
 %bcond_with watchdog
 %bcond_with monitoring
 %bcond_with snmp
@@ -82,9 +81,6 @@ Requires: libxslt
 %endif
 
 %{configure} \
-%if %{with testagents}
-	--enable-testagents \
-%endif
 %if %{with watchdog}
 	--enable-watchdog \
 %endif
@@ -212,29 +208,6 @@ fi
 %{_mandir}/man5/votequorum.5*
 %{_mandir}/man8/cmap_keys.8*
 
-# optional testagent rpm
-#
-%if %{with testagents}
-
-%package -n corosync-testagents
-Summary: The Corosync Cluster Engine Test Agents
-Requires: %{name} = %{version}-%{release}
-
-%description -n corosync-testagents
-This package contains corosync test agents.
-
-%files -n corosync-testagents
-%defattr(755,root,root,-)
-%{_datadir}/corosync/tests/mem_leak_test.sh
-%{_datadir}/corosync/tests/net_breaker.sh
-%{_datadir}/corosync/tests/cmap-dispatch-deadlock.sh
-%{_datadir}/corosync/tests/shm_leak_audit.sh
-%{_bindir}/cpg_test_agent
-%{_bindir}/sam_test_agent
-%{_bindir}/votequorum_test_agent
-
-%endif
-
 # library
 #
 %package -n corosynclib

cts/.gitignore

@@ -1,3 +0,0 @@
-CTSvars.py
-*_test_agent
-*.pyc

cts/CTSvars.py.in

@@ -1,3 +0,0 @@
-class CTSvars(object):
-    CTS_home="@prefix@/share/pacemaker/tests/cts"
-    INITDIR="/etc/init.d"

cts/Makefile.am

@@ -1,34 +0,0 @@
-# Copyright (c) 2010 Red Hat, Inc.
-#
-# Authors: Angus Salkeld (asalkeld@redhat.com)
-#
-# This software licensed under BSD license, the text of which follows:
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# - Redistributions of source code must retain the above copyright notice,
-#   this list of conditions and the following disclaimer.
-# - Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-# - Neither the name of the MontaVista Software, Inc. nor the names of its
-#   contributors may be used to endorse or promote products derived from this
-#   software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-# THE POSSIBILITY OF SUCH DAMAGE.
-
-MAINTAINERCLEANFILES    = Makefile.in
-
-SUBDIRS = agents
-

cts/README

@@ -1,46 +0,0 @@
-Quick start guide.
-==================
-
-CTS: Cluster Test System
-The CTS uses a test driver node(TDN) to drive the execution of the test
-software.  The CTS also uses 2 or more test target nodes(TTN) to run the test
-cases.  The CTS software requires atleast 3 nodes 1 of which acts as a TDN and
-the remaining acting as TTNs.
-
-The dependencies of the TDN include Pacemaker and Augeas.
-
-On the single TDN, it is recommended at this time to install pacemaker from
-source so that the CTS from Pacemaker remains compatible with the current
-working version of the CTS components within Corosync (currently you will
-the devel branch http://hg.clusterlabs.org/pacemaker/devel/).
-
-2] ssh access
---------------------------------------------------------------------------------
-CTS requires login-less root access to the CNs.
-so if my CN is called "node32"
-# ssh node32
-should not ask for a password and result in a root shell.
-
-To enable this behavior, create a ssh key with the command ssh-keygen.  When it
-prompts for a password, enter an empty field.  It will create a file called
-~/.ssh/id_dsa.pub.  Copy that file to the TTNs into /root/.ssh/authorized_keys.
-Ensure permissions are 700 on /root/.ssh.
-
-Test ssh into the machine works without a password from the TDN.
-
-3] Install augeas on the TDN
---------------------------------------------------------------------------------
-yum install augeas
-
-4] Configuring corosync for CTSs testing
---------------------------------------------------------------------------------
-
-./configure --enable-testagents --enable-augeas --enable-debug
-
-and then install it on the TTN.
-
-5] run CTS
---------------------------------------------------------------------------------
-cd <your-corosync-src-dir>/cts
-sudo ./corolab.py --nodes "n1 n2 n3"
-

cts/agents/.gitignore

@@ -1,2 +0,0 @@
-cmap-dispatch-deadlock.sh
-shm_leak_audit.sh

cts/agents/Makefile.am

@@ -1,90 +0,0 @@
-#
-# Copyright (c) 2010 Red Hat, Inc.
-#
-# Authors: Angus Salkeld (asalkeld@redhat.com)
-#
-# This software licensed under BSD license, the text of which follows:
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# - Redistributions of source code must retain the above copyright notice,
-#   this list of conditions and the following disclaimer.
-# - Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-# - Neither the name of the MontaVista Software, Inc. nor the names of its
-#   contributors may be used to endorse or promote products derived from this
-#   software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-# THE POSSIBILITY OF SUCH DAMAGE.
-
-MAINTAINERCLEANFILES		= Makefile.in
-
-EXTRA_DIST			= cmap-dispatch-deadlock.sh.in \
-				  shm_leak_audit.sh.in
-
-TEST_AGENTS			= cpg_test_agent \
-				  sam_test_agent \
-				  votequorum_test_agent
-
-SHELL_TESTS			= mem_leak_test.sh \
-				  net_breaker.sh \
-				  cmap-dispatch-deadlock.sh \
-				  shm_leak_audit.sh
-
-noinst_HEADERS			= common_test_agent.h
-
-if INSTALL_TESTAGENTS
-agentdir			= $(datadir)/$(PACKAGE)/tests
-bin_PROGRAMS			= $(TEST_AGENTS)
-dist_agent_SCRIPTS		= $(SHELL_TESTS)
-else
-noinst_PROGRAMS			= $(TEST_AGENTS)
-noinst_SCRIPTS			= $(SHELL_TESTS)
-endif
-
-cmap-dispatch-deadlock.sh: cmap-dispatch-deadlock.sh.in
-	sed -e 's#@''BASHPATH@#${BASHPATH}#g' $< > $@
-	chmod 755 $@
-
-shm_leak_audit.sh: shm_leak_audit.sh.in
-	sed -e 's#@''BASHPATH@#${BASHPATH}#g' $< > $@
-	chmod 755 $@
-
-cpg_test_agent_SOURCES		= cpg_test_agent.c common_test_agent.c
-cpg_test_agent_CFLAGS		= $(nss_CFLAGS)
-cpg_test_agent_LDADD		= $(LIBQB_LIBS) $(nss_LIBS) \
-				  $(top_builddir)/lib/libcfg.la \
-				  $(top_builddir)/lib/libcpg.la \
-				  $(top_builddir)/common_lib/libcorosync_common.la
-
-sam_test_agent_SOURCES		= sam_test_agent.c common_test_agent.c
-sam_test_agent_LDADD		= $(LIBQB_LIBS) \
-				  $(top_builddir)/lib/libsam.la \
-				  $(top_builddir)/lib/libcmap.la \
-				  $(top_builddir)/lib/libquorum.la \
-				  $(top_builddir)/common_lib/libcorosync_common.la
-
-votequorum_test_agent_SOURCES	= votequorum_test_agent.c common_test_agent.c
-votequorum_test_agent_LDADD	= $(LIBQB_LIBS) \
-				  $(top_builddir)/lib/libquorum.la \
-				  $(top_builddir)/lib/libvotequorum.la \
-				  $(top_builddir)/common_lib/libcorosync_common.la
-
-clean-local:
-	rm -f shm_leak_audit.sh cmap-dispatch-deadlock.sh
-
-lint:
-	-splint $(LINT_FLAGS) $(CFLAGS) *.c
-

cts/agents/cmap-dispatch-deadlock.sh.in

@@ -1,57 +0,0 @@
-#!@BASHPATH@
-
-export TIMEOUT=600
-export PID=$$
-up_to=200
-
-rec_plist() {
-    if [ "$2" == "" ];then
-	pl="`ps ax -o pid= -o ppid= -o comm=`"
-    else
-	pl=$2
-    fi
-
-    list=`echo "$pl" | egrep "^ *[0-9]+ +$1" | awk '{ print $1 }'`
-    tmplist=$list
-    for i in $tmplist;do
-	[ "$i" != "$1" ] && [ "$i" != "$$" ] && list="$list "`rec_plist $i "$pl"`
-    done
-
-    echo $list
-}
-
-rec_pkill() {
-    kill -9 `rec_plist "$1"` 2> /dev/null
-}
-
-exit_timeout() {
-    echo "ERR: Timeout. Test failed $PID"
-    rec_pkill "$$"
-    exit 1
-}
-
-corosync-cmapctl -s test.abd "str" "test" || exit 2
-
-trap exit_timeout SIGUSR1
-(sleep $TIMEOUT ; kill -SIGUSR1 $PID) &
-
-wait_list=""
-
-for e in {1..40};do
-    (for a in `seq 1 $up_to`;do corosync-cmapctl -s test.abd "str" $a ; done) &
-    wait_list="$wait_list $!"
-done
-
-notify_list=""
-
-for i in {1..2};do
-    sleep 600000 | corosync-cmapctl -t test > /dev/null &
-    notify_list="$notify_list $!"
-done
-
-wait $wait_list
-
-rec_pkill "$$"
-
-echo "OK"
-exit 0

cts/agents/common_test_agent.c

@@ -1,332 +0,0 @@
-/*
- * Copyright (c) 2010 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Angus Salkeld (asalkeld@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <poll.h>
-#include <fcntl.h>
-
-#include "common_test_agent.h"
-
-#define MAX_CLIENTS	64
-
-static char big_and_buf_rx[HOW_BIG_AND_BUF];
-ta_do_command_fn do_command;
-static qb_loop_t *poll_handle;
-static pre_exit_fn pre_exit = NULL;
-static int client_fds[MAX_CLIENTS];
-static int client_fds_pos = 0;
-
-qb_loop_t *ta_poll_handle_get(void)
-{
-	return poll_handle;
-}
-
-static void shut_me_down(void)
-{
-	if (pre_exit) {
-		pre_exit();
-	}
-	qb_loop_stop(poll_handle);
-}
-
-
-static void ta_handle_command (int sock, char* msg)
-{
-	int num_args;
-	char *saveptr = NULL;
-	char *str = strdup (msg);
-	char *str_len;
-	char *str_arg;
-	char *args[5];
-	int i = 0;
-	int a = 0;
-	char* func = NULL;
-
-	qb_log(LOG_DEBUG,"(MSG:%s)", msg);
-
-	str_len = strtok_r (str, ":", &saveptr);
-	assert (str_len);
-
-	num_args = atoi (str_len) * 2;
-	for (i = 0; i < num_args / 2; i++) {
-		str_len = strtok_r (NULL, ":", &saveptr);
-		str_arg = strtok_r (NULL, ":", &saveptr);
-		if (func == NULL) {
-			/* first "arg" is the function */
-			qb_log (LOG_TRACE, "(LEN:%s, FUNC:%s)", str_len, str_arg);
-			func = str_arg;
-			a = 0;
-		} else {
-			args[a] = str_arg;
-			a++;
-			qb_log (LOG_TRACE, "(LEN:%s, ARG:%s)", str_len, str_arg);
-		}
-	}
-	do_command (sock, func, args, a+1);
-
-	free (str);
-}
-
-static int server_process_data_fn (
-	int fd,
-	int revents,
-	void *data)
-{
-	char *saveptr;
-	char *msg;
-	char *cmd;
-	int32_t nbytes;
-
-	if (revents & POLLHUP || revents & POLLERR) {
-		qb_log (LOG_INFO, "command sockect got POLLHUP exiting...");
-		shut_me_down();
-		return -1;
-	}
-
-	if ((nbytes = recv (fd, big_and_buf_rx, sizeof (big_and_buf_rx), 0)) <= 0) {
-		/* got error or connection closed by client */
-		if (nbytes == 0) {
-			/* connection closed */
-			qb_log (LOG_WARNING, "socket %d hung up: exiting...", fd);
-		} else {
-			qb_perror(LOG_ERR, "recv() failed");
-		}
-		shut_me_down();
-		return -1;
-	} else {
-		big_and_buf_rx[nbytes] = '\0';
-
-		msg = strtok_r (big_and_buf_rx, ";", &saveptr);
-		assert (msg);
-		while (msg) {
-			cmd = strdup (msg);
-			ta_handle_command (fd, cmd);
-			free (cmd);
-			msg = strtok_r (NULL, ";", &saveptr);
-		}
-	}
-
-	return 0;
-}
-
-static int server_accept_fn (
-	int fd, int revents, void *data)
-{
-	socklen_t addrlen;
-	struct sockaddr_in in_addr;
-	int new_fd;
-	int res;
-
-	if (revents & POLLHUP || revents & POLLERR) {
-		qb_log (LOG_INFO, "command sockect got POLLHUP exiting...");
-		shut_me_down();
-		return -1;
-	}
-
-	addrlen = sizeof (struct sockaddr_in);
-
-retry_accept:
-	new_fd = accept (fd, (struct sockaddr *)&in_addr, &addrlen);
-	if (new_fd == -1 && errno == EINTR) {
-		goto retry_accept;
-	}
-
-	if (new_fd == -1) {
-		qb_log (LOG_ERR,
-			"Could not accept connection: %s", strerror (errno));
-		return (0); /* This is an error, but -1 would indicate disconnect from poll loop */
-	}
-
-	res = fcntl (new_fd, F_SETFL, O_NONBLOCK);
-	if (res == -1) {
-		qb_log (LOG_ERR,
-			"Could not set non-blocking operation on connection: %s",
-			strerror (errno));
-		close (new_fd);
-		return (0); /* This is an error, but -1 would indicate disconnect from poll loop */
-	}
-
-	client_fds[client_fds_pos] = new_fd;
-	client_fds_pos++;
-	assert(client_fds_pos < MAX_CLIENTS);
-
-	qb_loop_poll_add (poll_handle,
-			QB_LOOP_MED,
-			new_fd,
-			POLLIN|POLLNVAL,
-			NULL,
-			server_process_data_fn);
-	return 0;
-}
-
-
-static int create_server_sockect (int server_port)
-{
-	int listener;
-	int yes = 1;
-	int rv;
-	struct addrinfo hints, *ai, *p;
-	char server_port_str[16];
-	char addr_str[INET_ADDRSTRLEN];
-	void *ptr = NULL;
-
-	/* get a socket and bind it
-	 */
-	sprintf(server_port_str, "%d", server_port);
-	memset (&hints, 0, sizeof hints);
-	hints.ai_family = AF_UNSPEC;
-	hints.ai_socktype = SOCK_STREAM;
-	hints.ai_flags = AI_PASSIVE;
-	if ((rv = getaddrinfo (NULL, server_port_str, &hints, &ai)) != 0) {
-		qb_log (LOG_ERR, "%s", gai_strerror (rv));
-		exit (1);
-	}
-
-	for (p = ai; p != NULL; p = p->ai_next) {
-		listener = socket (p->ai_family, p->ai_socktype, p->ai_protocol);
-		if (listener < 0) {
-			continue;
-		}
-
-		/* lose the pesky "address already in use" error message
-		 */
-		if (setsockopt (listener, SOL_SOCKET, SO_REUSEADDR,
-				&yes, sizeof(int)) < 0) {
-			qb_log (LOG_ERR, "setsockopt() failed: %s", strerror (errno));
-		}
-
-		switch (p->ai_family)
-		{
-		case AF_INET:
-			ptr = &((struct sockaddr_in *) p->ai_addr)->sin_addr;
-			break;
-		case AF_INET6:
-			ptr = &((struct sockaddr_in6 *) p->ai_addr)->sin6_addr;
-			break;
-		default:
-			qb_log (LOG_ERR, "address family wrong");
-			exit (4);
-			
-		}
-
-		if (inet_ntop(p->ai_family, ptr, addr_str, INET_ADDRSTRLEN) == NULL) {
-			qb_log (LOG_ERR, "inet_ntop() failed: %s", strerror (errno));
-		}
-
-		if (bind (listener, p->ai_addr, p->ai_addrlen) < 0) {
-			qb_log (LOG_ERR, "bind(%s) failed: %s", addr_str, strerror (errno));
-			close (listener);
-			continue;
-		}
-
-		break;
-	}
-
-	if (p == NULL) {
-		qb_log (LOG_ERR, "failed to bind");
-		exit (2);
-	}
-
-	freeaddrinfo (ai);
-
-	if (listen (listener, 10) == -1) {
-		qb_log (LOG_ERR, "listen() failed: %s", strerror(errno));
-		exit (3);
-	}
-
-	return listener;
-}
-
-static int32_t sig_exit_handler (int num, void *data)
-{
-	qb_log (LOG_INFO, "got signal %d, exiting", num);
-	shut_me_down();
-	return 0;
-}
-
-int
-test_agent_run(const char * prog_name, int server_port,
-		ta_do_command_fn func, pre_exit_fn exit_fn)
-{
-	int listener;
-	int i;
-
-	qb_log_init(prog_name, LOG_DAEMON, LOG_DEBUG);
-	qb_log_format_set(QB_LOG_SYSLOG, "%n() [%p] %b");
-
-	qb_log (LOG_INFO, "STARTING");
-
-	do_command = func;
-	pre_exit = exit_fn;
-	poll_handle = qb_loop_create ();
-
-	if (exit_fn) {
-		qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
-			SIGINT, NULL, sig_exit_handler, NULL);
-		qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
-			SIGQUIT, NULL, sig_exit_handler, NULL);
-		qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
-			SIGTERM, NULL, sig_exit_handler, NULL);
-	}
-
-	listener = create_server_sockect (server_port);
-	qb_loop_poll_add (poll_handle,
-			  QB_LOOP_MED,
-			  listener,
-			  POLLIN|POLLNVAL,
-			  NULL, server_accept_fn);
-
-	qb_loop_run (poll_handle);
-
-	close(listener);
-
-	for (i = 0; i < client_fds_pos; i++) {
-		close(client_fds[client_fds_pos]);
-	}
-
-	qb_log (LOG_INFO, "EXITING");
-	qb_log_fini();
-	return 0;
-}
-

cts/agents/common_test_agent.h

@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2010 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Angus Salkeld (asalkeld@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef CORO_COMMON_TEST_AGNET_H_DEFINED
-#define CORO_COMMON_TEST_AGNET_H_DEFINED
-
-#include <corosync/hdb.h>
-#include <qb/qbloop.h>
-#include <qb/qblog.h>
-
-#define OK_STR "OK"
-#define FAIL_STR "FAIL"
-#define NOT_SUPPORTED_STR "NOT_SUPPORTED"
-
-#define HOW_BIG_AND_BUF 4096
-
-typedef void (*ta_do_command_fn) (int sock, char* func, char*args[], int num_args);
-typedef void (*pre_exit_fn) (void);
-
-int test_agent_run(const char * prog_name, int server_port,
-		   ta_do_command_fn func, pre_exit_fn exit_fn);
-
-qb_loop_t *ta_poll_handle_get(void);
-
-#endif /* CORO_COMMON_TEST_AGNET_H_DEFINED */
-

cts/agents/cpg_test_agent.c

@@ -1,804 +0,0 @@
-/*
- * Copyright (c) 2010 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Angus Salkeld (asalkeld@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/uio.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <poll.h>
-#include <fcntl.h>
-
-#include <qb/qblist.h>
-#include <qb/qbdefs.h>
-#include <qb/qbutil.h>
-#include <qb/qbloop.h>
-#include <qb/qblog.h>
-#include <corosync/cpg.h>
-#include <corosync/cfg.h>
-#include "common_test_agent.h"
-
-#include <nss.h>
-#include <pk11pub.h>
-
-typedef enum {
-	MSG_OK,
-	MSG_NODEID_ERR,
-	MSG_PID_ERR,
-	MSG_SEQ_ERR,
-	MSG_SIZE_ERR,
-	MSG_SHA1_ERR,
-} msg_status_t;
-
-typedef struct {
-	uint32_t nodeid;
-	pid_t   pid;
-	unsigned char sha1[20];
-	uint32_t seq;
-	size_t size;
-	unsigned char payload[0];
-} msg_t;
-
-#define LOG_STR_SIZE 80
-typedef struct {
-	char log[LOG_STR_SIZE];
-	struct qb_list_head list;
-} log_entry_t;
-
-static char big_and_buf[HOW_BIG_AND_BUF];
-static int32_t record_config_events_g = 0;
-static int32_t record_messages_g = 0;
-static cpg_handle_t cpg_handle = 0;
-static corosync_cfg_handle_t cfg_handle = 0;
-static int32_t cpg_fd = -1;
-static int32_t cfg_fd = -1;
-static struct qb_list_head config_chg_log_head;
-static struct qb_list_head msg_log_head;
-static pid_t my_pid;
-static uint32_t my_nodeid;
-static int32_t my_seq;
-static int32_t use_zcb = QB_FALSE;
-static int32_t my_msgs_to_send;
-static int32_t my_msgs_sent;
-static int32_t total_stored_msgs = 0;
-static int32_t total_msgs_revd = 0;
-static int32_t in_cnchg = 0;
-static int32_t pcmk_test = 0;
-PK11Context* sha1_context;
-
-static void send_some_more_messages (void * unused);
-
-static char*
-err_status_string (char * buf, size_t buf_len, msg_status_t status)
-{
-	switch (status) {
-	case MSG_OK:
-		strncpy (buf, "OK", buf_len);
-		break;
-	case MSG_NODEID_ERR:
-		strncpy (buf, "NODEID_ERR", buf_len);
-		break;
-	case MSG_PID_ERR:
-		strncpy (buf, "PID_ERR", buf_len);
-		break;
-	case MSG_SEQ_ERR:
-		strncpy (buf, "SEQ_ERR", buf_len);
-		break;
-	case MSG_SIZE_ERR:
-		strncpy (buf, "SIZE_ERR", buf_len);
-		break;
-	case MSG_SHA1_ERR:
-		strncpy (buf, "SHA1_ERR", buf_len);
-		break;
-	default:
-		strncpy (buf, "UNKNOWN_ERR", buf_len);
-		break;
-	}
-	if (buf_len > 0) {
-		buf[buf_len - 1] = '\0';
-	}
-	return buf;
-}
-
-static void delivery_callback (
-	cpg_handle_t handle,
-	const struct cpg_name *groupName,
-	uint32_t nodeid,
-	uint32_t pid,
-	void *msg,
-	size_t msg_len)
-{
-	log_entry_t *log_pt;
-	msg_t *msg_pt = (msg_t*)msg;
-	msg_status_t status = MSG_OK;
-	char status_buf[20];
-	unsigned char sha1_compare[20];
-	unsigned int sha1_len;
-
-	if (record_messages_g == 0) {
-		return;
-	}
-
-	if (nodeid != msg_pt->nodeid) {
-		status = MSG_NODEID_ERR;
-	}
-	if (pid != msg_pt->pid) {
-		status = MSG_PID_ERR;
-	}
-	if (msg_len != msg_pt->size) {
-		status = MSG_SIZE_ERR;
-	}
-	PK11_DigestBegin(sha1_context);
-	PK11_DigestOp(sha1_context, msg_pt->payload, (msg_pt->size - sizeof (msg_t)));
-        PK11_DigestFinal(sha1_context, sha1_compare, &sha1_len, sizeof(sha1_compare));
-	if (memcmp (sha1_compare, msg_pt->sha1, 20) != 0) {
-		qb_log (LOG_ERR, "msg seq:%d; incorrect hash",
-			msg_pt->seq);
-		status = MSG_SHA1_ERR;
-	}
-
-	log_pt = malloc (sizeof(log_entry_t));
-	qb_list_init (&log_pt->list);
-
-	snprintf (log_pt->log, LOG_STR_SIZE, "%u:%d:%d:%s;",
-		msg_pt->nodeid, msg_pt->seq, my_seq,
-		err_status_string (status_buf, 20, status));
-	qb_list_add_tail (&log_pt->list, &msg_log_head);
-	total_stored_msgs++;
-	total_msgs_revd++;
-	my_seq++;
-
-	if ((total_msgs_revd % 1000) == 0) {
-		qb_log (LOG_INFO, "rx %d", total_msgs_revd);
-	}
-}
-
-static void config_change_callback (
-	cpg_handle_t handle,
-	const struct cpg_name *groupName,
-	const struct cpg_address *member_list, size_t member_list_entries,
-	const struct cpg_address *left_list, size_t left_list_entries,
-	const struct cpg_address *joined_list, size_t joined_list_entries)
-{
-	int i;
-	log_entry_t *log_pt;
-
-	/* group_name,ip,pid,join|leave */
-	if (record_config_events_g > 0) {
-		qb_log (LOG_INFO, "got cpg event[recording] for group %s", groupName->value);
-	} else {
-		qb_log (LOG_INFO, "got cpg event[ignoring] for group %s", groupName->value);
-	}
-
-	for (i = 0; i < left_list_entries; i++) {
-		if (record_config_events_g > 0) {
-			log_pt = malloc (sizeof(log_entry_t));
-			qb_list_init (&log_pt->list);
-			assert(snprintf (log_pt->log, LOG_STR_SIZE, "%s,%u,%u,left",
-			    groupName->value, left_list[i].nodeid,left_list[i].pid) < LOG_STR_SIZE);
-			qb_list_add_tail(&log_pt->list, &config_chg_log_head);
-			qb_log (LOG_INFO, "cpg event %s", log_pt->log);
-		}
-	}
-	for (i = 0; i < joined_list_entries; i++) {
-		if (record_config_events_g > 0) {
-			log_pt = malloc (sizeof(log_entry_t));
-			qb_list_init (&log_pt->list);
-			assert(snprintf (log_pt->log, LOG_STR_SIZE, "%s,%u,%u,join",
-			    groupName->value, joined_list[i].nodeid,joined_list[i].pid) < LOG_STR_SIZE);
-			qb_list_add_tail (&log_pt->list, &config_chg_log_head);
-			qb_log (LOG_INFO, "cpg event %s", log_pt->log);
-		}
-	}
-	if (pcmk_test == 1) {
-		in_cnchg = 1;
-		send_some_more_messages (NULL);
-		in_cnchg = 0;
-	}
-}
-
-static void my_shutdown_callback (corosync_cfg_handle_t handle,
-	corosync_cfg_shutdown_flags_t flags)
-{
-	qb_log (LOG_CRIT, "flags:%d", flags);
-	if (flags == COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) {
-		corosync_cfg_replyto_shutdown (cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_YES);
-	}
-}
-
-
-static corosync_cfg_callbacks_t cfg_callbacks = {
-	.corosync_cfg_shutdown_callback = my_shutdown_callback,
-};
-static cpg_callbacks_t callbacks = {
-	.cpg_deliver_fn = delivery_callback,
-	.cpg_confchg_fn = config_change_callback,
-};
-
-static void record_messages (void)
-{
-	record_messages_g = 1;
-	qb_log (LOG_INFO, "record:%d", record_messages_g);
-}
-
-static void record_config_events (int sock)
-{
-	char response[100];
-	ssize_t rc;
-	size_t send_len;
-
-	record_config_events_g = 1;
-	qb_log (LOG_INFO, "record:%d", record_config_events_g);
-
-	snprintf (response, 100, "%s", OK_STR);
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-static void read_config_event (int sock)
-{
-	const char *empty = "None";
-	log_entry_t *entry;
-	ssize_t rc;
-	size_t send_len;
-
-	if (qb_list_empty(&config_chg_log_head) == 0) {
-		entry = qb_list_first_entry (&config_chg_log_head, log_entry_t, list);
-		send_len = strlen (entry->log);
-		rc = send (sock, entry->log, send_len, 0);
-		qb_list_del (&entry->list);
-		free (entry);
-	} else {
-		qb_log (LOG_DEBUG, "no events in list");
-		send_len = strlen (empty);
-		rc = send (sock, empty, send_len, 0);
-	}
-	assert(rc == send_len);
-}
-
-static void read_messages (int sock, char* atmost_str)
-{
-	struct qb_list_head *iter, *tmp_iter;
-
-	log_entry_t *entry;
-	int atmost = atoi (atmost_str);
-	int packed = 0;
-	ssize_t rc;
-
-	if (atmost == 0)
-		atmost = 1;
-	if (atmost > (HOW_BIG_AND_BUF / LOG_STR_SIZE))
-		atmost = (HOW_BIG_AND_BUF / LOG_STR_SIZE);
-
-	big_and_buf[0] = '\0';
-
-	qb_list_for_each_safe(iter, tmp_iter, &msg_log_head) {
-
-		if (packed >= atmost)
-			break;
-
-		entry = qb_list_entry (iter, log_entry_t, list);
-
-		strcat (big_and_buf, entry->log);
-		packed++;
-
-		qb_list_del (&entry->list);
-		free (entry);
-
-		total_stored_msgs--;
-	}
-	if (packed == 0) {
-		strcpy (big_and_buf, "None");
-	} else {
-		if ((total_stored_msgs % 1000) == 0) {
-			qb_log(LOG_INFO, "sending %d; total_stored_msgs:%d; len:%d",
-				packed, total_stored_msgs, (int)strlen (big_and_buf));
-		}
-	}
-	rc = send (sock, big_and_buf, strlen (big_and_buf), 0);
-	assert(rc == strlen (big_and_buf));
-}
-
-static qb_loop_timer_handle more_messages_timer_handle;
-static void send_some_more_messages_later (void)
-{
-	cpg_dispatch (cpg_handle, CS_DISPATCH_ALL);
-	qb_loop_timer_add (
-		ta_poll_handle_get(),
-		QB_LOOP_MED,
-		300*QB_TIME_NS_IN_MSEC, NULL,
-		send_some_more_messages,
-		&more_messages_timer_handle);
-}
-
-static void send_some_more_messages_zcb (void)
-{
-	msg_t *my_msg;
-	int i;
-	int send_now;
-	size_t payload_size;
-	size_t total_size;
-	unsigned int sha1_len;
-	cs_error_t res;
-	cpg_flow_control_state_t fc_state;
-	void *zcb_buffer;
-
-	if (cpg_fd < 0)
-		return;
-
-	send_now = my_msgs_to_send;
-	payload_size = (rand() % 100000);
-	total_size = payload_size + sizeof (msg_t);
-	cpg_zcb_alloc (cpg_handle, total_size, &zcb_buffer);
-
-	my_msg = (msg_t*)zcb_buffer;
-
-	qb_log(LOG_DEBUG, "send_now:%d", send_now);
-	my_msg->pid = my_pid;
-	my_msg->nodeid = my_nodeid;
-	my_msg->size = sizeof (msg_t) + payload_size;
-	my_msg->seq = my_msgs_sent;
-	for (i = 0; i < payload_size; i++) {
-		my_msg->payload[i] = i;
-	}
-	PK11_DigestBegin(sha1_context);
-	PK11_DigestOp(sha1_context,  my_msg->payload, payload_size);
-	PK11_DigestFinal(sha1_context, my_msg->sha1, &sha1_len, sizeof(my_msg->sha1));
-
-	for (i = 0; i < send_now; i++) {
-
-		res = cpg_flow_control_state_get (cpg_handle, &fc_state);
-		if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
-			/* lets do this later */
-			send_some_more_messages_later ();
-			qb_log (LOG_INFO, "flow control enabled.");
-			goto free_buffer;
-		}
-
-		res = cpg_zcb_mcast_joined (cpg_handle, CPG_TYPE_AGREED, zcb_buffer, total_size);
-		if (res == CS_ERR_TRY_AGAIN) {
-			/* lets do this later */
-			send_some_more_messages_later ();
-			goto free_buffer;
-		} else if (res != CS_OK) {
-			qb_log (LOG_ERR, "cpg_mcast_joined error:%d, exiting.",
-				res);
-			exit (-2);
-		}
-
-		my_msgs_sent++;
-		my_msgs_to_send--;
-	}
-free_buffer:
-	cpg_zcb_free (cpg_handle, zcb_buffer);
-}
-
-#define cs_repeat(counter, max, code) do {		\
-	code;						\
-	if (res == CS_ERR_TRY_AGAIN) {			\
-		counter++;				\
-		sleep(counter);				\
-	}						\
-} while (res == CS_ERR_TRY_AGAIN && counter < max)
-
-static unsigned char buffer[200000];
-static void send_some_more_messages_normal (void)
-{
-	msg_t my_msg;
-	struct iovec iov[2];
-	int i;
-	int send_now;
-	size_t payload_size;
-	cs_error_t res;
-	cpg_flow_control_state_t fc_state;
-	int retries = 0;
-	time_t before;
-	unsigned int sha1_len;
-
-	if (cpg_fd < 0)
-		return;
-
-	send_now = my_msgs_to_send;
-
-	qb_log (LOG_TRACE, "send_now:%d", send_now);
-
-	my_msg.pid = my_pid;
-	my_msg.nodeid = my_nodeid;
-	payload_size = (rand() % 10000);
-	my_msg.size = sizeof (msg_t) + payload_size;
-	my_msg.seq = my_msgs_sent;
-	for (i = 0; i < payload_size; i++) {
-		buffer[i] = i;
-	}
-	PK11_DigestBegin(sha1_context);
-	PK11_DigestOp(sha1_context,  buffer, payload_size);
-	PK11_DigestFinal(sha1_context, my_msg.sha1, &sha1_len, sizeof(my_msg.sha1));
-
-	iov[0].iov_len = sizeof (msg_t);
-	iov[0].iov_base = &my_msg;
-	iov[1].iov_len = payload_size;
-	iov[1].iov_base = buffer;
-
-	for (i = 0; i < send_now; i++) {
-		if (in_cnchg && pcmk_test) {
-			retries = 0;
-			before = time(NULL);
-			cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
-			if (retries > 20) {
-				qb_log (LOG_ERR, "cs_repeat: blocked for :%lu secs.",
-					(unsigned long)(time(NULL) - before));
-			}
-			if (res != CS_OK) {
-				qb_log (LOG_ERR, "cpg_mcast_joined error:%d.",
-					res);
-				return;
-			}
-		} else {
-			res = cpg_flow_control_state_get (cpg_handle, &fc_state);
-			if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
-				/* lets do this later */
-				send_some_more_messages_later ();
-				qb_log (LOG_INFO, "flow control enabled.");
-				return;
-			}
-
-			res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
-			if (res == CS_ERR_TRY_AGAIN) {
-				/* lets do this later */
-				send_some_more_messages_later ();
-				if (i > 0) {
-					qb_log (LOG_INFO, "TRY_AGAIN %d to send.",
-						my_msgs_to_send);
-				}
-				return;
-			} else if (res != CS_OK) {
-				qb_log (LOG_ERR, "cpg_mcast_joined error:%d, exiting.",
-					res);
-				exit (-2);
-			}
-		}
-		my_msgs_sent++;
-		my_msg.seq = my_msgs_sent;
-		my_msgs_to_send--;
-	}
-	qb_log (LOG_TRACE, "sent %d; to send %d.",
-		my_msgs_sent, my_msgs_to_send);
-}
-
-static void send_some_more_messages (void * unused)
-{
-	if (use_zcb) {
-		send_some_more_messages_zcb ();
-	} else {
-		send_some_more_messages_normal ();
-	}
-}
-
-static void msg_blaster (int sock, char* num_to_send_str)
-{
-	my_msgs_to_send = atoi (num_to_send_str);
-	my_msgs_sent = 0;
-	my_seq = 1;
-	my_pid = getpid();
-
-	use_zcb = QB_FALSE;
-	total_stored_msgs = 0;
-
-	cpg_local_get (cpg_handle, &my_nodeid);
-
-	/* control the limits */
-	if (my_msgs_to_send <= 0)
-		my_msgs_to_send = 1;
-	if (my_msgs_to_send > 10000)
-		my_msgs_to_send = 10000;
-
-	send_some_more_messages_normal ();
-}
-
-
-static void context_test (int sock)
-{
-	char response[100];
-	char *cmp;
-	ssize_t rc;
-	size_t send_len;
-
-	cpg_context_set (cpg_handle, response);
-	cpg_context_get (cpg_handle, (void**)&cmp);
-	if (response != cmp) {
-		snprintf (response, 100, "%s", FAIL_STR);
-	}
-	else {
-		snprintf (response, 100, "%s", OK_STR);
-	}
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-static void msg_blaster_zcb (int sock, char* num_to_send_str)
-{
-	my_msgs_to_send = atoi (num_to_send_str);
-	my_seq = 1;
-	my_pid = getpid();
-
-	use_zcb = QB_TRUE;
-	total_stored_msgs = 0;
-
-	cpg_local_get (cpg_handle, &my_nodeid);
-
-	/* control the limits */
-	if (my_msgs_to_send <= 0)
-		my_msgs_to_send = 1;
-	if (my_msgs_to_send > 10000)
-		my_msgs_to_send = 10000;
-
-	send_some_more_messages_zcb ();
-}
-
-static int cfg_dispatch_wrapper_fn (
-	int fd,
-	int revents,
-	void *data)
-{
-	cs_error_t error;
-
-	if (revents & POLLHUP || revents & POLLERR) {
-		qb_log (LOG_ERR, "got POLLHUP disconnecting from CFG");
-		corosync_cfg_finalize(cfg_handle);
-		cfg_handle = 0;
-		return -1;
-	}
-
-	error = corosync_cfg_dispatch (cfg_handle, CS_DISPATCH_ALL);
-	if (error == CS_ERR_LIBRARY) {
-		qb_log (LOG_ERR, "got LIB error disconnecting from CFG.");
-		corosync_cfg_finalize(cfg_handle);
-		cfg_handle = 0;
-		return -1;
-	}
-	return 0;
-}
-
-static int cpg_dispatch_wrapper_fn (
-	int fd,
-	int revents,
-	void *data)
-{
-	cs_error_t error;
-
-	if (revents & POLLHUP || revents & POLLERR) {
-		qb_log (LOG_ERR, "got POLLHUP disconnecting from CPG");
-		cpg_finalize(cpg_handle);
-		cpg_handle = 0;
-		return -1;
-	}
-
-	error = cpg_dispatch (cpg_handle, CS_DISPATCH_ALL);
-	if (error == CS_ERR_LIBRARY) {
-		qb_log (LOG_ERR, "got LIB error disconnecting from CPG");
-		cpg_finalize(cpg_handle);
-		cpg_handle = 0;
-		return -1;
-	}
-	return 0;
-}
-
-static void do_command (int sock, char* func, char*args[], int num_args)
-{
-	int result;
-	char response[100];
-	struct cpg_name group_name;
-	ssize_t rc;
-	size_t send_len;
-
-	qb_log (LOG_TRACE, "RPC:%s() called.", func);
-
-	if (strcmp ("cpg_mcast_joined",func) == 0) {
-		struct iovec iov[5];
-		int a;
-
-		for (a = 0; a < num_args; a++) {
-			iov[a].iov_base = args[a];
-			iov[a].iov_len = strlen(args[a])+1;
-		}
-		cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, num_args);
-
-	} else if (strcmp ("cpg_join",func) == 0) {
-		if (strlen(args[0]) >= CPG_MAX_NAME_LENGTH) {
-			qb_log (LOG_ERR, "Invalid group name");
-			exit (1);
-		}
-		strcpy (group_name.value, args[0]);
-		group_name.length = strlen(args[0]);
-		result = cpg_join (cpg_handle, &group_name);
-		if (result != CS_OK) {
-			qb_log (LOG_ERR,
-				"Could not join process group, error %d", result);
-			exit (1);
-		}
-		qb_log (LOG_INFO, "called cpg_join(%s)!", group_name.value);
-
-	} else if (strcmp ("cpg_leave",func) == 0) {
-
-		strcpy (group_name.value, args[0]);
-		group_name.length = strlen(args[0]);
-
-		result = cpg_leave (cpg_handle, &group_name);
-		if (result != CS_OK) {
-			qb_log (LOG_ERR,
-				"Could not leave process group, error %d", result);
-			exit (1);
-		}
-		qb_log (LOG_INFO, "called cpg_leave(%s)!", group_name.value);
-
-	} else if (strcmp ("cpg_initialize",func) == 0) {
-		int retry_count = 0;
-
-		result = cpg_initialize (&cpg_handle, &callbacks);
-		while (result != CS_OK) {
-			qb_log (LOG_ERR,
-				"cpg_initialize error %d (attempt %d)",
-				result, retry_count);
-			if (retry_count >= 3) {
-				exit (1);
-			}
-			sleep(1);
-			retry_count++;
-			result = cpg_initialize (&cpg_handle, &callbacks);
-		}
-
-		cpg_fd_get (cpg_handle, &cpg_fd);
-		qb_loop_poll_add (ta_poll_handle_get(),
-			QB_LOOP_MED,
-			cpg_fd,
-			POLLIN|POLLNVAL,
-			NULL,
-			cpg_dispatch_wrapper_fn);
-
-	} else if (strcmp ("cpg_local_get", func) == 0) {
-		unsigned int local_nodeid;
-
-		cpg_local_get (cpg_handle, &local_nodeid);
-		snprintf (response, 100, "%u",local_nodeid);
-		send_len = strlen (response);
-		rc = send (sock, response, send_len, 0);
-		assert(rc == send_len);
-	} else if (strcmp ("cpg_finalize", func) == 0) {
-
-		if (cpg_handle > 0) {
-			cpg_finalize (cpg_handle);
-			cpg_handle = 0;
-		}
-
-	} else if (strcmp ("record_config_events", func) == 0) {
-		record_config_events (sock);
-	} else if (strcmp ("record_messages", func) == 0) {
-		record_messages ();
-	} else if (strcmp ("read_config_event", func) == 0) {
-		read_config_event (sock);
-	} else if (strcmp ("read_messages", func) == 0) {
-		read_messages (sock, args[0]);
-	} else if (strcmp ("msg_blaster_zcb", func) == 0) {
-		msg_blaster_zcb (sock, args[0]);
-	} else if (strcmp ("pcmk_test", func) == 0) {
-		pcmk_test = 1;
-	} else if (strcmp ("msg_blaster", func) == 0) {
-		msg_blaster (sock, args[0]);
-	} else if (strcmp ("context_test", func) == 0) {
-		context_test (sock);
-	} else if (strcmp ("are_you_ok_dude", func) == 0) {
-		snprintf (response, 100, "%s", OK_STR);
-		send_len = strlen (response);
-		rc = send (sock, response, strlen (response), 0);
-		assert(rc == send_len);
-	} else if (strcmp ("cfg_shutdown", func) == 0) {
-
-		qb_log (LOG_INFO, "calling %s() called!", func);
-		result = corosync_cfg_try_shutdown (cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST);
-		qb_log (LOG_INFO,"%s() returned %d!", func, result);
-
-	} else if (strcmp ("cfg_initialize",func) == 0) {
-		int retry_count = 0;
-
-		qb_log (LOG_INFO,"%s() called!", func);
-		result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
-		while (result != CS_OK) {
-			qb_log (LOG_ERR,
-				"cfg_initialize error %d (attempt %d)",
-				result, retry_count);
-			if (retry_count >= 3) {
-				exit (1);
-			}
-			sleep(1);
-			retry_count++;
-			result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
-		}
-		qb_log (LOG_INFO,"corosync_cfg_initialize() == %d", result);
-
-		result = corosync_cfg_fd_get (cfg_handle, &cfg_fd);
-		qb_log (LOG_INFO,"corosync_cfg_fd_get() == %d", result);
-
-		qb_loop_poll_add (ta_poll_handle_get(),
-			QB_LOOP_MED,
-			cfg_fd,
-			POLLIN|POLLNVAL,
-			NULL,
-			cfg_dispatch_wrapper_fn);
-	} else {
-		qb_log(LOG_ERR, "RPC:%s not supported!", func);
-	}
-}
-
-static void my_pre_exit(void)
-{
-	qb_log (LOG_INFO, "%s PRE EXIT", __FILE__);
-	if (cpg_handle > 0) {
-		cpg_finalize (cpg_handle);
-		cpg_handle = 0;
-	}
-	if (cfg_handle > 0) {
-		corosync_cfg_finalize (cfg_handle);
-		cfg_handle = 0;
-	}
-
-	PK11_DestroyContext(sha1_context, PR_TRUE);
-}
-
-int
-main(int argc, char *argv[])
-{
-	qb_list_init (&msg_log_head);
-	qb_list_init (&config_chg_log_head);
-
-	if (NSS_NoDB_Init(NULL) != SECSuccess) {
-		qb_log(LOG_ERR, "Couldn't initialize nss");
-		exit (0);
-	}
-
-	if ((sha1_context = PK11_CreateDigestContext(SEC_OID_SHA1)) == NULL) {
-		qb_log(LOG_ERR, "Couldn't initialize nss");
-		exit (0);
-	}
-
-	return test_agent_run ("cpg_test_agent", 9034, do_command, my_pre_exit);
-}
-

cts/agents/mem_leak_test.sh

@@ -1,114 +0,0 @@
-#!/bin/sh
-
-_usage_()
-{
-  echo bla bla
-
-  exit 0
-}
-
-get_mem()
-{
-  if [ -z "$1" ]
-  then
-    type=Data
-  else
-    type=$1
-  fi
-  MEM=$(cat /proc/$(pidof corosync)/status | grep Vm$type | sed "s/Vm$type:\(.*\) kB/\1/")
-  echo $MEM
-}
-
-#
-# create and destroy a lot of objects
-#
-_object_test_()
-{
-  TYPE=RSS
-  temp_file=/tmp/object.txt
-  COUNT=1
-
-  corosync-cmapctl -s usr.angus u32 456
-  corosync-cmapctl -s usr.angus u32 4123
-  corosync-cmapctl -d usr.angus
-
-  BEFORE=$(get_mem $TYPE)
-  # this loop is just to ignore the first iteration
-  for f in /usr/share/man /usr/lib /usr/bin /usr/local ;
-  do
-    rm -f $temp_file
-
-    find $f | sed "s|\.|_|g" | sed "s|/|.|g" | while read l
-    do 
-      echo $l.count u64 $COUNT >> $temp_file
-      let COUNT="$COUNT+1"
-    done
-
-    corosync-cmapctl -p $temp_file
-    corosync-cmapctl -D usr
-  done
-  AFTER=$(get_mem $TYPE)
-  let DIFF="$AFTER - $BEFORE"
-  rm -f $temp_file
-  #echo $f diff $TYPE $DIFF
-  echo $DIFF
-
-  exit 0
-}
-
-#
-# run the corosync tools to cause IPC sessions to created/destroyed
-#
-_session_test_()
-{
-  echo _session_test_
-  COUNT=1
-
-  corosync-cmap -h >/dev/null
-  corosync-cfgtool -h >/dev/null
-  corosync-quorumtool -h >/dev/null
-
-  BEFORE=$(get_mem $TYPE)
-  corosync-cfgtool -a >/dev/null
-  corosync-quorumtool -s >/dev/null
-  corosync-quorumtool -l >/dev/null
-
-  find /usr/bin | sed "s|\.|_|g" | sed "s|/|.|g" | while read l
-  do 
-    corosync-cmapctl -s $l u32 $COUNT
-    let COUNT="$COUNT+1"
-  done
-  corosync-cmapctl -D usr
-  AFTER=$(get_mem $TYPE)
-  let DIFF="$AFTER - $BEFORE"
-  echo $DIFF
-
-  exit 0
-}
-
-# Note that we use `"$@"' to let each command-line parameter expand to a 
-# separate word. The quotes around `$@' are essential!
-# We need TEMP as the `eval set --' would nuke the return value of getopt.
-TEMP=`getopt -o u12 --long help,object,session \
-     -n '$0' -- "$@"`
-
-if [ $? != 0 ] ; then echo "Incorrect arguments..." >&2 ; _usage_ ; exit 1 ; fi
-
-# Note the quotes around `$TEMP': they are essential!
-eval set -- "$TEMP"
-
-while true ; do
-        case "$1" in
-                -u|--help) _usage_ ;;
-                -1|--object) _object_test_ ;;
-                -2|--session) _session_test_ ;;
-                --) shift ; break ;;
-                *) echo "Internal error!" ; exit 1 ;;
-        esac
-done
-echo "Remaining arguments:"
-for arg do echo '--> '"\`$arg'" ; done
-
-
-
-

cts/agents/net_breaker.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-set -e
-
-if [ $1 = "BreakCommCmd" ]
-then
-  iptables -A INPUT -s $2 -j DROP >/dev/null 2>&1
-  iptables -A OUTPUT -s $2 -j DROP >/dev/null 2>&1
-  iptables -A INPUT -m pkttype --pkt-type multicast -j DROP
-fi
-if [ $1 = "FixCommCmd" ]
-then
-  iptables -F >/dev/null 2>&1
-fi
-
-exit 0
-

cts/agents/sam_test_agent.c

@@ -1,1251 +0,0 @@
-/*
- * Copyright (c) 2009-2011 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Jan Friesse (jfriesse@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the Red Hat, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Provides test of SAM API
- */
-
-#include <config.h>
-
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <pthread.h>
-#include <limits.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <sys/un.h>
-#include <corosync/corotypes.h>
-#include <corosync/sam.h>
-#include <signal.h>
-#include <string.h>
-#include <sys/wait.h>
-#include <corosync/cmap.h>
-
-#include "common_test_agent.h"
-
-extern const char *__progname;
-
-static int test2_sig_delivered = 0;
-static int test5_hc_cb_count = 0;
-static int test6_sig_delivered = 0;
-
-/*
- * First test will just register SAM, with policy restart. First instance will
- * sleep one second, send hc and sleep another 3 seconds. This should force restart.
- * Second instance will sleep one second, send hc, stop hc and sleep 3 seconds.
- * Then start hc again and sleep 3 seconds. This should force restart again.
- * Last instance just calls initialize again. This should end with error.
- * Then call start, followed by stop and start again. Finally, we will call finalize
- * twice. One should succeed, second should fail. After this, we will call every function
- * (none should succeed).
- */
-static int test1 (void)
-{
-	cs_error_t error;
-	unsigned int instance_id;
-	int i;
-
-	qb_enter();
-	error = sam_initialize (2000, SAM_RECOVERY_POLICY_RESTART);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", error);
-		return 1;
-	}
-	qb_log (LOG_INFO, "register");
-	error = sam_register (&instance_id);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", error);
-		return 1;
-	}
-
-	if (instance_id == 1 || instance_id == 2) {
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		for (i = 0; i < 10; i++) {
-			qb_log (LOG_INFO, "iid %d: sleep 1", instance_id);
-			sleep (1);
-
-			qb_log (LOG_INFO, "iid %d: hc send", instance_id);
-			error = sam_hc_send ();
-			if (error != CS_OK) {
-				qb_log (LOG_ERR, "Can't send hc. Error %d", error);
-				return 1;
-			}
-		}
-
-		if (instance_id == 2) {
-			qb_log (LOG_INFO, "iid %d: stop", instance_id);
-			error = sam_stop ();
-
-			if (error != CS_OK) {
-				qb_log (LOG_ERR, "Can't send hc. Error %d", error);
-				return 1;
-			}
-		}
-
-		qb_log (LOG_INFO, "iid %d: sleep 3", instance_id);
-		sleep (3);
-
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sleep 3", instance_id);
-		sleep (3);
-		return 0;
-	}
-
-	if (instance_id == 3) {
-		error = sam_initialize (2000, SAM_RECOVERY_POLICY_RESTART);
-		if (error == CS_OK) {
-			qb_log (LOG_ERR, "Can initialize SAM API after initialization");
-			return 1;
-		}
-
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-		error = sam_stop ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't stop hc. Error %d", error);
-			return 1;
-		}
-		error = sam_finalize ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't finalize sam. Error %d", error);
-			return 1;
-		}
-		error = sam_finalize ();
-		if (error == CS_OK) {
-			qb_log (LOG_ERR, "Can finalize sam after finalization!");
-			return 1;
-		}
-
-		if (sam_initialize (2, SAM_RECOVERY_POLICY_RESTART) == CS_OK ||
-			sam_start () == CS_OK || sam_stop () == CS_OK ||
-			sam_register (NULL) == CS_OK || sam_hc_send () == CS_OK ||
-			sam_hc_callback_register (NULL) == CS_OK) {
-
-			qb_log (LOG_ERR, "Can call one of function after finalization!");
-
-			return 1;
-		}
-
-		return 0;
-	}
-
-	return 1;
-}
-
-
-static void test2_signal (int sig)
-{
-	qb_enter();
-
-	test2_sig_delivered = 1;
-}
-
-/*
- * This tests recovery policy quit and callback.
- */
-static int test2 (void) {
-	cs_error_t error;
-	unsigned int instance_id;
-
-	qb_enter();
-	error = sam_initialize (2000, SAM_RECOVERY_POLICY_QUIT);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", error);
-		return 1;
-	}
-	qb_log (LOG_INFO, "register");
-	error = sam_register (&instance_id);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", error);
-		return 1;
-	}
-
-	if (instance_id == 1) {
-		signal (SIGTERM, test2_signal);
-
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sleep 1", instance_id);
-		sleep (1);
-
-		qb_log (LOG_INFO, "iid %d: hc send", instance_id);
-		error = sam_hc_send ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't send hc. Error %d", error);
-			return 1;
-		}
-
-
-		qb_log (LOG_INFO, "iid %d: wait for delivery of signal", instance_id);
-		while (!test2_sig_delivered) {
-			sleep (1);
-		}
-
-		qb_log (LOG_INFO, "iid %d: wait for real kill", instance_id);
-
-		sleep (3);
-	}
-
-	return 1;
-
-}
-
-/*
- * Smoke test. Better to turn off coredump ;) This has no time limit, just restart process
- * when it dies.
- */
-static int test3 (void) {
-	cs_error_t error;
-	unsigned int instance_id;
-
-	qb_log (LOG_INFO, "initialize");
-	error = sam_initialize (0, SAM_RECOVERY_POLICY_RESTART);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", error);
-		return 1;
-	}
-	qb_log (LOG_INFO, "register");
-	error = sam_register (&instance_id);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", error);
-		return 1;
-	}
-
-	if (instance_id < 100) {
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: Sending signal", instance_id);
-		kill(getpid(), SIGSEGV);
-		return 1;
-	}
-
-	return 0;
-
-}
-
-/*
- * Test sam_data_store, sam_data_restore and sam_data_getsize
- */
-static int test4 (void)
-{
-	size_t size;
-	cs_error_t err;
-	int i;
-	unsigned int instance_id;
-	char saved_data[128];
-	char saved_data2[128];
-
-	qb_log (LOG_INFO, "sam_data_getsize 1");
-	err = sam_data_getsize (&size);
-	if (err != CS_ERR_BAD_HANDLE) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_BAD_HANDLE. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, "sam_data_getsize 2");
-	err = sam_data_getsize (NULL);
-	if (err != CS_ERR_INVALID_PARAM) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_INVALID_PARAM. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, "sam_data_store 1");
-	err = sam_data_store (NULL, 0);
-	if (err != CS_ERR_BAD_HANDLE) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_BAD_HANDLE. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, "sam_data_restore 1");
-	err = sam_data_restore (saved_data, sizeof (saved_data));
-	if (err != CS_ERR_BAD_HANDLE) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_BAD_HANDLE. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, "sam_initialize");
-	err = sam_initialize (0, SAM_RECOVERY_POLICY_RESTART);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, "sam_data_getsize 3");
-	err = sam_data_getsize (&size);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_BAD_HANDLE. Error returned %d", err);
-		return 1;
-	}
-	if (size != 0) {
-		qb_log (LOG_ERR, "Test should return size of 0. Returned %zx", size);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, "sam_data_restore 2");
-	err = sam_data_restore (NULL, sizeof (saved_data));
-	if (err != CS_ERR_INVALID_PARAM) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_INVALID_PARAM. Error returned %d", err);
-		return 1;
-	}
-
-	/*
-	 * Store some real data
-	 */
-	for (i = 0; i < sizeof (saved_data); i++) {
-		saved_data[i] = (char)(i + 5);
-	}
-
-	qb_log (LOG_INFO, "sam_data_store 2");
-	err = sam_data_store (saved_data, sizeof (saved_data));
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, " sam_data_getsize 4");
-	err = sam_data_getsize (&size);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-		return 1;
-	}
-	if (size != sizeof (saved_data)) {
-		qb_log (LOG_ERR, "Test should return size of 0. Returned %zx", size);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, " sam_data_restore 3");
-	err = sam_data_restore (saved_data2, sizeof (saved_data2) - 1);
-	if (err != CS_ERR_INVALID_PARAM) {
-		qb_log (LOG_ERR, "Test should return CS_ERR_INVALID_PARAM. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, " sam_data_restore 4");
-	err = sam_data_restore (saved_data2, sizeof (saved_data2));
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-		return 1;
-	}
-
-	if (memcmp (saved_data, saved_data2, sizeof (saved_data2)) != 0) {
-		qb_log (LOG_ERR, "Retored data are not same");
-		return 1;
-	}
-
-	memset (saved_data2, 0, sizeof (saved_data2));
-
-	qb_log (LOG_INFO, " sam_data_store 3");
-	err = sam_data_store (NULL, 1);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, " sam_data_getsize 5");
-	err = sam_data_getsize (&size);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-		return 1;
-	}
-	if (size != 0) {
-		qb_log (LOG_ERR, "Test should return size of 0. Returned %zx", size);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, " sam_data_store 4");
-	err = sam_data_store (saved_data, sizeof (saved_data));
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-		return 1;
-	}
-
-	qb_log (LOG_INFO, " register");
-	err = sam_register (&instance_id);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", err);
-		return 1;
-	}
-
-	if (instance_id == 1) {
-		qb_log (LOG_INFO, "iid %d: sam_start", instance_id);
-		err = sam_start ();
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", err);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sam_data_getsize 6", instance_id);
-		err = sam_data_getsize (&size);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-		if (size != sizeof (saved_data2)) {
-			qb_log (LOG_ERR, "Test should return size of 0. Returned %zx", size);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sam_data_restore 5", instance_id);
-		err = sam_data_restore (saved_data2, sizeof (saved_data2));
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-
-		if (memcmp (saved_data, saved_data2, sizeof (saved_data2)) != 0) {
-			qb_log (LOG_ERR, "Retored data are not same");
-			return 1;
-		}
-
-		for (i = 0; i < sizeof (saved_data); i++) {
-			saved_data[i] = (char)(i - 5);
-		}
-
-		qb_log (LOG_INFO, "iid %d: sam_data_store 5", instance_id);
-		err = sam_data_store (saved_data, sizeof (saved_data) - 7);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-
-		exit (1);
-	}
-
-	if (instance_id == 2) {
-		qb_log (LOG_INFO, "iid %d: sam_start", instance_id);
-		err = sam_start ();
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", err);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sam_data_getsize 7", instance_id);
-		err = sam_data_getsize (&size);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-		if (size != sizeof (saved_data2) - 7) {
-			qb_log (LOG_ERR, "Test should return size of 0. Returned %zx", size);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sam_data_restore 6", instance_id);
-		err = sam_data_restore (saved_data2, sizeof (saved_data2));
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-
-		for (i = 0; i < sizeof (saved_data); i++) {
-			saved_data[i] = (char)(i - 5);
-		}
-
-		if (memcmp (saved_data, saved_data2, sizeof (saved_data2) - 7) != 0) {
-			qb_log (LOG_ERR, "Retored data are not same");
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sam_data_store 6", instance_id);
-		err = sam_data_store (NULL, 0);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-
-		exit (1);
-	}
-
-	if (instance_id == 3) {
-		qb_log (LOG_INFO, "iid %d: sam_data_getsize 8", instance_id);
-		err = sam_data_getsize (&size);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Test should return CS_OK. Error returned %d", err);
-			return 1;
-		}
-		if (size != 0) {
-			qb_log (LOG_ERR, "Test should return size of 0. Returned %zx", size);
-			return 1;
-		}
-	}
-
-	return (0);
-}
-
-static int test5_hc_cb (void)
-{
-	cs_error_t res;
-
-	qb_log (LOG_INFO, "%d", ++test5_hc_cb_count);
-
-	res = sam_data_store (&test5_hc_cb_count, sizeof (test5_hc_cb_count));
-
-	if (res != CS_OK)
-		return 1;
-
-	if (test5_hc_cb_count > 10)
-		return 1;
-
-	return 0;
-}
-/*
- * Test event driven healtchecking.
- */
-static int test5 (void)
-{
-	cs_error_t error;
-	unsigned int instance_id;
-	int hc_cb_count;
-
-	qb_log (LOG_INFO, " initialize");
-	error = sam_initialize (100, SAM_RECOVERY_POLICY_RESTART);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", error);
-		return 1;
-	}
-	qb_log (LOG_INFO, " register");
-	error = sam_register (&instance_id);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", error);
-		return 1;
-	}
-
-	if (instance_id == 1) {
-		qb_log (LOG_INFO, "iid %d: hc callback register", instance_id);
-		error = sam_hc_callback_register (test5_hc_cb);
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't register hc cb. Error %d", error);
-			return 1;
-		}
-
-
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		sleep (2);
-
-		qb_log (LOG_INFO, "iid %d: Failed. Wasn't killed.", instance_id);
-		return 1;
-	}
-
-	if (instance_id == 2) {
-		error = sam_data_restore (&hc_cb_count, sizeof (hc_cb_count));
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "sam_data_restore should return CS_OK. Error returned %d", error);
-			return 1;
-		}
-
-		if (hc_cb_count != 11) {
-			qb_log (LOG_ERR, "iid %d: Premature killed. hc_cb_count should be 11 and it is %d",
-				__FUNCTION__, instance_id - 1, hc_cb_count);
-			return 1;
-
-		}
-		return 0;
-	}
-
-	return 1;
-}
-
-static void test6_signal (int sig) {
-	cs_error_t error;
-
-	qb_enter();
-	test6_sig_delivered++;
-
-	if ((error = sam_data_store (&test6_sig_delivered, sizeof (test6_sig_delivered))) != CS_OK) {
-		qb_log (LOG_ERR, "Can't store data! Error : %d", error);
-	}
-}
-
-/*
- * Test warn signal set.
- */
-static int test6 (void) {
-	cs_error_t error;
-	unsigned int instance_id;
-	int test6_sig_del;
-
-	qb_log (LOG_INFO, " initialize");
-	error = sam_initialize (2000, SAM_RECOVERY_POLICY_RESTART);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", error);
-		return 1;
-	}
-	qb_log (LOG_INFO, " register");
-	error = sam_register (&instance_id);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", error);
-		return 1;
-	}
-
-	if (instance_id == 1) {
-		error = sam_warn_signal_set (SIGUSR1);
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't set warn signal. Error %d", error);
-			return 1;
-		}
-
-		signal (SIGUSR1, test6_signal);
-
-		qb_log (LOG_INFO, " iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sleep 1", instance_id);
-		sleep (1);
-
-		qb_log (LOG_INFO, "iid %d: hc send", instance_id);
-		error = sam_hc_send ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't send hc. Error %d", error);
-			return 1;
-		}
-
-
-		qb_log (LOG_INFO, "iid %d: wait for delivery of signal", instance_id);
-		while (!test6_sig_delivered) {
-			sleep (1);
-		}
-
-		qb_log (LOG_INFO, "iid %d: wait for real kill", instance_id);
-
-		sleep (3);
-
-		qb_log (LOG_INFO, "iid %d: wasn't killed", instance_id);
-		return (1);
-	}
-
-	if (instance_id == 2) {
-		error = sam_data_restore (&test6_sig_del, sizeof (test6_sig_del));
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't restore data. Error %d", error);
-			return 1;
-		}
-
-		if (test6_sig_del != 1) {
-			qb_log (LOG_ERR, "Previous test failed. Signal was not delivered");
-			return 1;
-		}
-
-		error = sam_warn_signal_set (SIGKILL);
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't set warn signal. Error %d", error);
-			return 1;
-		}
-
-		signal (SIGUSR1, test6_signal);
-
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		error = sam_start ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", error);
-			return 1;
-		}
-
-		qb_log (LOG_INFO, "iid %d: sleep 1", instance_id);
-		sleep (1);
-
-		qb_log (LOG_INFO, "iid %d: hc send", instance_id);
-		error = sam_hc_send ();
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't send hc. Error %d", error);
-			return 1;
-		}
-
-
-		qb_log (LOG_INFO, "iid %d: wait for delivery of signal", instance_id);
-		while (!test6_sig_delivered) {
-			sleep (1);
-		}
-
-		qb_log (LOG_INFO, "iid %d: wasn't killed", instance_id);
-		return (1);
-	}
-
-	if (instance_id == 3) {
-		error = sam_data_restore (&test6_sig_del, sizeof (test6_sig_del));
-		if (error != CS_OK) {
-			qb_log (LOG_ERR, "Can't restore data. Error %d", error);
-			return 1;
-		}
-
-		if (test6_sig_del != 1) {
-			qb_log (LOG_ERR, "Previous test failed. Signal WAS delivered");
-			return 1;
-		}
-
-		return (0);
-	}
-
-	return 1;
-}
-
-/*
- * Test cmap integration + quit policy
- */
-static int test8 (pid_t pid, pid_t old_pid, int test_n) {
-	cmap_handle_t cmap_handle;
-	cs_error_t err;
-	uint64_t tstamp1, tstamp2;
-	int32_t msec_diff;
-	unsigned int instance_id;
-	char key_name[CMAP_KEYNAME_MAXLEN];
-	char *str;
-
-	err = cmap_initialize (&cmap_handle);
-	if (err != CS_OK) {
-		qb_log (LOG_INFO, "Could not initialize Cluster Map API instance error %d. Test skipped", err);
-		return (1);
-	}
-
-	qb_log (LOG_INFO, "test %d", test_n);
-
-	if (test_n == 2) {
-		/*
-		 * Object should not exist
-		 */
-		qb_log (LOG_INFO, "Testing if object exists (it shouldn't)");
-
-		snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.state", pid);
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err == CS_OK) {
-			qb_log (LOG_INFO, "Could find key \"%s\": %d.", key_name, err);
-			free(str);
-			return (2);
-		}
-	}
-
-	if (test_n == 1 || test_n == 2) {
-		qb_log (LOG_INFO, " initialize");
-		err = sam_initialize (2000, SAM_RECOVERY_POLICY_QUIT | SAM_RECOVERY_POLICY_CMAP);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", err);
-			return 2;
-		}
-
-		qb_log (LOG_INFO, " register");
-		err = sam_register (&instance_id);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't register. Error %d", err);
-			return 2;
-		}
-
-		snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.recovery", pid);
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"recovery\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "quit") != 0) {
-			qb_log (LOG_INFO, "Recovery key \"%s\" is not \"quit\".", key_name);
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.state", pid);
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "stopped") != 0) {
-			qb_log (LOG_INFO, "State key is not \"stopped\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		qb_log (LOG_INFO, "iid %d: start", instance_id);
-		err = sam_start ();
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", err);
-			return 2;
-		}
-
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "running") != 0) {
-			qb_log (LOG_INFO, "State key is not \"running\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		qb_log (LOG_INFO, "iid %d: stop", instance_id);
-		err = sam_stop ();
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't stop hc. Error %d", err);
-			return 2;
-		}
-
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "stopped") != 0) {
-			qb_log (LOG_INFO, "State key is not \"stopped\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		qb_log (LOG_INFO, "iid %d: sleeping 5", instance_id);
-		sleep (5);
-
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "stopped") != 0) {
-			qb_log (LOG_INFO, "State key is not \"stopped\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		qb_log (LOG_INFO, "iid %d: start 2", instance_id);
-		err = sam_start ();
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't start hc. Error %d", err);
-			return 2;
-		}
-
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "running") != 0) {
-			qb_log (LOG_INFO, "State key is not \"running\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		if (test_n == 2) {
-			qb_log (LOG_INFO, "iid %d: sleeping 5. Should be killed", instance_id);
-			sleep (5);
-
-			return (2);
-		} else {
-			qb_log (LOG_INFO, "iid %d: Test HC", instance_id);
-			err = sam_hc_send ();
-			if (err != CS_OK) {
-				qb_log (LOG_ERR, "Can't send hc. Error %d", err);
-				return 2;
-			}
-
-			snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.last_updated", pid);
-			err = cmap_get_uint64(cmap_handle, key_name, &tstamp1);
-			if (err != CS_OK) {
-				qb_log (LOG_INFO, "Could not get \"last_updated\" key: %d.", err);
-				return (2);
-			}
-			qb_log (LOG_INFO, "iid %d: Sleep 1", instance_id);
-			sleep (1);
-			err = sam_hc_send ();
-			if (err != CS_OK) {
-				qb_log (LOG_ERR, "Can't send hc. Error %d", err);
-				return 2;
-			}
-			sleep (1);
-			err = cmap_get_uint64(cmap_handle, key_name, &tstamp2);
-			if (err != CS_OK) {
-				qb_log (LOG_INFO, "Could not get \"last_updated\" key: %d.", err);
-				return (2);
-			}
-			msec_diff = (tstamp2 - tstamp1)/CS_TIME_NS_IN_MSEC;
-
-			if (msec_diff < 500 || msec_diff > 2000) {
-				qb_log (LOG_INFO, "Difference %d is not within <500, 2000> interval.", msec_diff);
-				return (2);
-			}
-
-			qb_log (LOG_INFO, "iid %d: stop 2", instance_id);
-			err = sam_stop ();
-			if (err != CS_OK) {
-				qb_log (LOG_ERR, "Can't stop hc. Error %d", err);
-				return 2;
-			}
-
-			snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.state", pid);
-			err = cmap_get_string(cmap_handle, key_name, &str);
-			if (err != CS_OK) {
-				qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-				return (2);
-			}
-
-			if (strcmp(str, "stopped") != 0) {
-				qb_log (LOG_INFO, "State key is not \"stopped\".");
-				free(str);
-				return (2);
-			}
-			free(str);
-
-			qb_log (LOG_INFO, "iid %d: exiting", instance_id);
-			return (0);
-		}
-	}
-
-	if (test_n == 3) {
-		qb_log (LOG_INFO, "Testing if status is failed");
-
-		/*
-		 * Previous should be FAILED
-		 */
-
-		snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.state", pid);
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "failed") != 0) {
-			qb_log (LOG_INFO, "State key is not \"failed\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		return (0);
-	}
-
-	return (2);
-}
-
-/*
- * Test cmap integration + restart policy
- */
-static int test9 (pid_t pid, pid_t old_pid, int test_n) {
-	cs_error_t err;
-	cmap_handle_t cmap_handle;
-	unsigned int instance_id;
-	char *str;
-	char key_name[CMAP_KEYNAME_MAXLEN];
-
-	err = cmap_initialize (&cmap_handle);
-	if (err != CS_OK) {
-		qb_log (LOG_INFO, "Could not initialize Cluster Map API instance error %d. Test skipped", err);
-		return (1);
-	}
-
-	qb_log (LOG_INFO, "test %d", test_n);
-
-	if (test_n == 1) {
-		qb_log (LOG_INFO, " initialize");
-		err = sam_initialize (2000, SAM_RECOVERY_POLICY_RESTART | SAM_RECOVERY_POLICY_CMAP);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", err);
-			return 2;
-		}
-
-		qb_log (LOG_INFO, " register");
-		err = sam_register (&instance_id);
-		if (err != CS_OK) {
-			qb_log (LOG_ERR, "Can't register. Error %d", err);
-			return 2;
-		}
-		qb_log (LOG_INFO, " iid %d", instance_id);
-
-		if (instance_id < 3) {
-			snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.recovery", pid);
-			err = cmap_get_string(cmap_handle, key_name, &str);
-			if (err != CS_OK) {
-				qb_log (LOG_INFO, "Could not get \"recovery\" key: %d.", err);
-				return (2);
-			}
-
-			if (strcmp(str, "restart") != 0) {
-				qb_log (LOG_INFO, "Recovery key \"%s\" is not \"restart\".", str);
-				free(str);
-				return (2);
-			}
-			free(str);
-
-			snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.state", pid);
-			err = cmap_get_string(cmap_handle, key_name, &str);
-			if (err != CS_OK) {
-				qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-				return (2);
-			}
-
-			if (strcmp(str, "stopped") != 0) {
-				qb_log (LOG_INFO, "State key is not \"stopped\".");
-				free(str);
-				return (2);
-			}
-			free(str);
-
-			qb_log (LOG_INFO, "iid %d: start", instance_id);
-			err = sam_start ();
-			if (err != CS_OK) {
-				qb_log (LOG_ERR, "Can't start hc. Error %d", err);
-				return 2;
-			}
-
-			err = cmap_get_string(cmap_handle, key_name, &str);
-			if (err != CS_OK) {
-				qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-				return (2);
-			}
-
-			if (strcmp(str, "running") != 0) {
-				qb_log (LOG_INFO, "State key is not \"running\".");
-				free(str);
-				return (2);
-			}
-			free(str);
-
-			qb_log (LOG_INFO, "iid %d: waiting for kill", instance_id);
-			sleep (10);
-
-			return (2);
-		}
-
-		if (instance_id == 3) {
-			qb_log (LOG_INFO, "iid %d: mark failed", instance_id);
-			err = sam_mark_failed ();
-			if (err != CS_OK) {
-				qb_log (LOG_ERR, "Can't mark failed. Error %d", err);
-				return 2;
-			}
-
-			sleep (10);
-
-			return (2);
-		}
-
-		return (2);
-	}
-
-	if (test_n == 2) {
-		qb_log (LOG_INFO, "Testing if status is failed");
-
-		/*
-		 * Previous should be FAILED
-		 */
-		snprintf(key_name, CMAP_KEYNAME_MAXLEN, "resources.process.%d.state", pid);
-		err = cmap_get_string(cmap_handle, key_name, &str);
-		if (err != CS_OK) {
-			qb_log (LOG_INFO, "Could not get \"state\" key: %d.", err);
-			return (2);
-		}
-
-		if (strcmp(str, "failed") != 0) {
-			qb_log (LOG_INFO, "State key is not \"failed\".");
-			free(str);
-			return (2);
-		}
-		free(str);
-
-		return (0);
-	}
-
-	return (2);
-}
-
-static int hc_allways_respond_cb(void)
-{
-	qb_log (LOG_INFO, "health check OK.");
-	return 0;
-}
-
-static int setup_hc (void)
-{
-	cs_error_t err;
-	unsigned int instance_id;
-
-	err = sam_initialize (1000, SAM_RECOVERY_POLICY_QUIT | SAM_RECOVERY_POLICY_CMAP);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Can't initialize SAM API. Error %d", err);
-		return 2;
-	}
-
-	qb_log (LOG_INFO, " register");
-	err = sam_register (&instance_id);
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Can't register. Error %d", err);
-		return 2;
-	}
-	err = sam_hc_callback_register (hc_allways_respond_cb);
-
-	qb_log (LOG_INFO, "instance id %d: start", instance_id);
-	err = sam_start ();
-	if (err != CS_OK) {
-		qb_log (LOG_ERR, "Can't start hc. Error %d", err);
-		return 2;
-	}
-
-	return (0);
-}
-
-static int do_test_command(int sock, char* func, char*args[], int num_args)
-{
-	int err = 0;
-	pid_t pid;
-	int stat;
-
-	pid = fork ();
-
-	if (pid == -1) {
-		qb_log (LOG_ERR, "Can't fork");
-		return -1;
-	}
-
-	if (pid == 0) {
-		if (strcmp ("test1", func) == 0) {
-			err = test1 ();
-		} else if (strcmp ("test2", func) == 0) {
-			err = test2 ();
-		} else if (strcmp ("test3", func) == 0) {
-			err = test3 ();
-		} else if (strcmp ("test4", func) == 0) {
-			err = test4 ();
-		} else if (strcmp ("test5", func) == 0) {
-			err = test5 ();
-		} else if (strcmp ("test6", func) == 0) {
-			err = test6 ();
-		} else if (strcmp ("test8", func) == 0) {
-			err = test8 (getpid(), 0, 1);
-		} else if (strcmp ("test9", func) == 0) {
-			err = test9 (getpid(), 0, 1);
-		}
-		sam_finalize ();
-		exit(err);
-	}
-
-	if (pid > 0) {
-		waitpid (pid, &stat, 0);
-
-		return WEXITSTATUS (stat);
-	}
-	return -1;
-}
-
-
-static void do_command (int sock, char* func, char*args[], int num_args)
-{
-	char response[100];
-	int err = 0;
-	ssize_t rc;
-	size_t send_len;
-
-	qb_log (LOG_INFO, "RPC:%s() called.", func);
-	if (strncmp ("test", func, 4) == 0) {
-		err = do_test_command(sock, func, args, num_args);
-	} else if (strcmp ("setup_hc", func) == 0) {
-		err = setup_hc ();
-	} else if (strcmp ("sam_stop", func) == 0) {
-		err = sam_stop ();
-		if (err != CS_OK) {
-			qb_log (LOG_ERR,"RPC:%s sam_stop failed!", func);
-			snprintf (response, 100, "%s", FAIL_STR);
-		}
-		err = sam_finalize();
-	} else {
-		err = -1;
-		qb_log (LOG_ERR,"RPC:%s not supported!", func);
-		snprintf (response, 100, "%s", NOT_SUPPORTED_STR);
-	}
-
-	if (err == 0) {
-		snprintf (response, 100, "%s", OK_STR);
-	} else if (err == 1) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "%s() test skipped?! (%d).", func, err);
-	} else {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "%s() failed (%d).", func, err);
-	}
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-int
-main (int argc, char *argv[])
-{
-	return test_agent_run ("sam_test_agent", 9036, do_command, NULL);
-}

cts/agents/shm_leak_audit.sh.in

@@ -1,55 +0,0 @@
-#!@BASHPATH@
-
-
-service corosync status >/dev/null
-CS_STATUS=$?
-
-if [ $CS_STATUS -eq 0 ]
-then
-    # corosync running
-    active=$(corosync-cmapctl runtime.connections.active | cut -d= -f2)
-    if [ $active -lt 2 ]
-    then
-        FILES=$(ls /dev/shm/qb-*)
-        for f in $FILES
-        do
-            if [[ "$f" =~ "blackbox" ]]
-            then
-                true
-            else
-                echo $f
-            fi
-        done
-    else
-        pids=$(corosync-cmapctl runtime.connections. | grep client_pid | cut -d= -f2)
-
-        FILES=$(ls /dev/shm/qb-*)
-        for f in $FILES
-        do
-            found=0
-            for p in $pids
-            do
-                if [[ "$f" =~ "$p" ]]
-                then
-                    found=1
-                elif [[ "$f" =~ "blackbox" ]]
-                then
-                    found=1
-                fi
-            done
-            if [ $found -eq 0 ]
-            then
-                echo $f
-            fi
-        done
-    fi
-else
-    FILES=$(ls /dev/shm/qb-*)
-    for f in $FILES
-    do
-        echo $f
-    done
-fi
-
-exit 0
-

cts/agents/votequorum_test_agent.c

@@ -1,360 +0,0 @@
-/*
- * Copyright (c) 2010 Red Hat Inc
- *
- * All rights reserved.
- *
- * Author: Angus Salkeld <asalkeld@redhat.com>
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <sys/un.h>
-#include <poll.h>
-
-#include <qb/qbloop.h>
-#include <corosync/corotypes.h>
-#include <corosync/votequorum.h>
-#include <corosync/quorum.h>
-#include "common_test_agent.h"
-#include "../../lib/util.h"
-
-static quorum_handle_t q_handle = 0;
-static votequorum_handle_t vq_handle = 0;
-
-static void votequorum_notification_fn(
-	votequorum_handle_t handle,
-	uint64_t context,
-	uint32_t quorate,
-	uint32_t node_list_entries,
-	votequorum_node_t node_list[])
-{
-	qb_log (LOG_INFO, "VQ notification quorate: %d", quorate);
-}
-
-static void quorum_notification_fn(
-	quorum_handle_t handle,
-	uint32_t quorate,
-	uint64_t ring_id,
-	uint32_t view_list_entries,
-	uint32_t *view_list)
-{
-	qb_log (LOG_INFO, "NQ notification quorate: %d", quorate);
-}
-
-
-static int vq_dispatch_wrapper_fn (
-	int fd,
-	int revents,
-	void *data)
-{
-	cs_error_t error = votequorum_dispatch (vq_handle, CS_DISPATCH_ALL);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "got %s error, disconnecting.",
-			cs_strerror(error));
-		votequorum_finalize(vq_handle);
-		vq_handle = 0;
-		return -1;
-	}
-	return 0;
-}
-
-static int q_dispatch_wrapper_fn (
-	int fd,
-	int revents,
-	void *data)
-{
-	cs_error_t error = quorum_dispatch (q_handle, CS_DISPATCH_ALL);
-	if (error != CS_OK) {
-		qb_log (LOG_ERR, "got %s error, disconnecting.",
-			cs_strerror(error));
-		quorum_finalize(q_handle);
-		q_handle = 0;
-		return -1;
-	}
-	return 0;
-}
-
-static int q_lib_init(void)
-{
-	votequorum_callbacks_t vq_callbacks;
-	quorum_callbacks_t q_callbacks;
-	int ret = 0;
-	int retry = 3;
-	int fd;
-
-	if (vq_handle == 0) {
-		qb_log (LOG_INFO, "votequorum_initialize");
-		vq_callbacks.votequorum_quorum_notify_fn = votequorum_notification_fn;
-		vq_callbacks.votequorum_expectedvotes_notify_fn = NULL;
-		ret = CS_ERR_NOT_EXIST;
-		while (ret == CS_ERR_NOT_EXIST && retry > 0) {
-			ret = votequorum_initialize (&vq_handle, &vq_callbacks);
-			if (ret == CS_ERR_NOT_EXIST) {
-				sleep (1);
-				retry--;
-			}
-		}
-		if (ret != CS_OK) {
-			qb_log (LOG_ERR, "votequorum_initialize FAILED: %d", ret);
-			vq_handle = 0;
-		}
-		else {
-			ret = votequorum_trackstart (vq_handle, vq_handle, CS_TRACK_CHANGES);
-			if (ret != CS_OK) {
-				qb_log (LOG_ERR, "votequorum_trackstart FAILED: %d", ret);
-			}
-
-			votequorum_fd_get (vq_handle, &fd);
-			qb_loop_poll_add (ta_poll_handle_get(), QB_LOOP_MED, fd,
-				POLLIN|POLLNVAL, NULL, vq_dispatch_wrapper_fn);
-		}
-	}
-	if (q_handle == 0) {
-		uint32_t q_type;
-		qb_log (LOG_INFO, "quorum_initialize");
-		q_callbacks.quorum_notify_fn = quorum_notification_fn;
-		ret = quorum_initialize (&q_handle, &q_callbacks, &q_type);
-		if (ret != CS_OK) {
-			qb_log (LOG_ERR, "quorum_initialize FAILED: %d", ret);
-			q_handle = 0;
-		}
-		else {
-			ret = quorum_trackstart (q_handle, CS_TRACK_CHANGES);
-			if (ret != CS_OK) {
-				qb_log (LOG_ERR, "quorum_trackstart FAILED: %d", ret);
-			}
-			quorum_fd_get (q_handle, &fd);
-			qb_loop_poll_add (ta_poll_handle_get(), QB_LOOP_MED, fd,
-				POLLIN|POLLNVAL, NULL, q_dispatch_wrapper_fn);
-		}
-	}
-	return ret;
-}
-
-static void getinfo (int sock)
-{
-	int ret;
-	struct votequorum_info info;
-	char response[100];
-	ssize_t rc;
-	size_t send_len;
-
-	q_lib_init ();
-
-	ret = votequorum_getinfo(vq_handle, 0, &info);
-	if (ret != CS_OK) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "votequorum_getinfo FAILED: %d", ret);
-		goto send_response;
-	}
-
-	snprintf (response, 100, "%d:%d:%d:%d:%d",
-		info.node_votes,
-		info.node_expected_votes,
-		info.highest_expected,
-		info.total_votes,
-		info.quorum);
-
-send_response:
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-
-static void setexpected (int sock, char *arg)
-{
-	int ret;
-	char response[100];
-	ssize_t rc;
-	size_t send_len;
-
-	q_lib_init ();
-
-	ret = votequorum_setexpected (vq_handle, atoi(arg));
-	if (ret != CS_OK) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "set expected votes FAILED: %d", ret);
-		goto send_response;
-	}
-
-	snprintf (response, 100, "%s", OK_STR);
-
-send_response:
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-static void setvotes (int sock, char *arg)
-{
-	int ret;
-	char response[100];
-	ssize_t rc;
-	size_t send_len;
-
-	q_lib_init ();
-
-	ret = votequorum_setvotes (vq_handle, 0, atoi(arg));
-	if (ret != CS_OK) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "set votes FAILED: %d", ret);
-		goto send_response;
-	}
-
-	snprintf (response, 100, "%s", OK_STR);
-
-send_response:
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-
-static void getquorate (int sock)
-{
-	int ret;
-	int quorate;
-	char response[100];
-	ssize_t rc;
-	size_t send_len;
-
-	q_lib_init ();
-
-	ret = quorum_getquorate (q_handle, &quorate);
-	if (ret != CS_OK) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "getquorate FAILED: %d", ret);
-		goto send_response;
-	}
-
-	snprintf (response, 100, "%d", quorate);
-
-send_response:
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-static void context_test (int sock)
-{
-	char response[100];
-	char *cmp;
-	cs_error_t rc1;
-	cs_error_t rc2;
-	ssize_t send_rc;
-	size_t send_len;
-
-	snprintf (response, 100, "%s", OK_STR);
-
-	rc1 = votequorum_context_set (vq_handle, response);
-	rc2 = votequorum_context_get (vq_handle, (void**)&cmp);
-	if (response != cmp) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "votequorum context not the same %d %d",
-			rc1, rc2);
-	}
-
-	rc1 = quorum_context_set (q_handle, response);
-	rc2 = quorum_context_get (q_handle, (const void**)&cmp);
-	if (response != cmp) {
-		snprintf (response, 100, "%s", FAIL_STR);
-		qb_log (LOG_ERR, "quorum context not the same %d %d",
-			rc1, rc2);
-	}
-	send_len = strlen (response);
-	send_rc = send (sock, response, send_len, 0);
-	assert(send_rc == send_len);
-}
-
-static void do_command (int sock, char* func, char*args[], int num_args)
-{
-	char response[100];
-	ssize_t rc;
-	size_t send_len;
-
-	q_lib_init ();
-
-	qb_log (LOG_INFO,"RPC:%s() called.", func);
-
-	if (strcmp ("votequorum_getinfo", func) == 0) {
-		getinfo (sock);
-	} else if (strcmp ("votequorum_setvotes", func) == 0) {
-		setvotes (sock, args[0]);
-	} else if (strcmp ("votequorum_setexpected", func) == 0) {
-		setexpected (sock, args[0]);
-	} else if (strcmp ("quorum_getquorate", func) == 0) {
-		getquorate (sock);
-	} else if (strcmp ("context_test", func) == 0) {
-		context_test (sock);
-	} else if (strcmp ("are_you_ok_dude", func) == 0 ||
-	           strcmp ("init", func) == 0) {
-		snprintf (response, 100, "%s", OK_STR);
-		goto send_response;
-	} else {
-		qb_log (LOG_ERR,"%s RPC:%s not supported!", __func__, func);
-		snprintf (response, 100, "%s", NOT_SUPPORTED_STR);
-		goto send_response;
-	}
-
-	return ;
-send_response:
-	send_len = strlen (response);
-	rc = send (sock, response, send_len, 0);
-	assert(rc == send_len);
-}
-
-static void my_pre_exit(void)
-{
-	qb_log (LOG_INFO, "PRE EXIT");
-	if (vq_handle) {
-		votequorum_finalize(vq_handle);
-		vq_handle = 0;
-	}
-	if (q_handle) {
-		quorum_finalize(q_handle);
-		q_handle = 0;
-	}
-}
-
-int
-main(int argc, char *argv[])
-{
-	return test_agent_run ("quorum_test_agent", 9037, do_command, my_pre_exit);
-}

cts/corolab.py

@@ -1,337 +0,0 @@
-#!/usr/bin/python
-
-'''CTS: Cluster Testing System: Lab environment module
- '''
-
-from __future__ import print_function
-
-__copyright__='''
-Copyright (c) 2010 Red Hat, Inc.
-
-'''
-
-# All rights reserved.
-# 
-# Author: Angus Salkeld <asalkeld@redhat.com>
-#
-# This software licensed under BSD license, the text of which follows:
-# 
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-# 
-# - Redistributions of source code must retain the above copyright notice,
-#   this list of conditions and the following disclaimer.
-# - Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-# - Neither the name of the MontaVista Software, Inc. nor the names of its
-#   contributors may be used to endorse or promote products derived from this
-#   software without specific prior written permission.
-# 
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-# THE POSSIBILITY OF SUCH DAMAGE.
-
-import sys
-from cts.CTSscenarios import *
-from cts.logging import *
-from corotests import CoroTestList
-from corosync import *
-
-
-sys.path.append("/usr/share/pacemaker/tests/cts") # So that things work from the source directory
-
-try:
-    from CTSlab import *
-
-except ImportError:
-    sys.stderr.write("abort: couldn't find CTSLab in [%s]\n" %
-                     ' '.join(sys.path))
-    sys.stderr.write("(check your install and PYTHONPATH)\n")
-    sys.exit(-1)
-
-tests = None
-cm = None
-old_handler = None
-DefaultFacility = "daemon"
-
-
-def usage(arg):
-    print("Illegal argument " + arg)
-    print("usage: " + sys.argv[0] +" [options] number-of-iterations")
-    print("\nCommon options: ")
-    print("\t [--at-boot (1|0)],         does the cluster software start at boot time")
-    print("\t [--nodes 'node list'],     list of cluster nodes separated by whitespace")
-    print("\t [--limit-nodes max],       only use the first 'max' cluster nodes supplied with --nodes")
-    print("\t [--logfile path],          where should the test software look for logs from cluster nodes")
-    print("\t [--rrp-bindaddr addr],       extra interface used for rrp, provide the bindaddr")
-    print("\t [--outputfile path],       optional location for the test software to write logs to")
-    print("\t [--syslog-facility name],  which syslog facility should the test software log to")
-    print("\t [--choose testcase-name],  run only the named test")
-    print("\t [--list-tests],            list the valid tests")
-    print("\t [--benchmark],             add the timing information")
-    print("\t ")
-    print("Additional (less common) options: ")
-    print("\t [--trunc (truncate logfile before starting)]")
-    print("\t [--xmit-loss lost-rate(0.0-1.0)]")
-    print("\t [--recv-loss lost-rate(0.0-1.0)]")
-    print("\t [--standby (1 | 0 | yes | no)]")
-    print("\t [--fencing (1 | 0 | yes | no)]")
-    print("\t [--once],                 run all valid tests once")
-    print("\t [--no-loop-tests],        dont run looping/time-based tests")
-    print("\t [--no-unsafe-tests],      dont run tests that are unsafe for use with ocfs2/drbd")
-    print("\t [--valgrind-tests],       include tests using valgrind")
-    print("\t [--experimental-tests],   include experimental tests")
-    print("\t [--oprofile 'node list'], list of cluster nodes to run oprofile on]")
-    print("\t [--qarsh]                 Use the QARSH backdoor to access nodes instead of SSH")
-    print("\t [--seed random_seed]")
-    print("\t [--set option=value]")
-    sys.exit(1)
-
-class CoroLabEnvironment(CtsLab):
-
-    def __init__(self):
-        CtsLab.__init__(self)
-
-        #  Get a random seed for the random number generator.
-        self["DoStonith"] = 0
-        self["DoStandby"] = 0
-        self["DoFencing"] = 0
-        self["XmitLoss"] = "0.0"
-        self["RecvLoss"] = "0.0"
-        self["IPBase"] = "127.0.0.10"
-        self["ClobberCIB"] = 0
-        self["CIBfilename"] = None
-        self["CIBResource"] = 0
-        self["DoBSC"]    = 0
-        self["use_logd"] = 0
-        self["oprofile"] = []
-        self["RrpBindAddr"] = None
-        self["warn-inactive"] = 0
-        self["ListTests"] = 0
-        self["benchmark"] = 0
-        self["logrestartcmd"] = "systemctl restart rsyslog.service 2>&1 > /dev/null"
-        self["syslogd"] ="rsyslog"
-        self["Schema"] = "corosync 2.0"
-        self["Stack"] = "corosync"
-        self['CMclass'] = corosync_needle
-        self["stonith-type"] = "external/ssh"
-        self["stonith-params"] = "hostlist=all,livedangerously=yes"
-        self["at-boot"] = 0  # Does the cluster software start automatically when the node boot 
-        self["logger"] = ([StdErrLog(self, 'corosync.log')])
-        self["loop-minutes"] = 60
-        self["valgrind-prefix"] = None
-        self["valgrind-procs"] = "corosync"
-        self["valgrind-opts"] = """--leak-check=full --show-reachable=yes --trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp"""
-
-        self["experimental-tests"] = 0
-        self["valgrind-tests"] = 0
-        self["unsafe-tests"] = 0
-        self["loop-tests"] = 0
-        self["all-once"] = 0
-        self["LogWatcher"] = "remote"
-        self["SyslogFacility"] = DefaultFacility
-        self["stats"] = 0
-        self.log = LogFactory().log
-
-#
-# Main entry into the test system.
-#
-if __name__ == '__main__': 
-    Environment = CoroLabEnvironment()
-
-    NumIter = 0
-    Version = 1
-    LimitNodes = 0
-    TestCase = None
-    TruncateLog = 0
-    ListTests = 0
-    HaveSeed = 0
-    node_list = ''
-
-    #
-    # The values of the rest of the parameters are now properly derived from
-    # the configuration files.
-    #
-    # Set the signal handler
-    signal.signal(15, sig_handler)
-    signal.signal(10, sig_handler)
-    
-    # Process arguments...
-
-    skipthis=None
-    args=sys.argv[1:]
-    for i in range(0, len(args)):
-       if skipthis:
-           skipthis=None
-           continue
-
-       elif args[i] == "-l" or args[i] == "--limit-nodes":
-           skipthis=1
-           LimitNodes = int(args[i+1])
-
-       elif args[i] == "-L" or args[i] == "--logfile":
-           skipthis=1
-           Environment["LogFileName"] = args[i+1]
-
-       elif args[i] == "--outputfile":
-           skipthis=1
-           Environment["OutputFile"] = args[i+1]
-
-       elif args[i] == "--rrp-bindaddr":
-           skipthis=1
-           Environment["RrpBindAddr"] = args[i+1]
-
-       elif args[i] == "--oprofile":
-           skipthis=1
-           Environment["oprofile"] = args[i+1].split(' ')
-
-       elif args[i] == "--trunc":
-           Environment["TruncateLog"]=1
-
-       elif args[i] == "--list-tests":
-           Environment["ListTests"]=1
-
-       elif args[i] == "--benchmark":
-           Environment["benchmark"]=1
-
-       elif args[i] == "--qarsh":
-           Environment.rsh.enable_qarsh()
-
-       elif args[i] == "--fencing":
-           skipthis=1
-           if args[i+1] == "1" or args[i+1] == "yes":
-               Environment["DoFencing"] = 1
-           elif args[i+1] == "0" or args[i+1] == "no":
-               Environment["DoFencing"] = 0
-           else:
-               usage(args[i+1])
-
-       elif args[i] == "--xmit-loss":
-           try:
-               float(args[i+1])
-           except ValueError:
-               print ("--xmit-loss parameter should be float")
-               usage(args[i+1])
-           skipthis=1
-           Environment["XmitLoss"] = args[i+1]
-
-       elif args[i] == "--recv-loss":
-           try:
-               float(args[i+1])
-           except ValueError:
-               print ("--recv-loss parameter should be float")
-               usage(args[i+1])
-           skipthis=1
-           Environment["RecvLoss"] = args[i+1]
-
-       elif args[i] == "--choose":
-           skipthis=1
-           TestCase = args[i+1]
-
-       elif args[i] == "--nodes":
-           skipthis=1
-           node_list = args[i+1].split(' ')
-
-       elif args[i] == "--at-boot" or args[i] == "--cluster-starts-at-boot":
-           skipthis=1
-           if args[i+1] == "1" or args[i+1] == "yes":
-               Environment["at-boot"] = 1
-           elif args[i+1] == "0" or args[i+1] == "no":
-               Environment["at-boot"] = 0
-           else:
-               usage(args[i+1])
-
-       elif args[i] == "--set":
-           skipthis=1
-           (name, value) = args[i+1].split('=')
-           Environment[name] = value
-
-       elif args[i] == "--once":
-           skipthis=1
-           Environment["all-once"]=1
-
-       elif args[i] == "--stack":
-           skipthis=1
-           Environment["Stack"] = args[i+1]
-
-       else:
-           try:
-               NumIter=int(args[i])
-           except ValueError:
-               usage(args[i])
-
-    if Environment["OutputFile"]:
-        Environment["logger"].append(FileLog(Environment, Environment["OutputFile"]))
-
-    if len(node_list) < 1:
-        print("No nodes specified!")
-        sys.exit(1)
-
-    if LimitNodes > 0:
-        if len(node_list) > LimitNodes:
-            print("Limiting the number of nodes configured=%d (max=%d)"
-                  %(len(node_list), LimitNodes))
-            while len(node_list) > LimitNodes:
-                node_list.pop(len(node_list)-1)
-
-    Environment["nodes"] = node_list
-
-    # Create the Cluster Manager object
-    cm = Environment['CMclass'](Environment)
-    Audits = CoroAuditList(cm)
-        
-    if Environment["ListTests"] == 1 :
-        Tests = CoroTestList(cm, Audits)
-        Environment.log("Total %d tests"%len(Tests))
-        for test in Tests :
-            Environment.log(str(test.name));
-        sys.exit(0)
-
-    if TruncateLog:
-        Environment.log("Truncating %s" % LogFile)
-        lf = open(LogFile, "w");
-        if lf != None:
-            lf.truncate(0)
-            lf.close()
-
-    if TestCase != None:
-        for test in CoroTestList(cm, Audits):
-            if test.name == TestCase:
-                Tests.append(test)
-        if Tests == []:
-            usage("--choose: No applicable/valid tests chosen")        
-
-    else:
-        Tests = CoroTestList(cm, Audits)
-    
-    # Scenario selection
-    if Environment["DoBSC"]:
-        scenario = RandomTests(cm, [ BasicSanityCheck(Environment) ], Audits, Tests)
-
-    elif Environment["all-once"] or NumIter == 0: 
-        NumIter = len(Tests)
-        scenario = AllOnce(
-            cm, [ BootCluster(Environment), TestAgentComponent(Environment), PacketLoss(Environment) ], Audits, Tests)
-    else:
-        scenario = RandomTests(
-            cm, [ BootCluster(Environment), TestAgentComponent(Environment), PacketLoss(Environment) ], Audits, Tests)
-
-    Environment.log(">>>>>>>>>>>>>>>> BEGINNING " + repr(NumIter) + " TESTS ")
-    Environment.log("Stack:            %s" % Environment["Stack"])
-    Environment.log("Schema:           %s" % Environment["Schema"])
-    Environment.log("Scenario:         %s" % scenario.__doc__)
-    Environment.log("Random Seed:      %s" % Environment["RandSeed"])
-    Environment.log("System log files: %s" % Environment["LogFileName"])
-
-    Environment.dump()
-    rc = Environment.run(scenario, NumIter)
-    sys.exit(rc)

cts/corosync.py

@@ -1,679 +0,0 @@
-'''CTS: Cluster Testing System: corosync...
-'''
-
-__copyright__='''
-Copyright (c) 2010 Red Hat, Inc.
-'''
-
-# All rights reserved.
-# 
-# Author: Angus Salkeld <asalkeld@redhat.com>
-#
-# This software licensed under BSD license, the text of which follows:
-# 
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-# 
-# - Redistributions of source code must retain the above copyright notice,
-#   this list of conditions and the following disclaimer.
-# - Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-# - Neither the name of the MontaVista Software, Inc. nor the names of its
-#   contributors may be used to endorse or promote products derived from this
-#   software without specific prior written permission.
-# 
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-# THE POSSIBILITY OF SUCH DAMAGE.
-
-import os
-import sys
-import time
-import socket
-import shutil
-import string
-
-import augeas
-from cts.CTS import ClusterManager
-from cts.CTSscenarios import ScenarioComponent
-from cts.remote import RemoteExec
-from cts.remote import RemoteFactory
-from cts.logging import *
-from cts.CTSvars import CTSvars
-from cts.CTSaudits import ClusterAudit
-from cts.CTSaudits import LogAudit
-
-
-###################################################################
-class CoroConfig(object):
-    def __init__(self, corobase=None):
-        self.base = "/files/etc/corosync/corosync.conf/"
-        self.new_root = "/tmp/aug-root/"
-        if corobase == None: 
-            self.corobase = os.getcwd() + "/.."
-        else:
-            self.corobase = corobase
-        example = self.corobase + "/conf/corosync.conf.example"
-
-        if os.path.isdir(self.new_root):
-            shutil.rmtree (self.new_root)
-        os.makedirs (self.new_root + "/etc/corosync")
-        shutil.copy (example, self.new_root + "/etc/corosync/corosync.conf")
-
-        self.aug = augeas.Augeas (root=self.new_root,
-                loadpath=self.corobase + "/conf/lenses")
-
-        self.original = {}
-        # store the original values (of totem), so we can restore them in
-        # apply_default_config()
-        totem = self.aug.match('/files/etc/corosync/corosync.conf/totem/*')
-        for c in totem:
-            # /files/etc/corosync/corosync.conf/
-            short_name = c[len(self.base):]
-            self.original[short_name] = self.aug.get(c)
-        interface = self.aug.match('/files/etc/corosync/corosync.conf/totem/interface/*')
-        for c in interface:
-            short_name = c[len(self.base):]
-            self.original[short_name] = self.aug.get(c)
-
-    def get (self, name):
-        return self.aug.get (self.base + name)
-
-    def set (self, name, value):
-        token = self.aug.set (self.base + name, str(value))
-
-    def save (self):
-        self.aug.save()
-
-    def get_filename(self):
-        return self.new_root + "/etc/corosync/corosync.conf"
-
-###################################################################
-class corosync_needle(ClusterManager):
-    '''
-     bla
-    '''
-    def __init__(self, Environment, randseed=None):
-        ClusterManager.__init__(self, Environment, randseed)
-
-        self.update({
-            "Name"           : "corosync(needle)",
-            "StartCmd"       : "service corosync start",
-            "StopCmd"        : "service corosync stop",
-            "RereadCmd"      : "service corosync reload",
-            "StatusCmd"      : "service corosync status",
-            "DeadTime"       : 30,
-            "StartTime"      : 15,        # Max time to start up
-            "StableTime"     : 10,
-            "BreakCommCmd"   : "/usr/share/corosync/tests/net_breaker.sh BreakCommCmd %s",
-            "FixCommCmd"     : "/usr/share/corosync/tests/net_breaker.sh FixCommCmd %s",
-            "QuorumCmd"      : "corosync-quorumtool -s",
-
-            "Pat:We_stopped"   : "%s.*Corosync Cluster Engine exiting.*",
-            "Pat:They_stopped" : "%s.*Member left:.*%s.*",
-            "Pat:They_dead"    : "corosync:.*Node %s is now: lost",
-            "Pat:Local_starting" : "%s.*Initializing transport",
-            "Pat:Local_started" :  "%s.*Initializing transport",
-            "Pat:Master_started" : "%s.*Completed service synchronization, ready to provide service.",
-            "Pat:Slave_started" :  "%s.*Completed service synchronization, ready to provide service.",
-            "Pat:ChildKilled"  : "%s corosync.*Child process %s terminated with signal 9",
-            "Pat:ChildRespawn" : "%s corosync.*Respawning failed child process: %s",
-            "Pat:ChildExit"    : "Child process .* exited",
-            "Pat:DC_IDLE"      : ".*A new membership.*was formed.",
-            # Bad news Regexes.  Should never occur.
-            "BadRegexes"   : (
-                r"ERROR:",
-                r"CRIT:",
-                r"Shutting down\.",
-                r"Forcing shutdown\.",
-                r"core dump",
-                r"Could not bind AF_UNIX",
-                r"Too many open files",
-		r"Address already in use",
-            ),
-            "LogFileName"    : Environment["LogFileName"],
-            })
-        self.start_cpg = True
-        self.cpg_agent = {}
-        self.sam_agent = {}
-        self.votequorum_agent = {}
-        self.config = CoroConfig ()
-        self.node_to_ip = {}
-        
-        self.new_config = {}
-        self.applied_config = {}
-        for n in self.Env["nodes"]:
-            ip = socket.gethostbyname(n)
-            ips = ip.split('.')
-            ips[3] = '0'
-            ip_mask = '.'.join(ips)
-            self.new_config['totem/interface/bindnetaddr'] = str(ip_mask)
-            return
-
-    def apply_default_config(self):
-
-        for c in self.applied_config:
-            if 'bindnetaddr' in c:
-                continue
-            elif c not in self.config.original:
-                # new config option (non default)
-                pass
-            elif self.applied_config[c] is not self.config.original[c]:
-                # reset to the original
-                self.new_config[c] = self.config.original[c]
-
-        if len(self.new_config) > 0:
-            self.debug('applying default config')
-            self.stopall()
-
-    def apply_new_config(self, need_all_up=True):
-
-        if len(self.new_config) > 0:
-            self.debug('applying new config')
-            self.stopall()
-            if need_all_up:
-                self.startall()
-
-    def install_all_config(self):
-        tmp1 = {}
-        sorted_keys = sorted(self.new_config.keys())
-        for c in sorted_keys:
-            self.log('configuring: ' + c + ' = '+ str(self.new_config[c]))
-            self.config.set (c, self.new_config[c])
-            self.applied_config[c] = self.new_config[c]
-            tmp1[c] = self.new_config[c]
-
-        for c in tmp1:
-            del self.new_config[c]
-
-        self.config.save()
-        src_file = self.config.get_filename()
-        for node in self.Env["nodes"]:
-            self.rsh.cp(src_file, "%s:%s" % (node, "/etc/corosync/"))
-
-    def install_config(self, node):
-        # install gets new_config and installs it, then moves the
-        # config to applied_config
-        if len(self.new_config) > 0:
-            self.install_all_config()
-
-    def key_for_node(self, node):
-        if node not in self.node_to_ip:
-            self.node_to_ip[node] = socket.gethostbyname (node)
-        return self.node_to_ip[node]
-
-    def StartaCM(self, node, verbose=False):
-
-        if node not in self.ShouldBeStatus:
-            self.ShouldBeStatus[node] = "down"
-
-        if self.ShouldBeStatus[node] != "down":
-            return 1
-
-        self.debug('starting corosync on : ' + node)
-        ret = ClusterManager.StartaCM(self, node)
-        if self.start_cpg:
-            if node in self.cpg_agent:
-                self.cpg_agent[node].restart()
-            else:
-                self.cpg_agent[node] = CpgTestAgent(node, self.Env)
-                self.cpg_agent[node].start()
-
-        if node in self.sam_agent:
-            self.sam_agent[node].restart()
-
-        # votequorum agent started as needed.
-        if 'quorum/provider' in self.applied_config:
-            if self.applied_config['quorum/provider'] is 'corosync_votequorum':
-                if node in self.votequorum_agent:
-                    self.votequorum_agent[node].restart()
-                else:
-                    self.votequorum_agent[node] = VoteQuorumTestAgent(node, self.Env)
-                    self.votequorum_agent[node].start()
-
-        return ret
-
-    def StopaCM(self, node, verbose=False):
-        if self.ShouldBeStatus[node] != "up":
-            return 1
-
-        self.debug('stoping corosync on : ' + node)
-        if node in self.cpg_agent:
-            self.cpg_agent[node].stop()
-        if node in self.sam_agent:
-            self.sam_agent[node].stop()
-        if node in self.votequorum_agent:
-            self.votequorum_agent[node].stop()
-        return ClusterManager.StopaCM(self, node)
-
-    def test_node_CM(self, node):
-        # 2 - up and stable
-        # 1 - unstable
-        # 0 - down
-        (rc, lines) = self.rsh(node, self["StatusCmd"], stdout=2)
-        out = str(lines)
-
-        if 'systemd' in out:
-            if 'running' in out:
-                ret = 2
-            else:
-                ret = 0
-        else:
-            is_stopped = string.find(out, 'stopped')
-            is_dead = string.find(out, 'dead')
-            ret = (is_dead is -1 and is_stopped is -1)
-
-        try:
-            if ret:
-                ret = 2
-                if self.ShouldBeStatus[node] == "down":
-                    self.log(
-                    "Node status for %s is %s but we think it should be %s"
-                    %        (node, "up", self.ShouldBeStatus[node]))
-            else:
-                if self.ShouldBeStatus[node] == "up":
-                    self.log(
-                    "Node status for %s is %s but we think it should be %s"
-                    %        (node, "down", self.ShouldBeStatus[node]))
-        except KeyError:        pass
-
-        if ret:        self.ShouldBeStatus[node] = "up"
-        else:        self.ShouldBeStatus[node] = "down"
-        return ret
-
-    def StataCM(self, node):
-
-        '''Report the status of corosync on a given node'''
-        if self.test_node_CM(node) > 0:
-            return 1
-        else:
-            return None
-
-    def RereadCM(self, node):
-        self.log('reloading corosync on : ' + node)
-        return ClusterManager.RereadCM(self, node)
-
-    def find_partitions(self):
-        ccm_partitions = []
-        return ccm_partitions
-
-    def prepare(self):
-        '''Finish the Initialization process. Prepare to test...'''
-
-        self.partitions_expected = 1
-        for node in self.Env["nodes"]:
-            self.ShouldBeStatus[node] = ""
-            self.unisolate_node(node)
-            self.StataCM(node)
-
-    def HasQuorum(self, node_list):
-        # If we are auditing a partition, then one side will
-        #   have quorum and the other not.
-        # So the caller needs to tell us which we are checking
-        # If no value for node_list is specified... assume all nodes  
-        if not node_list:
-            node_list = self.Env["nodes"]
-
-        for node in node_list:
-            if self.ShouldBeStatus[node] == "up":
-                (quorum, qout) = self.rsh(node, self["QuorumCmd"], stdout=2)
-                if quorum == 1:
-                    return 1
-                elif quorum == 0:
-                    return 0
-                else:
-                    self.log("WARN: Unexpected quorum test result from %s : %d" % (node, quorum))
-
-        return 0
-
-    def Components(self):    
-        return None
-
-class ShmLeakAudit(ClusterAudit):
-
-    def __init__(self, cm):
-        self.CM = cm
-
-    def name(self):
-        return "ShmLeakAudit"
-
-    def is_applicable(self):
-        return 1
-
-    def __call__(self):
-        rc = 1
-
-        for node in self.CM.Env["nodes"]:
-            (res, lines) = self.CM.rsh(node, "/usr/share/corosync/tests/shm_leak_audit.sh", None)
-            for line in lines:
-                self.CM.log("%s leaked %s" % (node, line))
-                rc = 0
-
-        return rc
-
-
-###################################################################
-class TestAgentComponent(ScenarioComponent):
-    def __init__(self, Env):
-        self.Env = Env
-
-    def IsApplicable(self):
-        '''Return TRUE if the current ScenarioComponent is applicable
-        in the given LabEnvironment given to the constructor.
-        '''
-        return True
-
-    def SetUp(self, CM):
-        '''Set up the given ScenarioComponent'''
-        self.CM = CM
-        
-        for node in self.Env["nodes"]:
-            if not CM.StataCM(node):
-                raise RuntimeError ("corosync not up")
-
-            if self.CM.start_cpg:
-                if node in self.CM.cpg_agent:
-                    self.CM.cpg_agent[node].clean_start()
-                else:
-                    self.CM.cpg_agent[node] = CpgTestAgent(node, CM.Env)
-                    self.CM.cpg_agent[node].start()
-            if node in self.CM.sam_agent:
-                self.CM.sam_agent[node].clean_start()
-            else:
-                self.CM.sam_agent[node] = SamTestAgent(node, CM.Env)
-                self.CM.sam_agent[node].start()
-            # votequorum agent started as needed.
-            if 'quorum/provider' in self.CM.applied_config:
-                if CM.applied_config['quorum/provider'] is 'corosync_votequorum':
-                    self.CM.votequorum_agent[node] = VoteQuorumTestAgent(node, CM.Env)
-                    self.CM.votequorum_agent[node].start()
-        return 1
-
-    def TearDown(self, CM):
-        '''Tear down (undo) the given ScenarioComponent'''
-        self.CM = CM
-        for node in self.Env["nodes"]:
-            if node in self.CM.cpg_agent:
-                self.CM.cpg_agent[node].stop()
-            self.CM.sam_agent[node].stop()
-            if node in self.CM.votequorum_agent:
-                self.CM.votequorum_agent[node].stop()
-
-###################################################################
-class TestAgent(object):
-
-    def __init__(self, binary, node, port, Env=None):
-        self.node = node
-        self.node_address = None
-        self.port = port
-        self.sock = None
-        self.binary = binary
-        self.started = False
-        resh = RemoteFactory.rsh
-        self.rsh = RemoteExec(resh)
-        self.__name__ = None
-        self.used = False
-        self.env = Env
-        self.send_recv = False
-
-    def restart(self):
-        LogFactory().debug('%s:%s restarting' % (self.node, self.binary))
-        self.stop()
-        self.start()
-
-    def clean_start(self):
-        if self.used or not self.status():
-            LogFactory().debug('%s:%s cleaning' % (self.node, self.binary))
-            self.stop()
-            self.start()
-
-    def status(self):
-        if not self.started:
-            return False
-
-        try:
-            self.send_internal(["are_you_ok_dude"])
-            self.read()
-            self.started = True
-            return True
-        except RuntimeError as msg:
-            self.started = False
-            return False
-
-    def start(self):
-        '''Set up the given ScenarioComponent'''
-        if self.status():
-            return
-        LogFactory().debug('%s:%s starting ' % (self.node, self.binary))
-
-        self.rsh(self.node, self.binary, synchronous=False)
-        self.sock = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
-        ip = socket.gethostbyname(self.node)
-        is_connected = False
-        retries = 0
-        while not is_connected:
-            try:
-                retries = retries + 1
-                self.sock.connect ((ip, self.port))
-                is_connected = True
-            except socket.error as msg:
-                if retries > 10:
-                    LogFactory().log("%s:%s Tried connecting %d times. %s" % (self.node, self.binary, retries, str(msg)))
-                if retries > 30:
-                    raise RuntimeError("%s:%s can't connect" % (self.node, self.binary))
-                time.sleep(1)
-        self.started = True
-        self.used = False
-
-    def stop(self):
-        '''Tear down (undo) the given ScenarioComponent'''
-        LogFactory().debug('%s:%s stopping' % (self.binary, self.node))
-        self.rsh(self.node, "killall " + self.binary + " 2>/dev/null")
-        if self.sock:
-            self.sock.close ()
-            del self.sock
-            self.sock = None
-        while self.getpid() != '':
-            time.sleep(1)
-        self.started = False
-
-    def kill(self):
-        '''Tear down (undo) the given ScenarioComponent'''
-        LogFactory().log('%s:%s killing' % (self.node, self.binary))
-        self.rsh(self.node, "killall -9 " + self.binary + " 2>/dev/null")
-        self.started = False
-
-    def getpid(self):
-        return self.rsh(self.node, 'pidof ' + self.binary, 1)
-
-    def send_internal(self, args):
-
-        real_msg = str (len (args))
-        for a in args:
-            a_str = str(a)
-            real_msg += ":" + str (len (a_str)) + ":" + a_str
-        real_msg += ";"
-        if sys.version_info > (3,):
-            real_msg = real_msg.encode("utf8")
-        try:
-            return self.sock.send (real_msg)
-        except socket.error as msg:
-            LogFactory().log("send(%s): %s; error: %s" % (self.node, real_msg, msg))
-            return 0
-
-    def send (self, args):
-        if not self.started:
-            self.start()
-
-        sent = self.send_internal(args)
-
-        if sent == 0:
-            raise RuntimeError ("socket connection broken")
-        self.used = True
-
-    def __getattribute__(self,name):
-
-        try:
-            return object.__getattribute__(self, name)
-        except:
-            self.__name__ = name
-            if self.send_recv:
-                return self.send_recv_dynamic
-            else:
-                return self.send_dynamic
-
-    def send_recv_dynamic (self, *args):
-        self.send_dynamic (args)
-
-        try:
-            res = self.read ()
-        except RuntimeError as msg:
-            res = None
-            LogFactory().log("send_recv_dynamic: %s(); error: %s" % (self.__name__, msg))
-
-        return res
-
-
-    def send_dynamic (self, *args):
-        if not self.started:
-            raise RuntimeError ("agent not started")
-
-        # number of args+func
-        real_msg = str (len (args) + 1) + ":" + str(len(self.__name__)) + ":" + self.__name__
-        for a in args:
-            a_str = str(a)
-            real_msg += ":" + str (len (a_str)) + ":" + a_str
-        real_msg += ";"
-        sent = 0
-        if sys.version_info > (3,):
-            real_msg = bytes(real_msg, encoding = "utf8")
-        try:
-            sent = self.sock.send (real_msg)
-        except socket.error as msg:
-            LogFactory().log("send_dynamic(%s): %s; error: %s" % (self.node, real_msg, msg))
-
-        if sent == 0:
-            raise RuntimeError ("socket connection broken")
-        self.used = True
-
-    def read (self):
-
-        try:
-            msg = self.sock.recv (4096)
-        except socket.error as msg:
-            raise RuntimeError(msg)
-        if sys.version_info > (3,):
-            msg = msg.decode("utf8")
-
-        if msg == '':
-            raise RuntimeError("socket connection broken")
-        return msg
-
-
-class CpgConfigEvent(object):
-    def __init__(self, msg):
-        info = msg.split(',')
-        self.group_name = info[0]
-        self.node_id = info[1]
-        self.node = None
-        self.pid = info[2]
-        if "left" in info[3]:
-            self.is_member = False
-        else:
-            self.is_member = True
-
-    def __str__ (self):
-
-        str = self.group_name + "," + self.node_id + "," + self.pid + ","
-        if self.is_member:
-            return str + "joined"
-        else:
-            return str + "left"
-
-###################################################################
-class CpgTestAgent(TestAgent):
-
-    def __init__(self, node, Env=None):
-        TestAgent.__init__(self, "cpg_test_agent", node, 9034, Env)
-        self.nodeid = None
-
-    def start(self):
-        if not self.status():
-            TestAgent.start(self)
-            self.cpg_initialize()
-            self.used = False
-
-    def cpg_local_get(self):
-        if self.nodeid == None:
-            self.send (["cpg_local_get"])
-            self.nodeid = self.read ()
-        return self.nodeid
-
-    def record_config_events(self, truncate=True):
-        if truncate:
-            self.send (["record_config_events", "truncate"])
-        else:
-            self.send (["record_config_events", "append"])
-        return self.read ()
-
-    def read_config_event(self):
-        self.send (["read_config_event"])
-        msg = self.read ()
-
-        if "None" in msg:
-            return None
-        else:
-            return CpgConfigEvent(msg)
-
-    def read_messages(self, atmost):
-        self.send (["read_messages", atmost])
-        msg = self.read ()
-
-        if "None" in msg:
-            return None
-        else:
-            return msg
-
-    def context_test(self):
-        self.send (["context_test"])
-        return self.read ()
-
-###################################################################
-class SamTestAgent(TestAgent):
-
-    def __init__(self, node, Env=None):
-        TestAgent.__init__(self, "sam_test_agent", node, 9036, Env)
-        self.nodeid = None
-        self.send_recv = True
-
-###################################################################
-class VoteQuorumTestAgent(TestAgent):
-
-    def __init__(self, node, Env=None):
-        TestAgent.__init__(self, "votequorum_test_agent", node, 9037, Env)
-        self.nodeid = None
-        self.send_recv = True
-
-AllAuditClasses = []
-AllAuditClasses.append(LogAudit)
-AllAuditClasses.append(ShmLeakAudit)
-
-def CoroAuditList(cm):
-    result = []
-    for auditclass in AllAuditClasses:
-        a = auditclass(cm)
-        if a.is_applicable():
-            result.append(a)
-    return result
-
-
-

cts/corotests.py

@@ -1,1624 +0,0 @@
-from __future__ import division
-from __future__ import print_function
-
-__copyright__='''
-Copyright (c) 2010 Red Hat, Inc.
-'''
-
-# All rights reserved.
-#
-# Author: Angus Salkeld <asalkeld@redhat.com>
-#
-# This software licensed under BSD license, the text of which follows:
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# - Redistributions of source code must retain the above copyright notice,
-#   this list of conditions and the following disclaimer.
-# - Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-# - Neither the name of the MontaVista Software, Inc. nor the names of its
-#   contributors may be used to endorse or promote products derived from this
-#   software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-# THE POSSIBILITY OF SUCH DAMAGE.
-
-import random
-import socket
-import sys
-if sys.version_info < (3,):
-    from UserDict import UserDict
-else:
-    from collections import UserDict
-from cts.CTStests import *
-from corosync import CpgTestAgent
-
-###################################################################
-class CoroTest(CTSTest):
-    '''
-    basic class to make sure that new configuration is applied
-    and old configuration is removed.
-    '''
-    def __init__(self, cm):
-        CTSTest.__init__(self,cm)
-        self.start = StartTest(cm)
-        self.stop = StopTest(cm)
-        self.config = {}
-        self.config['logging/logger_subsys[1]/subsys'] = 'MAIN'
-        self.config['logging/logger_subsys[1]/debug'] = 'on'
-        self.need_all_up = True
-        self.CM.start_cpg = True
-        self.cpg_name = 'cts_group'
-
-    def setup(self, node):
-        ret = CTSTest.setup(self, node)
-
-        # setup the authkey
-        localauthkey = '/tmp/authkey'
-        if not os.path.exists(localauthkey):
-            self.CM.rsh(node, 'corosync-keygen -l')
-            self.CM.rsh.cp("%s:%s" % (node, "/etc/corosync/authkey"), localauthkey)
-
-        for n in self.CM.Env["nodes"]:
-            if n is not node:
-                #copy key onto other nodes
-                self.CM.rsh.cp(localauthkey, "%s:%s" % (n, "/etc/corosync/authkey"))
-
-        # copy over any new config
-        for c in self.config:
-            self.CM.new_config[c] = self.config[c]
-
-        # apply the config
-        self.CM.apply_new_config(self.need_all_up)
-
-        # start/stop all corosyncs'
-        for n in self.CM.Env["nodes"]:
-            if self.need_all_up and not self.CM.StataCM(n):
-                self.incr("started")
-                self.start(n)
-            if self.need_all_up and self.CM.start_cpg:
-                self.CM.cpg_agent[n].clean_start()
-                self.CM.cpg_agent[n].cpg_join(self.cpg_name)
-                self.CM.cpg_agent[n].cfg_initialize()
-            if not self.need_all_up and self.CM.StataCM(n):
-                self.incr("stopped")
-                self.stop(n)
-        return ret
-
-    def config_valid(self, config):
-        return True
-
-    def teardown(self, node):
-        self.CM.apply_default_config()
-        return CTSTest.teardown(self, node)
-
-###################################################################
-class CpgContextTest(CoroTest):
-    def __init__(self, cm):
-        self.name="CpgContextTest"
-        CoroTest.__init__(self, cm)
-        self.CM.start_cpg = True
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.cpg_agent[node].context_test()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure('context_test failed')
-
-###################################################################
-class CpgConfigChangeBase(CoroTest):
-    '''
-    join a cpg group on each node, and test that the following
-    causes a leave event:
-    - a call to cpg_leave()
-    - app exit
-    - node leave
-    - node leave (with large token timeout)
-    '''
-
-    def setup(self, node):
-        ret = CoroTest.setup(self, node)
-
-        self.listener = None
-        self.wobbly = None
-        for n in self.CM.Env["nodes"]:
-            if self.wobbly is None:
-                self.wobbly = n
-            elif self.listener is None:
-                self.listener = n
-
-        if self.wobbly in self.CM.cpg_agent:
-            self.wobbly_id = self.CM.cpg_agent[self.wobbly].cpg_local_get()
-        if self.listener in self.CM.cpg_agent:
-            self.CM.cpg_agent[self.listener].record_config_events(truncate=True)
-
-        return ret
-
-    def wait_for_config_change(self):
-        found = False
-        max_timeout = 60 * 15
-        waited = 0
-        printit = 0
-        self.CM.log("Waiting for config change on " + self.listener)
-        while not found:
-            try:
-                event = self.CM.cpg_agent[self.listener].read_config_event()
-            except:
-                return self.failure('connection to test cpg_agent failed.')
-            if not event == None:
-                self.CM.debug("RECEIVED: " + str(event))
-            if event == None:
-                if waited >= max_timeout:
-                    return self.failure("timedout(" + str(waited) + " sec) == no event!")
-                else:
-                    time.sleep(1)
-                    waited = waited + 1
-                    printit = printit + 1
-                    if printit is 60:
-                        print('waited ' + str(waited) + ' seconds')
-                        printit = 0
-
-            elif str(event.node_id) in str(self.wobbly_id) and not event.is_member:
-                self.CM.log("Got the config change in " + str(waited) + " seconds")
-                found = True
-            else:
-                self.CM.debug("No match")
-                self.CM.debug("wobbly nodeid:" + str(self.wobbly_id))
-                self.CM.debug("event nodeid:" + str(event.node_id))
-                self.CM.debug("event.is_member:" + str(event.is_member))
-
-        if found:
-            return self.success()
-
-
-###################################################################
-class CpgCfgChgOnGroupLeave(CpgConfigChangeBase):
-
-    def __init__(self, cm):
-        CpgConfigChangeBase.__init__(self,cm)
-        self.name="CpgCfgChgOnGroupLeave"
-
-    def failure_action(self):
-        self.CM.log("calling cpg_leave() on " + self.wobbly)
-        self.CM.cpg_agent[self.wobbly].cpg_leave(self.cpg_name)
-
-    def __call__(self, node):
-        self.incr("calls")
-        self.failure_action()
-        return self.wait_for_config_change()
-
-###################################################################
-class CpgCfgChgOnNodeLeave(CpgConfigChangeBase):
-
-    def __init__(self, cm):
-        CpgConfigChangeBase.__init__(self,cm)
-        self.name="CpgCfgChgOnNodeLeave"
-
-    def failure_action(self):
-        self.CM.log("stopping corosync on " + self.wobbly)
-        self.stop(self.wobbly)
-
-    def __call__(self, node):
-        self.incr("calls")
-        self.failure_action()
-        return self.wait_for_config_change()
-
-###################################################################
-class CpgCfgChgOnLowestNodeJoin(CTSTest):
-    '''
-    1) stop all nodes
-    2) start all but the node with the smallest ip address
-    3) start recording events
-    4) start the last node
-    '''
-    def __init__(self, cm):
-        CTSTest.__init__(self, cm)
-        self.name="CpgCfgChgOnLowestNodeJoin"
-        self.start = StartTest(cm)
-        self.stop = StopTest(cm)
-        self.config = {}
-        self.need_all_up = False
-
-    def config_valid(self, config):
-        return True
-
-    def lowest_ip_set(self):
-        self.lowest = None
-        for n in self.CM.Env["nodes"]:
-            if self.lowest is None:
-                self.lowest = n
-
-        self.CM.log("lowest node is " + self.lowest)
-
-    def setup(self, node):
-        # stop all nodes
-        for n in self.CM.Env["nodes"]:
-            self.CM.StopaCM(n)
-
-        self.lowest_ip_set()
-
-        # copy over any new config
-        for c in self.config:
-            self.CM.new_config[c] = self.config[c]
-
-        # install the config
-        self.CM.install_all_config()
-
-        # start all but lowest
-        self.listener = None
-        for n in self.CM.Env["nodes"]:
-            if n is not self.lowest:
-                if self.listener is None:
-                    self.listener = n
-                self.incr("started")
-                self.CM.log("starting " + n)
-                self.start(n)
-                self.CM.cpg_agent[n].clean_start()
-                self.CM.cpg_agent[n].cpg_join(self.cpg_name)
-
-        # start recording events
-        pats = []
-        pats.append("%s .*sync: node joined.*" % self.listener)
-        pats.append("%s .*sync: activate correctly.*" % self.listener)
-        self.sync_log = self.create_watch(pats, 60)
-        self.sync_log.setwatch()
-
-        self.CM.log("setup done")
-
-        return CTSTest.setup(self, node)
-
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        self.start(self.lowest)
-        self.CM.cpg_agent[self.lowest].clean_start()
-        self.CM.cpg_agent[self.lowest].cpg_join(self.cpg_name)
-        self.wobbly_id = self.CM.cpg_agent[self.lowest].cpg_local_get()
-
-        self.CM.log("waiting for sync events")
-        if not self.sync_log.lookforall():
-            return self.failure("Patterns not found: " + repr(self.sync_log.unmatched))
-        else:
-            return self.success()
-
-
-###################################################################
-class CpgCfgChgOnExecCrash(CpgConfigChangeBase):
-
-    def __init__(self, cm):
-        CpgConfigChangeBase.__init__(self,cm)
-        self.name="CpgCfgChgOnExecCrash"
-
-    def failure_action(self):
-        self.CM.log("sending KILL to corosync on " + self.wobbly)
-        self.CM.rsh(self.wobbly, "killall -9 corosync")
-        self.CM.rsh(self.wobbly, "rm -f /var/run/corosync.pid")
-        self.CM.rsh(self.wobbly, "rm -f /dev/shm/qb-corosync-blackbox*")
-        self.CM.ShouldBeStatus[self.wobbly] = "down"
-
-    def __call__(self, node):
-        self.incr("calls")
-        self.failure_action()
-        return self.wait_for_config_change()
-
-
-###################################################################
-class CpgCfgChgOnNodeIsolate(CpgConfigChangeBase):
-
-    def __init__(self, cm):
-        CpgConfigChangeBase.__init__(self,cm)
-        self.name="CpgCfgChgOnNodeIsolate"
-
-    def config_valid(self, config):
-        if 'totem/rrp_mode' in config:
-            return False
-        else:
-            return True
-
-    def failure_action(self):
-        self.CM.log("isolating node " + self.wobbly)
-        self.CM.isolate_node(self.wobbly)
-
-    def __call__(self, node):
-        self.incr("calls")
-        self.failure_action()
-        return self.wait_for_config_change()
-
-    def teardown(self, node):
-        self.CM.unisolate_node (self.wobbly)
-        return CpgConfigChangeBase.teardown(self, node)
-
-###################################################################
-class CpgCfgChgOnNodeRestart(CpgConfigChangeBase):
-
-    def __init__(self, cm):
-        CpgConfigChangeBase.__init__(self,cm)
-        self.name="CpgCfgChgOnNodeRestart"
-        self.CM.start_cpg = False
-
-    def config_valid(self, config):
-        if 'totem/secauth' in config:
-            if config['totem/secauth'] is 'on':
-                return False
-            else:
-                return True
-        if 'totem/rrp_mode' in config:
-            return False
-        else:
-            return True
-
-    def failure_action(self):
-        self.CM.log("2: isolating node " + self.wobbly)
-        self.CM.isolate_node(self.wobbly)
-        self.CM.log("3: Killing corosync on " + self.wobbly)
-        self.CM.rsh(self.wobbly, "killall -9 corosync")
-        self.CM.rsh(self.wobbly, "rm -f /var/run/corosync.pid")
-        self.CM.ShouldBeStatus[self.wobbly] = "down"
-        self.CM.log("4: unisolating node " + self.wobbly)
-        self.CM.unisolate_node (self.wobbly)
-        self.CM.log("5: starting corosync on " + self.wobbly)
-        self.CM.StartaCM(self.wobbly)
-        time.sleep(5)
-        self.CM.log("6: starting cpg on all nodes")
-        self.CM.start_cpg = True
-        for node in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[node] = CpgTestAgent(node, self.CM.Env)
-            self.CM.cpg_agent[node].start()
-            self.CM.cpg_agent[node].cpg_join(self.cpg_name)
-
-        self.wobbly_id = self.CM.cpg_agent[self.wobbly].cpg_local_get()
-        self.CM.cpg_agent[self.listener].record_config_events(truncate=True)
-
-        self.CM.log("7: isolating node " + self.wobbly)
-        self.CM.isolate_node(self.wobbly)
-        self.CM.log("8: Killing corosync on " + self.wobbly)
-        self.CM.rsh(self.wobbly, "killall -9 corosync")
-        self.CM.rsh(self.wobbly, "rm -f /var/run/corosync.pid")
-        self.CM.ShouldBeStatus[self.wobbly] = "down"
-        self.CM.log("9: unisolating node " + self.wobbly)
-        self.CM.unisolate_node (self.wobbly)
-        self.CM.log("10: starting corosync on " + self.wobbly)
-        self.CM.StartaCM(self.wobbly)
-
-    def __call__(self, node):
-        self.incr("calls")
-        self.failure_action()
-        return self.wait_for_config_change()
-
-    def teardown(self, node):
-        self.CM.unisolate_node (self.wobbly)
-        return CpgConfigChangeBase.teardown(self, node)
-
-###################################################################
-class CpgMsgOrderBase(CoroTest):
-
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.num_msgs_per_node = 0
-        self.total_num_msgs = 0
-
-    def setup(self, node):
-        ret = CoroTest.setup(self, node)
-
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].clean_start()
-            self.CM.cpg_agent[n].cpg_join(self.cpg_name)
-            self.CM.cpg_agent[n].record_messages()
-
-        time.sleep(1)
-        return ret
-
-    def cpg_msg_blaster(self):
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].msg_blaster(self.num_msgs_per_node)
-
-    def wait_and_validate_order(self):
-        msgs = {}
-
-        self.total_num_msgs = 0
-        for n in self.CM.Env["nodes"]:
-            self.total_num_msgs = self.total_num_msgs + self.num_msgs_per_node
-
-        for n in self.CM.Env["nodes"]:
-            msgs[n] = []
-            stopped = False
-            waited = 0
-
-            while len(msgs[n]) < self.total_num_msgs and waited < 360:
-
-                try:
-                    msg = self.CM.cpg_agent[n].read_messages(50)
-                except:
-                    return self.failure('connection to test cpg_agent failed.')
-
-                if not msg == None:
-                    msgl = msg.split(";")
-
-                    # remove empty entries
-                    not_done=True
-                    while not_done:
-                        try:
-                            msgl.remove('')
-                        except:
-                            not_done = False
-
-                    msgs[n].extend(msgl)
-                elif msg == None:
-                    time.sleep(2)
-                    waited = waited + 2
-
-            if len(msgs[n]) < self.total_num_msgs:
-                return self.failure("expected %d messages from %s got %d" % (self.total_num_msgs, n, len(msgs[n])))
-
-        fail = False
-        error_message = ''
-        for i in range(0, self.total_num_msgs):
-            first = None
-            for n in self.CM.Env["nodes"]:
-                # first test for errors
-                params = msgs[n][i].split(":")
-                if not 'OK' in params[3]:
-                    fail = True
-                    error_message = 'error: ' + params[3] + ' in received message'
-                    self.CM.log(str(params))
-
-                # then look for out of order messages
-                if first == None:
-                    first = n
-                else:
-                    if not msgs[first][i] == msgs[n][i]:
-                        # message order not the same!
-                        fail = True
-                        error_message = 'message out of order'
-                        self.CM.log(msgs[first][i] + " != " + msgs[n][i])
-
-        if fail:
-            return self.failure(error_message)
-        else:
-            return self.success()
-
-###################################################################
-class CpgMsgOrderBasic(CpgMsgOrderBase):
-    '''
-    each sends & logs lots of messages
-    '''
-    def __init__(self, cm):
-        CpgMsgOrderBase.__init__(self,cm)
-        self.name="CpgMsgOrderBasic"
-        self.num_msgs_per_node = 9000
-
-    def __call__(self, node):
-        self.incr("calls")
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].msg_blaster(self.num_msgs_per_node)
-
-        return self.wait_and_validate_order()
-
-###################################################################
-class CpgMsgOrderZcb(CpgMsgOrderBase):
-    '''
-    each sends & logs lots of messages
-    '''
-    def __init__(self, cm):
-        CpgMsgOrderBase.__init__(self,cm)
-        self.name="CpgMsgOrderZcb"
-        self.num_msgs_per_node = 9000
-
-    def __call__(self, node):
-        self.incr("calls")
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].msg_blaster_zcb(self.num_msgs_per_node)
-        return self.wait_and_validate_order()
-
-###################################################################
-class MemLeakObject(CoroTest):
-    '''
-    run mem_leak_test.sh -1
-    '''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="MemLeakObject"
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        mem_leaked = self.CM.rsh(node, "/usr/share/corosync/tests/mem_leak_test.sh -1")
-        if mem_leaked is 0:
-            return self.success()
-        else:
-            return self.failure(str(mem_leaked) + 'kB memory leaked.')
-
-###################################################################
-class MemLeakSession(CoroTest):
-    '''
-    run mem_leak_test.sh -2
-    '''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="MemLeakSession"
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        mem_leaked = self.CM.rsh(node, "/usr/share/corosync/tests/mem_leak_test.sh -2")
-        if mem_leaked is 0:
-            return self.success()
-        else:
-            return self.failure(str(mem_leaked) + 'kB memory leaked.')
-
-###################################################################
-class CMapDispatchDeadlock(CoroTest):
-    '''
-    run cmap-dispatch-deadlock.sh
-    '''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="CMapDispatchDeadlock"
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        result = self.CM.rsh(node, "/usr/share/corosync/tests/cmap-dispatch-deadlock.sh")
-        if result is 0:
-            return self.success()
-        else:
-            return self.failure('Deadlock detected')
-
-###################################################################
-class SamTest1(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest1"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test1()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-###################################################################
-class SamTest2(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest2"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test2()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-###################################################################
-class SamTest4(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest4"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test4()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-###################################################################
-class SamTest5(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest5"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test5()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-###################################################################
-class SamTest6(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest6"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test6()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-###################################################################
-class SamTest8(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest8"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test8()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-###################################################################
-class SamTest9(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="SamTest9"
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.sam_agent[node].test9()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure(self.name + ' failed')
-
-class QuorumState(object):
-    def __init__(self, cm, node):
-        self.node = node
-        self.CM = cm
-        self.CM.votequorum_agent[self.node].init()
-
-    def refresh(self):
-        info = self.CM.votequorum_agent[self.node].votequorum_getinfo()
-        assert(info != 'FAIL')
-        assert(info != 'NOT_SUPPORTED')
-
-        #self.CM.log('refresh: ' + info)
-        if info is None:
-            return
-        params = info.split(':')
-
-        self.node_votes = int(params[0])
-        self.expected_votes = int(params[1])
-        self.highest_expected = int(params[2])
-        self.total_votes = int(params[3])
-        self.quorum = int(params[4])
-        self.quorate = self.CM.votequorum_agent[self.node].quorum_getquorate()
-        assert(self.quorate != 'FAIL')
-        assert(self.quorate != 'NOT_SUPPORTED')
-        #self.CM.log('quorate: ' + str(self.quorate))
-
-###################################################################
-class VoteQuorumBase(CoroTest):
-
-    def setup(self, node):
-        ret = CoroTest.setup(self, node)
-        self.listener = None
-        for n in self.CM.Env["nodes"]:
-            if self.listener is None:
-                self.listener = n
-
-        return ret
-
-    def config_valid(self, config):
-        if 'totem/rrp_mode' in config:
-            return False
-        if 'quorum/provider' in config:
-            return False
-        return True
-
-
-###################################################################
-class VoteQuorumGoDown(VoteQuorumBase):
-# all up
-# calc min expected votes to get Q
-# bring nodes down one-by-one
-# confirm cluster looses Q when V < EV
-#
-
-    def __init__(self, cm):
-        VoteQuorumBase.__init__(self, cm)
-        self.name="VoteQuorumGoDown"
-        self.victims = []
-        self.expected = len(self.CM.Env["nodes"])
-        self.config['quorum/provider'] = 'corosync_votequorum'
-        self.config['quorum/expected_votes'] = self.expected
-        #self.CM.log('set expected to %d' % (self.expected))
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        self.victims = []
-        pats = []
-        pats.append("%s .*VQ notification quorate: 0" % self.listener)
-        pats.append("%s .*NQ notification quorate: 0" % self.listener)
-        quorum = self.create_watch(pats, 30)
-        quorum.setwatch()
-
-        state = QuorumState(self.CM, self.listener)
-        state.refresh()
-        for n in self.CM.Env["nodes"]:
-            if n is self.listener:
-                continue
-
-            self.victims.append(n)
-            self.CM.StopaCM(n)
-
-            #if not self.wait_for_quorum_change():
-            #    return self.failure(self.error_message)
-            nodes_alive = len(self.CM.Env["nodes"]) - len(self.victims)
-            state.refresh()
-            #self.expected = self.expected - 1
-
-            if state.node_votes != 1:
-                self.failure('unexpected number of node_votes')
-
-            if state.expected_votes != self.expected:
-                self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected))
-                self.failure('unexpected number of expected_votes')
-
-            if state.total_votes != nodes_alive:
-                self.failure('unexpected number of total votes:%d, nodes_alive:%d' % (state.total_votes, nodes_alive))
-
-            min = int((len(self.CM.Env["nodes"]) + 2) / 2)
-            if min != state.quorum:
-                self.failure('we should have %d (not %d) as quorum' % (min, state.quorum))
-
-            if nodes_alive < state.quorum:
-                if state.quorate == 1:
-                    self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate, state.quorum, nodes_alive))
-            else:
-                if state.quorate == 0:
-                    self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive))
-
-        if not quorum.lookforall():
-            self.CM.log("Patterns not found: " + repr(quorum.unmatched))
-            return self.failure('quorm event not found')
-
-        return self.success()
-
-###################################################################
-class VoteQuorumGoUp(VoteQuorumBase):
-# all down
-# calc min expected votes to get Q
-# bring nodes up one-by-one
-# confirm cluster gains Q when V >= EV
-
-    def __init__(self, cm):
-        VoteQuorumBase.__init__(self, cm)
-        self.name="VoteQuorumGoUp"
-        self.need_all_up = False
-        self.expected = len(self.CM.Env["nodes"])
-        self.config['quorum/provider'] = 'corosync_votequorum'
-        self.config['quorum/expected_votes'] = self.expected
-        #self.CM.log('set expected to %d' % (self.expected))
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        pats = []
-        pats.append("%s .*VQ notification quorate: 1" % self.listener)
-        pats.append("%s .*NQ notification quorate: 1" % self.listener)
-        quorum = self.create_watch(pats, 30)
-        quorum.setwatch()
-
-        self.CM.StartaCM(self.listener)
-        nodes_alive = 1
-        state = QuorumState(self.CM, self.listener)
-        state.refresh()
-
-        for n in self.CM.Env["nodes"]:
-            if n is self.listener:
-                continue
-
-            #if not self.wait_for_quorum_change():
-            #    return self.failure(self.error_message)
-
-            if state.node_votes != 1:
-                self.failure('unexpected number of node_votes')
-
-            if state.expected_votes != self.expected:
-                self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected))
-                self.failure('unexpected number of expected_votes')
-
-            if state.total_votes != nodes_alive:
-                self.failure('unexpected number of total votes')
-
-            min = ((len(self.CM.Env["nodes"]) + 2) / 2)
-            if min != state.quorum:
-                self.failure('we should have %d (not %d) as quorum' % (min, state.quorum))
-
-            if nodes_alive < state.quorum:
-                if state.quorate == 1:
-                    self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate, state.quorum, nodes_alive))
-            else:
-                if state.quorate == 0:
-                    self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive))
-
-            self.CM.StartaCM(n)
-            nodes_alive = nodes_alive + 1
-            state.refresh()
-
-        if not quorum.lookforall():
-            self.CM.log("Patterns not found: " + repr(quorum.unmatched))
-            return self.failure('quorm event not found')
-
-        return self.success()
-
-###################################################################
-class VoteQuorumWaitForAll(VoteQuorumBase):
-# all down
-# bring nodes up one-by-one
-# confirm cluster gains Q when V == num nodes
-
-    def __init__(self, cm):
-        VoteQuorumBase.__init__(self, cm)
-        self.name="VoteQuorumWaitForAll"
-        self.need_all_up = False
-        self.expected = len(self.CM.Env["nodes"])
-        self.config['quorum/provider'] = 'corosync_votequorum'
-        self.config['quorum/expected_votes'] = self.expected
-        self.config['quorum/wait_for_all'] = '1'
-
-    def __call__(self, node):
-        self.incr("calls")
-
-        pats = []
-        pats.append("%s .*VQ notification quorate: 1" % self.listener)
-        pats.append("%s .*NQ notification quorate: 1" % self.listener)
-        quorum = self.create_watch(pats, 30)
-        quorum.setwatch()
-
-        # make absolutly all are stopped
-        for n in self.CM.Env["nodes"]:
-            self.CM.StopaCM(n)
-
-        # start the listener
-        self.CM.StartaCM(self.listener)
-        nodes_alive = 1
-        state = QuorumState(self.CM, self.listener)
-        state.refresh()
-
-        for n in self.CM.Env["nodes"]:
-            if n is self.listener:
-                continue
-
-            self.CM.StartaCM(n)
-            nodes_alive = nodes_alive + 1
-            state.refresh()
-
-            if state.node_votes != 1:
-                self.failure('unexpected number of node_votes')
-
-            if state.expected_votes != self.expected:
-                self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected))
-                self.failure('unexpected number of expected_votes')
-
-            if state.total_votes != nodes_alive:
-                self.failure('unexpected number of total votes')
-
-            if nodes_alive < len(self.CM.Env["nodes"]):
-                if state.quorate == 1:
-                    self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate,
-                        len(self.CM.Env["nodes"]), nodes_alive))
-            else:
-                if state.quorate == 0:
-                    self.failure('we should have quorum(%d) %d <= %d' % (state.quorate,
-                        len(self.CM.Env["nodes"]), nodes_alive))
-
-        if not quorum.lookforall():
-            self.CM.log("Patterns not found: " + repr(quorum.unmatched))
-            return self.failure('quorm event not found')
-
-        return self.success()
-
-###################################################################
-class VoteQuorumContextTest(CoroTest):
-
-    def __init__(self, cm):
-        CoroTest.__init__(self, cm)
-        self.name="VoteQuorumContextTest"
-        self.expected = len(self.CM.Env["nodes"])
-        self.config['quorum/provider'] = 'corosync_votequorum'
-        self.config['quorum/expected_votes'] = self.expected
-
-    def __call__(self, node):
-        self.incr("calls")
-        res = self.CM.votequorum_agent[node].context_test()
-        if 'OK' in res:
-            return self.success()
-        else:
-            return self.failure('context_test failed')
-
-
-###################################################################
-class GenSimulStart(CoroTest):
-    '''Start all the nodes ~ simultaneously'''
-
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenSimulStart"
-        self.need_all_up = False
-        self.stopall = SimulStopLite(cm)
-        self.startall = SimulStartLite(cm)
-
-    def __call__(self, dummy):
-        '''Perform the 'SimulStart' test. '''
-        self.incr("calls")
-
-        #        We ignore the "node" parameter...
-
-        #        Shut down all the nodes...
-        ret = self.stopall(None)
-        if not ret:
-            return self.failure("Setup failed")
-        #clear_all_caches was removed
-        #self.CM.clear_all_caches()
-
-        if not self.startall(None):
-            return self.failure("Startall failed")
-
-        return self.success()
-
-###################################################################
-class GenSimulStop(CoroTest):
-    '''Stop all the nodes ~ simultaneously'''
-
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenSimulStop"
-        self.startall = SimulStartLite(cm)
-        self.stopall = SimulStopLite(cm)
-        self.need_all_up = True
-
-    def __call__(self, dummy):
-        '''Perform the 'GenSimulStop' test. '''
-        self.incr("calls")
-
-        #     We ignore the "node" parameter...
-
-        #     Start up all the nodes...
-        ret = self.startall(None)
-        if not ret:
-            return self.failure("Setup failed")
-
-        if not self.stopall(None):
-            return self.failure("Stopall failed")
-
-        return self.success()
-
-
-class GenFlipTest(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenFlipTest"
-        self.test = FlipTest(cm)
-
-    def __call__(self, dummy):
-        '''Perform the test. '''
-        self.incr("calls")
-        return self.test.__call__(dummy)
-
-class GenRestartTest(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenRestartTest"
-        self.test = RestartTest(cm)
-
-    def __call__(self, dummy):
-        '''Perform the test. '''
-        self.incr("calls")
-        return self.test.__call__(dummy)
-
-class GenStartOnebyOne(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenStartOnebyOne"
-        self.test = RestartOnebyOne(cm)
-
-    def __call__(self, dummy):
-        '''Perform the test. '''
-        self.incr("calls")
-        return self.test.__call__(dummy)
-
-class GenStopOnebyOne(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenStopOnebyOne"
-        self.test = StopOnebyOne(cm)
-
-    def __call__(self, dummy):
-        '''Perform the test. '''
-        self.incr("calls")
-        return self.test.__call__(dummy)
-
-class GenRestartOnebyOne(CoroTest):
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenRestartOnebyOne"
-        self.test = RestartOnebyOne(cm)
-
-    def __call__(self, dummy):
-        '''Perform the test. '''
-        self.incr("calls")
-        return self.test.__call__(dummy)
-
-
-
-###################################################################
-class GenStopAllBeekhof(CoroTest):
-    '''Stop all the nodes ~ simultaneously'''
-
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="GenStopAllBeekhof"
-        self.need_all_up = True
-        self.config['logging/logger_subsys[2]/subsys'] = 'CFG'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-
-    def __call__(self, node):
-        '''Perform the 'GenStopAllBeekhof' test. '''
-        self.incr("calls")
-
-        stopping = int(time.time())
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].pcmk_test()
-
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].msg_blaster(1000)
-
-        for n in self.CM.Env["nodes"]:
-            self.CM.cpg_agent[n].cfg_shutdown()
-            self.CM.ShouldBeStatus[n] = "down"
-
-        waited = 0
-        max_wait = 60 * 15
-
-        still_up = list(self.CM.Env["nodes"])
-        while len(still_up) > 0:
-            waited = int(time.time()) - stopping
-            self.CM.log("%s still up %s; waited %d secs" % (self.name, str(still_up), waited))
-            if waited > max_wait:
-                break
-            time.sleep(3)
-            for v in self.CM.Env["nodes"]:
-                if v in still_up:
-                    self.CM.ShouldBeStatus[n] = "down"
-                    if not self.CM.StataCM(v):
-                        still_up.remove(v)
-
-        waited = int(time.time()) - stopping
-        if waited > max_wait:
-            return self.failure("Waited %d secs for nodes: %s to stop" % (waited, str(still_up)))
-
-        self.CM.log("%s ALL good            (waited %d secs)" % (self.name, waited))
-        return self.success()
-
-###################################################################
-class NoWDConfig(CoroTest):
-    '''Assertion: no config == no watchdog
-Setup: no config, kmod inserted
-1] make sure watchdog is not enabled
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="NoWDConfig"
-        self.need_all_up = False
-
-    def config_valid(self, config):
-        return 'resources' not in config
-
-    def __call__(self, node):
-        '''Perform the 'NoWDConfig' test. '''
-        self.incr("calls")
-
-        self.CM.StopaCM(node)
-        pats = []
-        pats.append("%s .*no resources configured." % node)
-        w = self.create_watch(pats, 60)
-        w.setwatch()
-
-        self.CM.StartaCM(node)
-        if not w.lookforall():
-            return self.failure("Patterns not found: " + repr(w.unmatched))
-        else:
-            return self.success()
-
-###################################################################
-class WDConfigNoWd(CoroTest):
-    '''Assertion: watchdog config but no watchdog kmod will emit a log
-Setup: config watchdog, but no kmod
-1] look in the log for warning that there is no kmod
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="WDConfigNoWd"
-        self.need_all_up = False
-
-    def __call__(self, node):
-        '''Perform the 'WDConfigNoWd' test. '''
-        self.incr("calls")
-
-        self.CM.StopaCM(node)
-        self.CM.rsh(node, 'rmmod softdog')
-        pats = []
-        pats.append("%s .*No Watchdog, try modprobe.*" % node)
-        w = self.create_watch(pats, 60)
-        w.setwatch()
-
-        self.CM.StartaCM(node)
-        if not w.lookforall():
-            return self.failure("Patterns not found: " + repr(w.unmatched))
-        else:
-            return self.success()
-
-
-###################################################################
-class NoWDOnCorosyncStop(CoroTest):
-    '''Configure WD then /etc/init.d/corosync stop
-must stay up for > 60 secs
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="NoWDOnCorosyncStop"
-        self.need_all_up = False
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-
-        self.CM.StopaCM(node)
-        self.CM.rsh(node, 'modprobe softdog')
-        self.CM.StartaCM(node)
-        pats = []
-        pats.append("%s .*Unexpected close, not stopping watchdog.*" % node)
-        w = self.create_watch(pats, 60)
-        w.setwatch()
-        self.CM.StopaCM(node)
-
-        if w.lookforall():
-            return self.failure("Should have closed the WD better: " + repr(w.matched))
-        else:
-            return self.success()
-
-
-###################################################################
-class WDOnForkBomb(CoroTest):
-    '''Configure memory resource
-run memory leaker / forkbomb
-confirm watchdog action
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="WDOnForkBomb"
-        self.need_all_up = False
-        self.config['logging/logger_subsys[2]/subsys'] = 'WD'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-        self.config['resources/system/memory_used/recovery'] = 'watchdog'
-        self.config['resources/system/memory_used/max'] = '80'
-        self.config['resources/system/memory_used/poll_period'] = '800'
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-
-        # get the uptime
-        up_before = self.CM.rsh(node, 'cut -d. -f1 /proc/uptime', 1).rstrip()
-        self.CM.StopaCM(node)
-        self.CM.rsh(node, 'modprobe softdog')
-        self.CM.StartaCM(node)
-
-        self.CM.rsh(node, ':(){ :|:& };:', synchronous=0)
-
-        self.CM.log("wait for it to watchdog")
-        time.sleep(60 * 5)
-
-        ping_able = False
-        while not ping_able:
-            if self.CM.rsh("localhost", "ping -nq -c10 -w10 %s" % node) == 0:
-                ping_able = True
-                self.CM.log("can ping 10 in 10secs.")
-            else:
-                self.CM.log("not yet responding to pings.")
-
-        self.CM.ShouldBeStatus[node] = "down"
-        # wait for the node to come back up
-        self.CM.log("waiting for node to come back up.")
-        if self.CM.ns.WaitForNodeToComeUp(node):
-            up_after = self.CM.rsh(node, 'cut -d. -f1 /proc/uptime', 1).rstrip()
-            if int(up_after) < int(up_before):
-                return self.success()
-            else:
-                return self.failure("node didn't seem to watchdog uptime 1 %s; 2 %s" %(up_before, up_after))
-        else:
-            return self.failure("node didn't seem to come back up")
-
-
-###################################################################
-class SamWdIntegration1(CoroTest):
-    '''start sam hc
-kill agent
-confirm action
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="SamWdIntegration1"
-        self.need_all_up = True
-        self.config['logging/logger_subsys[2]/subsys'] = 'WD'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-        self.CM.sam_agent[node].setup_hc()
-        pids = self.CM.sam_agent[node].getpid().rstrip().split(" ")
-
-        pats = []
-        for pid in pids:
-            pats.append('%s .*resource "%s" failed!' % (node, pid))
-
-        w = self.create_watch(pats, 60)
-        w.setwatch()
-
-        self.CM.sam_agent[node].kill()
-
-        look_result = w.look()
-        if not look_result:
-            return self.failure("Patterns not found: " + repr(w.regexes))
-        else:
-            return self.success()
-
-###################################################################
-class SamWdIntegration2(CoroTest):
-    '''start sam hc
-call sam_stop()
-confirm resource "stopped" and no watchdog action.
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="SamWdIntegration2"
-        self.need_all_up = True
-        self.config['logging/logger_subsys[2]/subsys'] = 'WD'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-        self.CM.sam_agent[node].setup_hc()
-        pids = self.CM.sam_agent[node].getpid().rstrip().split(" ")
-
-        no_pats = []
-        yes_pats = []
-        for pid in pids:
-            no_pats.append('%s .*resource "%s" failed!' % (node, pid))
-            yes_pats.append('%s .*Fsm:%s event "config_changed", state "running" --> "stopped"' % (node, pid))
-
-        yes_w = self.create_watch(yes_pats, 10)
-        no_w = self.create_watch(no_pats, 10)
-        yes_w.setwatch()
-        no_w.setwatch()
-        time.sleep(2)
-
-        self.CM.sam_agent[node].sam_stop()
-
-        yes_matched = yes_w.look()
-        no_matched = no_w.look()
-        if no_matched:
-            return self.failure("Patterns found: " + repr(no_matched))
-        else:
-            if not yes_matched:
-                return self.failure("Patterns NOT found: " + repr(yes_w.regexes))
-
-        return self.success()
-
-###################################################################
-class WdDeleteResource(CoroTest):
-    '''config resource & start corosync
-check that it is getting checked
-delete the object resource object
-check that we do NOT get watchdog'ed
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="WdDeleteResource"
-        self.need_all_up = True
-        self.config['logging/logger_subsys[2]/subsys'] = 'MON'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-        self.config['logging/logger_subsys[3]/subsys'] = 'WD'
-        self.config['logging/logger_subsys[3]/debug'] = 'on'
-        self.config['resources/system/memory_used/recovery'] = 'watchdog'
-        self.config['resources/system/memory_used/max'] = '80'
-        self.config['resources/system/memory_used/poll_period'] = '800'
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-
-        no_pats = []
-        yes_pats = []
-        no_pats.append('%s .*resource "memory_used" failed!' % node)
-        yes_pats.append('%s .*resource "memory_used" deleted from cmap!' % node)
-        yes_w = self.create_watch(yes_pats, 10)
-        no_w = self.create_watch(no_pats, 10)
-        yes_w.setwatch()
-        no_w.setwatch()
-        time.sleep(2)
-
-        self.CM.rsh(node, 'corosync-cmapctl -D resources.system.memory_used')
-
-        yes_matched = yes_w.look()
-        no_matched = no_w.look()
-        if no_matched:
-            return self.failure("Patterns found: " + repr(no_matched))
-        else:
-            if not yes_matched:
-                return self.failure("Patterns NOT found: " + repr(yes_w.regexes))
-
-        return self.success()
-
-
-###################################################################
-class ResourcePollAdjust(CoroTest):
-    '''config resource & start corosync
-change the poll_period
-check that we do NOT get watchdog'ed
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="ResourcePollAdjust"
-        self.need_all_up = True
-        self.config['logging/logger_subsys[2]/subsys'] = 'MON'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-        self.config['logging/logger_subsys[3]/subsys'] = 'WD'
-        self.config['logging/logger_subsys[3]/debug'] = 'on'
-        self.config['resources/system/memory_used/recovery'] = 'none'
-        self.config['resources/system/memory_used/max'] = '80'
-        self.config['resources/system/memory_used/poll_period'] = '800'
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-
-        no_pats = []
-        no_pats.append('%s .*resource "memory_used" failed!' % node)
-        no_pats.append('%s .*Could NOT use poll_period.*' % node)
-        no_w = self.create_watch(no_pats, 10)
-        no_w.setwatch()
-        changes = 0
-        while changes < 50:
-            changes = changes + 1
-            poll_period = int(random.random() * 5000)
-            if poll_period < 500:
-                poll_period = 500
-            self.CM.log("setting poll_period to: %d" % poll_period)
-            self.CM.rsh(node, 'corosync-cmapctl -s resources.system.memory_used.poll_period str %d' % poll_period)
-            sleep_time = poll_period * 2 / 1000
-            if sleep_time < 1:
-                sleep_time = 1
-            time.sleep(sleep_time)
-
-        no_matched = no_w.look()
-        if no_matched:
-            return self.failure("Patterns found: " + repr(no_matched))
-
-        return self.success()
-
-
-###################################################################
-class RebootOnHighMem(CoroTest):
-    '''Configure memory resource
-run memory leaker / forkbomb
-confirm reboot action
-'''
-    def __init__(self, cm):
-        CoroTest.__init__(self,cm)
-        self.name="RebootOnHighMem"
-        self.need_all_up = True
-        self.config['logging/logger_subsys[2]/subsys'] = 'WD'
-        self.config['logging/logger_subsys[2]/debug'] = 'on'
-        self.config['resources/system/memory_used/recovery'] = 'reboot'
-        self.config['resources/system/memory_used/max'] = '80'
-        self.config['resources/system/memory_used/poll_period'] = '800'
-
-    def __call__(self, node):
-        '''Perform the test. '''
-        self.incr("calls")
-
-        # get the uptime
-        up_before = self.CM.rsh(node, 'cut -d. -f1 /proc/uptime', 1).rstrip()
-        cmd = 'corosync-cmapctl resources.system.memory_used. | grep current | cut -d= -f2'
-        mem_current_str = self.CM.rsh(node, cmd, 1).rstrip()
-        mem_new_max = int(mem_current_str) + 5
-
-        self.CM.log("current mem usage: %s, new max:%d" % (mem_current_str, mem_new_max))
-        cmd = 'corosync-cmapctl -s resources.system.memory_used.max str ' + str(mem_new_max)
-        self.CM.rsh(node, cmd)
-
-        self.CM.rsh(node, 'memhog -r10000 200m', synchronous=0)
-
-        self.CM.log("wait for it to reboot")
-        time.sleep(60 * 3)
-        cmd = 'corosync-cmapctl resources.system.memory_used. | grep current | cut -d= -f2'
-        mem_current_str = self.CM.rsh(node, cmd, 1).rstrip()
-        self.CM.log("current mem usage: %s" % (mem_current_str))
-
-        ping_able = False
-        while not ping_able:
-            if self.CM.rsh("localhost", "ping -nq -c10 -w10 %s" % node) == 0:
-                ping_able = True
-                self.CM.log("can ping 10 in 10secs.")
-            else:
-                self.CM.log("not yet responding to pings.")
-
-        self.CM.ShouldBeStatus[node] = "down"
-        # wait for the node to come back up
-        self.CM.log("waiting for node to come back up.")
-        if self.CM.ns.WaitForNodeToComeUp(node):
-            up_after = self.CM.rsh(node, 'cut -d. -f1 /proc/uptime', 1).rstrip()
-            if int(up_after) < int(up_before):
-                return self.success()
-            else:
-                return self.failure("node didn't seem to watchdog uptime 1 %s; 2 %s" %(up_before, up_after))
-        else:
-            return self.failure("node didn't seem to come back up")
-
-
-GenTestClasses = []
-GenTestClasses.append(GenSimulStart)
-GenTestClasses.append(GenSimulStop)
-GenTestClasses.append(GenFlipTest)
-GenTestClasses.append(GenRestartTest)
-GenTestClasses.append(GenStartOnebyOne)
-GenTestClasses.append(GenStopOnebyOne)
-GenTestClasses.append(GenRestartOnebyOne)
-GenTestClasses.append(GenStopAllBeekhof)
-GenTestClasses.append(CpgMsgOrderBasic)
-GenTestClasses.append(CpgMsgOrderZcb)
-GenTestClasses.append(CpgCfgChgOnExecCrash)
-GenTestClasses.append(CpgCfgChgOnGroupLeave)
-GenTestClasses.append(CpgCfgChgOnNodeLeave)
-GenTestClasses.append(CpgCfgChgOnNodeIsolate)
-#GenTestClasses.append(CpgCfgChgOnNodeRestart)
-
-AllTestClasses = []
-AllTestClasses.append(CpgContextTest)
-AllTestClasses.append(SamTest1)
-AllTestClasses.append(SamTest2)
-AllTestClasses.append(SamTest4)
-AllTestClasses.append(SamTest5)
-AllTestClasses.append(SamTest6)
-AllTestClasses.append(SamTest8)
-AllTestClasses.append(SamTest9)
-AllTestClasses.append(SamWdIntegration1)
-AllTestClasses.append(SamWdIntegration2)
-AllTestClasses.append(NoWDConfig)
-AllTestClasses.append(WDConfigNoWd)
-AllTestClasses.append(NoWDOnCorosyncStop)
-#AllTestClasses.append(WDOnForkBomb)
-AllTestClasses.append(WdDeleteResource)
-#AllTestClasses.append(RebootOnHighMem)
-AllTestClasses.append(ResourcePollAdjust)
-AllTestClasses.append(MemLeakObject)
-AllTestClasses.append(MemLeakSession)
-#AllTestClasses.append(CMapDispatchDeadlock)
-
-# quorum tests
-AllTestClasses.append(VoteQuorumContextTest)
-GenTestClasses.append(VoteQuorumGoDown)
-GenTestClasses.append(VoteQuorumGoUp)
-GenTestClasses.append(VoteQuorumWaitForAll)
-
-# FIXME need log messages in sync
-#GenTestClasses.append(CpgCfgChgOnLowestNodeJoin)
-
-
-class ConfigContainer(UserDict):
-    def __init__ (self, name):
-        self.name = name
-        UserDict.__init__(self)
-
-def CoroTestList(cm, audits):
-    result = []
-    configs = []
-
-    for testclass in AllTestClasses:
-        bound_test = testclass(cm)
-        if bound_test.is_applicable():
-            bound_test.Audits = audits
-            result.append(bound_test)
-
-    default = ConfigContainer('default')
-    default['logging/fileline'] = 'on'
-    default['logging/function_name'] = 'off'
-    default['logging/logfile_priority'] = 'info'
-    default['logging/syslog_priority'] = 'info'
-    default['logging/syslog_facility'] = 'daemon'
-    default['uidgid/uid'] = '0'
-    default['uidgid/gid'] = '0'
-    configs.append(default)
-
-    a = ConfigContainer('none_5min')
-    a['totem/token'] = (5 * 60 * 1000)
-    a['totem/consensus'] = int(5 * 60 * 1000 * 1.2) + 1
-    configs.append(a)
-
-    b = ConfigContainer('pcmk_basic')
-    b['totem/token'] = 5000
-    b['totem/token_retransmits_before_loss_const'] = 10
-    b['totem/join'] = 1000
-    b['totem/consensus'] = 7500
-    configs.append(b)
-
-    c = ConfigContainer('pcmk_sec_nss')
-    c['totem/secauth'] = 'on'
-    c['totem/crypto_type'] = 'nss'
-    c['totem/token'] = 5000
-    c['totem/token_retransmits_before_loss_const'] = 10
-    c['totem/join'] = 1000
-    c['totem/consensus'] = 7500
-    configs.append(c)
-#
-#    s = ConfigContainer('pcmk_vq')
-#    s['quorum/provider'] = 'corosync_votequorum'
-#    s['quorum/expected_votes'] = len(cm.Env["nodes"])
-#    s['totem/token'] = 5000
-#    s['totem/token_retransmits_before_loss_const'] = 10
-#    s['totem/join'] = 1000
-#    s['totem/vsftype'] = 'none'
-#    s['totem/consensus'] = 7500
-#    s['totem/max_messages'] = 20
-#    configs.append(s)
-#
-    d = ConfigContainer('sec_nss')
-    d['totem/secauth'] = 'on'
-    d['totem/crypto_type'] = 'nss'
-    configs.append(d)
-
-    if not cm.Env["RrpBindAddr"] is None:
-        g = ConfigContainer('rrp_passive')
-        g['totem/rrp_mode'] = 'passive'
-        g['totem/interface[2]/ringnumber'] = '1'
-        g['totem/interface[2]/bindnetaddr'] = cm.Env["RrpBindAddr"]
-        g['totem/interface[2]/mcastaddr'] = '226.94.1.2'
-        g['totem/interface[2]/mcastport'] = '5405'
-        configs.append(g)
-
-        h = ConfigContainer('rrp_active')
-        h['totem/rrp_mode'] = 'active'
-        h['totem/interface[2]/ringnumber'] = '1'
-        h['totem/interface[2]/bindnetaddr'] = cm.Env["RrpBindAddr"]
-        h['totem/interface[2]/mcastaddr'] = '226.94.1.2'
-        h['totem/interface[2]/mcastport'] = '5405'
-        configs.append(h)
-    else:
-        print('Not including rrp tests. Use --rrp-binaddr to enable them.')
-
-    num=1
-    for cfg in configs:
-        for testclass in GenTestClasses:
-            bound_test = testclass(cm)
-            if bound_test.is_applicable() and bound_test.config_valid(cfg):
-                bound_test.Audits = audits
-                for c in list(cfg.keys()):
-                    bound_test.config[c] = cfg[c]
-                bound_test.name = bound_test.name + '_' + cfg.name
-                result.append(bound_test)
-        num = num + 1
-
-    return result
-