CTS: make test agent shutdown more gracefully

Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
Reviewed-by: Steven Dake <sdake@redhat.com>

cts/agents/common_test_agent.c

@@ -55,13 +55,22 @@ int32_t parse_debug = 0;
 static char big_and_buf_rx[HOW_BIG_AND_BUF];
 ta_do_command_fn do_command;
 static qb_loop_t *poll_handle;
-
+static pre_exit_fn pre_exit = NULL;
 
 qb_loop_t *ta_poll_handle_get(void)
 {
 	return poll_handle;
 }
 
+static void shut_me_down(void)
+{
+	if (pre_exit) {
+		pre_exit();
+	}
+	qb_loop_stop(poll_handle);
+}
+
+
 static void ta_handle_command (int sock, char* msg)
 {
 	int num_args;
@@ -112,6 +121,11 @@ static int server_process_data_fn (
 	char *cmd;
 	int32_t nbytes;
 
+	if (revents & POLLHUP) {
+		shut_me_down();
+		return -1;
+	}
+
 	if ((nbytes = recv (fd, big_and_buf_rx, sizeof (big_and_buf_rx), 0)) <= 0) {
 		/* got error or connection closed by client */
 		if (nbytes == 0) {
@@ -120,8 +134,8 @@ static int server_process_data_fn (
 		} else {
 			syslog (LOG_ERR,"recv() failed: %s", strerror(errno));
 		}
-		close (fd);
-		qb_loop_stop (ta_poll_handle_get());
+		shut_me_down();
+		return -1;
 	} else {
 		big_and_buf_rx[nbytes] = '\0';
 
@@ -146,6 +160,11 @@ static int server_accept_fn (
 	int new_fd;
 	int res;
 
+	if (revents & POLLHUP) {
+		shut_me_down();
+		return -1;
+	}
+
 	addrlen = sizeof (struct sockaddr_in);
 
 retry_accept:
@@ -254,23 +273,26 @@ static int create_server_sockect (int server_port)
 
 static int32_t sig_exit_handler (int num, void *data)
 {
-	qb_loop_stop(poll_handle);
+	shut_me_down();
 	return 0;
 }
 
-int test_agent_run(int server_port, ta_do_command_fn func)
+int test_agent_run(int server_port, ta_do_command_fn func, pre_exit_fn exit_fn)
 {
 	int listener;
 
 	do_command = func;
+	pre_exit = exit_fn;
 	poll_handle = qb_loop_create ();
 
-	qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
-		SIGINT, NULL, sig_exit_handler, NULL);
-	qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
-		SIGQUIT, NULL, sig_exit_handler, NULL);
-	qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
-		SIGTERM, NULL, sig_exit_handler, NULL);
+	if (exit_fn) {
+		qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
+			SIGINT, NULL, sig_exit_handler, NULL);
+		qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
+			SIGQUIT, NULL, sig_exit_handler, NULL);
+		qb_loop_signal_add(poll_handle, QB_LOOP_HIGH,
+			SIGTERM, NULL, sig_exit_handler, NULL);
+	}
 
 	listener = create_server_sockect (server_port);
 	qb_loop_poll_add (poll_handle,

cts/agents/common_test_agent.h

@@ -46,8 +46,9 @@ extern int32_t parse_debug;
 #define HOW_BIG_AND_BUF 4096
 
 typedef void (*ta_do_command_fn) (int sock, char* func, char*args[], int num_args);
+typedef void (*pre_exit_fn) (void);
 
-int test_agent_run(int server_port, ta_do_command_fn func);
+int test_agent_run(int server_port, ta_do_command_fn func, pre_exit_fn exit_fn);
 
 qb_loop_t *ta_poll_handle_get(void);
 

cts/agents/confdb_test_agent.c

@@ -621,17 +621,21 @@ static void do_command (int sock, char* func, char*args[], int num_args)
 	}
 }
 
+static void my_pre_exit(void)
+{
+	syslog (LOG_INFO, "%s PRE EXIT", __FILE__);
+}
 
 int main (int argc, char *argv[])
 {
 	int ret;
 
 	openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON);
-	syslog (LOG_ERR, "confdb_test_agent STARTING");
+	syslog (LOG_ERR, "%s STARTING", __FILE__);
 
 	parse_debug = 1;
-	ret = test_agent_run (9035, do_command);
-	syslog (LOG_ERR, "confdb_test_agent EXITING");
+	ret = test_agent_run (9035, do_command, my_pre_exit);
+	syslog (LOG_ERR, "%s EXITING", __FILE__);
 
 	return ret;
 }

cts/agents/cpg_test_agent.c

@@ -570,6 +570,8 @@ static int cfg_dispatch_wrapper_fn (
 	cs_error_t error;
 	if (revents & POLLHUP || revents & POLLERR) {
 		syslog (LOG_ERR, "%s() got POLLHUP disconnecting from CFG", __func__);
+		corosync_cfg_finalize(cfg_handle);
+		cfg_handle = 0;
 		qb_loop_poll_del (ta_poll_handle_get(), cfg_fd);
 		close (cfg_fd);
 		cfg_fd = -1;
@@ -578,6 +580,8 @@ static int cfg_dispatch_wrapper_fn (
 	error = corosync_cfg_dispatch (cfg_handle, CS_DISPATCH_ALL);
 	if (error == CS_ERR_LIBRARY) {
 		syslog (LOG_ERR, "%s() got LIB error disconnecting from CFG.", __func__);
+		corosync_cfg_finalize(cfg_handle);
+		cfg_handle = 0;
 		qb_loop_poll_del (ta_poll_handle_get(), cfg_fd);
 		close (cfg_fd);
 		cfg_fd = -1;
@@ -594,6 +598,8 @@ static int cpg_dispatch_wrapper_fn (
 	cs_error_t error;
 	if (revents & POLLHUP || revents & POLLERR) {
 		syslog (LOG_ERR, "%s() got POLLHUP disconnecting from CPG", __func__);
+		cpg_finalize(cpg_handle);
+		cpg_handle = 0;
 		qb_loop_poll_del (ta_poll_handle_get(), cpg_fd);
 		close (cpg_fd);
 		cpg_fd = -1;
@@ -602,6 +608,8 @@ static int cpg_dispatch_wrapper_fn (
 	error = cpg_dispatch (cpg_handle, CS_DISPATCH_ALL);
 	if (error == CS_ERR_LIBRARY) {
 		syslog (LOG_ERR, "%s() got LIB error disconnecting from CPG", __func__);
+		cpg_finalize(cpg_handle);
+		cpg_handle = 0;
 		qb_loop_poll_del (ta_poll_handle_get(), cpg_fd);
 		close (cpg_fd);
 		cpg_fd = -1;
@@ -687,9 +695,14 @@ static void do_command (int sock, char* func, char*args[], int num_args)
 		send (sock, response, strlen (response), 0);
 	} else if (strcmp ("cpg_finalize", func) == 0) {
 
-		cpg_finalize (cpg_handle);
-		qb_loop_poll_del (ta_poll_handle_get(), cpg_fd);
-		cpg_fd = -1;
+		if (cpg_handle > 0) {
+			cpg_finalize (cpg_handle);
+			cpg_handle = 0;
+		}
+		if (cpg_fd > 0) {
+			qb_loop_poll_del (ta_poll_handle_get(), cpg_fd);
+			cpg_fd = -1;
+		}
 
 	} else if (strcmp ("record_config_events", func) == 0) {
 		record_config_events (sock);
@@ -760,15 +773,27 @@ static void cs_ipcs_libqb_log_fn(const char *file_name,
 	syslog(severity, "%s:%d %s() %s", file_name, file_line, __func__, msg);
 }
 
+static void my_pre_exit(void)
+{
+	syslog (LOG_INFO, "%s PRE EXIT", __FILE__);
+	if (cpg_handle > 0) {
+		cpg_finalize (cpg_handle);
+		cpg_handle = 0;
+	}
+}
 
 int main (int argc, char *argv[])
 {
+	int res = 0;
 	openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON);
+	syslog (LOG_INFO, "%s STARTING", __FILE__);
 
 	qb_util_set_log_function (cs_ipcs_libqb_log_fn);
 	list_init (&msg_log_head);
 	list_init (&config_chg_log_head);
 
-	return test_agent_run (9034, do_command);
+	res = test_agent_run (9034, do_command, my_pre_exit);
+	syslog (LOG_INFO, "%s EXITING", __FILE__);
+	return res;
 }
 

cts/agents/sam_test_agent.c

@@ -1495,11 +1495,11 @@ int main (int argc, char *argv[])
 	int ret;
 
 	openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON);
-	syslog (LOG_ERR, "sam_test_agent STARTING");
+	syslog (LOG_INFO, "%s STARTNG", __FILE__);
 
 	parse_debug = 1;
-	ret = test_agent_run (9036, do_command);
-	syslog (LOG_ERR, "sam_test_agent EXITING");
+	ret = test_agent_run (9036, do_command, NULL);
+	syslog (LOG_INFO, "%s EXITING", __FILE__);
 
 	return ret;
 }

cts/agents/votequorum_test_agent.c

@@ -53,6 +53,7 @@
 #include <corosync/votequorum.h>
 #include <corosync/quorum.h>
 #include "common_test_agent.h"
+#include "../../lib/util.h"
 
 static quorum_handle_t q_handle = 0;
 static votequorum_handle_t vq_handle = 0;
@@ -84,10 +85,14 @@ static int vq_dispatch_wrapper_fn (
 	void *data)
 {
 	cs_error_t error = votequorum_dispatch (vq_handle, CS_DISPATCH_ALL);
-	if (error == CS_ERR_LIBRARY) {
-		syslog (LOG_ERR, "%s() got LIB error disconnecting from corosync.", __func__);
+	if (error != CS_OK) {
+		syslog (LOG_ERR, "%s() got %s error, disconnecting.",
+			__func__, cs_strerror(error));
+		votequorum_finalize(vq_handle);
 		qb_loop_poll_del (ta_poll_handle_get(), fd);
 		close (fd);
+		vq_handle = 0;
+		return -1;
 	}
 	return 0;
 }
@@ -98,10 +103,14 @@ static int q_dispatch_wrapper_fn (
 	void *data)
 {
 	cs_error_t error = quorum_dispatch (q_handle, CS_DISPATCH_ALL);
-	if (error == CS_ERR_LIBRARY) {
-		syslog (LOG_ERR, "%s() got LIB error disconnecting from corosync.", __func__);
+	if (error != CS_OK) {
+		syslog (LOG_ERR, "%s() got %s error, disconnecting.",
+			__func__, cs_strerror(error));
+		quorum_finalize(q_handle);
 		qb_loop_poll_del (ta_poll_handle_get(), fd);
 		close (fd);
+		q_handle = 0;
+		return -1;
 	}
 	return 0;
 }
@@ -222,7 +231,6 @@ static void setexpected (int sock, char *arg)
 	snprintf (response, 100, "%s", OK_STR);
 
 send_response:
-	votequorum_finalize (vq_handle);
 	send (sock, response, strlen (response) + 1, 0);
 }
 
@@ -243,7 +251,6 @@ static void setvotes (int sock, char *arg)
 	snprintf (response, 100, "%s", OK_STR);
 
 send_response:
-	votequorum_finalize (vq_handle);
 	send (sock, response, strlen (response), 0);
 }
 
@@ -321,17 +328,29 @@ static void do_command (int sock, char* func, char*args[], int num_args)
 	}
 }
 
+static void my_pre_exit(void)
+{
+	syslog (LOG_INFO, "PRE EXIT");
+	if (vq_handle) {
+		votequorum_finalize(vq_handle);
+		vq_handle = 0;
+	}
+	if (q_handle) {
+		quorum_finalize(q_handle);
+		q_handle = 0;
+	}
+}
 
 int main (int argc, char *argv[])
 {
 	int ret;
 
 	openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON);
-	syslog (LOG_ERR, "votequorum_test_agent STARTING");
+	syslog (LOG_INFO, "%s STARTING", __FILE__);
 
 	parse_debug = 1;
-	ret = test_agent_run (9037, do_command);
-	syslog (LOG_ERR, "votequorum_test_agent EXITING");
+	ret = test_agent_run (9037, do_command, my_pre_exit);
+	syslog (LOG_INFO, "EXITING");
 
 	return ret;
 }