libqb: use the main loop to shutdown

Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
Reviewed-by: Steven Dake <sdake@redhat.com>

cts/agents/syncv2.c

@@ -91,6 +91,8 @@ static void tst_sv2_sync_activate (void);
 
 static void tst_sv2_sync_abort (void);
 
+static int tst_sv2_exec_exit_fn (void);
+
 struct corosync_service_engine tst_sv2_service_engine = {
 	.name			= "corosync test synv2 service",
 	.id			= TST_SV2_SERVICE,
@@ -105,6 +107,7 @@ struct corosync_service_engine tst_sv2_service_engine = {
 	.exec_engine_count	= 0,
 	.confchg_fn		= tst_sv2_confchg_fn,
 	.exec_init_fn		= tst_sv2_exec_init_fn,
+	.exec_exit_fn		= tst_sv2_exec_exit_fn,
 	.exec_dump_fn		= NULL,
 	.sync_mode		= CS_SYNC_V2,
 	.sync_init              = tst_sv2_sync_init_v2,
@@ -181,6 +184,16 @@ static int tst_sv2_exec_init_fn (
 	return 0;
 }
 
+static int32_t exit_count = 0;
+static int tst_sv2_exec_exit_fn (void)
+{
+	exit_count++;
+	log_printf (LOGSYS_LEVEL_INFO, "exit_count:%d", exit_count);
+	if (exit_count < 4) {
+		return -1;
+	}
+}
+
 static void tst_sv2_confchg_fn (
 	enum totem_configuration_type configuration_type,
 	const unsigned int *member_list, size_t member_list_entries,

exec/ipc_glue.c

@@ -191,7 +191,10 @@ static const char* cs_ipcs_serv_short_name(int32_t service_id)
 
 int32_t cs_ipcs_service_destroy(int32_t service_id)
 {
-	qb_ipcs_destroy(ipcs_mapper[service_id].inst);
+	if (ipcs_mapper[service_id].inst) {
+		qb_ipcs_destroy(ipcs_mapper[service_id].inst);
+		ipcs_mapper[service_id].inst = NULL;
+	}
 	return 0;
 }
 

exec/main.c

@@ -172,10 +172,6 @@ static hdb_handle_t object_memb_handle;
 
 static corosync_timer_handle_t corosync_stats_timer_handle;
 
-static pthread_t corosync_exit_thread;
-
-static sem_t corosync_exit_sem;
-
 static const char *corosync_lock_file = LOCALSTATEDIR"/run/corosync.pid";
 
 static void serialize_unlock (void);
@@ -232,54 +228,20 @@ static void unlink_all_completed (void)
 
 void corosync_shutdown_request (void)
 {
-	static int called = 0;
-	if (called) {
-		return;
-	}
-	if (called == 0) {
-		called = 1;
-	}
-
-	sem_post (&corosync_exit_sem);
-}
-
-static void *corosync_exit_thread_handler (void *arg)
-{
-	totempg_stats_t * stats;
-
-	sem_wait (&corosync_exit_sem);
-
-	stats = api->totem_get_stats();
-	if (stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ||
-	    stats->mrp->srp->operational_entered == 0) {
-		unlink_all_completed ();
-		/* NOTREACHED */
-	}
-
 	corosync_service_unlink_all (api, unlink_all_completed);
-
-	return arg;
 }
 
-static void sigusr2_handler (int num)
+static int32_t sig_diag_handler (int num, void *data)
 {
 	corosync_state_dump ();
 	logsys_log_rec_store (LOCALSTATEDIR "/lib/corosync/fdata");
+	return 0;
 }
 
-static void sigterm_handler (int num)
-{
-	corosync_shutdown_request ();
-}
-
-static void sigquit_handler (int num)
+static int32_t sig_exit_handler (int num, void *data)
 {
-	corosync_shutdown_request ();
-}
-
-static void sigintr_handler (int num)
-{
-	corosync_shutdown_request ();
+	corosync_service_unlink_all (api, unlink_all_completed);
+	return 0;
 }
 
 static void sigsegv_handler (int num)
@@ -1351,13 +1313,18 @@ int main (int argc, char **argv, char **envp)
 	log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine ('%s'): started and ready to provide service.\n", VERSION);
 	log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES "\n");
 
+	corosync_poll_handle = qb_loop_create ();
 
-	(void)signal (SIGINT, sigintr_handler);
-	(void)signal (SIGUSR2, sigusr2_handler);
+	qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
+		SIGUSR2, NULL, sig_diag_handler, NULL);
+	qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
+		SIGINT, NULL, sig_exit_handler, NULL);
+	qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
+		SIGQUIT, NULL, sig_exit_handler, NULL);
+	qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
+		SIGTERM, NULL, sig_exit_handler, NULL);
 	(void)signal (SIGSEGV, sigsegv_handler);
 	(void)signal (SIGABRT, sigabrt_handler);
-	(void)signal (SIGQUIT, sigquit_handler);
-	(void)signal (SIGTERM, sigterm_handler);
 #if MSG_NOSIGNAL != 0
 	(void)signal (SIGPIPE, SIG_IGN);
 #endif
@@ -1525,31 +1492,6 @@ int main (int argc, char **argv, char **envp)
 		corosync_exit_error (flock_err);
 	}
 
-	corosync_poll_handle = qb_loop_create ();
-
-	/*
-	 * Sleep for a while to let other nodes in the cluster
-	 * understand that this node has been away (if it was
-	 * an corosync restart).
-	 */
-
-// TODO what is this hack for?	usleep(totem_config.token_timeout * 2000);
-
-	/*
-	 * Create semaphore and start "exit" thread
-	 */
-	res = sem_init (&corosync_exit_sem, 0, 0);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create exit thread.\n");
-		corosync_exit_error (AIS_DONE_FATAL_ERR);
-	}
-
-	res = pthread_create (&corosync_exit_thread, NULL, corosync_exit_thread_handler, NULL);
-	if (res != 0) {
-		log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create exit thread.\n");
-		corosync_exit_error (AIS_DONE_FATAL_ERR);
-	}
-
 	/*
 	 * if totempg_initialize doesn't have root priveleges, it cannot
 	 * bind to a specific interface.  This only matters if

exec/service.c

@@ -55,6 +55,7 @@
 #include "service.h"
 
 #include <qb/qbipcs.h>
+#include <qb/qbloop.h>
 
 LOGSYS_DECLARE_SUBSYS ("SERV");
 
@@ -123,9 +124,6 @@ static hdb_handle_t object_stats_services_handle;
 
 static void (*service_unlink_all_complete) (void) = NULL;
 
-static hdb_handle_t swrk_service_exit_handle;
-static hdb_handle_t swrk_service_unlink_handle;
-
 static unsigned int default_services_requested (struct corosync_api_v1 *corosync_api)
 {
 	hdb_handle_t object_service_handle;
@@ -473,6 +471,8 @@ static unsigned int service_unlink_and_exit (
 				(void *)&found_service_handle,
 				NULL);
 
+			cs_ipcs_service_destroy (*service_id);
+
 			lcr_ifact_release (*found_service_handle);
 
 			corosync_api->object_destroy (object_service_handle);
@@ -572,17 +572,17 @@ unsigned int corosync_service_defaults_link_and_init (struct corosync_api_v1 *co
  * Declaration of exit_schedwrk_handler, because of cycle
  * (service_exit_schedwrk_handler calls service_unlink_schedwrk_handler, and vice-versa)
  */
-static int service_exit_schedwrk_handler (const void *data);
+static void service_exit_schedwrk_handler (void *data);
 
-static int service_unlink_schedwrk_handler (const void *data) {
+static void service_unlink_schedwrk_handler (void *data) {
 	struct seus_handler_data *cb_data = (struct seus_handler_data *)data;
-	struct corosync_api_v1 *api = (struct corosync_api_v1 *)cb_data->api;
 
 	/*
 	 * Exit all ipc connections dependent on this service
 	 */
-	if (cs_ipcs_service_destroy (cb_data->service_engine) == -1)
-		return -1;
+	if (cs_ipcs_service_destroy (cb_data->service_engine) == -1) {
+		goto redo_this_function;
+	}
 
 	log_printf(LOGSYS_LEVEL_NOTICE,
 		"Service engine unloaded: %s\n",
@@ -592,15 +592,22 @@ static int service_unlink_schedwrk_handler (const void *data) {
 
 	lcr_ifact_release (cb_data->service_handle);
 
-	api->schedwrk_create (
-		&swrk_service_exit_handle,
-		&service_exit_schedwrk_handler,
-		data);
+	qb_loop_job_add(corosync_poll_handle_get(),
+		QB_LOOP_HIGH,
+		data,
+		service_exit_schedwrk_handler);
+
+	return;
+
+ redo_this_function:
+	qb_loop_job_add(corosync_poll_handle_get(),
+		QB_LOOP_HIGH,
+		data,
+		service_unlink_schedwrk_handler);
 
-	return 0;
 }
 
-static int service_exit_schedwrk_handler (const void *data) {
+static void service_exit_schedwrk_handler (void *data) {
 	int res;
 	static int current_priority = 0;
 	static int current_service_engine = 0;
@@ -624,24 +631,26 @@ static int service_exit_schedwrk_handler (const void *data) {
 		&service_handle);
 	if (res == 0) {
 		service_unlink_all_complete();
-		return (res);
+		return;
 	}
 
 	if (res == 1) {
 		cb_data->service_engine = current_service_engine;
 		cb_data->service_handle = service_handle;
 
-		api->schedwrk_create_nolock (
-			&swrk_service_unlink_handle,
-			&service_unlink_schedwrk_handler,
-			data);
-
-		return (0);
+		qb_loop_job_add(corosync_poll_handle_get(),
+			QB_LOOP_HIGH,
+			data,
+			service_unlink_schedwrk_handler);
+		return;
 	}
 
-	return (res);
+	qb_loop_job_add(corosync_poll_handle_get(),
+		QB_LOOP_HIGH,
+		data,
+		service_exit_schedwrk_handler);
 }
-		
+
 void corosync_service_unlink_all (
 	struct corosync_api_v1 *api,
 	void (*unlink_all_complete) (void))
@@ -662,10 +671,10 @@ void corosync_service_unlink_all (
 
 	cb_data.api = api;
 
-	api->schedwrk_create (
-		&swrk_service_exit_handle,
-		&service_exit_schedwrk_handler,
-		&cb_data);
+	qb_loop_job_add(corosync_poll_handle_get(),
+		QB_LOOP_HIGH,
+		&cb_data,
+		service_exit_schedwrk_handler);
 }
 
 struct service_unlink_and_exit_data {
@@ -675,7 +684,7 @@ struct service_unlink_and_exit_data {
 	unsigned int ver;
 };
 
-static int service_unlink_and_exit_schedwrk_handler (void *data)
+static void service_unlink_and_exit_schedwrk_handler (void *data)
 {
 	struct service_unlink_and_exit_data *service_unlink_and_exit_data =
 		data;
@@ -688,8 +697,12 @@ static int service_unlink_and_exit_schedwrk_handler (void *data)
 
 	if (res == 0) {
 		free (service_unlink_and_exit_data);
+	} else {
+		qb_loop_job_add(corosync_poll_handle_get(),
+			QB_LOOP_HIGH,
+			data,
+			service_unlink_and_exit_schedwrk_handler);
 	}
-	return (res);
 }
 
 typedef int (*schedwrk_cast) (const void *);
@@ -706,10 +719,10 @@ unsigned int corosync_service_unlink_and_exit (
 	service_unlink_and_exit_data->api = api;
 	service_unlink_and_exit_data->name = strdup (service_name);
 	service_unlink_and_exit_data->ver = service_ver;
-	
-	api->schedwrk_create (
-		&service_unlink_and_exit_data->handle,
-		(schedwrk_cast)service_unlink_and_exit_schedwrk_handler,
-		service_unlink_and_exit_data);
+
+	qb_loop_job_add(corosync_poll_handle_get(),
+		QB_LOOP_HIGH,
+		service_unlink_and_exit_data,
+		service_unlink_and_exit_schedwrk_handler);
 	return (0);
 }