qdevice: Port qdevice to use pr-poll-loop

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/Makefile.am

@@ -110,7 +110,6 @@ corosync_qdevice_SOURCES = corosync-qdevice.c \
                            pr-poll-array.c pr-poll-array.h dynar-simple-lex.c dynar-simple-lex.h \
                            dynar-str.c dynar-str.h qdevice-ipc-cmd.c qdevice-ipc-cmd.h \
                            qdevice-net-ipc-cmd.c qdevice-net-ipc-cmd.h \
-                           qdevice-net-poll.c qdevice-net-poll.h \
                            qdevice-net-send.c qdevice-net-send.h \
                            qdevice-net-votequorum.c qdevice-net-votequorum.h \
                            qdevice-net-socket.c qdevice-net-socket.h \
@@ -123,7 +122,6 @@ corosync_qdevice_SOURCES = corosync-qdevice.c \
                            qdevice-net-algo-ffsplit.c qdevice-net-algo-ffsplit.h \
                            qdevice-net-algo-2nodelms.c qdevice-net-algo-2nodelms.h \
                            qdevice-net-algo-lms.c qdevice-net-algo-lms.h \
-                           qdevice-net-poll-array-user-data.h \
                            qdevice-config.h qnet-config.h qdevice-net-disconnect-reason.h \
                            qdevice-model-type.h qdevice-advanced-settings.c \
                            qdevice-advanced-settings.h dynar-getopt-lex.c dynar-getopt-lex.h \

qdevices/corosync-qdevice.c

@@ -102,6 +102,54 @@ signal_handlers_register(void)
 	sigaction(SIGPIPE, &act, NULL);
 }
 
+static int
+qdevice_run_main_loop(struct qdevice_instance *instance)
+{
+	int res;
+	enum qdevice_model_post_poll_loop_exit_reason exit_reason;
+	int restart_loop;
+
+	log(LOG_DEBUG, "Running QDevice main loop");
+
+	restart_loop = 1;
+
+	while (restart_loop) {
+		res = qdevice_model_pre_poll_loop(instance);
+		if (res == -1) {
+			return (-1);
+		}
+
+		while ((res = pr_poll_loop_exec(&instance->main_poll_loop)) == 0) {
+		}
+
+		if (res == -2) {
+			log(LOG_CRIT, "pr_poll_loop_exec returned -2 - internal error");
+			return (-1);
+		} else if (res == -3) {
+			log_nss(LOG_CRIT, "pr_poll_loop_exec returned -3 - PR_Poll error");
+			return (-1);
+		}
+
+		exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_MODEL;
+		if (instance->votequorum_closed) {
+			exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_VOTEQUORUM_CLOSED;
+		} else if (instance->cmap_closed) {
+			exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_CMAP_CLOSED;
+		} else if (instance->heuristics_closed) {
+			exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_HEURISTICS_CLOSED;
+		} else if (qdevice_ipc_is_closed(instance)) {
+			exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_IPC_SOCKET_CLOSED;
+		}
+
+		res = qdevice_model_post_poll_loop(instance, exit_reason);
+		if (res == 0 || res == -1) {
+			restart_loop = 0;
+		}
+	}
+
+	return (res);
+}
+
 static void
 usage(void)
 {
@@ -187,7 +235,7 @@ main(int argc, char * const argv[])
 	int bump_log_priority;
 	int lock_file;
 	int another_instance_running;
-	int model_run_res;
+	int main_loop_res;
 
 	if (qdevice_advanced_settings_init(&advanced_settings) != 0) {
 		errx(EXIT_FAILURE, "Can't alloc memory for advanced settings");
@@ -269,19 +317,20 @@ main(int argc, char * const argv[])
 		return (EXIT_FAILURE);
 	}
 
+	global_instance = &instance;
+	signal_handlers_register();
+
 	log(LOG_DEBUG, "Waiting for initial heuristics exec result");
 	if (qdevice_pr_poll_loop_wait_for_initial_heuristics_exec_result(&instance) != 0) {
 		return (EXIT_FAILURE);
 	}
 
-	global_instance = &instance;
-	signal_handlers_register();
-
-	log(LOG_DEBUG, "Running qdevice model");
+	log(LOG_DEBUG, "Qdevice ready to provide service");
 #ifdef HAVE_LIBSYSTEMD
 	sd_notify (0, "READY=1");
 #endif
-	model_run_res = qdevice_model_run(&instance);
+
+	main_loop_res = qdevice_run_main_loop(&instance);
 
 	log(LOG_DEBUG, "Removing cmap tracking");
 	/*
@@ -309,5 +358,5 @@ main(int argc, char * const argv[])
 
 	qdevice_advanced_settings_destroy(&advanced_settings);
 
-	return (model_run_res == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
+	return (main_loop_res == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
 }

qdevices/qdevice-instance.h

@@ -123,9 +123,10 @@ struct qdevice_instance {
 	struct pr_poll_loop main_poll_loop;
 
 	/*
-	 * Set by poll handler when votequorum connection is closed
+	 * Set by poll handler when votequorum/cmap connection is closed
 	 */
 	int votequorum_closed;
+	int cmap_closed;
 	/*
 	 * Set by poll handler when one of the heuristics pipes becomes closed
 	 */

qdevices/qdevice-ipc.c

@@ -40,9 +40,139 @@
 #include "dynar-str.h"
 #include "qdevice-ipc-cmd.h"
 
+/*
+ * Callbacks
+ */
+
+/*
+ * IPC server socket
+ */
+static int
+ipc_socket_poll_loop_set_events_cb(int fd, short *events, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+
+	if (qdevice_ipc_is_closed(instance)) {
+		log(LOG_DEBUG, "Listening socket is closed");
+
+		return (-2);
+	}
+
+	return (0);
+}
+
+static int
+ipc_socket_poll_loop_read_cb(int fd, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+	struct unix_socket_client *ipc_client;
+
+	qdevice_ipc_accept(instance, &ipc_client);
+
+	return (0);
+}
+
+static int
+ipc_socket_poll_loop_err_cb(int fd, short revents, void *user_data1, void *user_data2)
+{
+
+	if (revents != POLLNVAL) {
+		/*
+		 * Poll ERR on listening socket is fatal error.
+		 * POLL_NVAL is used as a signal to quit poll loop.
+		 */
+		log(LOG_CRIT, "POLL_ERR (%u) on listening socket", revents);
+	} else {
+		log(LOG_DEBUG, "Listening socket is closed");
+	}
+
+	return (-1);
+}
+
+/*
+ * IPC client sockets
+ */
+static int
+ipc_client_socket_poll_loop_set_events_cb(int fd, short *events, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+	struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2;
+
+	if (ipc_client->schedule_disconnect) {
+		qdevice_ipc_client_disconnect(instance, ipc_client);
+
+		if (pr_poll_loop_del_fd(&instance->main_poll_loop, fd) == -1) {
+			log(LOG_ERR, "pr_poll_loop_del_fd for ipc client socket failed");
+
+			return (-2);
+		}
+
+		return (-1);
+	}
+
+	if (!ipc_client->reading_line && !ipc_client->writing_buffer) {
+		return (-1);
+	}
+
+	if (ipc_client->reading_line) {
+		*events |= POLLIN;
+	}
+
+	if (ipc_client->writing_buffer) {
+		*events |= POLLOUT;
+	}
+
+	return (0);
+}
+
+static int
+ipc_client_socket_poll_loop_read_cb(int fd, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+	struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2;
+
+	if (!ipc_client->schedule_disconnect) {
+		qdevice_ipc_io_read(instance, ipc_client);
+	}
+
+	return (0);
+}
+
+static int
+ipc_client_socket_poll_loop_write_cb(int fd, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+	struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2;
+
+	if (!ipc_client->schedule_disconnect) {
+		qdevice_ipc_io_write(instance, ipc_client);
+	}
+
+	return (0);
+}
+
+static int
+ipc_client_socket_poll_loop_err_cb(int fd, short revents, void *user_data1, void *user_data2)
+{
+	struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2;
+
+	if (!ipc_client->schedule_disconnect) {
+		log(LOG_DEBUG, "POLL_ERR (%u) on ipc client socket."
+		    " Disconnecting.", revents);
+
+		ipc_client->schedule_disconnect = 1;
+	}
+
+	return (0);
+}
+
+/*
+ * Exported functions
+ */
 int
 qdevice_ipc_init(struct qdevice_instance *instance)
 {
+
 	if (unix_socket_ipc_init(&instance->local_ipc,
 	    instance->advanced_settings->local_socket_file,
 	    instance->advanced_settings->local_socket_backlog,
@@ -54,6 +184,16 @@ qdevice_ipc_init(struct qdevice_instance *instance)
 		return (-1);
 	}
 
+	if (pr_poll_loop_add_fd(&instance->main_poll_loop, instance->local_ipc.socket, POLLIN,
+	    ipc_socket_poll_loop_set_events_cb,
+	    ipc_socket_poll_loop_read_cb,
+	    NULL,
+	    ipc_socket_poll_loop_err_cb, instance, NULL) == -1) {
+		log(LOG_ERR, "Can't add IPC socket to main poll loop");
+
+		return (-1);
+	}
+
 	return (0);
 }
 
@@ -137,6 +277,19 @@ qdevice_ipc_accept(struct qdevice_instance *instance, struct unix_socket_client
 		memset((*res_client)->user_data, 0, sizeof(struct qdevice_ipc_user_data));
 	}
 
+	if (pr_poll_loop_add_fd(&instance->main_poll_loop, (*res_client)->socket, 0,
+	    ipc_client_socket_poll_loop_set_events_cb,
+	    ipc_client_socket_poll_loop_read_cb,
+	    ipc_client_socket_poll_loop_write_cb,
+	    ipc_client_socket_poll_loop_err_cb, instance, *res_client) == -1) {
+		log(LOG_ERR, "Can't add IPC client socket to main poll loop");
+		res = -1;
+
+		qdevice_ipc_client_disconnect(instance, *res_client);
+
+		goto return_res;
+	}
+
 return_res:
 	return (res);
 }

qdevices/qdevice-ipc.h

@@ -42,7 +42,6 @@ extern "C" {
 #endif
 
 struct qdevice_ipc_user_data {
-	void *model_data;
 	int shutdown_requested;
 };
 

qdevices/qdevice-model-net.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2019 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -42,12 +42,47 @@
 #include "qdevice-net-ipc-cmd.h"
 #include "qdevice-net-algorithm.h"
 #include "qdevice-net-heuristics.h"
-#include "qdevice-net-poll.h"
 #include "qdevice-net-send.h"
+#include "qdevice-net-socket.h"
 #include "qdevice-net-votequorum.h"
 #include "qnet-config.h"
 #include "nss-sock.h"
 
+/*
+ * Callbacks
+ */
+static int
+check_schedule_disconnect_cb(void *user_data1, void *user_data2)
+{
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	if (instance->schedule_disconnect) {
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+qdevice_model_net_timer_connect_timeout(void *data1, void *data2)
+{
+	struct qdevice_net_instance *instance;
+
+	instance = (struct qdevice_net_instance *)data1;
+
+	log(LOG_ERR, "Connect timeout");
+
+	instance->schedule_disconnect = 1;
+
+	instance->connect_timer = NULL;
+	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER;
+
+	return (0);
+}
+
+/*
+ * Exported functions
+ */
 int
 qdevice_model_net_init(struct qdevice_instance *instance)
 {
@@ -120,22 +155,6 @@ qdevice_model_net_destroy(struct qdevice_instance *instance)
 	return (0);
 }
 
-static int
-qdevice_model_net_timer_connect_timeout(void *data1, void *data2)
-{
-	struct qdevice_net_instance *instance;
-
-	instance = (struct qdevice_net_instance *)data1;
-
-	log(LOG_ERR, "Connect timeout");
-
-	instance->schedule_disconnect = 1;
-
-	instance->connect_timer = NULL;
-	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER;
-
-	return (0);
-}
 
 static PRIntn
 qdevice_model_net_get_af(const struct qdevice_net_instance *instance)
@@ -154,134 +173,213 @@ qdevice_model_net_get_af(const struct qdevice_net_instance *instance)
 	return (af);
 }
 
+/*
+ *  0 - Continue
+ * -1 - End loop
+ */
 int
-qdevice_model_net_run(struct qdevice_instance *instance)
+qdevice_model_net_pre_poll_loop(struct qdevice_instance *instance)
 {
 	struct qdevice_net_instance *net_instance;
-	int try_connect;
 	int res;
-	enum tlv_vote vote;
-	int delay_before_reconnect;
-	int ret_val;
 
 	net_instance = instance->model_data;
 
-	log(LOG_DEBUG, "Executing qdevice-net");
+	net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT;
+	net_instance->socket = NULL;
 
-	ret_val = -1;
+	net_instance->connect_timer = timer_list_add(pr_poll_loop_get_timer_list(&instance->main_poll_loop),
+		net_instance->connect_timeout, qdevice_model_net_timer_connect_timeout,
+		(void *)net_instance, NULL);
 
-	try_connect = 1;
-	while (try_connect) {
-		net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT;
-		net_instance->socket = NULL;
+	if (net_instance->connect_timer == NULL) {
+		log(LOG_CRIT, "Can't schedule connect timer");
 
-		net_instance->connect_timer = timer_list_add(&net_instance->main_timer_list,
-			net_instance->connect_timeout, qdevice_model_net_timer_connect_timeout,
-			(void *)net_instance, NULL);
+		return (-1);
+	}
 
-		if (net_instance->connect_timer == NULL) {
-			log(LOG_CRIT, "Can't schedule connect timer");
+	log(LOG_DEBUG, "Trying connect to qnetd server %s:%u (timeout = %ums)",
+	    net_instance->host_addr, net_instance->host_port, net_instance->connect_timeout);
 
-			try_connect = 0;
-			break;
-		}
+	res = nss_sock_non_blocking_client_init(net_instance->host_addr,
+	    net_instance->host_port, qdevice_model_net_get_af(net_instance),
+	    &net_instance->non_blocking_client);
+	if (res == -1) {
+		log_nss(LOG_ERR, "Can't initialize non blocking client connection");
+	}
 
-		log(LOG_DEBUG, "Trying connect to qnetd server %s:%u (timeout = %ums)",
-		    net_instance->host_addr, net_instance->host_port, net_instance->connect_timeout);
+	res = nss_sock_non_blocking_client_try_next(&net_instance->non_blocking_client);
+	if (res == -1) {
+		log_nss(LOG_ERR, "Can't connect to qnetd host");
+	}
 
-		res = nss_sock_non_blocking_client_init(net_instance->host_addr,
-		    net_instance->host_port, qdevice_model_net_get_af(net_instance),
-		    &net_instance->non_blocking_client);
-		if (res == -1) {
-			log_nss(LOG_ERR, "Can't initialize non blocking client connection");
-		}
+	res = qdevice_net_socket_add_to_main_poll_loop(net_instance);
+	if (res == -1) {
+		goto error_free_non_blocking_client;
+	}
 
-		res = nss_sock_non_blocking_client_try_next(&net_instance->non_blocking_client);
-		if (res == -1) {
-			log_nss(LOG_ERR, "Can't connect to qnetd host");
-			nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client);
-		}
+	res = pr_poll_loop_add_pre_poll_cb(&instance->main_poll_loop, check_schedule_disconnect_cb,
+	    net_instance, NULL);
+	if (res == -1) {
+		log(LOG_CRIT, "Can't add pre poll callback to main loop");
+		goto error_del_from_main_poll_loop;
+	}
 
-		while (qdevice_net_poll(net_instance) == 0) {
-		};
+	return (0);
 
-		if (net_instance->connect_timer != NULL) {
-			timer_list_delete(&net_instance->main_timer_list, net_instance->connect_timer);
-			net_instance->connect_timer = NULL;
-		}
+error_del_from_main_poll_loop:
+	(void)qdevice_net_socket_del_from_main_poll_loop(net_instance);
 
-		if (net_instance->echo_request_timer != NULL) {
-			timer_list_delete(&net_instance->main_timer_list, net_instance->echo_request_timer);
-			net_instance->echo_request_timer = NULL;
-		}
+error_free_non_blocking_client:
+	nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client);
+	return (-1);
+}
 
-		try_connect = qdevice_net_disconnect_reason_try_reconnect(net_instance->disconnect_reason);
+/*
+ *  1 - Restart loop
+ *  0 - End loop with no error
+ * -1 - End loop with error -1
+ */
+int
+qdevice_model_net_post_poll_loop(struct qdevice_instance *instance,
+    enum qdevice_model_post_poll_loop_exit_reason exit_reason)
+{
+	struct qdevice_net_instance *net_instance;
+	int restart_loop;
+	int ret_val;
+	enum tlv_vote vote;
+	int delay_before_reconnect;
 
+	net_instance = instance->model_data;
+
+	restart_loop = 1;
+	ret_val = -1;
+
+	if (net_instance->connect_timer != NULL) {
+		timer_list_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop),
+		    net_instance->connect_timer);
+		net_instance->connect_timer = NULL;
+	}
+
+	if (net_instance->echo_request_timer != NULL) {
+		timer_list_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop),
+		    net_instance->echo_request_timer);
+		net_instance->echo_request_timer = NULL;
+	}
+
+	/*
+	 * Map qdevice exit_reason to qdevice-net disconnect reason
+	 */
+	switch (exit_reason) {
+	case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_MODEL:
 		/*
-		 * Unpause cast vote timer, because if it is paused we cannot remove tracking
+		 * Disconnect reason should be already set
 		 */
-		qdevice_net_cast_vote_timer_set_paused(net_instance, 0);
+		break;
+	case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_VOTEQUORUM_CLOSED:
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
+		break;
+	case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_CMAP_CLOSED:
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
+		break;
+	case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_HEURISTICS_CLOSED:
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED;
+		break;
+	case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_IPC_SOCKET_CLOSED:
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED;
+		break;
+	/*
+	 * Default not handled intentionally. Compiler shows warning when new exit reason is added
+	 */
+	}
 
-		vote = TLV_VOTE_NO_CHANGE;
+	restart_loop = qdevice_net_disconnect_reason_try_reconnect(net_instance->disconnect_reason);
 
-		if (qdevice_net_algorithm_disconnected(net_instance,
-		    net_instance->disconnect_reason, &try_connect, &vote) != 0) {
-			log(LOG_ERR, "Algorithm returned error, force exit");
-			return (-1);
-		} else {
-			log(LOG_DEBUG, "Algorithm result vote is %s",
-			    tlv_vote_to_str(vote));
-		}
+	/*
+	 * Unpause cast vote timer, because if it is paused we cannot remove tracking
+	 */
+	qdevice_net_cast_vote_timer_set_paused(net_instance, 0);
 
-		if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
-			log(LOG_ERR, "qdevice_model_net_run fatal error. "
-			    " Can't update cast vote timer vote");
-		}
+	vote = TLV_VOTE_NO_CHANGE;
 
-		if (qdevice_net_disconnect_reason_force_disconnect(net_instance->disconnect_reason)) {
-			try_connect = 0;
-		}
+	if (qdevice_net_algorithm_disconnected(net_instance,
+	    net_instance->disconnect_reason, &restart_loop, &vote) != 0) {
+		log(LOG_ERR, "Algorithm returned error, force exit");
+		return (-1);
+	} else {
+		log(LOG_DEBUG, "Algorithm result vote is %s",
+		    tlv_vote_to_str(vote));
+	}
 
-		/*
-		 * Return 0 only when local socket was closed -> regular exit
-		 */
-		if (net_instance->disconnect_reason == QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED) {
-			ret_val = 0;
-		}
+	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
+		log(LOG_ERR, "qdevice_model_net_run fatal error. "
+		    " Can't update cast vote timer vote");
+	}
 
-		if (net_instance->socket != NULL) {
-			if (PR_Close(net_instance->socket) != PR_SUCCESS) {
-				log_nss(LOG_WARNING, "Unable to close connection");
-			}
-			net_instance->socket = NULL;
-		}
+	if (qdevice_net_disconnect_reason_force_disconnect(net_instance->disconnect_reason)) {
+		restart_loop = 0;
+	}
 
-		if (!net_instance->non_blocking_client.destroyed) {
-			nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client);
-		}
+	/*
+	 * Return 0 only when local socket was closed -> regular exit
+	 */
+	if (net_instance->disconnect_reason == QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED) {
+		ret_val = 0;
+	}
+
+	/*
+	 * Remove pre poll cb
+	 */
+	if (pr_poll_loop_del_pre_poll_cb(&instance->main_poll_loop, check_schedule_disconnect_cb) == -1) {
+		log(LOG_ERR, "Can't delete pre poll callback from main loop");
+		restart_loop = 0;
+		ret_val = -1;
+	}
+
+	/*
+	 * Remove socket from loop
+	 */
+	if (qdevice_net_socket_del_from_main_poll_loop(net_instance) == -1) {
+		restart_loop = 0;
+		ret_val = -1;
+	}
 
-		if (net_instance->non_blocking_client.socket != NULL) {
-			if (PR_Close(net_instance->non_blocking_client.socket) != PR_SUCCESS) {
-				log_nss(LOG_WARNING, "Unable to close non-blocking client connection");
-			}
-			net_instance->non_blocking_client.socket = NULL;
+	if (net_instance->socket != NULL) {
+		if (PR_Close(net_instance->socket) != PR_SUCCESS) {
+			log_nss(LOG_WARNING, "Unable to close connection");
 		}
+		net_instance->socket = NULL;
+	}
 
-		if (try_connect &&
-		    net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
-			/*
-			 * Give qnetd server a little time before reconnect
-			 */
-			delay_before_reconnect = random() %
-			    (int)(net_instance->cast_vote_timer_interval * 0.9);
+	if (!net_instance->non_blocking_client.destroyed) {
+		nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client);
+	}
 
-			log(LOG_DEBUG, "Sleeping for %u ms before reconnect",
-			    delay_before_reconnect);
-			(void)poll(NULL, 0, delay_before_reconnect);
+	if (net_instance->non_blocking_client.socket != NULL) {
+		if (PR_Close(net_instance->non_blocking_client.socket) != PR_SUCCESS) {
+			log_nss(LOG_WARNING, "Unable to close non-blocking client connection");
 		}
+		net_instance->non_blocking_client.socket = NULL;
+	}
+
+	if (restart_loop &&
+	    net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
+		/*
+		 * Give qnetd server a little time before reconnect
+		 */
+		delay_before_reconnect = random() %
+		    (int)(net_instance->cast_vote_timer_interval * 0.9);
+
+		log(LOG_DEBUG, "Sleeping for %u ms before reconnect",
+		    delay_before_reconnect);
+		(void)poll(NULL, 0, delay_before_reconnect);
+	}
 
 
-		qdevice_net_instance_clean(net_instance);
+	qdevice_net_instance_clean(net_instance);
+
+	if (restart_loop) {
+		return (1);
 	}
 
 	return (ret_val);
@@ -679,7 +777,8 @@ static struct qdevice_model qdevice_model_net = {
 	.name					= "net",
 	.init					= qdevice_model_net_init,
 	.destroy				= qdevice_model_net_destroy,
-	.run					= qdevice_model_net_run,
+	.pre_poll_loop				= qdevice_model_net_pre_poll_loop,
+	.post_poll_loop				= qdevice_model_net_post_poll_loop,
 	.get_config_node_list_failed		= qdevice_model_net_get_config_node_list_failed,
 	.config_node_list_changed		= qdevice_model_net_config_node_list_changed,
 	.votequorum_quorum_notify		= qdevice_model_net_votequorum_quorum_notify,
@@ -693,5 +792,6 @@ static struct qdevice_model qdevice_model_net = {
 int
 qdevice_model_net_register(void)
 {
+
 	return (qdevice_model_register(QDEVICE_MODEL_TYPE_NET, &qdevice_model_net));
 }

qdevices/qdevice-model-net.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -46,7 +46,10 @@ extern int	qdevice_model_net_init(struct qdevice_instance *instance);
 
 extern int	qdevice_model_net_destroy(struct qdevice_instance *instance);
 
-extern int	qdevice_model_net_run(struct qdevice_instance *instance);
+extern int	qdevice_model_net_pre_poll_loop(struct qdevice_instance *instance);
+
+extern int	qdevice_model_net_post_poll_loop(struct qdevice_instance *instance,
+    enum qdevice_model_post_poll_loop_exit_reason exit_reason);
 
 extern int	qdevice_model_net_get_config_node_list_failed(struct qdevice_instance *instance);
 

qdevices/qdevice-model.c

@@ -65,16 +65,30 @@ qdevice_model_destroy(struct qdevice_instance *instance)
 }
 
 int
-qdevice_model_run(struct qdevice_instance *instance)
+qdevice_model_pre_poll_loop(struct qdevice_instance *instance)
 {
 
 	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
 	    qdevice_model_array[instance->model_type] == NULL) {
-		log(LOG_CRIT, "qdevice_model_run unhandled model");
+		log(LOG_CRIT, "qdevice_model_pre_poll_loop unhandled model");
 		exit(EXIT_FAILURE);
 	}
 
-	return (qdevice_model_array[instance->model_type]->run(instance));
+	return (qdevice_model_array[instance->model_type]->pre_poll_loop(instance));
+}
+
+int
+qdevice_model_post_poll_loop(struct qdevice_instance *instance,
+    enum qdevice_model_post_poll_loop_exit_reason exit_reason)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		log(LOG_CRIT, "qdevice_model_post_poll_loop unhandled model");
+		exit(EXIT_FAILURE);
+	}
+
+	return (qdevice_model_array[instance->model_type]->post_poll_loop(instance, exit_reason));
 }
 
 int
@@ -83,7 +97,7 @@ qdevice_model_get_config_node_list_failed(struct qdevice_instance *instance)
 
 	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
 	    qdevice_model_array[instance->model_type] == NULL) {
-		log(LOG_CRIT, "qdevice_model_run unhandled model");
+		log(LOG_CRIT, "qdevice_model_get_config_node_list_failed unhandled model");
 		exit(EXIT_FAILURE);
 	}
 
@@ -97,7 +111,7 @@ qdevice_model_config_node_list_changed(struct qdevice_instance *instance,
 
 	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
 	    qdevice_model_array[instance->model_type] == NULL) {
-		log(LOG_CRIT, "qdevice_model_run unhandled model");
+		log(LOG_CRIT, "qdevice_model_config_node_list_changed unhandled model");
 		exit(EXIT_FAILURE);
 	}
 

qdevices/qdevice-model.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -44,11 +44,33 @@
 extern "C" {
 #endif
 
+enum qdevice_model_post_poll_loop_exit_reason {
+	/* Some of model callbacks made poll loop exit */
+	QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_MODEL,
+	/* Votequorum connection closed */
+	QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_VOTEQUORUM_CLOSED,
+	/* Cmap connection closed */
+	QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_CMAP_CLOSED,
+	/* One of heuristics pipes closed */
+	QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_HEURISTICS_CLOSED,
+	/* IPC socket closed (user initiated shutdown) */
+	QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_IPC_SOCKET_CLOSED
+};
+
 extern int	qdevice_model_init(struct qdevice_instance *instance);
 
 extern int	qdevice_model_destroy(struct qdevice_instance *instance);
 
-extern int	qdevice_model_run(struct qdevice_instance *instance);
+/*
+ *  Return codes: 0 - Continue, -1 - End loop
+ */
+extern int	qdevice_model_pre_poll_loop(struct qdevice_instance *instance);
+
+/*
+ * Return codes:  1 - Restart loop, 0 - End loop with no error, -1 - End loop with error -1
+ */
+extern int	qdevice_model_post_poll_loop(struct qdevice_instance *instance,
+    enum qdevice_model_post_poll_loop_exit_reason exit_reason);
 
 extern int	qdevice_model_get_config_node_list_failed(struct qdevice_instance *instance);
 
@@ -78,7 +100,9 @@ struct qdevice_model {
 	const char *name;
 	int (*init)(struct qdevice_instance *instance);
 	int (*destroy)(struct qdevice_instance *instance);
-	int (*run)(struct qdevice_instance *instance);
+	int (*pre_poll_loop)(struct qdevice_instance *instance);
+	int (*post_poll_loop)(struct qdevice_instance *instance,
+	    enum qdevice_model_post_poll_loop_exit_reason exit_reason);
 	int (*get_config_node_list_failed)(struct qdevice_instance *instance);
 	int (*config_node_list_changed)(struct qdevice_instance *instance,
 	    const struct node_list *nlist, int config_version_set, uint64_t config_version);

qdevices/qdevice-net-cast-vote-timer.c

@@ -136,7 +136,8 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t
 
 	if (timer_needs_running) {
 		if (instance->cast_vote_timer == NULL) {
-			instance->cast_vote_timer = timer_list_add(&instance->main_timer_list,
+			instance->cast_vote_timer = timer_list_add(
+			    pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop),
 			    instance->cast_vote_timer_interval,
 			    qdevice_net_cast_vote_timer_callback, (void *)instance, NULL);
 
@@ -161,7 +162,9 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t
 		}
 	} else {
 		if (instance->cast_vote_timer != NULL) {
-			timer_list_delete(&instance->main_timer_list, instance->cast_vote_timer);
+			timer_list_delete(
+			    pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop),
+			    instance->cast_vote_timer);
 			instance->cast_vote_timer = NULL;
 			log(LOG_DEBUG, "Cast vote timer is now stopped.");
 		} else {

qdevices/qdevice-net-disconnect-reason.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -121,7 +121,6 @@ enum qdevice_net_disconnect_reason {
 	QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_REPLY_ERR,
 
 	QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED,
-	QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG,
 };
 
 #define qdevice_net_disconnect_reason_try_reconnect(reason) (						\
@@ -142,8 +141,7 @@ enum qdevice_net_disconnect_reason {
     reason == QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER ||		\
     reason == QDEVICE_NET_DISCONNECT_REASON_CANT_REGISTER_VOTEQUORUM_CALLBACK ||	\
     reason == QDEVICE_NET_DISCONNECT_REASON_CANT_REGISTER_CMAP_CALLBACK ||		\
-    reason == QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED ||			\
-    reason == QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG)
+    reason == QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED)
 
 #ifdef __cplusplus
 }

qdevices/qdevice-net-echo-request-timer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2019 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -85,12 +85,15 @@ qdevice_net_echo_request_timer_schedule(struct qdevice_net_instance *instance)
 	instance->echo_reply_received_msg_seq_num = 0;
 
 	if (instance->echo_request_timer != NULL) {
-		timer_list_delete(&instance->main_timer_list, instance->echo_request_timer);
+		timer_list_delete(
+		    pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop),
+		    instance->echo_request_timer);
 		instance->echo_request_timer = NULL;
 	}
 
 	log(LOG_DEBUG, "Scheduling send of heartbeat every %"PRIu32"ms", instance->heartbeat_interval);
-	instance->echo_request_timer = timer_list_add(&instance->main_timer_list,
+	instance->echo_request_timer = timer_list_add(
+	    pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop),
 	    instance->heartbeat_interval, qdevice_net_echo_request_timer_callback,
 	    (void *)instance, NULL);
 

qdevices/qdevice-net-heuristics.c

@@ -350,7 +350,8 @@ qdevice_net_heuristics_stop_timer(struct qdevice_net_instance *net_instance)
 	if (net_instance->regular_heuristics_timer != NULL) {
 		log(LOG_DEBUG, "Regular heuristics timer stopped");
 
-		timer_list_delete(&net_instance->main_timer_list, net_instance->regular_heuristics_timer);
+		timer_list_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop),
+		    net_instance->regular_heuristics_timer);
 		net_instance->regular_heuristics_timer = NULL;
 
 		if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
@@ -397,7 +398,8 @@ qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance *net_instance)
 
 	log(LOG_DEBUG, "Scheduling next regular heuristics in %"PRIu32"ms", interval);
 
-	net_instance->regular_heuristics_timer = timer_list_add(&net_instance->main_timer_list,
+	net_instance->regular_heuristics_timer = timer_list_add(
+		pr_poll_loop_get_timer_list(&instance->main_poll_loop),
 		interval,
 		qdevice_net_heuristics_timer_callback,
 	        (void *)net_instance, NULL);

qdevices/qdevice-net-instance.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2019 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -39,14 +39,8 @@
 #include "qdevice-net-instance.h"
 #include "qnet-config.h"
 #include "utils.h"
-#include "qdevice-net-poll-array-user-data.h"
 #include "qdevice-ipc.h"
 
-/*
- * Needed for creating nspr handle from unix fd
- */
-#include <private/pprio.h>
-
 int
 qdevice_net_instance_init(struct qdevice_net_instance *instance,
     enum tlv_tls_supported tls_supported,
@@ -86,45 +80,8 @@ qdevice_net_instance_init(struct qdevice_net_instance *instance,
 	send_buffer_list_init(&instance->send_buffer_list, advanced_settings->net_max_send_buffers,
 	    advanced_settings->net_initial_msg_send_size);
 
-	timer_list_init(&instance->main_timer_list);
-
-	pr_poll_array_init(&instance->poll_array, sizeof(struct qdevice_net_poll_array_user_data));
-
 	instance->tls_supported = tls_supported;
 
-	if ((instance->cmap_poll_fd = PR_CreateSocketPollFd(cmap_fd)) == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR cmap poll fd");
-		return (-1);
-	}
-
-	if ((instance->votequorum_poll_fd = PR_CreateSocketPollFd(votequorum_fd)) == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR votequorum poll fd");
-		return (-1);
-	}
-
-	if ((instance->ipc_socket_poll_fd = PR_CreateSocketPollFd(local_socket_fd)) == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR IPC socket poll fd");
-		return (-1);
-	}
-
-	if ((instance->heuristics_pipe_cmd_send_poll_fd =
-	    PR_CreateSocketPollFd(heuristics_pipe_cmd_send_fd)) == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe command send poll fd");
-		return (-1);
-	}
-
-	if ((instance->heuristics_pipe_cmd_recv_poll_fd =
-	    PR_CreateSocketPollFd(heuristics_pipe_cmd_recv_fd)) == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe command recv poll fd");
-		return (-1);
-	}
-
-	if ((instance->heuristics_pipe_log_recv_poll_fd =
-	    PR_CreateSocketPollFd(heuristics_pipe_log_recv_fd)) == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe log recv poll fd");
-		return (-1);
-	}
-
 	return (0);
 }
 
@@ -152,60 +109,14 @@ qdevice_net_instance_clean(struct qdevice_net_instance *instance)
 int
 qdevice_net_instance_destroy(struct qdevice_net_instance *instance)
 {
-	struct unix_socket_client *ipc_client;
-	const struct unix_socket_client_list *ipc_client_list;
-	struct qdevice_ipc_user_data *qdevice_ipc_user_data;
-	PRFileDesc *prfd;
-
-	ipc_client_list = &instance->qdevice_instance_ptr->local_ipc.clients;
-
-	TAILQ_FOREACH(ipc_client, ipc_client_list, entries) {
-		qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data;
-		prfd = (PRFileDesc *)qdevice_ipc_user_data->model_data;
-
-		if (PR_DestroySocketPollFd(prfd) != PR_SUCCESS) {
-			log_nss(LOG_WARNING, "Unable to destroy client IPC poll socket fd");
-		}
-	}
 
 	dynar_destroy(&instance->receive_buffer);
 
 	send_buffer_list_free(&instance->send_buffer_list);
 
-	pr_poll_array_destroy(&instance->poll_array);
-
-	timer_list_free(&instance->main_timer_list);
-
 	free((void *)instance->cluster_name);
 	free((void *)instance->host_addr);
 
-	if (PR_DestroySocketPollFd(instance->votequorum_poll_fd) != PR_SUCCESS) {
-		log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
-	}
-
-	if (PR_DestroySocketPollFd(instance->cmap_poll_fd) != PR_SUCCESS) {
-		log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
-	}
-
-	if (PR_DestroySocketPollFd(instance->ipc_socket_poll_fd) != PR_SUCCESS) {
-		log_nss(LOG_WARNING, "Unable to close local socket poll fd");
-	}
-
-	if (PR_DestroySocketPollFd(instance->heuristics_pipe_cmd_send_poll_fd) != PR_SUCCESS) {
-		log_nss(LOG_WARNING, "Unable to close heuristics pipe command send poll fd");
-		return (-1);
-	}
-
-	if (PR_DestroySocketPollFd(instance->heuristics_pipe_cmd_recv_poll_fd) != PR_SUCCESS) {
-		log_nss(LOG_WARNING, "Unable to close heuristics pipe command recv poll fd");
-		return (-1);
-	}
-
-	if (PR_DestroySocketPollFd(instance->heuristics_pipe_log_recv_poll_fd) != PR_SUCCESS) {
-		log_nss(LOG_WARNING, "Unable to close heuristics pipe log recv poll fd");
-		return (-1);
-	}
-
 	return (0);
 }
 

qdevices/qdevice-net-instance.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -46,7 +46,6 @@
 
 #include "dynar.h"
 #include "node-list.h"
-#include "pr-poll-array.h"
 #include "qdevice-net-disconnect-reason.h"
 #include "send-buffer-list.h"
 #include "tlv.h"
@@ -89,12 +88,8 @@ struct qdevice_net_instance {
 	uint16_t host_port;
 	const char *cluster_name;
 	enum tlv_decision_algorithm_type decision_algorithm;
-	struct timer_list main_timer_list;
 	struct timer_list_entry *echo_request_timer;
 	int schedule_disconnect;
-	PRFileDesc *votequorum_poll_fd;
-	PRFileDesc *cmap_poll_fd;
-	PRFileDesc *ipc_socket_poll_fd;
 	struct tlv_ring_id last_sent_ring_id;
 	struct tlv_tie_breaker tie_breaker;
 	void *algorithm_data;
@@ -103,13 +98,9 @@ struct qdevice_net_instance {
 	struct nss_sock_non_blocking_client non_blocking_client;
 	struct timer_list_entry *connect_timer;
 	int force_ip_version;
-	struct pr_poll_array poll_array;
 	time_t last_echo_reply_received_time;
 	time_t connected_since_time;
 	const struct qdevice_advanced_settings *advanced_settings;
-	PRFileDesc *heuristics_pipe_cmd_send_poll_fd;
-	PRFileDesc *heuristics_pipe_cmd_recv_poll_fd;
-	PRFileDesc *heuristics_pipe_log_recv_poll_fd;
 	struct timer_list_entry *regular_heuristics_timer;
 	int server_supports_heuristics;
 	enum tlv_heuristics latest_regular_heuristics_result;

qdevices/qdevice-net-msg-received.c

@@ -377,7 +377,9 @@ qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
 	 * Finally fully connected so it's possible to remove connection timer
 	 */
 	if (instance->connect_timer != NULL) {
-		timer_list_delete(&instance->main_timer_list, instance->connect_timer);
+		timer_list_delete(
+		    pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop),
+		    instance->connect_timer);
 		instance->connect_timer = NULL;
 	}
 

qdevices/qdevice-net-poll-array-user-data.h

@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2015-2017 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Jan Friesse (jfriesse@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the Red Hat, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _QDEVICE_NET_POLL_ARRAY_USER_DATA_H_
-#define _QDEVICE_NET_POLL_ARRAY_USER_DATA_H_
-
-#include "unix-socket-client.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum qdevice_net_poll_array_user_data_type {
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV,
-	QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV,
-};
-
-struct qdevice_net_poll_array_user_data {
-	enum qdevice_net_poll_array_user_data_type type;
-	struct unix_socket_client *ipc_client;
-};
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _QDEVICE_NET_POLL_ARRAY_USER_DATA_H_ */

qdevices/qdevice-net-poll.c

@@ -1,546 +0,0 @@
-/*
- * Copyright (c) 2015-2020 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Jan Friesse (jfriesse@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the Red Hat, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "log.h"
-#include "qdevice-cmap.h"
-#include "qdevice-net-poll.h"
-#include "qdevice-net-send.h"
-#include "qdevice-net-socket.h"
-#include "qdevice-votequorum.h"
-#include "qdevice-ipc.h"
-#include "qdevice-net-poll-array-user-data.h"
-#include "qdevice-heuristics.h"
-#include "qdevice-heuristics-cmd.h"
-
-/*
- * Needed for creating nspr handle from unix fd
- */
-#include <private/pprio.h>
-
-static void
-qdevice_net_poll_read_socket(struct qdevice_net_instance *instance)
-{
-
-	if (qdevice_net_socket_read(instance) == -1) {
-		instance->schedule_disconnect = 1;
-	}
-}
-
-static void
-qdevice_net_poll_read_votequorum(struct qdevice_net_instance *instance)
-{
-
-	if (qdevice_votequorum_dispatch(instance->qdevice_instance_ptr) == -1) {
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
-	}
-}
-
-static void
-qdevice_net_poll_read_cmap(struct qdevice_net_instance *instance)
-{
-
-	if (qdevice_cmap_dispatch(instance->qdevice_instance_ptr) == -1) {
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
-	}
-}
-
-static void
-qdevice_net_poll_write_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd)
-{
-	int res;
-
-	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
-		res = nss_sock_non_blocking_client_succeeded(pfd);
-		if (res == -1) {
-			/*
-			 * Connect failed -> try next
-			 */
-			res = nss_sock_non_blocking_client_try_next(&instance->non_blocking_client);
-			if (res == -1) {
-				log_nss(LOG_ERR, "Can't connect to qnetd host.");
-				nss_sock_non_blocking_client_destroy(&instance->non_blocking_client);
-			}
-		} else if (res == 0) {
-			/*
-			 * Poll again
-			 */
-		} else if (res == 1) {
-			/*
-			 * Connect success
-			 */
-			instance->socket = instance->non_blocking_client.socket;
-			nss_sock_non_blocking_client_destroy(&instance->non_blocking_client);
-			instance->non_blocking_client.socket = NULL;
-
-			instance->state = QDEVICE_NET_INSTANCE_STATE_SENDING_PREINIT_REPLY;
-
-			log(LOG_DEBUG, "Sending preinit msg to qnetd");
-			if (qdevice_net_send_preinit(instance) != 0) {
-				instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-				instance->schedule_disconnect = 1;
-			}
-		} else {
-			log(LOG_CRIT, "Unhandled nss_sock_non_blocking_client_succeeded");
-			exit(EXIT_FAILURE);
-		}
-	} else {
-		if (qdevice_net_socket_write(instance) == -1) {
-			instance->schedule_disconnect = 1;
-		}
-	}
-}
-
-static void
-qdevice_net_poll_err_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd)
-{
-
-	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
-		/*
-		 * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work
-		 * only with poll, select works as expected!???).
-		 * So test if client is still valid and if pollout was not already called (ensured
-		 * by default because of order in PR_Poll).
-		 * If both applies it's possible to emulate pollout set by calling poll_write.
-		 */
-		if (!instance->non_blocking_client.destroyed) {
-			qdevice_net_poll_write_socket(instance, pfd);
-		}
-	} else {
-		log(LOG_ERR, "POLL_ERR (%u) on main socket", pfd->out_flags);
-
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
-	}
-}
-
-static void
-qdevice_net_poll_read_heuristics_log(struct qdevice_net_instance *instance)
-{
-	int res;
-
-	res = qdevice_heuristics_log_read_from_pipe(&instance->qdevice_instance_ptr->heuristics_instance);
-	if (res == -1) {
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED;
-	}
-}
-
-static void
-qdevice_net_poll_read_heuristics_cmd(struct qdevice_net_instance *instance)
-{
-	int res;
-
-	res = qdevice_heuristics_cmd_read_from_pipe(&instance->qdevice_instance_ptr->heuristics_instance);
-	if (res == -1) {
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED;
-	}
-}
-
-static void
-qdevice_net_poll_write_heuristics_cmd(struct qdevice_net_instance *instance)
-{
-	int res;
-
-	res = qdevice_heuristics_cmd_write(&instance->qdevice_instance_ptr->heuristics_instance);
-	if (res == -1) {
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason =
-		    QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG;
-	}
-}
-
-static void
-qdevice_net_poll_read_ipc_socket(struct qdevice_net_instance *instance)
-{
-	struct unix_socket_client *client;
-	PRFileDesc *prfd;
-	struct qdevice_ipc_user_data *user_data;
-
-	if (qdevice_ipc_accept(instance->qdevice_instance_ptr, &client) != 0) {
-		return ;
-	}
-
-	prfd = PR_CreateSocketPollFd(client->socket);
-	if (prfd == NULL) {
-		log_nss(LOG_CRIT, "Can't create NSPR poll fd for IPC client. "
-		    "Disconnecting client");
-		qdevice_ipc_client_disconnect(instance->qdevice_instance_ptr, client);
-
-		return ;
-	}
-
-	user_data = (struct qdevice_ipc_user_data *)client->user_data;
-	user_data->model_data = (void *)prfd;
-}
-
-static PRPollDesc *
-qdevice_net_pr_poll_array_create(struct qdevice_net_instance *instance)
-{
-	struct pr_poll_array *poll_array;
-	PRPollDesc *poll_desc;
-	struct qdevice_net_poll_array_user_data *user_data;
-	struct unix_socket_client *ipc_client;
-	const struct unix_socket_client_list *ipc_client_list;
-	struct qdevice_ipc_user_data *qdevice_ipc_user_data;
-
-	poll_array = &instance->poll_array;
-	ipc_client_list = &instance->qdevice_instance_ptr->local_ipc.clients;
-
-	if (qdevice_ipc_is_closed(instance->qdevice_instance_ptr)) {
-		log(LOG_DEBUG, "Local socket is closed");
-		instance->schedule_disconnect = 1;
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED;
-
-		return (NULL);
-	}
-
-	pr_poll_array_clean(poll_array);
-
-	if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-		return (NULL);
-	}
-	poll_desc->fd = instance->votequorum_poll_fd;
-	poll_desc->in_flags = PR_POLL_READ;
-	user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM;
-
-	if (!instance->qdevice_instance_ptr->sync_in_progress) {
-		if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-			return (NULL);
-		}
-		poll_desc->fd = instance->cmap_poll_fd;
-		poll_desc->in_flags = PR_POLL_READ;
-		user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP;
-	}
-
-	if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-		return (NULL);
-	}
-	poll_desc->fd = instance->ipc_socket_poll_fd;
-	poll_desc->in_flags = PR_POLL_READ;
-	user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET;
-
-	if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-		return (NULL);
-	}
-
-	poll_desc->fd = instance->heuristics_pipe_log_recv_poll_fd;
-	poll_desc->in_flags = PR_POLL_READ;
-	user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV;
-
-	if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-		return (NULL);
-	}
-
-	poll_desc->fd = instance->heuristics_pipe_cmd_recv_poll_fd;
-	poll_desc->in_flags = PR_POLL_READ;
-	user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV;
-
-	if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-		return (NULL);
-	}
-
-	if (!send_buffer_list_empty(
-	    &instance->qdevice_instance_ptr->heuristics_instance.cmd_out_buffer_list)) {
-		poll_desc->fd = instance->heuristics_pipe_cmd_send_poll_fd;
-		poll_desc->in_flags = PR_POLL_WRITE;
-		user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND;
-
-		if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-			return (NULL);
-		}
-	}
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT ||
-	    !instance->non_blocking_client.destroyed) {
-		if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-			return (NULL);
-		}
-
-		user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET;
-
-		if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
-			poll_desc->fd = instance->non_blocking_client.socket;
-			poll_desc->in_flags = PR_POLL_WRITE | PR_POLL_EXCEPT;
-		} else {
-			poll_desc->fd = instance->socket;
-			poll_desc->in_flags = PR_POLL_READ;
-
-			if (!send_buffer_list_empty(&instance->send_buffer_list)) {
-				poll_desc->in_flags |= PR_POLL_WRITE;
-			}
-		}
-	}
-
-	TAILQ_FOREACH(ipc_client, ipc_client_list, entries) {
-		if (!ipc_client->reading_line && !ipc_client->writing_buffer) {
-			continue;
-		}
-
-		if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
-			return (NULL);
-		}
-
-		qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data;
-		poll_desc->fd = (PRFileDesc *)qdevice_ipc_user_data->model_data;
-		if (ipc_client->reading_line) {
-			poll_desc->in_flags |= PR_POLL_READ;
-		}
-
-		if (ipc_client->writing_buffer) {
-			poll_desc->in_flags |= PR_POLL_WRITE;
-		}
-
-		user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT;
-		user_data->ipc_client = ipc_client;
-	}
-
-	pr_poll_array_gc(poll_array);
-
-	return (poll_array->array);
-}
-
-int
-qdevice_net_poll(struct qdevice_net_instance *instance)
-{
-	PRPollDesc *pfds;
-	PRFileDesc *prfd;
-	PRInt32 poll_res;
-	ssize_t i;
-	struct qdevice_net_poll_array_user_data *user_data;
-	struct unix_socket_client *ipc_client;
-	struct qdevice_ipc_user_data *qdevice_ipc_user_data;
-	int case_processed;
-
-	pfds = qdevice_net_pr_poll_array_create(instance);
-	if (pfds == NULL) {
-		return (-1);
-	}
-
-	instance->schedule_disconnect = 0;
-
-	if ((poll_res = PR_Poll(pfds, pr_poll_array_size(&instance->poll_array),
-	    timer_list_time_to_expire(&instance->main_timer_list))) > 0) {
-		for (i = 0; i < pr_poll_array_size(&instance->poll_array); i++) {
-			user_data = pr_poll_array_get_user_data(&instance->poll_array, i);
-
-			ipc_client = user_data->ipc_client;
-
-			if (pfds[i].out_flags & PR_POLL_READ) {
-				case_processed = 0;
-
-				switch (user_data->type) {
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
-					case_processed = 1;
-					qdevice_net_poll_read_socket(instance);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM:
-					case_processed = 1;
-					qdevice_net_poll_read_votequorum(instance);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP:
-					case_processed = 1;
-					qdevice_net_poll_read_cmap(instance);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET:
-					case_processed = 1;
-					qdevice_net_poll_read_ipc_socket(instance);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT:
-					case_processed = 1;
-					qdevice_ipc_io_read(instance->qdevice_instance_ptr, ipc_client);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND:
-					/*
-					 * Read on heuristics cmd send fd shouldn't happen
-					 */
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV:
-					case_processed = 1;
-					qdevice_net_poll_read_heuristics_cmd(instance);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV:
-					case_processed = 1;
-					qdevice_net_poll_read_heuristics_log(instance);
-					break;
-				/*
-				 * Default is not defined intentionally. Compiler shows warning when
-				 * new poll_array_user_data_type is added
-				 */
-				}
-
-				if (!case_processed) {
-					log(LOG_CRIT, "Unhandled read on poll descriptor %zu", i);
-					exit(EXIT_FAILURE);
-				}
-			}
-
-			if (!instance->schedule_disconnect && pfds[i].out_flags & PR_POLL_WRITE) {
-				case_processed = 0;
-
-				switch (user_data->type) {
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
-					case_processed = 1;
-					qdevice_net_poll_write_socket(instance, &pfds[i]);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT:
-					case_processed = 1;
-					qdevice_ipc_io_write(instance->qdevice_instance_ptr, ipc_client);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND:
-					case_processed = 1;
-					qdevice_net_poll_write_heuristics_cmd(instance);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV:
-					/*
-					 * Write on votequorum, cmap, ipc socket and
-					 * heuristics log shouldn't happen.
-					 */
-					break;
-				/*
-				 * Default is not defined intentionally. Compiler shows warning when
-				 * new poll_array_user_data_type is added
-				 */
-				}
-
-				if (!case_processed) {
-					log(LOG_CRIT, "Unhandled write on poll descriptor %zu", i);
-					exit(EXIT_FAILURE);
-				}
-			}
-
-			if (!instance->schedule_disconnect &&
-			    (pfds[i].out_flags & (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) &&
-			    !(pfds[i].out_flags & (PR_POLL_READ|PR_POLL_WRITE))) {
-				case_processed = 0;
-
-				switch (user_data->type) {
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
-					case_processed = 1;
-					qdevice_net_poll_err_socket(instance, &pfds[i]);
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET:
-					case_processed = 1;
-					if (pfds[i].out_flags != PR_POLL_NVAL) {
-						log(LOG_CRIT, "POLLERR (%u) on local socket",
-						    pfds[i].out_flags);
-						exit(EXIT_FAILURE);
-					} else {
-						log(LOG_DEBUG, "Local socket is closed");
-						instance->schedule_disconnect = 1;
-						instance->disconnect_reason =
-						    QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED;
-					}
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT:
-					case_processed = 1;
-					log(LOG_DEBUG, "POLL_ERR (%u) on ipc client socket. "
-					    "Disconnecting.",  pfds[i].out_flags);
-					ipc_client->schedule_disconnect = 1;
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP:
-					case_processed = 1;
-					log(LOG_DEBUG, "POLL_ERR (%u) on corosync socket. "
-					    "Disconnecting.",  pfds[i].out_flags);
-
-					instance->schedule_disconnect = 1;
-					instance->disconnect_reason =
-						QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
-					break;
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV:
-				case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND:
-					case_processed = 1;
-
-					/*
-					 * Closed pipe doesn't mean return of PR_POLL_READ. To display
-					 * better log message, we call read log as if POLL_READ would
-					 * be set.
-					 */
-					qdevice_net_poll_read_heuristics_log(instance);
-
-					log(LOG_DEBUG, "POLL_ERR (%u) on heuristics pipe. "
-					    "Disconnecting.",  pfds[i].out_flags);
-
-					instance->schedule_disconnect = 1;
-					instance->disconnect_reason =
-						QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED;
-					break;
-				/*
-				 * Default is not defined intentionally. Compiler shows warning when
-				 * new poll_array_user_data_type is added
-				 */
-				}
-
-				if (!case_processed) {
-					log(LOG_CRIT, "Unhandled error on poll descriptor %zu", i);
-					exit(EXIT_FAILURE);
-				}
-			}
-
-			if (user_data->type == QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT &&
-			    ipc_client->schedule_disconnect) {
-				qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data;
-				prfd = (PRFileDesc *)qdevice_ipc_user_data->model_data;
-
-				if (PR_DestroySocketPollFd(prfd) != PR_SUCCESS) {
-					log_nss(LOG_WARNING, "Unable to destroy client IPC poll socket fd");
-				}
-
-				qdevice_ipc_client_disconnect(instance->qdevice_instance_ptr, ipc_client);
-			}
-		}
-	}
-
-	if (!instance->schedule_disconnect) {
-		timer_list_expire(&instance->main_timer_list);
-	}
-
-	if (instance->schedule_disconnect) {
-		/*
-		 * Schedule disconnect can be set by this function, by some timer_list callback
-		 * or cmap/votequorum callbacks
-		 */
-		return (-1);
-	}
-
-	return (0);
-}

qdevices/qdevice-net-poll.h

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Jan Friesse (jfriesse@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the Red Hat, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _QDEVICE_NET_POLL_H_
-#define _QDEVICE_NET_POLL_H_
-
-
-#include "qdevice-net-instance.h"
-
-extern int		qdevice_net_poll(struct qdevice_net_instance *instance);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _QDEVICE_NET_POLL_H_ */

qdevices/qdevice-net-socket.c

@@ -41,6 +41,150 @@
 #include "qdevice-net-send.h"
 #include "qdevice-net-socket.h"
 
+/*
+ * Socket callbacks
+ */
+static int
+socket_set_events_cb(PRFileDesc *prfd, short *events, void *user_data1, void *user_data2)
+{
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	if (!send_buffer_list_empty(&instance->send_buffer_list)) {
+		*events |= POLLOUT;
+	}
+
+	return (0);
+}
+
+static int
+socket_read_cb(PRFileDesc *prfd, const PRPollDesc *pd, void *user_data1, void *user_data2)
+{
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	if (qdevice_net_socket_read(instance) == -1) {
+		instance->schedule_disconnect = 1;
+
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+socket_write_cb(PRFileDesc *prfd, const PRPollDesc *pd, void *user_data1, void *user_data2)
+{
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	if (qdevice_net_socket_write(instance) == -1) {
+		instance->schedule_disconnect = 1;
+
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+non_blocking_client_socket_write_cb(PRFileDesc *prfd, const PRPollDesc *pd, void *user_data1,
+    void *user_data2)
+{
+	int res;
+
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	res = nss_sock_non_blocking_client_succeeded(pd);
+	if (res == -1) {
+		/*
+		 * Connect failed -> remove this fd from main loop and try next
+		 */
+		res = qdevice_net_socket_del_from_main_poll_loop(instance);
+		if (res == -1) {
+			return (-1);
+		}
+
+		res = nss_sock_non_blocking_client_try_next(&instance->non_blocking_client);
+		if (res == -1) {
+			log_nss(LOG_ERR, "Can't connect to qnetd host.");
+			nss_sock_non_blocking_client_destroy(&instance->non_blocking_client);
+		}
+
+		res = qdevice_net_socket_add_to_main_poll_loop(instance);
+		if (res == -1) {
+			return (-1);
+		}
+	} else if (res == 0) {
+		/*
+		 * Poll again
+		 */
+	} else if (res == 1) {
+		/*
+		 * Connect success -> delete socket from main loop and add final one
+		 */
+		res = qdevice_net_socket_del_from_main_poll_loop(instance);
+		if (res == -1) {
+			return (-1);
+		}
+
+		instance->socket = instance->non_blocking_client.socket;
+		nss_sock_non_blocking_client_destroy(&instance->non_blocking_client);
+		instance->non_blocking_client.socket = NULL;
+
+		instance->state = QDEVICE_NET_INSTANCE_STATE_SENDING_PREINIT_REPLY;
+
+		res = qdevice_net_socket_add_to_main_poll_loop(instance);
+		if (res == -1) {
+			return (-1);
+		}
+
+		log(LOG_DEBUG, "Sending preinit msg to qnetd");
+		if (qdevice_net_send_preinit(instance) != 0) {
+			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+			return (-1);
+		}
+	} else {
+		log(LOG_CRIT, "Unhandled nss_sock_non_blocking_client_succeeded");
+		exit(EXIT_FAILURE);
+	}
+
+	return (0);
+}
+
+static int
+socket_err_cb(PRFileDesc *prfd, short revents, const PRPollDesc *pd, void *user_data1, void *user_data2)
+{
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	log(LOG_ERR, "POLL_ERR (%u) on main socket", revents);
+
+	instance->schedule_disconnect = 1;
+	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
+
+	return (-1);
+}
+
+static int
+non_blocking_client_socket_err_cb(PRFileDesc *prfd, short revents, const PRPollDesc *pd,
+    void *user_data1, void *user_data2)
+{
+	struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1;
+
+	/*
+	 * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work
+	 * only with poll, select works as expected!???).
+	 * So test if client is still valid and if pollout was not already called (ensured
+	 * by default because of order in PR_Poll).
+	 * If both applies it's possible to emulate pollout set by calling poll_write.
+	 */
+	if (!instance->non_blocking_client.destroyed) {
+		return (non_blocking_client_socket_write_cb(prfd, pd, user_data1, user_data2));
+	}
+
+	return (0);
+}
+/*
+ * Exported functions
+ */
+
 /*
  * -1 means end of connection (EOF) or some other unhandled error. 0 = success
  */
@@ -211,3 +355,63 @@ qdevice_net_socket_write(struct qdevice_net_instance *instance)
 
 	return (0);
 }
+
+int
+qdevice_net_socket_add_to_main_poll_loop(struct qdevice_net_instance *instance)
+{
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT ||
+	    !instance->non_blocking_client.destroyed) {
+		if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
+			if (pr_poll_loop_add_prfd(&instance->qdevice_instance_ptr->main_poll_loop,
+			    instance->non_blocking_client.socket,
+			    POLLOUT|POLLPRI,
+			    NULL, NULL, non_blocking_client_socket_write_cb,
+			    non_blocking_client_socket_err_cb,
+			    instance, NULL) != 0) {
+				log(LOG_ERR, "Can't add net socket (non_blocking_client) "
+				    "fd to main poll loop");
+
+				return (-1);
+			}
+		} else {
+			if (pr_poll_loop_add_prfd(&instance->qdevice_instance_ptr->main_poll_loop,
+			    instance->socket,
+			    POLLIN,
+			    socket_set_events_cb, socket_read_cb, socket_write_cb, socket_err_cb,
+			    instance, NULL) != 0) {
+				log(LOG_ERR, "Can't add net socket fd to main poll loop");
+
+				return (-1);
+			}
+		}
+	}
+
+	return (0);
+}
+
+int
+qdevice_net_socket_del_from_main_poll_loop(struct qdevice_net_instance *instance)
+{
+
+	if (!instance->non_blocking_client.destroyed) {
+		if (pr_poll_loop_del_prfd(&instance->qdevice_instance_ptr->main_poll_loop,
+		    instance->non_blocking_client.socket) != 0) {
+			log(LOG_ERR, "Can't remove net socket (non_blocking_client) "
+			    "fd from main poll loop");
+
+			return (-1);
+		}
+	}
+
+	if (instance->socket != NULL) {
+		if (pr_poll_loop_del_prfd(&instance->qdevice_instance_ptr->main_poll_loop,
+		    instance->socket) != 0) {
+			log(LOG_ERR, "Can't remove net socket fd from main poll loop");
+
+			return (-1);
+		}
+	}
+
+	return (0);
+}

qdevices/qdevice-net-socket.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016 Red Hat, Inc.
+ * Copyright (c) 2015-2020 Red Hat, Inc.
  *
  * All rights reserved.
  *
@@ -45,6 +45,10 @@ extern int		qdevice_net_socket_read(struct qdevice_net_instance *instance);
 
 extern int		qdevice_net_socket_write(struct qdevice_net_instance *instance);
 
+extern int		qdevice_net_socket_add_to_main_poll_loop(struct qdevice_net_instance *instance);
+
+extern int		qdevice_net_socket_del_from_main_poll_loop(struct qdevice_net_instance *instance);
+
 #ifdef __cplusplus
 }
 #endif

qdevices/qdevice-pr-poll-loop-cb.c

@@ -33,6 +33,7 @@
  */
 
 #include "log.h"
+#include "qdevice-cmap.h"
 #include "qdevice-heuristics-cmd.h"
 #include "qdevice-heuristics-log.h"
 #include "qdevice-pr-poll-loop-cb.h"
@@ -149,6 +150,49 @@ votequorum_err_cb(int fd, short revents, void *user_data1, void *user_data2)
 	return (-1);
 }
 
+static int
+cmap_set_events_cb(int fd, short *events, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+
+	if (instance->sync_in_progress) {
+		/*
+		 * During sync cmap is blocked -> don't add fd
+		 */
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+cmap_read_cb(int fd, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+	int res;
+
+	res = qdevice_cmap_dispatch(instance);
+	if (res == -1) {
+		instance->cmap_closed = 1;
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+cmap_err_cb(int fd, short revents, void *user_data1, void *user_data2)
+{
+	struct qdevice_instance *instance = (struct qdevice_instance *)user_data1;
+
+	instance->cmap_closed = 1;
+
+	log(LOG_DEBUG, "POLL_ERR (%u) on corosync socket. "
+	     "Disconnecting.",  revents);
+
+	return (-1);
+}
+
 int
 qdevice_pr_poll_loop_cb_register(struct qdevice_instance *instance)
 {
@@ -186,5 +230,13 @@ qdevice_pr_poll_loop_cb_register(struct qdevice_instance *instance)
 		return (-1);
 	}
 
+	if (pr_poll_loop_add_fd(&instance->main_poll_loop, instance->cmap_poll_fd,
+	    POLLIN, cmap_set_events_cb, cmap_read_cb, NULL, cmap_err_cb,
+	    instance, NULL) != 0) {
+		log(LOG_ERR, "Can't add votequorum fd to main poll loop");
+
+		return (-1);
+	}
+
 	return (0);
 }