enhancement 1088

This replaces MSG_DONTWAIT with an fcntl to O_NONBLOCK to be more portable
for the BSD port since BSD doesn't support MSG_DONTWAIT properly.


git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@923 fd59a12c-fef9-0310-b244-a6a79926bd2f

exec/main.c

@@ -344,7 +344,7 @@ static int cleanup_send_response (struct conn_info *conn_info) {
 		iov_send.iov_len = queue_item->mlen - conn_info->byte_start;
 
 retry_sendmsg:
-		res = sendmsg (conn_info->fd, &msg_send, MSG_DONTWAIT | MSG_NOSIGNAL);
+		res = sendmsg (conn_info->fd, &msg_send, MSG_NOSIGNAL);
 		if (res == -1 && errno == EINTR) {
 			goto retry_sendmsg;
 		}
@@ -421,7 +421,7 @@ extern int openais_conn_send_response (
 		iov_send.iov_len = queue_item->mlen - conn_info->byte_start;
 
 retry_sendmsg:
-		res = sendmsg (conn_info->fd, &msg_send, MSG_DONTWAIT | MSG_NOSIGNAL);
+		res = sendmsg (conn_info->fd, &msg_send, MSG_NOSIGNAL);
 		if (res == -1 && errno == EINTR) {
 			goto retry_sendmsg;
 		}
@@ -455,7 +455,7 @@ retry_sendmsg:
 		iov_send.iov_base = msg;
 		iov_send.iov_len = mlen;
 retry_sendmsg_two:
-		res = sendmsg (conn_info->fd, &msg_send, MSG_DONTWAIT | MSG_NOSIGNAL);
+		res = sendmsg (conn_info->fd, &msg_send, MSG_NOSIGNAL);
 		if (res == -1 && errno == EINTR) {
 			goto retry_sendmsg_two;
 		}
@@ -509,6 +509,7 @@ static int poll_handler_libais_accept (
 	struct sockaddr_un un_addr;
 	int new_fd;
 	int on = 1;
+	int res;
 
 	addrlen = sizeof (struct sockaddr_un);
 
@@ -523,6 +524,13 @@ retry_accept:
 		return (0); /* This is an error, but -1 would indicate disconnect from poll loop */
 	}
 		
+	res = fcntl (new_fd, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		log_printf (LOG_LEVEL_ERROR, "Could not set non-blocking operation on library connection: %s\n", strerror (errno));
+		close (new_fd);
+		return (0); /* This is an error, but -1 would indicate disconnect from poll loop */
+	}
+
 	/*
 	 * Valid accept
 	 */
@@ -707,7 +715,7 @@ static int poll_handler_libais_deliver (poll_handle handle, int fd, int revent,
 	assert (iov_recv.iov_len != 0);
 
 retry_recv:
-	res = recvmsg (fd, &msg_recv, MSG_DONTWAIT | MSG_NOSIGNAL);
+	res = recvmsg (fd, &msg_recv, MSG_NOSIGNAL);
 	if (res == -1 && errno == EINTR) {
 		goto retry_recv;
 	} else
@@ -1037,6 +1045,12 @@ static void aisexec_libais_bind (int *server_fd)
 		ais_done (AIS_DONE_LIBAIS_SOCKET);
 	};
 
+	res = fcntl (libais_server_fd, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		log_printf (LOG_LEVEL_ERROR, "Could not set non-blocking operation on server socket: %s\n", strerror (errno));
+		ais_done (AIS_DONE_LIBAIS_SOCKET);
+	}
+
 	memset (&un_addr, 0, sizeof (struct sockaddr_un));
 	un_addr.sun_family = AF_UNIX;
 	strcpy (un_addr.sun_path + 1, socketname);

exec/totemnet.c

@@ -435,7 +435,7 @@ void totemnet_iovec_send (
 	 * An error here is recovered by totemnet
 	 */
 	res = sendmsg (instance->totemnet_sockets.mcast_send, &msg_mcast,
-		MSG_NOSIGNAL | MSG_DONTWAIT);
+		MSG_NOSIGNAL);
 }
 
 void totemnet_msg_send (
@@ -509,7 +509,7 @@ void totemnet_msg_send (
 	 * Transmit token or multicast message
 	 * An error here is recovered by totemnet
 	 */
-	res = sendmsg (fd, &msg_mcast, MSG_NOSIGNAL | MSG_DONTWAIT);
+	res = sendmsg (fd, &msg_mcast, MSG_NOSIGNAL);
 }
 
 static void totemnet_mcast_thread_state_constructor (
@@ -581,7 +581,7 @@ static void totemnet_mcast_worker_fn (void *thread_state, void *work_item_in)
 	 * An error here is recovered by totemnet
 	 */
 	res = sendmsg (instance->totemnet_sockets.mcast_send, &msg_mcast,
-		MSG_NOSIGNAL | MSG_DONTWAIT);
+		MSG_NOSIGNAL);
 	if (res > 0) {
 		instance->stats_sent += res;
 	}
@@ -649,7 +649,7 @@ static int net_deliver_fn (
 	msg_recv.msg_controllen = 0;
 	msg_recv.msg_flags = 0;
 
-	bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
+	bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL);
 	if (bytes_received == -1) {
 		return (0);
 	} else {
@@ -953,6 +953,10 @@ static int totemnet_build_sockets_ipv4 (
 		perror ("socket");
 		return (-1);
 	}
+	res = fcntl (sockets->mcast_send, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		instance->totemnet_log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on multicast socket: %s\n", strerror (errno));
+	}
 
 	/*
 	 * Bind to multicast socket used for multicast send/receives
@@ -972,6 +976,10 @@ static int totemnet_build_sockets_ipv4 (
 		perror ("socket2");
 		return (-1);
 	}
+	res = fcntl (sockets->mcast_send, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		instance->totemnet_log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on token socket: %s\n", strerror (errno));
+	}
 
 	/*
 	 * Bind to unicast socket used for token send/receives
@@ -1077,6 +1085,11 @@ static int totemnet_build_sockets_ipv6 (
 		perror ("socket");
 		return (-1);
 	}
+	res = fcntl (sockets->mcast_recv, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		instance->totemnet_log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on multicast recv socket: %s\n", strerror (errno));
+	}
+
 
 	/*
 	 * Bind to multicast socket used for multicast receives
@@ -1098,6 +1111,10 @@ static int totemnet_build_sockets_ipv6 (
 		perror ("socket");
 		return (-1);
 	}
+	res = fcntl (sockets->mcast_send, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		instance->totemnet_log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on multicast send socket: %s\n", strerror (errno));
+	}
 
 	totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_config->ip_port - 1,
 		&sockaddr, &addrlen);
@@ -1115,6 +1132,11 @@ static int totemnet_build_sockets_ipv6 (
 		perror ("socket2");
 		return (-1);
 	}
+	res = fcntl (sockets->token, F_SETFL, O_NONBLOCK);
+	if (res == -1) {
+		instance->totemnet_log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on token socket: %s\n", strerror (errno));
+	}
+
 
 	/*
 	 * Bind to unicast socket used for token send/receives

lib/util.c

@@ -38,6 +38,7 @@
 #include <stdio.h>
 #include <unistd.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <sys/ioctl.h>
 #include <sys/types.h>
 #include <sys/uio.h>
@@ -79,6 +80,7 @@ saServiceConnect (
 	struct res_lib_response_init res_lib_response_init;
 	SaAisErrorT error;
 	gid_t egid;
+	int res;
 
 	/*
 	 * Allow set group id binaries to be authenticated
@@ -97,6 +99,7 @@ saServiceConnect (
 	if (result == -1) {
 		return (SA_AIS_ERR_TRY_AGAIN);
 	}
+	res = fcntl (fd, F_SETFL, O_NONBLOCK);
 
 	req_lib_response_init.resdis_header.size = sizeof (req_lib_response_init);
 	req_lib_response_init.resdis_header.id = MESSAGE_REQ_RESPONSE_INIT;
@@ -144,6 +147,7 @@ saServiceConnectTwo (
 	struct res_lib_dispatch_init res_lib_dispatch_init;
 	SaAisErrorT error;
 	gid_t egid;
+	int res;
 
 	/*
 	 * Allow set group id binaries to be authenticated
@@ -160,6 +164,13 @@ saServiceConnectTwo (
 	}
 	result = connect (responseFD, (struct sockaddr *)&address, sizeof (address));
 	if (result == -1) {
+		close (responseFD);
+		return (SA_AIS_ERR_TRY_AGAIN);
+	}
+
+	result = fcntl (responseFD, F_SETFL, O_NONBLOCK);
+	if (result == -1) {
+		close (responseFD);
 		return (SA_AIS_ERR_TRY_AGAIN);
 	}
 
@@ -194,8 +205,15 @@ saServiceConnectTwo (
 	if (callbackFD == -1) {
 		return (SA_AIS_ERR_NO_RESOURCES);
 	}
+	result = fcntl (callbackFD, F_SETFL, O_NONBLOCK);
+	if (result == -1) {
+		close (callbackFD);
+		close (responseFD);
+		return (SA_AIS_ERR_TRY_AGAIN);
+	}
 	result = connect (callbackFD, (struct sockaddr *)&address, sizeof (address));
 	if (result == -1) {
+		close (responseFD);
 		return (SA_AIS_ERR_TRY_AGAIN);
 	}
 
@@ -260,10 +278,13 @@ retry_recv:
 	iov_recv.iov_base = (void *)&rbuf[processed];
 	iov_recv.iov_len = len - processed;
 
-	result = recvmsg (s, &msg_recv, MSG_NOSIGNAL);
+	result = recvmsg (s, &msg_recv, MSG_NOSIGNAL|MSG_WAITALL);
 	if (result == -1 && errno == EINTR) {
 		goto retry_recv;
 	}
+	if (result == -1 && errno == EAGAIN) {
+		goto retry_recv;
+	}
 	if (result == -1 || result == 0) {
 		error = SA_AIS_ERR_LIBRARY;
 		goto error_exit;