qnetd: Factorize qnetd-poll-array to generic unit

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/Makefile.am

@@ -41,7 +41,7 @@ sbin_SCRIPTS             = corosync-qnetd-certutil
 
 corosync_qnetd_SOURCES	= corosync-qnetd.c dynar.c msg.c msgio.c nss-sock.c  \
 			    qnetd-client.c qnetd-client-list.c qnetd-log.c \
-			    qnetd-poll-array.c timer-list.c tlv.c send-buffer-list.c \
+			    pr-poll-array.c timer-list.c tlv.c send-buffer-list.c \
 			    node-list.c qnetd-algo-test.c qnetd-algorithm.c qnetd-algo-utils.c \
 			    qnetd-algo-ffsplit.c qnetd-cluster-list.c qnetd-client-send.c \
 			    qnetd-algo-2nodelms.c qnetd-algo-lms.c utils.c qnetd-instance.c \
@@ -55,7 +55,7 @@ corosync_qdevice_SOURCES	= corosync-qdevice.c qdevice-cmap.c qdevice-instance.c
 				  qdevice-net-instance.c dynar.c send-buffer-list.c timer-list.c \
 				  msg.c msgio.c nss-sock.c tlv.c \
 				  unix-socket.c unix-socket-client.c unix-socket-client-list.c \
-				  unix-socket-ipc.c qdevice-ipc.c \
+				  unix-socket-ipc.c qdevice-ipc.c pr-poll-array.c \
 				  qdevice-net-poll.c qdevice-net-send.c qdevice-net-votequorum.c \
 				  qdevice-net-socket.c qdevice-net-nss.c qdevice-net-msg-received.c \
 				  qdevice-net-cast-vote-timer.c qdevice-net-echo-request-timer.c \

qdevices/corosync-qnetd.c

@@ -41,11 +41,13 @@
 #include "qnet-config.h"
 
 #include "nss-sock.h"
+#include "pr-poll-array.h"
 #include "qnetd-algorithm.h"
 #include "qnetd-instance.h"
 #include "qnetd-log.h"
 #include "qnetd-client-net.h"
 #include "qnetd-client-msg-received.h"
+#include "qnetd-poll-array-user-data.h"
 #include "utils.h"
 
 /*
@@ -75,81 +77,127 @@ qnetd_warn_nss(void) {
 	qnetd_log_nss(LOG_WARNING, "NSS warning");
 }
 
+static PRPollDesc *
+qnetd_pr_poll_array_create(struct qnetd_instance *instance)
+{
+	struct pr_poll_array *poll_array;
+	const struct qnetd_client_list *client_list;
+	struct qnetd_client *client;
+	PRPollDesc *poll_desc;
+	struct qnetd_poll_array_user_data *user_data;
+
+	poll_array = &instance->poll_array;
+	client_list = &instance->clients;
+
+	pr_poll_array_clean(poll_array);
+
+	if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
+		return (NULL);
+	}
+
+	poll_desc->fd = instance->server.socket;
+	poll_desc->in_flags = PR_POLL_READ;
+	poll_desc->out_flags = 0;
+
+	user_data->type = QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET;
+	user_data->client = NULL;
+
+	TAILQ_FOREACH(client, client_list, entries) {
+		if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) {
+			return (NULL);
+		}
+		poll_desc->fd = client->socket;
+		poll_desc->in_flags = PR_POLL_READ;
+		if (!send_buffer_list_empty(&client->send_buffer_list)) {
+			poll_desc->in_flags |= PR_POLL_WRITE;
+		}
+		poll_desc->out_flags = 0;
+
+		user_data->type = QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT;
+		user_data->client = client;
+	}
+
+	pr_poll_array_gc(poll_array);
+
+	return (poll_array->array);
+}
+
 static int
 qnetd_poll(struct qnetd_instance *instance)
 {
 	struct qnetd_client *client;
-	struct qnetd_client *client_next;
 	PRPollDesc *pfds;
 	PRInt32 poll_res;
-	int i;
+	ssize_t i;
 	int client_disconnect;
+	struct qnetd_poll_array_user_data *user_data;
 
 	client = NULL;
 	client_disconnect = 0;
 
-	pfds = qnetd_poll_array_create_from_client_list(&instance->poll_array,
-	    &instance->clients, instance->server.socket, PR_POLL_READ);
-
+	pfds = qnetd_pr_poll_array_create(instance);
 	if (pfds == NULL) {
 		return (-1);
 	}
 
-	if ((poll_res = PR_Poll(pfds, qnetd_poll_array_size(&instance->poll_array),
+	if ((poll_res = PR_Poll(pfds, pr_poll_array_size(&instance->poll_array),
 	    timer_list_time_to_expire(&instance->main_timer_list))) >= 0) {
 		timer_list_expire(&instance->main_timer_list);
 
 		/*
 		 * Walk thru pfds array and process events
 		 */
-		for (i = 0; i < qnetd_poll_array_size(&instance->poll_array); i++) {
-			/*
-			 * Also traverse clients list
-			 */
-			if (i > 0) {
-				if (i == 1) {
-					client = TAILQ_FIRST(&instance->clients);
-					client_next = TAILQ_NEXT(client, entries);
-				} else {
-					client = client_next;
-					client_next = TAILQ_NEXT(client, entries);
-				}
-				client_disconnect = client->schedule_disconnect;
-			} else {
+		for (i = 0; i < pr_poll_array_size(&instance->poll_array); i++) {
+			user_data = pr_poll_array_get_user_data(&instance->poll_array, i);
+
+			switch (user_data->type) {
+			case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
 				client_disconnect = 0;
+				break;
+			case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT:
+				client = user_data->client;
+				client_disconnect = client->schedule_disconnect;
+				break;
 			}
 
 			if (!client_disconnect && poll_res > 0 &&
 			    pfds[i].out_flags & PR_POLL_READ) {
-				if (i == 0) {
+				switch (user_data->type) {
+				case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
 					qnetd_client_net_accept(instance);
-				} else {
+					break;
+				case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT:
 					if (qnetd_client_net_read(instance, client) == -1) {
 						client_disconnect = 1;
 					}
+					break;
 				}
 			}
 
 			if (!client_disconnect && poll_res > 0 &&
 			    pfds[i].out_flags & PR_POLL_WRITE) {
-				if (i == 0) {
+				switch (user_data->type) {
+				case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
 					/*
 					 * Poll write on listen socket -> fatal error
 					 */
 					qnetd_log(LOG_CRIT, "POLL_WRITE on listening socket");
 
 					return (-1);
-				} else {
+					break;
+				case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT:
 					if (qnetd_client_net_write(instance, client) == -1) {
 						client_disconnect = 1;
 					}
+					break;
 				}
 			}
 
 			if (!client_disconnect && poll_res > 0 &&
 			    (pfds[i].out_flags & (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) &&
 			    !(pfds[i].out_flags & (PR_POLL_READ|PR_POLL_WRITE))) {
-				if (i == 0) {
+				switch (user_data->type) {
+				case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET:
 					if (pfds[i].out_flags != PR_POLL_NVAL) {
 						/*
 						 * Poll ERR on listening socket is fatal error.
@@ -162,12 +210,13 @@ qnetd_poll(struct qnetd_instance *instance)
 					}
 
 					return (-1);
-
-				} else {
+					break;
+				case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT:
 					qnetd_log(LOG_DEBUG, "POLL_ERR (%u) on client socket. "
 					    "Disconnecting.", pfds[i].out_flags);
 
 					client_disconnect = 1;
+					break;
 				}
 			}
 

qdevices/msgio.c

@@ -129,6 +129,8 @@ msgio_write(PRFileDesc *sock, const struct dynar *msg, size_t *already_sent_byte
 }
 
 /*
+ *  1 Full message received
+ *  0 Partial read (no error)
  * -1 End of connection
  * -2 Unhandled error
  * -3 Fatal error. Unable to store message header

qdevices/qnetd-poll-array.c → qdevices/pr-poll-array.c

@@ -39,35 +39,38 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "qnetd-poll-array.h"
+#include "pr-poll-array.h"
 
 void
-qnetd_poll_array_init(struct qnetd_poll_array *poll_array)
+pr_poll_array_init(struct pr_poll_array *poll_array, size_t user_data_size)
 {
 
 	memset(poll_array, 0, sizeof(*poll_array));
+	poll_array->user_data_size = user_data_size;
 }
 
 void
-qnetd_poll_array_destroy(struct qnetd_poll_array *poll_array)
+pr_poll_array_destroy(struct pr_poll_array *poll_array)
 {
 
 	free(poll_array->array);
-	qnetd_poll_array_init(poll_array);
+	free(poll_array->user_data_array);
+	pr_poll_array_init(poll_array, poll_array->user_data_size);
 }
 
 void
-qnetd_poll_array_clean(struct qnetd_poll_array *poll_array)
+pr_poll_array_clean(struct pr_poll_array *poll_array)
 {
 
 	poll_array->items = 0;
 }
 
 static int
-qnetd_poll_array_realloc(struct qnetd_poll_array *poll_array,
-    unsigned int new_array_size)
+pr_poll_array_realloc(struct pr_poll_array *poll_array,
+    ssize_t new_array_size)
 {
 	PRPollDesc *new_array;
+	char *new_user_data_array;
 
 	new_array = realloc(poll_array->array,
 	    sizeof(PRPollDesc) * new_array_size);
@@ -76,45 +79,59 @@ qnetd_poll_array_realloc(struct qnetd_poll_array *poll_array,
 		return (-1);
 	}
 
+	if (poll_array->user_data_size > 0) {
+		new_user_data_array = realloc(poll_array->user_data_array,
+		    poll_array->user_data_size * new_array_size);
+
+		if (new_user_data_array == NULL) {
+			return (-1);
+		}
+
+		poll_array->user_data_array = new_user_data_array;
+	}
+
 	poll_array->allocated = new_array_size;
 	poll_array->array = new_array;
 
 	return (0);
 }
 
-unsigned int
-qnetd_poll_array_size(struct qnetd_poll_array *poll_array)
+ssize_t
+pr_poll_array_size(struct pr_poll_array *poll_array)
 {
 
 	return (poll_array->items);
 }
 
-PRPollDesc *
-qnetd_poll_array_add(struct qnetd_poll_array *poll_array)
+ssize_t
+pr_poll_array_add(struct pr_poll_array *poll_array, PRPollDesc **pfds, void **user_data)
 {
 
-	if (qnetd_poll_array_size(poll_array) >= poll_array->allocated) {
-		if (qnetd_poll_array_realloc(poll_array, (poll_array->allocated * 2) + 1)) {
-			return (NULL);
+	if (pr_poll_array_size(poll_array) >= poll_array->allocated) {
+		if (pr_poll_array_realloc(poll_array, (poll_array->allocated * 2) + 1)) {
+			return (-1);
 		}
 	}
 
+	*pfds = &poll_array->array[pr_poll_array_size(poll_array)];
+	*user_data = poll_array->user_data_array + (poll_array->items * poll_array->user_data_size);
+
 	poll_array->items++;
 
-	return (&poll_array->array[qnetd_poll_array_size(poll_array) - 1]);
+	return (poll_array->items - 1);
 }
 
-static void
-qnetd_poll_array_gc(struct qnetd_poll_array *poll_array)
+void
+pr_poll_array_gc(struct pr_poll_array *poll_array)
 {
 
-	if (poll_array->allocated > (qnetd_poll_array_size(poll_array) * 3) + 1) {
-		qnetd_poll_array_realloc(poll_array, (qnetd_poll_array_size(poll_array) * 2) + 1);
+	if (poll_array->allocated > (pr_poll_array_size(poll_array) * 3) + 1) {
+		pr_poll_array_realloc(poll_array, (pr_poll_array_size(poll_array) * 2) + 1);
 	}
 }
 
 PRPollDesc *
-qnetd_poll_array_get(const struct qnetd_poll_array *poll_array, unsigned int pos)
+pr_poll_array_get(const struct pr_poll_array *poll_array, ssize_t pos)
 {
 
 	if (pos >= poll_array->items) {
@@ -124,41 +141,13 @@ qnetd_poll_array_get(const struct qnetd_poll_array *poll_array, unsigned int pos
 	return (&poll_array->array[pos]);
 }
 
-PRPollDesc *
-qnetd_poll_array_create_from_client_list(struct qnetd_poll_array *poll_array,
-    const struct qnetd_client_list *client_list,
-    PRFileDesc *extra_fd, PRInt16 extra_fd_in_flags)
+void *
+pr_poll_array_get_user_data(const struct pr_poll_array *poll_array, ssize_t pos)
 {
-	struct qnetd_client *client;
-	PRPollDesc *poll_desc;
-
-	qnetd_poll_array_clean(poll_array);
 
-	if (extra_fd != NULL) {
-		poll_desc = qnetd_poll_array_add(poll_array);
-		if (poll_desc == NULL) {
-			return (NULL);
-		}
-
-		poll_desc->fd = extra_fd;
-		poll_desc->in_flags = extra_fd_in_flags;
-		poll_desc->out_flags = 0;
-	}
-
-	TAILQ_FOREACH(client, client_list, entries) {
-		poll_desc = qnetd_poll_array_add(poll_array);
-		if (poll_desc == NULL) {
-			return (NULL);
-		}
-		poll_desc->fd = client->socket;
-		poll_desc->in_flags = PR_POLL_READ;
-		if (!send_buffer_list_empty(&client->send_buffer_list)) {
-			poll_desc->in_flags |= PR_POLL_WRITE;
-		}
-		poll_desc->out_flags = 0;
+	if (pos >= poll_array->items) {
+		return (NULL);
 	}
 
-	qnetd_poll_array_gc(poll_array);
-
-	return (poll_array->array);
+	return (poll_array->user_data_array + (pos * poll_array->user_data_size));
 }

qdevices/qnetd-poll-array.h → qdevices/pr-poll-array.h

@@ -32,47 +32,47 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef _QNETD_POLL_ARRAY_H_
-#define _QNETD_POLL_ARRAY_H_
+#ifndef _PR_POLL_ARRAY_H_
+#define _PR_POLL_ARRAY_H_
 
 #include <sys/types.h>
 #include <inttypes.h>
 
 #include <nspr.h>
 
-#include "qnetd-client.h"
-#include "qnetd-client-list.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-struct qnetd_poll_array {
+struct pr_poll_array {
 	PRPollDesc *array;
-	unsigned int allocated;
-	unsigned int items;
+	char *user_data_array;
+	size_t user_data_size;
+	ssize_t allocated;
+	ssize_t items;
 };
 
+extern void		 pr_poll_array_init(struct pr_poll_array *poll_array, size_t user_data_size);
 
-extern void		 qnetd_poll_array_init(struct qnetd_poll_array *poll_array);
+extern void		 pr_poll_array_destroy(struct pr_poll_array *poll_array);
 
-extern void		 qnetd_poll_array_destroy(struct qnetd_poll_array *poll_array);
+extern void		 pr_poll_array_clean(struct pr_poll_array *poll_array);
 
-extern void		 qnetd_poll_array_clean(struct qnetd_poll_array *poll_array);
+extern ssize_t		 pr_poll_array_size(struct pr_poll_array *poll_array);
 
-extern unsigned int	 qnetd_poll_array_size(struct qnetd_poll_array *poll_array);
+extern ssize_t		 pr_poll_array_add(struct pr_poll_array *poll_array,  PRPollDesc **pfds,
+    void **user_data);
 
-extern PRPollDesc	*qnetd_poll_array_add(struct qnetd_poll_array *poll_array);
+extern PRPollDesc 	*pr_poll_array_get(const struct pr_poll_array *poll_array,
+    ssize_t pos);
 
-extern PRPollDesc 	*qnetd_poll_array_get(const struct qnetd_poll_array *poll_array,
-    unsigned int pos);
+extern void		*pr_poll_array_get_user_data(const struct pr_poll_array *poll_array,
+    ssize_t pos);
 
-extern PRPollDesc	*qnetd_poll_array_create_from_client_list(
-    struct qnetd_poll_array *poll_array, const struct qnetd_client_list *client_list,
-    PRFileDesc *extra_fd, PRInt16 extra_fd_in_flags);
+extern void		 pr_poll_array_gc(struct pr_poll_array *poll_array);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* _QNETD_POLL_ARRAY_H_ */
+#endif /* _PR_POLL_ARRAY_H_ */

qdevices/qnetd-instance.c

@@ -40,6 +40,7 @@
 #include "qnetd-algorithm.h"
 #include "qnetd-log-debug.h"
 #include "qnetd-dpd-timer.h"
+#include "qnetd-poll-array-user-data.h"
 
 int
 qnetd_instance_init(struct qnetd_instance *instance, size_t max_client_receive_size,
@@ -49,7 +50,7 @@ qnetd_instance_init(struct qnetd_instance *instance, size_t max_client_receive_s
 
 	memset(instance, 0, sizeof(*instance));
 
-	qnetd_poll_array_init(&instance->poll_array);
+	pr_poll_array_init(&instance->poll_array, sizeof(struct qnetd_poll_array_user_data));
 	qnetd_client_list_init(&instance->clients);
 	qnetd_cluster_list_init(&instance->clusters);
 
@@ -88,7 +89,7 @@ qnetd_instance_destroy(struct qnetd_instance *instance)
 		client = client_next;
 	}
 
-	qnetd_poll_array_destroy(&instance->poll_array);
+	pr_poll_array_destroy(&instance->poll_array);
 	qnetd_cluster_list_free(&instance->clusters);
 	qnetd_client_list_free(&instance->clients);
 	timer_list_free(&instance->main_timer_list);

qdevices/qnetd-instance.h

@@ -43,7 +43,7 @@
 
 #include "qnetd-client-list.h"
 #include "qnetd-cluster-list.h"
-#include "qnetd-poll-array.h"
+#include "pr-poll-array.h"
 #include "qnet-config.h"
 #include "timer-list.h"
 
@@ -63,7 +63,7 @@ struct qnetd_instance {
 	size_t max_clients;
 	struct qnetd_client_list clients;
 	struct qnetd_cluster_list clusters;
-	struct qnetd_poll_array poll_array;
+	struct pr_poll_array poll_array;
 	enum tlv_tls_supported tls_supported;
 	int tls_client_cert_required;
 	const char *host_addr;

qdevices/qnetd-poll-array-user-data.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QNETD_POLL_ARRAY_USER_DATA_H_
+#define _QNETD_POLL_ARRAY_USER_DATA_H_
+
+#include "qnetd-client.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum qnetd_poll_array_user_data_type {
+	QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET,
+	QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT,
+};
+
+struct qnetd_poll_array_user_data {
+	enum qnetd_poll_array_user_data_type type;
+	struct qnetd_client *client;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QNETD_POLL_ARRAY_USER_DATA_H_ */