Refactor qdevice-net

- corosync-device-net as binary is gone. Replacement is
  corosync-qdevice
- corosync-qdevice has support for multiple models (only net is
  currently implemented)
- Completelly redesign qdevice-net main loop.
  - Connect is non blocking
  - Cmap and Votequorum events are handled even before connect to
    qnetd. Algorithm gets send_node_list and vote set so it's not needed
    to check connection status and also vote_timer is running and voting
    until something changes (configuration or votequorum node list)
  - If connect fails, algorithm_disconnected with new reason
    CANT_CONNECT_TO_THE_SERVER is called
- Logging for qdevice is based on libqb logging functions. Also
  logging configuration from corosync.conf is now used and dynamic
  changes of configuration are handled.
- Added qdevice_net_algorithm_config_node_list_changed
- Changed qdevice_net_algorithm_votequorum_node_list_notify in respect
  of adding  send_node_list so it's similar to other functions.

Signed-off-by: Jan Friesse <jfriesse@redhat.com>

qdevices/.gitignore

@@ -1,4 +1,5 @@
 corosync-qdevice-net
+corosync-qdevice
 corosync-qnetd-certutil
 corosync-qnetd
 *.test

qdevices/Makefile.am

@@ -35,7 +35,7 @@ if BUILD_QDEVICES
 
 SUBDIRS			=
 
-sbin_PROGRAMS		= corosync-qnetd corosync-qdevice-net
+sbin_PROGRAMS		= corosync-qnetd corosync-qdevice
 
 sbin_SCRIPTS             = corosync-qnetd-certutil
 
@@ -47,20 +47,24 @@ corosync_qnetd_SOURCES	= corosync-qnetd.c dynar.c msg.c msgio.c nss-sock.c  \
 			    qnetd-algo-2nodelms.c qnetd-algo-lms.c utils.c qnetd-instance.c \
 			    qnetd-client-net.c qnetd-client-msg-received.c qnetd-log-debug.c
 
-corosync_qdevice_net_SOURCES	= corosync-qdevice-net.c dynar.c msg.c msgio.c nss-sock.c  \
-			    qnetd-client.c qnetd-client-list.c qnetd-log.c \
-			    qnetd-poll-array.c timer-list.c tlv.c send-buffer-list.c \
-			    node-list.c qdevice-net-send.c qdevice-net-cmap.c \
-			    qdevice-net-votequorum.c qdevice-net-instance.c \
-			    qdevice-net-cast-vote-timer.c utils.c qdevice-net-algorithm.c \
-			    qdevice-net-algo-test.c qdevice-net-algo-ffsplit.c \
-			    qdevice-net-algo-2nodelms.c qdevice-net-algo-lms.c
+corosync_qdevice_SOURCES	= corosync-qdevice.c qdevice-cmap.c qdevice-instance.c node-list.c \
+				  utils.c qdevice-log.c qdevice-log-debug.c qdevice-votequorum.c \
+				  qdevice-model.c \
+				  qdevice-model-net.c \
+				  qdevice-net-instance.c dynar.c send-buffer-list.c timer-list.c \
+				  msg.c msgio.c nss-sock.c tlv.c \
+				  qdevice-net-poll.c qdevice-net-send.c qdevice-net-votequorum.c \
+				  qdevice-net-socket.c qdevice-net-nss.c qdevice-net-msg-received.c \
+				  qdevice-net-cast-vote-timer.c \
+				  qdevice-net-algorithm.c \
+				  qdevice-net-algo-test.c qdevice-net-algo-ffsplit.c \
+				  qdevice-net-algo-2nodelms.c qdevice-net-algo-lms.c
 
 corosync_qnetd_CFLAGS		= $(nss_CFLAGS)
 corosync_qnetd_LDADD		= $(nss_LIBS)
 
-corosync_qdevice_net_CFLAGS	= $(nss_CFLAGS)
-corosync_qdevice_net_LDADD	= $(nss_LIBS) $(LIBQB_LIBS) $(top_builddir)/lib/libcmap.la \
+corosync_qdevice_CFLAGS	= $(nss_CFLAGS)
+corosync_qdevice_LDADD	= $(nss_LIBS) $(LIBQB_LIBS) $(top_builddir)/lib/libcmap.la \
 				    $(top_builddir)/lib/libvotequorum.la
 
 corosync-qnetd-certutil: corosync-qnetd-certutil.sh

qdevices/corosync-qdevice-net.c

@@ -1,1569 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Red Hat, Inc.
- *
- * All rights reserved.
- *
- * Author: Jan Friesse (jfriesse@redhat.com)
- *
- * This software licensed under BSD license, the text of which follows:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * - Neither the name of the Red Hat, Inc. nor the names of its
- *   contributors may be used to endorse or promote products derived from this
- *   software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include <stdio.h>
-#include <nss.h>
-#include <secerr.h>
-#include <sslerr.h>
-#include <pk11func.h>
-#include <certt.h>
-#include <ssl.h>
-#include <prio.h>
-#include <prnetdb.h>
-#include <prerror.h>
-#include <prinit.h>
-#include <getopt.h>
-#include <err.h>
-#include <keyhi.h>
-#include <poll.h>
-
-/*
- * Needed for creating nspr handle from unix fd
- */
-#include <private/pprio.h>
-
-#include <cmap.h>
-#include <votequorum.h>
-
-#include "qnet-config.h"
-#include "dynar.h"
-#include "nss-sock.h"
-#include "tlv.h"
-#include "msg.h"
-#include "msgio.h"
-#include "qdevice-net-log.h"
-#include "timer-list.h"
-#include "send-buffer-list.h"
-#include "qdevice-net-instance.h"
-#include "qdevice-net-send.h"
-#include "qdevice-net-votequorum.h"
-#include "qdevice-net-cast-vote-timer.h"
-#include "utils.h"
-#include "qdevice-net-algorithm.h"
-
-static SECStatus
-qdevice_net_nss_bad_cert_hook(void *arg, PRFileDesc *fd) {
-	if (PR_GetError() == SEC_ERROR_EXPIRED_CERTIFICATE ||
-	    PR_GetError() == SEC_ERROR_EXPIRED_ISSUER_CERTIFICATE ||
-	    PR_GetError() == SEC_ERROR_CRL_EXPIRED ||
-	    PR_GetError() == SEC_ERROR_KRL_EXPIRED ||
-	    PR_GetError() == SSL_ERROR_EXPIRED_CERT_ALERT) {
-		qdevice_net_log(LOG_WARNING, "Server certificate is expired.");
-
-		return (SECSuccess);
-        }
-
-	qdevice_net_log_nss(LOG_ERR, "Server certificate verification failure.");
-
-	return (SECFailure);
-}
-
-static SECStatus
-qdevice_net_nss_get_client_auth_data(void *arg, PRFileDesc *sock, struct CERTDistNamesStr *caNames,
-    struct CERTCertificateStr **pRetCert, struct SECKEYPrivateKeyStr **pRetKey)
-{
-
-	qdevice_net_log(LOG_DEBUG, "Sending client auth data.");
-
-	return (NSS_GetClientAuthData(arg, sock, caNames, pRetCert, pRetKey));
-}
-
-static void
-qdevice_net_log_msg_decode_error(int ret)
-{
-
-	switch (ret) {
-	case -1:
-		qdevice_net_log(LOG_WARNING, "Received message with option with invalid length");
-		break;
-	case -2:
-		qdevice_net_log(LOG_CRIT, "Can't allocate memory");
-		break;
-	case -3:
-		qdevice_net_log(LOG_WARNING, "Received inconsistent msg (tlv len > msg size)");
-		break;
-	case -4:
-		qdevice_net_log(LOG_ERR, "Received message with option with invalid value");
-		break;
-	default:
-		qdevice_net_log(LOG_ERR, "Unknown error occured when decoding message");
-		break;
-	}
-}
-
-/*
- * -1 - Incompatible tls combination
- *  0 - Don't use TLS
- *  1 - Use TLS
- */
-static int
-qdevice_net_msg_received_check_tls_compatibility(enum tlv_tls_supported server_tls,
-    enum tlv_tls_supported client_tls)
-{
-	int res;
-
-	res = -1;
-
-	switch (server_tls) {
-	case TLV_TLS_UNSUPPORTED:
-		switch (client_tls) {
-		case TLV_TLS_UNSUPPORTED: res = 0; break;
-		case TLV_TLS_SUPPORTED: res = 0; break;
-		case TLV_TLS_REQUIRED: res = -1; break;
-		}
-		break;
-	case TLV_TLS_SUPPORTED:
-		switch (client_tls) {
-		case TLV_TLS_UNSUPPORTED: res = 0; break;
-		case TLV_TLS_SUPPORTED: res = 1; break;
-		case TLV_TLS_REQUIRED: res = 1; break;
-		}
-		break;
-	case TLV_TLS_REQUIRED:
-		switch (client_tls) {
-		case TLV_TLS_UNSUPPORTED: res = -1; break;
-		case TLV_TLS_SUPPORTED: res = 1; break;
-		case TLV_TLS_REQUIRED: res = 1; break;
-		}
-		break;
-	}
-
-	return (res);
-}
-
-static int
-qdevice_net_msg_received_unexpected_msg(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg, const char *msg_str)
-{
-
-	qdevice_net_log(LOG_ERR, "Received unexpected %s message. Disconnecting from server",
-	    msg_str);
-
-	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-
-	return (-1);
-}
-
-static int
-qdevice_net_msg_received_preinit(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "preinit"));
-}
-
-static int
-qdevice_net_msg_check_seq_number(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	if (!msg->seq_number_set || msg->seq_number != instance->last_msg_seq_num) {
-		qdevice_net_log(LOG_ERR, "Received message doesn't contain seq_number or "
-		    "it's not expected one.");
-
-		return (-1);
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_preinit_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-	int res;
-	struct send_buffer_list_entry *send_buffer;
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_PREINIT_REPLY) {
-		qdevice_net_log(LOG_ERR, "Received unexpected preinit reply message. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-
-		return (-1);
-	}
-
-	if (qdevice_net_msg_check_seq_number(instance, msg) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	/*
-	 * Check TLS support
-	 */
-	if (!msg->tls_supported_set || !msg->tls_client_cert_required_set) {
-		qdevice_net_log(LOG_ERR, "Required tls_supported or tls_client_cert_required "
-		    "option is unset");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	res = qdevice_net_msg_received_check_tls_compatibility(msg->tls_supported, instance->tls_supported);
-	if (res == -1) {
-		qdevice_net_log(LOG_ERR, "Incompatible tls configuration (server %u client %u)",
-		    msg->tls_supported, instance->tls_supported);
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_INCOMPATIBLE_TLS;
-
-		return (-1);
-	} else if (res == 1) {
-		/*
-		 * Start TLS
-		 */
-		send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
-		if (send_buffer == NULL) {
-			qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for "
-			    "starttls msg");
-
-			instance->disconnect_reason =
-			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-			return (-1);
-		}
-
-		instance->last_msg_seq_num++;
-		if (msg_create_starttls(&send_buffer->buffer, 1,
-		    instance->last_msg_seq_num) == 0) {
-			qdevice_net_log(LOG_ERR, "Can't allocate send buffer for starttls msg");
-
-			instance->disconnect_reason =
-			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-			return (-1);
-		}
-
-		send_buffer_list_put(&instance->send_buffer_list, send_buffer);
-
-		instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_STARTTLS_BEING_SENT;
-	} else if (res == 0) {
-		if (qdevice_net_send_init(instance) != 0) {
-			instance->disconnect_reason =
-			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-			return (-1);
-		}
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-	size_t zi;
-	int res;
-	struct send_buffer_list_entry *send_buffer;
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY) {
-		qdevice_net_log(LOG_ERR, "Received unexpected init reply message. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-
-		return (-1);
-	}
-
-	if (qdevice_net_msg_check_seq_number(instance, msg) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (!msg->reply_error_code_set) {
-		qdevice_net_log(LOG_ERR, "Received init reply message without error code."
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (msg->reply_error_code != TLV_REPLY_ERROR_CODE_NO_ERROR) {
-		qdevice_net_log(LOG_ERR, "Received init reply message with error code %"PRIu16". "
-		    "Disconnecting from server", msg->reply_error_code);
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_SENT_ERROR;
-		return (-1);
-	}
-
-	if (!msg->server_maximum_request_size_set || !msg->server_maximum_reply_size_set) {
-		qdevice_net_log(LOG_ERR, "Required maximum_request_size or maximum_reply_size "
-		    "option is unset");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (msg->supported_messages == NULL || msg->supported_options == NULL) {
-		qdevice_net_log(LOG_ERR, "Required supported messages or supported options "
-		    "option is unset");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (msg->supported_decision_algorithms == NULL) {
-		qdevice_net_log(LOG_ERR, "Required supported decision algorithms option is unset");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (msg->server_maximum_request_size < instance->min_send_size) {
-		qdevice_net_log(LOG_ERR,
-		    "Server accepts maximum %zu bytes message but this client minimum "
-		    "is %zu bytes.", msg->server_maximum_request_size, instance->min_send_size);
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_INCOMPATIBLE_MSG_SIZE;
-		return (-1);
-	}
-
-	if (msg->server_maximum_reply_size > instance->max_receive_size) {
-		qdevice_net_log(LOG_ERR,
-		    "Server may send message up to %zu bytes message but this client maximum "
-		    "is %zu bytes.", msg->server_maximum_reply_size, instance->max_receive_size);
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_INCOMPATIBLE_MSG_SIZE;
-		return (-1);
-	}
-
-	/*
-	 * Change buffer sizes
-	 */
-	dynar_set_max_size(&instance->receive_buffer, msg->server_maximum_reply_size);
-	send_buffer_list_set_max_buffer_size(&instance->send_buffer_list,
-	    msg->server_maximum_request_size);
-
-
-	/*
-	 * Check if server supports decision algorithm we need
-	 */
-	res = 0;
-
-	for (zi = 0; zi < msg->no_supported_decision_algorithms && !res; zi++) {
-		if (msg->supported_decision_algorithms[zi] == instance->decision_algorithm) {
-			res = 1;
-		}
-	}
-
-	if (!res) {
-		qdevice_net_log(LOG_ERR, "Server doesn't support required decision algorithm");
-
-		instance->disconnect_reason =
-		    QDEVICE_NET_DISCONNECT_REASON_SERVER_DOESNT_SUPPORT_REQUIRED_ALGORITHM;
-
-		return (-1);
-	}
-
-	/*
-	 * Send set options message
-	 */
-	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
-	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for set option msg");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-		return (-1);
-	}
-
-	instance->last_msg_seq_num++;
-
-	if (msg_create_set_option(&send_buffer->buffer, 1, instance->last_msg_seq_num,
-	    1, instance->heartbeat_interval, 1, &instance->tie_breaker) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for set option msg");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-		return (-1);
-	}
-
-	send_buffer_list_put(&instance->send_buffer_list, send_buffer);
-
-	instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_SET_OPTION_REPLY;
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_starttls(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "starttls"));
-}
-
-static int
-qdevice_net_msg_received_server_error(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	if (!msg->reply_error_code_set) {
-		qdevice_net_log(LOG_ERR, "Received server error without error code set. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-	} else {
-		qdevice_net_log(LOG_ERR, "Received server error %"PRIu16". "
-		    "Disconnecting from server", msg->reply_error_code);
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_SENT_ERROR;
-	}
-
-	return (-1);
-}
-
-static int
-qdevice_net_msg_received_set_option(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "set option"));
-}
-
-static int
-qdevice_net_timer_send_heartbeat(void *data1, void *data2)
-{
-	struct qdevice_net_instance *instance;
-
-	instance = (struct qdevice_net_instance *)data1;
-
-	if (instance->echo_reply_received_msg_seq_num !=
-	    instance->echo_request_expected_msg_seq_num) {
-		qdevice_net_log(LOG_ERR, "Server didn't send echo reply message on time");
-
-		if (qdevice_net_algorithm_echo_reply_not_received(instance) != 0) {
-			qdevice_net_log(LOG_DEBUG, "Algorithm decided to disconnect");
-
-			instance->schedule_disconnect = 1;
-			instance->disconnect_reason =
-			    QDEVICE_NET_DISCONNECT_REASON_ALGO_ECHO_REPLY_NOT_RECEIVED_ERR;
-
-			return (0);
-		} else {
-			qdevice_net_log(LOG_DEBUG, "Algorithm decided to continue send heartbeat");
-
-			return (-1);
-		}
-	}
-
-	if (qdevice_net_send_echo_request(instance) == -1) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-		instance->schedule_disconnect = 1;
-		return (0);
-	}
-
-	/*
-	 * Schedule this function callback again
-	 */
-	return (-1);
-}
-
-static int
-qdevice_net_register_votequorum_callbacks(struct qdevice_net_instance *instance)
-{
-	cs_error_t res;
-
-	if ((res = votequorum_trackstart(instance->votequorum_handle, 0,
-	    CS_TRACK_CHANGES)) != CS_OK) {
-		qdevice_net_log(LOG_ERR, "Can't start tracking votequorum changes. Error %s",
-		    cs_strerror(res));
-
-		return (-1);
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_set_option_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_SET_OPTION_REPLY) {
-		qdevice_net_log(LOG_ERR, "Received unexpected set option reply message. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-
-		return (-1);
-	}
-
-	if (qdevice_net_msg_check_seq_number(instance, msg) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (!msg->decision_algorithm_set || !msg->heartbeat_interval_set) {
-		qdevice_net_log(LOG_ERR, "Received set option reply message without "
-		    "required options. Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	if (msg->decision_algorithm != instance->decision_algorithm ||
-	    msg->heartbeat_interval != instance->heartbeat_interval) {
-		qdevice_net_log(LOG_ERR, "Server doesn't accept sent decision algorithm or "
-		    "heartbeat interval.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-
-		return (-1);
-	}
-
-	/*
-	 * Server accepted heartbeat interval -> schedule regular sending of echo request
-	 */
-	if (instance->heartbeat_interval > 0) {
-		instance->echo_request_expected_msg_seq_num = 0;
-		instance->echo_reply_received_msg_seq_num = 0;
-
-		instance->echo_request_timer = timer_list_add(&instance->main_timer_list,
-		    instance->heartbeat_interval, qdevice_net_timer_send_heartbeat,
-		    (void *)instance, NULL);
-
-		if (instance->echo_request_timer == NULL) {
-			qdevice_net_log(LOG_ERR, "Can't schedule regular sending of heartbeat.");
-
-			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_HB_TIMER;
-
-			return (-1);
-		}
-	}
-
-	if (qdevice_net_algorithm_connected(instance) != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR;
-		return (-1);
-	}
-
-	/*
-	 * Now we can finally really send node list and initialize qdevice
-	 */
-	if (qdevice_net_send_config_node_list(instance, 1, 1) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-		return (-1);
-	}
-
-	if (qdevice_net_register_votequorum_callbacks(instance) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_REGISTER_VOTEQUORUM_CALLBACK;
-		return (-1);
-	}
-
-	if (qdevice_net_cmap_add_track(instance) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_REGISTER_CMAP_CALLBACK;
-		return (-1);
-	}
-
-	if (qdevice_net_cast_vote_timer_update(instance, TLV_VOTE_WAIT_FOR_REPLY) != 0) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_msg_received_set_option_reply fatal error. "
-		    " Can't update cast vote timer vote");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
-	}
-
-	instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_echo_request(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "echo request"));
-}
-
-static int
-qdevice_net_msg_received_echo_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	if (!msg->seq_number_set) {
-		qdevice_net_log(LOG_ERR, "Received echo reply message doesn't contain seq_number.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-		return (-1);
-	}
-
-	if (msg->seq_number != instance->echo_request_expected_msg_seq_num) {
-		qdevice_net_log(LOG_WARNING, "Received echo reply message seq_number is not expected one.");
-	}
-
-	if (qdevice_net_algorithm_echo_reply_received(instance, msg->seq_number,
-	    msg->seq_number == instance->echo_request_expected_msg_seq_num) != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_ECHO_REPLY_RECEIVED_ERR;
-		return (-1);
-	}
-
-	instance->echo_reply_received_msg_seq_num = msg->seq_number;
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_node_list(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "node list"));
-}
-
-static int
-qdevice_net_msg_received_node_list_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-	const char *str;
-	enum tlv_vote result_vote;
-	int res;
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
-		qdevice_net_log(LOG_ERR, "Received unexpected node list reply message. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-		return (-1);
-	}
-
-	if (!msg->vote_set || !msg->seq_number_set || !msg->node_list_type_set) {
-		qdevice_net_log(LOG_ERR, "Received node list reply message without "
-		    "required options. Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-		return (-1);
-	}
-
-	if (msg->node_list_type == TLV_NODE_LIST_TYPE_MEMBERSHIP && !msg->ring_id_set) {
-		qdevice_net_log(LOG_ERR, "Received node list reply message with type membership "
-		    "without ring id set. Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-		return (-1);
-	}
-
-	switch (msg->node_list_type) {
-	case TLV_NODE_LIST_TYPE_INITIAL_CONFIG: str = "initial config"; break;
-	case TLV_NODE_LIST_TYPE_CHANGED_CONFIG: str = "changed config"; break;
-	case TLV_NODE_LIST_TYPE_MEMBERSHIP: str ="membership"; break;
-	case TLV_NODE_LIST_TYPE_QUORUM: str ="quorum"; break;
-	default:
-		qdevice_net_log(LOG_CRIT, "qdevice_net_msg_received_node_list_reply fatal error. "
-		    "Unhandled node_list_type (debug output)");
-		exit(1);
-		break;
-	}
-	qdevice_net_log(LOG_DEBUG, "Received %s node list reply", str);
-	qdevice_net_log(LOG_DEBUG, "  seq = %"PRIu32, msg->seq_number);
-	qdevice_net_log(LOG_DEBUG, "  vote = %s", tlv_vote_to_str(msg->vote));
-	if (msg->ring_id_set) {
-		qdevice_net_log(LOG_DEBUG, "  ring id = (%"PRIx32".%"PRIx64")",
-		    msg->ring_id.node_id, msg->ring_id.seq);
-	}
-
-	/*
-	 * Call algorithm
-	 */
-	result_vote = msg->vote;
-
-	switch (msg->node_list_type) {
-	case TLV_NODE_LIST_TYPE_INITIAL_CONFIG:
-	case TLV_NODE_LIST_TYPE_CHANGED_CONFIG:
-		res = qdevice_net_algorithm_config_node_list_reply_received(instance,
-		    msg->seq_number, (msg->node_list_type == TLV_NODE_LIST_TYPE_INITIAL_CONFIG),
-		    &result_vote);
-		break;
-	case TLV_NODE_LIST_TYPE_MEMBERSHIP:
-		res = qdevice_net_algorithm_membership_node_list_reply_received(instance,
-		    msg->seq_number, &msg->ring_id, &result_vote);
-		break;
-	case TLV_NODE_LIST_TYPE_QUORUM:
-		res = qdevice_net_algorithm_quorum_node_list_reply_received(instance,
-		    msg->seq_number, &result_vote);
-		break;
-	default:
-		qdevice_net_log(LOG_CRIT, "qdevice_net_msg_received_node_list_reply fatal error. "
-		    "Unhandled node_list_type (algorithm call)");
-		exit(1);
-		break;
-	}
-
-	if (res != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_NODE_LIST_REPLY_ERR;
-		return (-1);
-	} else {
-		qdevice_net_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(msg->vote));
-	}
-
-	if (result_vote != TLV_VOTE_NO_CHANGE) {
-		if (msg->node_list_type == TLV_NODE_LIST_TYPE_MEMBERSHIP &&
-		    !tlv_ring_id_eq(&msg->ring_id, &instance->last_sent_ring_id)) {
-			qdevice_net_log(LOG_INFO, "Received membership node list reply with "
-			    "old ring id. Not updating timer");
-		} else {
-			if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
-				instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
-				return (-1);
-			}
-		}
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_ask_for_vote(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "ask for vote"));
-}
-
-static int
-qdevice_net_msg_received_ask_for_vote_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-	enum tlv_vote result_vote;
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
-		qdevice_net_log(LOG_ERR, "Received unexpected ask for vote reply message. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-		return (-1);
-	}
-
-	if (!msg->vote_set || !msg->seq_number_set) {
-		qdevice_net_log(LOG_ERR, "Received node list reply message without "
-		    "required options. Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-		return (-1);
-	}
-
-	qdevice_net_log(LOG_DEBUG, "Received ask for vote reply");
-	qdevice_net_log(LOG_DEBUG, "  seq = %"PRIu32, msg->seq_number);
-	qdevice_net_log(LOG_DEBUG, "  vote = %s", tlv_vote_to_str(msg->vote));
-
-	result_vote = msg->vote;
-
-	if (qdevice_net_algorithm_ask_for_vote_reply_received(instance, msg->seq_number,
-	    &result_vote) != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_ASK_FOR_VOTE_REPLY_ERR;
-		return (-1);
-	} else {
-		qdevice_net_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(msg->vote));
-	}
-
-	if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
-		return (-1);
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_vote_info(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-	struct send_buffer_list_entry *send_buffer;
-	enum tlv_vote result_vote;
-
-	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
-		qdevice_net_log(LOG_ERR, "Received unexpected vote info message. "
-		    "Disconnecting from server");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-		return (-1);
-	}
-
-	if (!msg->vote_set || !msg->seq_number_set) {
-		qdevice_net_log(LOG_ERR, "Received node list reply message without "
-		    "required options. Disconnecting from server");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
-		return (-1);
-	}
-
-	qdevice_net_log(LOG_DEBUG, "Received vote info");
-	qdevice_net_log(LOG_DEBUG, "  seq = %"PRIu32, msg->seq_number);
-	qdevice_net_log(LOG_DEBUG, "  vote = %s", tlv_vote_to_str(msg->vote));
-
-	result_vote = msg->vote;
-	if (qdevice_net_algorithm_vote_info_received(instance, msg->seq_number,
-	    &result_vote) != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTE_INFO_ERR;
-		return (-1);
-	} else {
-		qdevice_net_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(result_vote));
-	}
-
-	if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
-		return (-1);
-	}
-
-	/*
-	 * Create reply message
-	 */
-	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
-	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for "
-		    "vote info reply msg");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-		return (-1);
-	}
-
-	if (msg_create_vote_info_reply(&send_buffer->buffer, msg->seq_number) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for "
-		    "vote info reply list msg");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-		return (-1);
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_msg_received_vote_info_reply(struct qdevice_net_instance *instance,
-    const struct msg_decoded *msg)
-{
-
-	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "vote info reply"));
-}
-
-static int
-qdevice_net_msg_received(struct qdevice_net_instance *instance)
-{
-	struct msg_decoded msg;
-	int res;
-	int ret_val;
-
-	msg_decoded_init(&msg);
-
-	res = msg_decode(&instance->receive_buffer, &msg);
-	if (res != 0) {
-		/*
-		 * Error occurred. Disconnect.
-		 */
-		qdevice_net_log_msg_decode_error(res);
-		qdevice_net_log(LOG_ERR, "Disconnecting from server");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_MSG_DECODE_ERROR;
-
-		return (-1);
-	}
-
-	ret_val = 0;
-
-	switch (msg.type) {
-	case MSG_TYPE_PREINIT:
-		ret_val = qdevice_net_msg_received_preinit(instance, &msg);
-		break;
-	case MSG_TYPE_PREINIT_REPLY:
-		ret_val = qdevice_net_msg_received_preinit_reply(instance, &msg);
-		break;
-	case MSG_TYPE_STARTTLS:
-		ret_val = qdevice_net_msg_received_starttls(instance, &msg);
-		break;
-	case MSG_TYPE_SERVER_ERROR:
-		ret_val = qdevice_net_msg_received_server_error(instance, &msg);
-		break;
-	case MSG_TYPE_INIT_REPLY:
-		ret_val = qdevice_net_msg_received_init_reply(instance, &msg);
-		break;
-	case MSG_TYPE_SET_OPTION:
-		ret_val = qdevice_net_msg_received_set_option(instance, &msg);
-		break;
-	case MSG_TYPE_SET_OPTION_REPLY:
-		ret_val = qdevice_net_msg_received_set_option_reply(instance, &msg);
-		break;
-	case MSG_TYPE_ECHO_REQUEST:
-		ret_val = qdevice_net_msg_received_echo_request(instance, &msg);
-		break;
-	case MSG_TYPE_ECHO_REPLY:
-		ret_val = qdevice_net_msg_received_echo_reply(instance, &msg);
-		break;
-	case MSG_TYPE_NODE_LIST:
-		ret_val = qdevice_net_msg_received_node_list(instance, &msg);
-		break;
-	case MSG_TYPE_NODE_LIST_REPLY:
-		ret_val = qdevice_net_msg_received_node_list_reply(instance, &msg);
-		break;
-	case MSG_TYPE_ASK_FOR_VOTE:
-		ret_val = qdevice_net_msg_received_ask_for_vote(instance, &msg);
-		break;
-	case MSG_TYPE_ASK_FOR_VOTE_REPLY:
-		ret_val = qdevice_net_msg_received_ask_for_vote_reply(instance, &msg);
-		break;
-	case MSG_TYPE_VOTE_INFO:
-		ret_val = qdevice_net_msg_received_vote_info(instance, &msg);
-		break;
-	case MSG_TYPE_VOTE_INFO_REPLY:
-		ret_val = qdevice_net_msg_received_vote_info_reply(instance, &msg);
-		break;
-	default:
-		qdevice_net_log(LOG_ERR, "Received unsupported message %u. "
-		    "Disconnecting from server", msg.type);
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
-
-		ret_val = -1;
-		break;
-	}
-
-	msg_decoded_destroy(&msg);
-
-	return (ret_val);
-}
-
-/*
- * -1 means end of connection (EOF) or some other unhandled error. 0 = success
- */
-static int
-qdevice_net_socket_read(struct qdevice_net_instance *instance)
-{
-	int res;
-	int ret_val;
-	int orig_skipping_msg;
-
-	orig_skipping_msg = instance->skipping_msg;
-
-	res = msgio_read(instance->socket, &instance->receive_buffer,
-	    &instance->msg_already_received_bytes, &instance->skipping_msg);
-
-	if (!orig_skipping_msg && instance->skipping_msg) {
-		qdevice_net_log(LOG_DEBUG, "msgio_read set skipping_msg");
-	}
-
-	ret_val = 0;
-
-	switch (res) {
-	case 0:
-		/*
-		 * Partial read
-		 */
-		break;
-	case -1:
-		qdevice_net_log(LOG_DEBUG, "Server closed connection");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
-		ret_val = -1;
-		break;
-	case -2:
-		qdevice_net_log_nss(LOG_ERR, "Unhandled error when reading from server. "
-		    "Disconnecting from server");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
-		ret_val = -1;
-		break;
-	case -3:
-		qdevice_net_log(LOG_ERR, "Can't store message header from server. "
-		    "Disconnecting from server");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
-		ret_val = -1;
-		break;
-	case -4:
-		qdevice_net_log(LOG_ERR, "Can't store message from server. "
-		    "Disconnecting from server");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
-		ret_val = -1;
-		break;
-	case -5:
-		qdevice_net_log(LOG_WARNING, "Server sent unsupported msg type %u. "
-		    "Disconnecting from server", msg_get_type(&instance->receive_buffer));
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNSUPPORTED_MSG;
-		ret_val = -1;
-		break;
-	case -6:
-		qdevice_net_log(LOG_WARNING,
-		    "Server wants to send too long message %u bytes. Disconnecting from server",
-		    msg_get_len(&instance->receive_buffer));
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
-		ret_val = -1;
-		break;
-	case 1:
-		/*
-		 * Full message received / skipped
-		 */
-		if (!instance->skipping_msg) {
-			if (qdevice_net_msg_received(instance) == -1) {
-				ret_val = -1;
-			}
-		} else {
-			qdevice_net_log(LOG_CRIT, "net_socket_read in skipping msg state");
-			exit(1);
-		}
-
-		instance->skipping_msg = 0;
-		instance->msg_already_received_bytes = 0;
-		dynar_clean(&instance->receive_buffer);
-		break;
-	default:
-		qdevice_net_log(LOG_CRIT, "qdevice_net_socket_read unhandled error %d", res);
-		exit(1);
-		break;
-	}
-
-	return (ret_val);
-}
-
-static int
-qdevice_net_socket_write_finished(struct qdevice_net_instance *instance)
-{
-	PRFileDesc *new_pr_fd;
-
-	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_STARTTLS_BEING_SENT) {
-		/*
-		 * StartTLS sent to server. Begin with TLS handshake
-		 */
-		if ((new_pr_fd = nss_sock_start_ssl_as_client(instance->socket, QNETD_NSS_SERVER_CN,
-		    qdevice_net_nss_bad_cert_hook,
-		    qdevice_net_nss_get_client_auth_data,
-		    (void *)QDEVICE_NET_NSS_CLIENT_CERT_NICKNAME, 0, NULL)) == NULL) {
-			qdevice_net_log_nss(LOG_ERR, "Can't start TLS");
-			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_TLS;
-			return (-1);
-		}
-
-		/*
-		 * And send init msg
-		 */
-		if (qdevice_net_send_init(instance) != 0) {
-			instance->disconnect_reason =
-			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-
-			return (-1);
-		}
-
-		instance->socket = new_pr_fd;
-	}
-
-	return (0);
-}
-
-static int
-qdevice_net_socket_write(struct qdevice_net_instance *instance)
-{
-	int res;
-	struct send_buffer_list_entry *send_buffer;
-	enum msg_type sent_msg_type;
-
-	send_buffer = send_buffer_list_get_active(&instance->send_buffer_list);
-	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_CRIT, "send_buffer_list_get_active returned NULL");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SEND_MESSAGE;
-
-		return (-1);
-	}
-
-	res = msgio_write(instance->socket, &send_buffer->buffer,
-	    &send_buffer->msg_already_sent_bytes);
-
-	if (res == 1) {
-		sent_msg_type = msg_get_type(&send_buffer->buffer);
-
-		send_buffer_list_delete(&instance->send_buffer_list, send_buffer);
-
-		if (sent_msg_type != MSG_TYPE_ECHO_REQUEST) {
-			if (qdevice_net_socket_write_finished(instance) == -1) {
-				return (-1);
-			}
-		}
-	}
-
-	if (res == -1) {
-		qdevice_net_log_nss(LOG_CRIT, "PR_Send returned 0");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
-		return (-1);
-	}
-
-	if (res == -2) {
-		qdevice_net_log_nss(LOG_ERR, "Unhandled error when sending message to server");
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SEND_MESSAGE;
-
-		return (-1);
-	}
-
-	return (0);
-}
-
-#define QDEVICE_NET_POLL_NO_FDS		3
-#define QDEVICE_NET_POLL_SOCKET		0
-#define QDEVICE_NET_POLL_VOTEQUORUM	1
-#define QDEVICE_NET_POLL_CMAP		2
-
-static int
-qdevice_net_poll(struct qdevice_net_instance *instance)
-{
-	PRPollDesc pfds[QDEVICE_NET_POLL_NO_FDS];
-	PRInt32 poll_res;
-	int i;
-
-	pfds[QDEVICE_NET_POLL_SOCKET].fd = instance->socket;
-	pfds[QDEVICE_NET_POLL_SOCKET].in_flags = PR_POLL_READ;
-	if (!send_buffer_list_empty(&instance->send_buffer_list)) {
-		pfds[QDEVICE_NET_POLL_SOCKET].in_flags |= PR_POLL_WRITE;
-	}
-	pfds[QDEVICE_NET_POLL_VOTEQUORUM].fd = instance->votequorum_poll_fd;
-	pfds[QDEVICE_NET_POLL_VOTEQUORUM].in_flags = PR_POLL_READ;
-
-	pfds[QDEVICE_NET_POLL_CMAP].fd = instance->cmap_poll_fd;
-	pfds[QDEVICE_NET_POLL_CMAP].in_flags = PR_POLL_READ;
-
-	instance->schedule_disconnect = 0;
-
-	if ((poll_res = PR_Poll(pfds, QDEVICE_NET_POLL_NO_FDS,
-	    timer_list_time_to_expire(&instance->main_timer_list))) > 0) {
-		for (i = 0; i < QDEVICE_NET_POLL_NO_FDS; i++) {
-			if (pfds[i].out_flags & PR_POLL_READ) {
-				switch (i) {
-				case QDEVICE_NET_POLL_SOCKET:
-					if (qdevice_net_socket_read(instance) == -1) {
-						instance->schedule_disconnect = 1;
-					}
-
-					break;
-				case QDEVICE_NET_POLL_VOTEQUORUM:
-					if (votequorum_dispatch(instance->votequorum_handle,
-					    CS_DISPATCH_ALL) != CS_OK) {
-						qdevice_net_log(LOG_ERR, "Can't dispatch votequorum messages");
-						instance->schedule_disconnect = 1;
-						instance->disconnect_reason =
-						    QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
-					}
-					break;
-				case QDEVICE_NET_POLL_CMAP:
-					if (cmap_dispatch(instance->cmap_handle,
-					    CS_DISPATCH_ALL) != CS_OK) {
-						qdevice_net_log(LOG_ERR, "Can't dispatch cmap messages");
-						instance->schedule_disconnect = 1;
-						instance->disconnect_reason =
-						    QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
-					}
-					break;
-				default:
-					qdevice_net_log(LOG_CRIT, "Unhandled read on poll descriptor %u", i);
-					exit(1);
-					break;
-				}
-			}
-
-			if (!instance->schedule_disconnect && pfds[i].out_flags & PR_POLL_WRITE) {
-				switch (i) {
-				case QDEVICE_NET_POLL_SOCKET:
-					if (qdevice_net_socket_write(instance) == -1) {
-						instance->schedule_disconnect = 1;
-					}
-
-					break;
-				default:
-					qdevice_net_log(LOG_CRIT, "Unhandled write on poll descriptor %u", i);
-					exit(1);
-					break;
-				}
-			}
-
-			if (!instance->schedule_disconnect &&
-			    pfds[i].out_flags &
-			    (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) {
-				switch (i) {
-				case QDEVICE_NET_POLL_SOCKET:
-					qdevice_net_log(LOG_ERR, "POLL_ERR (%u) on main socket",
-					    pfds[i].out_flags);
-
-					instance->schedule_disconnect = 1;
-					instance->disconnect_reason =
-					    QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
-
-					break;
-				default:
-					qdevice_net_log(LOG_CRIT, "Unhandled error on poll descriptor %u", i);
-					exit(1);
-					break;
-				}
-			}
-		}
-	}
-
-	if (!instance->schedule_disconnect) {
-		timer_list_expire(&instance->main_timer_list);
-	}
-
-	if (instance->schedule_disconnect) {
-		/*
-		 * Schedule disconnect can be set by this function, by some timer_list callback
-		 * or cmap/votequorum callbacks
-		 */
-		return (-1);
-	}
-
-	return (0);
-}
-
-static void
-qdevice_net_instance_init_from_cmap(struct qdevice_net_instance *instance,
-    cmap_handle_t cmap_handle)
-{
-	uint32_t node_id;
-	enum tlv_tls_supported tls_supported;
-	int i;
-	long int li;
-	char *str;
-	enum tlv_decision_algorithm_type decision_algorithm;
-	struct tlv_tie_breaker tie_breaker;
-	uint32_t heartbeat_interval;
-	uint32_t sync_heartbeat_interval;
-	uint32_t cast_vote_timer_interval;
-	char *host_addr;
-	int host_port;
-	char *ep;
-	char *cluster_name;
-	uint32_t connect_timeout;
-
-	/*
-	 * Check if provider is net
-	 */
-	if (cmap_get_string(cmap_handle, "quorum.device.model", &str) != CS_OK) {
-		errx(1, "Can't read quorum.device.model cmap key.");
-	}
-
-	if (strcmp(str, "net") != 0) {
-		free(str);
-		errx(1, "Configured device model is not net. "
-		    "This qdevice provider is only for net.");
-	}
-	free(str);
-
-	/*
-	 * Get nodeid
-	 */
-	if (cmap_get_uint32(cmap_handle, "runtime.votequorum.this_node_id", &node_id) != CS_OK) {
-		errx(1, "Unable to retrive this node nodeid.");
-	}
-
-	/*
-	 * Check tls
-	 */
-	if (cmap_get_string(cmap_handle, "quorum.device.net.tls", &str) == CS_OK) {
-		if ((i = utils_parse_bool_str(str)) == -1) {
-			free(str);
-			errx(1, "quorum.device.net.tls value is not valid.");
-		}
-
-		if (i == 1) {
-			tls_supported = TLV_TLS_SUPPORTED;
-		} else {
-			tls_supported = TLV_TLS_UNSUPPORTED;
-		}
-
-		free(str);
-	}
-
-	/*
-	 * Host
-	 */
-	if (cmap_get_string(cmap_handle, "quorum.device.net.host", &str) != CS_OK) {
-		errx(1, "Qdevice net daemon address is not defined (quorum.device.net.host)");
-	}
-	host_addr = str;
-
-	if (cmap_get_string(cmap_handle, "quorum.device.net.port", &str) == CS_OK) {
-		host_port = strtol(str, &ep, 10);
-
-
-		if (host_port <= 0 || host_port > ((uint16_t)~0) || *ep != '\0') {
-			errx(1, "quorum.device.net.port must be in range 0-65535");
-		}
-
-		free(str);
-	} else {
-		host_port = QNETD_DEFAULT_HOST_PORT;
-	}
-
-	/*
-	 * Cluster name
-	 */
-	if (cmap_get_string(cmap_handle, "totem.cluster_name", &str) != CS_OK) {
-		errx(1, "Cluster name (totem.cluster_name) has to be defined.");
-	}
-	cluster_name = str;
-
-	/*
-	 * Configure timeouts
-	 */
-	if (cmap_get_uint32(cmap_handle, "quorum.device.timeout", &heartbeat_interval) != CS_OK) {
-		heartbeat_interval = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT;
-	}
-	cast_vote_timer_interval = heartbeat_interval * 0.5;
-	heartbeat_interval = heartbeat_interval * 0.8;
-
-	if (cmap_get_uint32(cmap_handle, "quorum.device.sync_timeout",
-	    &sync_heartbeat_interval) != CS_OK) {
-		sync_heartbeat_interval = VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT;
-	}
-	sync_heartbeat_interval = sync_heartbeat_interval * 0.8;
-
-
-	/*
-	 * Choose decision algorithm
-	 */
-	if (cmap_get_string(cmap_handle, "quorum.device.net.algorithm", &str) != CS_OK) {
-		decision_algorithm = QDEVICE_NET_DEFAULT_ALGORITHM;
-	} else {
-		if (strcmp(str, "test") == 0) {
-			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_TEST;
-		} else if (strcmp(str, "ffsplit") == 0) {
-			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_FFSPLIT;
-		} else if (strcmp(str, "2nodelms") == 0) {
-			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_2NODELMS;
-		} else if (strcmp(str, "lms") == 0) {
-			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_LMS;
-		} else {
-			errx(1, "Unknown decision algorithm %s", str);
-		}
-
-		free(str);
-	}
-
-	/*
-	 * Load tie_breaker mode
-	 */
-	memset(&tie_breaker, 0, sizeof(tie_breaker));
-
-	if (cmap_get_string(cmap_handle, "quorum.device.net.tie_breaker", &str) != CS_OK) {
-		tie_breaker.mode = QDEVICE_NET_DEFAULT_TIE_BREAKER_MODE;
-	} else {
-		if (strcmp(str, "lowest") == 0) {
-			tie_breaker.mode = TLV_TIE_BREAKER_MODE_LOWEST;
-		} else if (strcmp(str, "highest") == 0) {
-			tie_breaker.mode = TLV_TIE_BREAKER_MODE_HIGHEST;
-		} else {
-			li = strtol(str, &ep, 10);
-			if (li <= 0 || li > ((uint32_t)~0) || *ep != '\0') {
-				errx(1, "tie_breaker must be lowest|highest|valid_node_id");
-			}
-
-			tie_breaker.mode = TLV_TIE_BREAKER_MODE_NODE_ID;
-			tie_breaker.node_id = li;
-		}
-
-		free(str);
-	}
-
-	/*
-	 * Get connect timeout
-	 */
-	if (cmap_get_string(cmap_handle, "quorum.device.net.connect_timeout", &str) != CS_OK) {
-		connect_timeout = QDEVICE_NET_CONNECT_TIMEOUT;
-	} else {
-		li = strtol(str, &ep, 10);
-		if (li < QDEVICE_NET_MIN_CONNECT_TIMEOUT || li > QDEVICE_NET_MAX_CONNECT_TIMEOUT || *ep != '\0') {
-			errx(1, "connect_timeout must be valid number in range <%lu,%lu>",
-			    QDEVICE_NET_MIN_CONNECT_TIMEOUT, QDEVICE_NET_MAX_CONNECT_TIMEOUT);
-		}
-
-		connect_timeout = li;
-
-		free(str);
-	}
-
-	/*
-	 * Really initialize instance
-	 */
-	if (qdevice_net_instance_init(instance,
-	    QDEVICE_NET_INITIAL_MSG_RECEIVE_SIZE, QDEVICE_NET_INITIAL_MSG_SEND_SIZE,
-	    QDEVICE_NET_MIN_MSG_SEND_SIZE, QDEVICE_NET_MAX_SEND_BUFFERS,
-	    QDEVICE_NET_MAX_MSG_RECEIVE_SIZE,
-	    tls_supported, node_id, decision_algorithm,
-	    heartbeat_interval, sync_heartbeat_interval, cast_vote_timer_interval,
-	    host_addr, host_port, cluster_name, &tie_breaker, connect_timeout) == -1) {
-		errx(1, "Can't initialize qdevice-net");
-	}
-
-	instance->cmap_handle = cmap_handle;
-	qdevice_net_cmap_init_fd(instance);
-}
-
-int
-main(void)
-{
-	struct qdevice_net_instance instance;
-	cmap_handle_t cmap_handle;
-	struct send_buffer_list_entry *send_buffer;
-	int try_connect;
-
-	/*
-	 * Init
-	 */
-	qdevice_net_cmap_init(&cmap_handle);
-	qdevice_net_instance_init_from_cmap(&instance, cmap_handle);
-
-	qdevice_net_log_init(QDEVICE_NET_LOG_TARGET_STDERR);
-        qdevice_net_log_set_debug(1);
-
-	qdevice_net_log(LOG_DEBUG, "Registering algorithms");
-	qdevice_net_algorithm_register_all();
-
-	if (nss_sock_init_nss((instance.tls_supported != TLV_TLS_UNSUPPORTED ?
-	    (char *)QDEVICE_NET_NSS_DB_DIR : NULL)) != 0) {
-		qdevice_net_log_nss(LOG_ERR, "Can't init nss");
-		exit(1);
-	}
-
-
-	if (qdevice_net_algorithm_init(&instance) != 0) {
-		qdevice_net_log(LOG_ERR, "Algorithm init failed");
-		exit(1);
-	}
-
-	try_connect = 1;
-	while (try_connect) {
-		qdevice_net_votequorum_init(&instance);
-
-		/*
-		 * Try to connect to qnetd host
-		 */
-		instance.socket = nss_sock_create_client_socket(instance.host_addr, instance.host_port,
-		    PR_AF_UNSPEC, PR_MillisecondsToInterval(instance.connect_timeout));
-		if (instance.socket == NULL) {
-			qdevice_net_log_nss(LOG_CRIT, "Can't connect to server");
-			poll(NULL, 0, QDEVICE_NET_PAUSE_BEFORE_RECONNECT);
-
-			continue ;
-		}
-
-		if (nss_sock_set_nonblocking(instance.socket) != 0) {
-			if (PR_Close(instance.socket) != PR_SUCCESS) {
-				qdevice_net_log_nss(LOG_WARNING, "Unable to close connection");
-			}
-
-			qdevice_net_log_nss(LOG_CRIT, "Can't set socket nonblocking");
-			poll(NULL, 0, QDEVICE_NET_PAUSE_BEFORE_RECONNECT);
-
-			continue ;
-		}
-
-		/*
-		 * Create and schedule send of preinit message to qnetd
-		 */
-		send_buffer = send_buffer_list_get_new(&instance.send_buffer_list);
-		if (send_buffer == NULL) {
-			errx(1, "Can't allocate send buffer list");
-		}
-
-		if (msg_create_preinit(&send_buffer->buffer, instance.cluster_name, 1,
-		    instance.last_msg_seq_num) == 0) {
-			errx(1, "Can't allocate buffer");
-		}
-
-		send_buffer_list_put(&instance.send_buffer_list, send_buffer);
-
-		instance.state = QDEVICE_NET_INSTANCE_STATE_WAITING_PREINIT_REPLY;
-
-		/*
-		 * Main loop
-		 */
-		while (qdevice_net_poll(&instance) == 0) {
-		}
-
-		try_connect = qdevice_net_disconnect_reason_try_reconnect(instance.disconnect_reason);
-
-		if (qdevice_net_algorithm_disconnected(&instance, instance.disconnect_reason,
-		    &try_connect) != 0) {
-			qdevice_net_log(LOG_ERR, "Algorithm returned error, force exit");
-			exit(2);
-		}
-
-		if (instance.disconnect_reason == QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED) {
-			try_connect = 0;
-		}
-
-		/*
-		 * Cleanup
-		 */
-		if (PR_Close(instance.socket) != PR_SUCCESS) {
-			qdevice_net_log_nss(LOG_WARNING, "Unable to close connection");
-		}
-
-		/*
-		 * Close cmap and votequorum connections
-		 */
-		qdevice_net_votequorum_destroy(&instance);
-		qdevice_net_cmap_del_track(&instance);
-
-		if (try_connect) {
-			/*
-			 * Reinit instance
-			 */
-			qdevice_net_instance_clean(&instance);
-		}
-	}
-
-	qdevice_net_algorithm_destroy(&instance);
-
-	qdevice_net_cmap_destroy(&instance);
-
-	qdevice_net_instance_destroy(&instance);
-
-	SSL_ClearSessionCache();
-
-	if (NSS_Shutdown() != SECSuccess) {
-		qdevice_net_log_nss(LOG_WARNING, "Can't shutdown NSS");
-	}
-
-	PR_Cleanup();
-
-	qdevice_net_log_close();
-
-	return (0);
-}

qdevices/corosync-qdevice.c

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include "qdevice-config.h"
+#include "qdevice-cmap.h"
+#include "qdevice-log.h"
+#include "qdevice-model.h"
+#include "qdevice-votequorum.h"
+#include "utils.h"
+
+int
+main(void)
+{
+	struct qdevice_instance instance;
+
+	qdevice_instance_init(&instance);
+	qdevice_cmap_init(&instance);
+	qdevice_log_init(&instance);
+	qdevice_votequorum_init(&instance);
+
+	qdevice_log(LOG_DEBUG, "Registering qdevice models");
+	qdevice_model_register_all();
+
+	qdevice_log(LOG_DEBUG, "Configuring qdevice");
+	if (qdevice_instance_configure_from_cmap(&instance) != 0) {
+		return (1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Getting configuration node list");
+	if (qdevice_cmap_store_config_node_list(&instance) != 0) {
+		return (1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Initializing qdevice model");
+	if (qdevice_model_init(&instance) != 0) {
+		return (1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Initializing cmap tracking");
+	if (qdevice_cmap_add_track(&instance) != 0) {
+		return (1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Running qdevice model");
+	if (qdevice_model_run(&instance) != 0) {
+		return (1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Removing cmap tracking");
+	if (qdevice_cmap_del_track(&instance) != 0) {
+		return (1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Destorying qdevice model");
+	qdevice_model_destroy(&instance);
+
+	qdevice_votequorum_destroy(&instance);
+	qdevice_cmap_destroy(&instance);
+	qdevice_log_close(&instance);
+	qdevice_instance_destroy(&instance);
+
+	return (0);
+}

qdevices/corosync-qnetd.c

@@ -142,8 +142,8 @@ qnetd_poll(struct qnetd_instance *instance)
 			}
 
 			if (!client_disconnect &&
-			    pfds[i].out_flags &
-			    (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) {
+			    (pfds[i].out_flags & (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) &&
+			    !(pfds[i].out_flags & (PR_POLL_READ|PR_POLL_WRITE))) {
 				if (i == 0) {
 					if (pfds[i].out_flags != PR_POLL_NVAL) {
 						/*
@@ -367,7 +367,7 @@ main(int argc, char *argv[])
 		qnetd_err_nss();
 	}
 
-	if (nss_sock_set_nonblocking(instance.server.socket) != 0) {
+	if (nss_sock_set_non_blocking(instance.server.socket) != 0) {
 		qnetd_err_nss();
 	}
 

qdevices/msg.h

@@ -68,48 +68,48 @@ enum msg_type {
 struct msg_decoded {
 	enum msg_type type;
 	uint8_t seq_number_set;
-	uint32_t seq_number;	// Only valid if seq_number_set != 0
+	uint32_t seq_number;	/* Only valid if seq_number_set != 0 */
 	size_t cluster_name_len;
-	// Valid only if != NULL. Trailing \0 is added but not counted in cluster_name_len
+	/* Valid only if != NULL. Trailing \0 is added but not counted in cluster_name_len */
 	char *cluster_name;
 	uint8_t tls_supported_set;
-	enum tlv_tls_supported tls_supported;	// Valid only if tls_supported_set != 0.
+	enum tlv_tls_supported tls_supported;	/* Valid only if tls_supported_set != 0. */
 	uint8_t tls_client_cert_required_set;
-	uint8_t tls_client_cert_required;	// Valid only if tls_client_cert_required_set != 0
+	uint8_t tls_client_cert_required;	/* Valid only if tls_client_cert_required_set != 0 */
 	size_t no_supported_messages;
-	enum msg_type *supported_messages;	// Valid only if != NULL
+	enum msg_type *supported_messages;	/* Valid only if != NULL */
 	size_t no_supported_options;
-	enum tlv_opt_type *supported_options;	// Valid only if != NULL
+	enum tlv_opt_type *supported_options;	/* Valid only if != NULL */
 	uint8_t reply_error_code_set;
-	enum tlv_reply_error_code reply_error_code;	// Valid only if reply_error_code_set != 0
+	enum tlv_reply_error_code reply_error_code;	/* Valid only if reply_error_code_set != 0 */
 	uint8_t server_maximum_request_size_set;
-	// Valid only if server_maximum_request_size_set != 0
+	/* Valid only if server_maximum_request_size_set != 0 */
 	size_t server_maximum_request_size;
 	uint8_t server_maximum_reply_size_set;
-	size_t server_maximum_reply_size;	// Valid only if server_maximum_reply_size_set != 0
+	size_t server_maximum_reply_size;	/* Valid only if server_maximum_reply_size_set != 0 */
 	uint8_t node_id_set;
 	uint32_t node_id;
 	size_t no_supported_decision_algorithms;
-	// Valid only if != NULL
+	/* Valid only if != NULL */
 	enum tlv_decision_algorithm_type *supported_decision_algorithms;
 	uint8_t decision_algorithm_set;
-	// Valid only if decision_algorithm_set != 0
+	/* Valid only if decision_algorithm_set != 0 */
 	enum tlv_decision_algorithm_type decision_algorithm;
 	uint8_t heartbeat_interval_set;
-	uint32_t heartbeat_interval;	// Valid only if heartbeat_interval_set != 0
+	uint32_t heartbeat_interval;	/* Valid only if heartbeat_interval_set != 0 */
 	uint8_t ring_id_set;
-	struct tlv_ring_id ring_id;	// Valid only if ring_id_set != 0
+	struct tlv_ring_id ring_id;	/* Valid only if ring_id_set != 0 */
 	uint8_t config_version_set;
-	uint64_t config_version;	// Valid only if config_version_set != 0
-	uint32_t data_center_id;	// Valid only if != 0
-	enum tlv_node_state node_state;	// Valid only if != TLV_NODE_STATE_NOT_SET
-	struct node_list nodes;		// Valid only if node_list_is_empty(nodes) != 0
+	uint64_t config_version;	/* Valid only if config_version_set != 0 */
+	uint32_t data_center_id;	/* Valid only if != 0 */
+	enum tlv_node_state node_state;	/* Valid only if != TLV_NODE_STATE_NOT_SET */
+	struct node_list nodes;		/* Valid only if node_list_is_empty(nodes) != 0 */
 	int node_list_type_set;
-	enum tlv_node_list_type node_list_type;	// Valid only if node_list_type_set != 0
+	enum tlv_node_list_type node_list_type;	/* Valid only if node_list_type_set != 0 */
 	int vote_set;
-	enum tlv_vote vote;	// Valid only if vote_set != 0
+	enum tlv_vote vote;	/* Valid only if vote_set != 0 */
 	int quorate_set;
-	enum tlv_quorate quorate;	// Valid only if quorate_set != 0
+	enum tlv_quorate quorate;	/* Valid only if quorate_set != 0 */
 	int tie_breaker_set;
 	struct tlv_tie_breaker tie_breaker;
 };

qdevices/nss-sock.c

@@ -32,9 +32,7 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <err.h>
-
-#include <prnetdb.h>
+#include <limits.h>
 
 #include "nss-sock.h"
 
@@ -62,7 +60,7 @@ nss_sock_init_nss(char *config_dir)
  * Set NSS socket non-blocking
  */
 int
-nss_sock_set_nonblocking(PRFileDesc *sock)
+nss_sock_set_non_blocking(PRFileDesc *sock)
 {
 	PRSocketOptionData sock_opt;
 
@@ -183,11 +181,6 @@ nss_sock_create_listen_socket(const char *hostname, uint16_t port, PRIntn af)
 	return (sock);
 }
 
-/*
- * Create listen socket and bind it to address. hostname can be NULL and then
- * any address is used. Address family (af) can be ether PR_AF_UNSPEC or
- * PR_AF_INET.
- */
 PRFileDesc *
 nss_sock_create_client_socket(const char *hostname, uint16_t port, PRIntn af,
     PRIntervalTime timeout)
@@ -236,6 +229,118 @@ nss_sock_create_client_socket(const char *hostname, uint16_t port, PRIntn af,
 	return (sock);
 }
 
+int
+nss_sock_non_blocking_client_init(const char *host_name, uint16_t port, PRIntn af,
+    struct nss_sock_non_blocking_client *client)
+{
+
+	client->destroyed = 1;
+
+	if ((client->host_name = strdup(host_name)) == NULL) {
+		PR_SetError(PR_OUT_OF_MEMORY_ERROR, 0);
+
+		return (-1);
+	}
+
+	client->port = port;
+	client->af = af;
+
+	client->addr_info = PR_GetAddrInfoByName(client->host_name, af, PR_AI_ADDRCONFIG);
+	if (client->addr_info == NULL) {
+		free(client->host_name);
+
+		return (-1);
+	}
+	client->addr_iter = NULL;
+	client->connect_attempts = 0;
+	client->socket = NULL;
+	client->destroyed = 0;
+
+	return (0);
+}
+
+int
+nss_sock_non_blocking_client_try_next(struct nss_sock_non_blocking_client *client)
+{
+	PRNetAddr addr;
+	PRStatus res;
+
+	if (client->socket != NULL) {
+		PR_Close(client->socket);
+		client->socket = NULL;
+	}
+
+	while ((client->addr_iter = PR_EnumerateAddrInfo(client->addr_iter, client->addr_info,
+	    client->port, &addr)) != NULL) {
+		client->socket = nss_sock_create_socket(addr.raw.family, 0);
+		if (client->socket == NULL) {
+			continue ;
+		}
+
+		if (nss_sock_set_non_blocking(client->socket) == -1) {
+			PR_Close(client->socket);
+			client->socket = NULL;
+			continue ;
+		}
+
+		res = PR_Connect(client->socket, &addr, PR_INTERVAL_NO_TIMEOUT);
+		if (res == PR_SUCCESS || PR_GetError() == PR_IN_PROGRESS_ERROR) {
+			return (0);
+		}
+
+		PR_Close(client->socket);
+		client->socket = NULL;
+
+		if (client->connect_attempts < INT_MAX) {
+			client->connect_attempts++;
+		}
+	}
+
+	if (client->connect_attempts == 0) {
+		PR_SetError(PR_ADDRESS_NOT_AVAILABLE_ERROR, 0);
+	}
+
+	return (-1);
+}
+
+void
+nss_sock_non_blocking_client_destroy(struct nss_sock_non_blocking_client *client)
+{
+
+	if (client->addr_info != NULL) {
+		PR_FreeAddrInfo(client->addr_info);
+	}
+
+	free(client->host_name);
+
+	client->destroyed = 1;
+}
+
+/*
+ * -1 = Client connect failed
+ *  0 = Client connect still in progress
+ *  1 = Client successfuly connected
+ */
+int
+nss_sock_non_blocking_client_succeeded(const PRPollDesc *pfd)
+{
+	int res;
+
+	res = -1;
+
+	if (PR_GetConnectStatus(pfd) == PR_SUCCESS) {
+		res = 1;
+	} else {
+		if (PR_GetError() == PR_IN_PROGRESS_ERROR) {
+			res = 0;
+		} else {
+			res = -1;
+		}
+	}
+
+	return (res);
+}
+
 /*
  * Start client side SSL connection. This can block.
  *

qdevices/nss-sock.h

@@ -37,17 +37,29 @@
 
 #include <nss.h>
 #include <ssl.h>
+#include <prnetdb.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+struct nss_sock_non_blocking_client {
+	char *host_name;
+	uint16_t port;
+	PRIntn af;
+	PRFileDesc *socket;
+	PRAddrInfo *addr_info;
+	void *addr_iter;
+	unsigned int connect_attempts;
+	int destroyed;
+};
+
 extern int		nss_sock_init_nss(char *config_dir);
 
 extern PRFileDesc	*nss_sock_create_listen_socket(const char *hostname, uint16_t port,
     PRIntn af);
 
-extern int		nss_sock_set_nonblocking(PRFileDesc *sock);
+extern int		nss_sock_set_non_blocking(PRFileDesc *sock);
 
 extern PRFileDesc 	*nss_sock_create_client_socket(const char *hostname, uint16_t port,
     PRIntn af, PRIntervalTime timeout);
@@ -60,6 +72,17 @@ extern PRFileDesc	*nss_sock_start_ssl_as_server(PRFileDesc *input_sock,
     CERTCertificate *server_cert, SECKEYPrivateKey *server_key, int require_client_cert,
     int force_handshake, int *reset_would_block);
 
+extern int		 nss_sock_non_blocking_client_init(const char *host_name,
+    uint16_t port, PRIntn af, struct nss_sock_non_blocking_client *client);
+
+extern int		 nss_sock_non_blocking_client_try_next(
+    struct nss_sock_non_blocking_client *client);
+
+extern void		 nss_sock_non_blocking_client_destroy(
+    struct nss_sock_non_blocking_client *client);
+
+extern int		 nss_sock_non_blocking_client_succeeded(const PRPollDesc *pfd);
+
 #ifdef __cplusplus
 }
 #endif

qdevices/qdevice-net-cmap.c → qdevices/qdevice-cmap.c

@@ -35,24 +35,20 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 
+#include <err.h>
+#include <poll.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <netdb.h>
-#include <err.h>
-#include <poll.h>
 
-/*
- * Needed for creating nspr handle from unix fd
- */
-#include <private/pprio.h>
-
-#include "qnet-config.h"
-#include "qdevice-net-cmap.h"
-#include "qdevice-net-log.h"
-#include "qdevice-net-send.h"
+#include "qdevice-config.h"
+#include "qdevice-cmap.h"
+#include "qdevice-log.h"
+#include "qdevice-log-debug.h"
+#include "qdevice-model.h"
 
 static uint32_t
-qdevice_net_cmap_autogenerate_node_id(const char *addr, int clear_node_high_byte)
+qdevice_cmap_autogenerate_node_id(const char *addr, int clear_node_high_byte)
 {
 	struct addrinfo *ainfo;
 	struct addrinfo ahints;
@@ -64,7 +60,7 @@ qdevice_net_cmap_autogenerate_node_id(const char *addr, int clear_node_high_byte
 	/*
 	 * Hardcoded AF_INET because autogenerated nodeid is valid only for ipv4
 	 */
-	ahints.ai_family   = AF_INET;
+	ahints.ai_family = AF_INET;
 
 	ret = getaddrinfo(addr, NULL, &ahints, &ainfo);
 	if (ret != 0)
@@ -77,7 +73,7 @@ qdevice_net_cmap_autogenerate_node_id(const char *addr, int clear_node_high_byte
 		return (0);
 	}
 
-        memcpy(&i, &((struct sockaddr_in *)ainfo->ai_addr)->sin_addr, sizeof(struct in_addr));
+	memcpy(&i, &((struct sockaddr_in *)ainfo->ai_addr)->sin_addr, sizeof(struct in_addr));
 	freeaddrinfo(ainfo);
 
 	ret = htonl(i);
@@ -90,7 +86,7 @@ qdevice_net_cmap_autogenerate_node_id(const char *addr, int clear_node_high_byte
 }
 
 int
-qdevice_net_cmap_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list)
+qdevice_cmap_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list)
 {
 	cs_error_t cs_err;
 	cmap_iter_handle_t iter_handle;
@@ -146,7 +142,7 @@ qdevice_net_cmap_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list)
 				return (-1);
 			}
 
-			node_id = qdevice_net_cmap_autogenerate_node_id(addr0_str,
+			node_id = qdevice_cmap_autogenerate_node_id(addr0_str,
 			    clear_node_high_byte);
 
 			free(addr0_str);
@@ -179,68 +175,151 @@ iter_finalize:
 }
 
 int
-qdevice_net_cmap_get_config_version(cmap_handle_t cmap_handle, uint64_t *config_version)
+qdevice_cmap_get_config_version(cmap_handle_t cmap_handle, uint64_t *config_version)
 {
 	int res;
 
 	if (cmap_get_uint64(cmap_handle, "totem.config_version", config_version) == CS_OK) {
-		res = 1;
+		res = 0;
 	} else {
 		*config_version = 0;
-		res = 0;
+		res = -1;
 	}
 
 	return (res);
 }
 
+int
+qdevice_cmap_store_config_node_list(struct qdevice_instance *instance)
+{
+	int res;
+
+	node_list_free(&instance->config_node_list);
+
+	if (qdevice_cmap_get_nodelist(instance->cmap_handle, &instance->config_node_list) != 0) {
+		qdevice_log(LOG_ERR, "Can't get configuration node list.");
+
+		return (-1);
+	}
+
+	res = qdevice_cmap_get_config_version(instance->cmap_handle, &instance->config_node_list_version);
+	instance->config_node_list_version_set = (res == 0);
+
+	return (0);
+}
+
 void
-qdevice_net_cmap_init(cmap_handle_t *handle)
+qdevice_cmap_init(struct qdevice_instance *instance)
 {
 	cs_error_t res;
 	int no_retries;
 
 	no_retries = 0;
 
-	while ((res = cmap_initialize(handle)) == CS_ERR_TRY_AGAIN &&
-	    no_retries++ < QDEVICE_NET_MAX_CS_TRY_AGAIN) {
-		poll(NULL, 0, 1000);
+	while ((res = cmap_initialize(&instance->cmap_handle)) == CS_ERR_TRY_AGAIN &&
+	    no_retries++ < QDEVICE_MAX_CS_TRY_AGAIN) {
+		(void)poll(NULL, 0, 1000);
 	}
 
         if (res != CS_OK) {
 		errx(1, "Failed to initialize the cmap API. Error %s", cs_strerror(res));
 	}
+
+	if ((res = cmap_context_set(instance->cmap_handle, (void *)instance)) != CS_OK) {
+		errx(1, "Can't set cmap context. Error %s", cs_strerror(res));
+	}
+
+	cmap_fd_get(instance->cmap_handle, &instance->cmap_poll_fd);
 }
 
-void
-qdevice_net_cmap_init_fd(struct qdevice_net_instance *instance)
+static void
+qdevice_cmap_node_list_event(struct qdevice_instance *instance)
 {
-	int fd;
-	cs_error_t res;
+	struct node_list nlist;
+	int config_version_set;
+	uint64_t config_version;
 
-	if ((res = cmap_context_set(instance->cmap_handle, (void *)instance)) != CS_OK) {
-		qdevice_net_log(LOG_ERR, "Can't set cmap context. Error %s", cs_strerror(res));
-		exit(1);
+	qdevice_log(LOG_DEBUG, "Node list configuration possibly changed");
+	if (qdevice_cmap_get_nodelist(instance->cmap_handle, &nlist) != 0) {
+		qdevice_log(LOG_ERR, "Can't get configuration node list.");
+
+		if (qdevice_model_get_config_node_list_failed(instance) != 0) {
+			qdevice_log(LOG_DEBUG, "qdevice_model_get_config_node_list_failed returned error -> exit");
+			exit(2);
+		}
+
+		return ;
 	}
 
-	cmap_fd_get(instance->cmap_handle, &fd);
-	if ((instance->cmap_poll_fd = PR_CreateSocketPollFd(fd)) == NULL) {
-		qdevice_net_log_nss(LOG_CRIT, "Can't create NSPR cmap poll fd");
-		exit(1);
+	config_version_set = (qdevice_cmap_get_config_version(instance->cmap_handle,
+	    &config_version) == 0);
+
+	if (node_list_eq(&instance->config_node_list, &nlist)) {
+		return ;
+	}
+
+	qdevice_log(LOG_DEBUG, "Node list changed");
+	if (config_version_set) {
+		qdevice_log(LOG_DEBUG, "  config_version = %"PRIu64, config_version);
+	}
+	qdevice_log_debug_dump_node_list(&nlist);
+
+	if (qdevice_model_config_node_list_changed(instance, &nlist,
+	    config_version_set, config_version) != 0) {
+		qdevice_log(LOG_DEBUG, "qdevice_model_config_node_list_changed returned error -> exit");
+		exit(2);
+	}
+
+	node_list_free(&instance->config_node_list);
+	if (node_list_clone(&instance->config_node_list, &nlist) != 0) {
+		qdevice_log(LOG_ERR, "Can't allocate instance->config_node_list clone");
+
+		node_list_free(&nlist);
+
+		if (qdevice_model_get_config_node_list_failed(instance) != 0) {
+			qdevice_log(LOG_DEBUG, "qdevice_model_get_config_node_list_failed returned error -> exit");
+			exit(2);
+		}
+
+		return ;
+	}
+
+	instance->config_node_list_version_set = config_version_set;
+
+	if (config_version_set) {
+		instance->config_node_list_version = config_version;
 	}
 }
 
 static void
-qdevice_net_cmap_nodelist_reload_cb(cmap_handle_t cmap_handle, cmap_track_handle_t cmap_track_handle,
+qdevice_cmap_logging_event(struct qdevice_instance *instance)
+{
+
+	qdevice_log(LOG_DEBUG, "Logging configuration possibly changed");
+	qdevice_log_configure(instance);
+}
+
+static void
+qdevice_cmap_reload_cb(cmap_handle_t cmap_handle, cmap_track_handle_t cmap_track_handle,
     int32_t event, const char *key_name,
     struct cmap_notify_value new_value, struct cmap_notify_value old_value,
     void *user_data)
 {
 	cs_error_t cs_res;
 	uint8_t reload;
-	struct qdevice_net_instance *instance;
+	struct qdevice_instance *instance;
+	int node_list_event;
+	int logging_event;
+	const char *node_list_prefix_str;
+	const char *logging_prefix_str;
+
+	node_list_event = 0;
+	logging_event = 0;
+	node_list_prefix_str = "nodelist.";
+	logging_prefix_str = "logging.";
 
 	if (cmap_context_get(cmap_handle, (const void **)&instance) != CS_OK) {
-		qdevice_net_log(LOG_ERR, "Fatal error. Can't get cmap context");
+		qdevice_log(LOG_ERR, "Fatal error. Can't get cmap context");
 		exit(1);
 	}
 
@@ -258,6 +337,8 @@ qdevice_net_cmap_nodelist_reload_cb(cmap_handle_t cmap_handle, cmap_track_handle
 			return ;
 		} else {
 			instance->cmap_reload_in_progress = 0;
+			node_list_event = 1;
+			logging_event = 1;
 		}
 	}
 
@@ -270,36 +351,54 @@ qdevice_net_cmap_nodelist_reload_cb(cmap_handle_t cmap_handle, cmap_track_handle
 		return ;
 	}
 
-	if (qdevice_net_send_config_node_list(instance, 0, 0) != 0) {
-		/*
-		 * Fatal error -> schedule disconnect
-		 */
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-		instance->schedule_disconnect = 1;
+	if (strncmp(key_name, node_list_prefix_str, strlen(node_list_prefix_str)) == 0) {
+		node_list_event = 1;
+	}
+
+	if (strncmp(key_name, logging_prefix_str, strlen(logging_prefix_str)) == 0) {
+		logging_event = 1;
+	}
+
+	if (logging_event) {
+		qdevice_cmap_logging_event(instance);
+	}
+
+	if (node_list_event) {
+		qdevice_cmap_node_list_event(instance);
 	}
 }
 
 int
-qdevice_net_cmap_add_track(struct qdevice_net_instance *instance)
+qdevice_cmap_add_track(struct qdevice_instance *instance)
 {
 	cs_error_t res;
 
 	res = cmap_track_add(instance->cmap_handle, "config.totemconfig_reload_in_progress",
-	    CMAP_TRACK_ADD | CMAP_TRACK_MODIFY, qdevice_net_cmap_nodelist_reload_cb,
+	    CMAP_TRACK_ADD | CMAP_TRACK_MODIFY, qdevice_cmap_reload_cb,
 	    NULL, &instance->cmap_reload_track_handle);
 
 	if (res != CS_OK) {
-		qdevice_net_log(LOG_ERR, "Can't initialize cmap totemconfig_reload_in_progress tracking");
+		qdevice_log(LOG_ERR, "Can't initialize cmap totemconfig_reload_in_progress tracking");
 		return (-1);
 	}
 
 	res = cmap_track_add(instance->cmap_handle, "nodelist.",
 	    CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX,
-	    qdevice_net_cmap_nodelist_reload_cb,
+	    qdevice_cmap_reload_cb,
 	    NULL, &instance->cmap_nodelist_track_handle);
 
 	if (res != CS_OK) {
-		qdevice_net_log(LOG_ERR, "Can't initialize cmap nodelist tracking");
+		qdevice_log(LOG_ERR, "Can't initialize cmap nodelist tracking");
+		return (-1);
+	}
+
+	res = cmap_track_add(instance->cmap_handle, "logging.",
+	    CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX,
+	    qdevice_cmap_reload_cb,
+	    NULL, &instance->cmap_logging_track_handle);
+
+	if (res != CS_OK) {
+		qdevice_log(LOG_ERR, "Can't initialize logging tracking");
 		return (-1);
 	}
 
@@ -307,39 +406,52 @@ qdevice_net_cmap_add_track(struct qdevice_net_instance *instance)
 }
 
 int
-qdevice_net_cmap_del_track(struct qdevice_net_instance *instance)
+qdevice_cmap_del_track(struct qdevice_instance *instance)
 {
 	cs_error_t res;
 
 	res = cmap_track_delete(instance->cmap_handle, instance->cmap_reload_track_handle);
-
 	if (res != CS_OK) {
-		qdevice_net_log(LOG_WARNING, "Can't delete cmap totemconfig_reload_in_progress tracking");
+		qdevice_log(LOG_WARNING, "Can't delete cmap totemconfig_reload_in_progress tracking");
 	}
-	instance->cmap_reload_track_handle = 0;
 
 	res = cmap_track_delete(instance->cmap_handle, instance->cmap_nodelist_track_handle);
-	instance->cmap_nodelist_track_handle = 0;
+	if (res != CS_OK) {
+		qdevice_log(LOG_WARNING, "Can't delete cmap nodelist tracking");
+	}
 
+	res = cmap_track_delete(instance->cmap_handle, instance->cmap_logging_track_handle);
 	if (res != CS_OK) {
-		qdevice_net_log(LOG_WARNING, "Can't delete cmap nodelist tracking");
+		qdevice_log(LOG_WARNING, "Can't delete cmap logging tracking");
 	}
 
 	return (0);
 }
 
 void
-qdevice_net_cmap_destroy(struct qdevice_net_instance *instance)
+qdevice_cmap_destroy(struct qdevice_instance *instance)
 {
 	cs_error_t res;
 
 	res = cmap_finalize(instance->cmap_handle);
 
         if (res != CS_OK) {
-		qdevice_net_log(LOG_WARNING, "Can't finalize cmap. Error %s", cs_strerror(res));
+		qdevice_log(LOG_WARNING, "Can't finalize cmap. Error %s", cs_strerror(res));
 	}
+}
+
+int
+qdevice_cmap_dispatch(struct qdevice_instance *instance)
+{
+	cs_error_t res;
 
-	if (PR_DestroySocketPollFd(instance->cmap_poll_fd) != PR_SUCCESS) {
-		qdevice_net_log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
+	res = cmap_dispatch(instance->cmap_handle, CS_DISPATCH_ALL);
+
+	if (res != CS_OK && res != CS_ERR_TRY_AGAIN) {
+		qdevice_log(LOG_ERR, "Can't dispatch cmap messages");
+
+		return (-1);
 	}
+
+	return (0);
 }

qdevices/qdevice-net-cmap.h → qdevices/qdevice-cmap.h

@@ -32,36 +32,38 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef _QDEVICE_NET_CMAP_H_
-#define _QDEVICE_NET_CMAP_H_
+#ifndef _QDEVICE_CMAP_H_
+#define _QDEVICE_CMAP_H_
 
 #include <cmap.h>
 
 #include "node-list.h"
-#include "qdevice-net-instance.h"
+#include "qdevice-instance.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-extern int		qdevice_net_cmap_get_nodelist(cmap_handle_t cmap_handle,
+extern int		qdevice_cmap_get_nodelist(cmap_handle_t cmap_handle,
     struct node_list *list);
 
-extern int		qdevice_net_cmap_get_config_version(cmap_handle_t cmap_handle,
+extern int		qdevice_cmap_get_config_version(cmap_handle_t cmap_handle,
     uint64_t *config_version);
 
-extern void		qdevice_net_cmap_init(cmap_handle_t *handle);
+extern void		qdevice_cmap_init(struct qdevice_instance *instance);
 
-extern void		qdevice_net_cmap_init_fd(struct qdevice_net_instance *instance);
+extern int		qdevice_cmap_add_track(struct qdevice_instance *instance);
 
-extern int		qdevice_net_cmap_add_track(struct qdevice_net_instance *instance);
+extern int		qdevice_cmap_del_track(struct qdevice_instance *instance);
 
-extern int		qdevice_net_cmap_del_track(struct qdevice_net_instance *instance);
+extern void		qdevice_cmap_destroy(struct qdevice_instance *instance);
 
-extern void		qdevice_net_cmap_destroy(struct qdevice_net_instance *instance);
+extern int		qdevice_cmap_dispatch(struct qdevice_instance *instance);
+
+extern int		qdevice_cmap_store_config_node_list(struct qdevice_instance *instance);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* _QDEVICE_NET_CMAP_H_ */
+#endif /* _QDEVICE_CMAP_H_ */

qdevices/qdevice-config.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_CONFIG_H_
+#define _QDEVICE_CONFIG_H_
+
+#include <qb/qbdefs.h>
+#include <qb/qblog.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * There are "hardcoded" defines for qdevice. It's not so good
+ * idea to change them as long as you are not 100% sure what you are doing.
+ */
+#define QDEVICE_LOCK_FILE			LOCALSTATEDIR"/run/corosync-qdevice.pid"
+
+#define QDEVICE_MAX_CS_TRY_AGAIN		10
+
+#define QDEVICE_PROGRAM_NAME			"qdevice-net"
+#define QDEVICE_LOG_SUBSYS			"QDEVICE"
+#define QDEVICE_LOG_DEFAULT_TO_STDERR		1
+#define QDEVICE_LOG_DEFAULT_TO_SYSLOG		1
+#define QDEVICE_LOG_DEFAULT_TO_LOGFILE		0
+#define QDEVICE_LOG_DEFAULT_SYSLOG_FACILITY	LOG_DAEMON
+#define QDEVICE_LOG_DEFAULT_SYSLOG_PRIORITY	LOG_INFO
+#define QDEVICE_LOG_DEFAULT_DEBUG		0
+#define QDEVICE_LOG_DEFAULT_FILELINE		0
+#define QDEVICE_LOG_DEFAULT_TIMESTAMP		0
+#define QDEVICE_LOG_DEFAULT_FUNCTION_NAME	0
+
+#define QDEVICE_VOTEQUORUM_DEVICE_NAME      "Qdevice"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_CONFIG_H_ */

qdevices/qdevice-instance.c

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qdevice-instance.h"
+#include "qdevice-log.h"
+#include "qdevice-model.h"
+
+int
+qdevice_instance_init(struct qdevice_instance *instance)
+{
+
+	memset(instance, 0, sizeof(*instance));
+
+	node_list_init(&instance->config_node_list);
+
+	return (0);
+}
+
+int
+qdevice_instance_destroy(struct qdevice_instance *instance)
+{
+
+	node_list_free(&instance->config_node_list);
+
+	return (0);
+}
+
+int
+qdevice_instance_configure_from_cmap(struct qdevice_instance *instance)
+{
+	char *str;
+
+	if (cmap_get_string(instance->cmap_handle, "quorum.device.model", &str) != CS_OK) {
+		qdevice_log(LOG_ERR, "Can't read quorum.device.model cmap key.");
+
+		return (-1);
+	}
+
+	if (qdevice_model_str_to_type(str, &instance->model_type) != 0) {
+		qdevice_log(LOG_ERR, "Configured device model %s is not supported.", str);
+		free(str);
+
+		return (-1);
+	}
+	free(str);
+
+	if (cmap_get_uint32(instance->cmap_handle, "runtime.votequorum.this_node_id",
+	    &instance->node_id) != CS_OK) {
+		qdevice_log(LOG_ERR, "Unable to retrive this node nodeid.");
+
+		return (-1);
+	}
+
+	if (cmap_get_uint32(instance->cmap_handle, "quorum.device.timeout", &instance->heartbeat_interval) != CS_OK) {
+		instance->heartbeat_interval = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT;
+	}
+
+	if (cmap_get_uint32(instance->cmap_handle, "quorum.device.sync_timeout",
+	    &instance->sync_heartbeat_interval) != CS_OK) {
+		instance->sync_heartbeat_interval = VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT;
+	}
+
+	return (0);
+}

qdevices/qdevice-instance.h

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_INSTANCE_H_
+#define _QDEVICE_INSTANCE_H_
+
+#include <sys/types.h>
+
+#include <stdlib.h>
+#include <stdint.h>
+
+#include <cmap.h>
+#include <votequorum.h>
+
+#include "qdevice-model-type.h"
+#include "node-list.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct qdevice_instance {
+	cmap_handle_t cmap_handle;
+	int cmap_poll_fd;
+	int cmap_reload_in_progress;
+	cmap_track_handle_t cmap_reload_track_handle;
+	cmap_track_handle_t cmap_nodelist_track_handle;
+	cmap_track_handle_t cmap_logging_track_handle;
+
+	votequorum_handle_t votequorum_handle;
+	int votequorum_poll_fd;
+
+	enum qdevice_model_type model_type;
+
+	uint32_t node_id;
+	uint32_t heartbeat_interval;		/* Heartbeat interval during normal operation */
+	uint32_t sync_heartbeat_interval;	/* Heartbeat interval during corosync sync */
+
+	struct node_list config_node_list;
+	int config_node_list_version_set;
+	uint64_t config_node_list_version;
+
+	/*
+	 * Copy of votequorum_quorum_notify_fn callback paramters.
+	 * Set after model callback is called.
+	 */
+	uint32_t vq_quorum_quorate;
+	uint32_t vq_quorum_node_list_entries;
+	votequorum_node_t *vq_quorum_node_list;
+
+	/*
+	 * Copy of votequorum_nodelist_notify_fn callback paramters.
+	 * Set after model callback is called.
+	 */
+	votequorum_ring_id_t vq_node_list_ring_id;
+	uint32_t vq_node_list_entries;
+	uint32_t *vq_node_list;
+
+	/*
+	 * Copy of votequorum_expectedvotes_notify_fn callback parameters.
+	 * Set after model callback is called.
+	 */
+	uint32_t vq_expected_votes;
+
+	void *model_data;
+};
+
+extern int	qdevice_instance_init(struct qdevice_instance *instance);
+
+extern int	qdevice_instance_destroy(struct qdevice_instance *instance);
+
+extern int	qdevice_instance_configure_from_cmap(struct qdevice_instance *instance);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_INSTANCE_H_ */

qdevices/qdevice-log-debug.c

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qdevice-log-debug.h"
+#include "qdevice-log.h"
+
+void
+qdevice_log_debug_dump_node_list(const struct node_list *nlist)
+{
+	struct node_list_entry *node_info;
+	size_t zi;
+
+	qdevice_log(LOG_DEBUG, "  Node list:");
+
+	zi = 0;
+
+	TAILQ_FOREACH(node_info, nlist, entries) {
+		qdevice_log(LOG_DEBUG, "    %zu node_id = %"PRIx32", "
+		    "data_center_id = %"PRIx32", node_state = %s",
+		    zi, node_info->node_id, node_info->data_center_id,
+		    tlv_node_state_to_str(node_info->node_state));
+		zi++;
+        }
+}

qdevices/qdevice-log-debug.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_LOG_DEBUG_H_
+#define _QDEVICE_LOG_DEBUG_H_
+
+#include "node-list.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern void		qdevice_log_debug_dump_node_list(const struct node_list *nlist);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_LOG_DEBUG_H_ */

qdevices/qdevice-log.c

@@ -0,0 +1,315 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qdevice-log.h"
+#include "qdevice-config.h"
+#include "utils.h"
+
+struct qdevice_log_syslog_names {
+	const char *prio_name;
+	int priority;
+};
+
+static struct qdevice_log_syslog_names qdevice_log_priority_names[] = {
+	{ "alert", LOG_ALERT },
+	{ "crit", LOG_CRIT },
+	{ "debug", LOG_DEBUG },
+	{ "emerg", LOG_EMERG },
+	{ "err", LOG_ERR },
+	{ "error", LOG_ERR },
+	{ "info", LOG_INFO },
+	{ "notice", LOG_NOTICE },
+	{ "warning", LOG_WARNING },
+	{ NULL, -1 }};
+
+static int
+qdevice_log_priority_str_to_int(const char *priority_str)
+{
+	unsigned int i;
+
+	for (i = 0; qdevice_log_priority_names[i].prio_name != NULL; i++) {
+		if (strcasecmp(priority_str, qdevice_log_priority_names[i].prio_name) == 0) {
+			return (qdevice_log_priority_names[i].priority);
+		}
+	}
+
+	return (-1);
+}
+
+void
+qdevice_log_configure(struct qdevice_instance *instance)
+{
+	int to_stderr;
+	int to_syslog;
+	int syslog_facility;
+	int syslog_priority;
+	int logfile_priority;
+	int debug;
+	char *str;
+	int i;
+	int fileline;
+	int timestamp;
+	int function_name;
+	char log_format_syslog[64];
+	char log_format_stderr[64];
+
+	to_stderr = QDEVICE_LOG_DEFAULT_TO_STDERR;
+	if (cmap_get_string(instance->cmap_handle, "logging.to_stderr", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING, "logging.to_stderr value is not valid");
+		} else {
+			to_stderr = i;
+		}
+		free(str);
+	}
+
+	if (cmap_get_string(instance->cmap_handle,
+	    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".to_stderr", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING,
+			    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".to_stderr value is not valid.");
+		} else {
+			to_stderr = i;
+		}
+		free(str);
+	}
+
+	to_syslog = QDEVICE_LOG_DEFAULT_TO_SYSLOG;
+	if (cmap_get_string(instance->cmap_handle, "logging.to_syslog", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING, "logging.to_syslog value is not valid");
+		} else {
+			to_syslog = i;
+		}
+		free(str);
+	}
+
+	if (cmap_get_string(instance->cmap_handle,
+	    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".to_syslog", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING,
+			    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".to_syslog value is not valid.");
+		} else {
+			to_syslog = i;
+		}
+		free(str);
+	}
+
+	syslog_facility = QDEVICE_LOG_DEFAULT_SYSLOG_FACILITY;
+	if (cmap_get_string(instance->cmap_handle, "logging.syslog_facility", &str) == CS_OK) {
+		if ((i = qb_log_facility2int(str)) < 0) {
+			qdevice_log(LOG_WARNING, "logging.syslog_facility value is not valid");
+		} else {
+			syslog_facility = i;
+		}
+
+		free(str);
+	}
+
+	if (cmap_get_string(instance->cmap_handle,
+	    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".syslog_facility", &str) == CS_OK) {
+		if ((i = qb_log_facility2int(str)) < 0) {
+			qdevice_log(LOG_WARNING,
+			    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".syslog_facility value is not valid.");
+		} else {
+			syslog_facility = i;
+		}
+		free(str);
+	}
+
+	syslog_priority = QDEVICE_LOG_DEFAULT_SYSLOG_PRIORITY;
+	if (cmap_get_string(instance->cmap_handle, "logging.syslog_priority", &str) == CS_OK) {
+		if ((i = qdevice_log_priority_str_to_int(str)) < 0) {
+			qdevice_log(LOG_WARNING, "logging.syslog_priority value is not valid");
+		} else {
+			syslog_priority = i;
+		}
+
+		free(str);
+	}
+
+	if (cmap_get_string(instance->cmap_handle,
+	    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".syslog_priority", &str) == CS_OK) {
+		if ((i = qdevice_log_priority_str_to_int(str)) < 0) {
+			qdevice_log(LOG_WARNING,
+			    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".syslog_priority value is not valid.");
+		} else {
+			syslog_priority = i;
+		}
+		free(str);
+	}
+
+	logfile_priority = QDEVICE_LOG_DEFAULT_SYSLOG_PRIORITY;
+	if (cmap_get_string(instance->cmap_handle, "logging.logfile_priority", &str) == CS_OK) {
+		if ((i = qdevice_log_priority_str_to_int(str)) < 0) {
+			qdevice_log(LOG_WARNING, "logging.logfile_priority value is not valid");
+		} else {
+			logfile_priority = i;
+		}
+
+		free(str);
+	}
+
+	if (cmap_get_string(instance->cmap_handle,
+	    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".logfile_priority", &str) == CS_OK) {
+		if ((i = qdevice_log_priority_str_to_int(str)) < 0) {
+			qdevice_log(LOG_WARNING,
+			    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".logfile_priority value is not valid.");
+		} else {
+			logfile_priority = i;
+		}
+		free(str);
+	}
+
+	debug = QDEVICE_LOG_DEFAULT_DEBUG;
+	if (cmap_get_string(instance->cmap_handle, "logging.debug", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			if (strcasecmp(str, "trace") == 0) {
+				debug = 1;
+			} else {
+				qdevice_log(LOG_WARNING, "logging.debug value is not valid");
+			}
+		} else {
+			debug = i;
+		}
+		free(str);
+	}
+
+	if (cmap_get_string(instance->cmap_handle,
+	    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".debug", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			if (strcasecmp(str, "trace") == 0) {
+				debug = 1;
+			} else {
+				qdevice_log(LOG_WARNING,
+				    "logging.logger_subsys." QDEVICE_LOG_SUBSYS ".debug value is not valid.");
+			}
+		} else {
+			debug = i;
+		}
+		free(str);
+	}
+
+	fileline = QDEVICE_LOG_DEFAULT_FILELINE;
+	if (cmap_get_string(instance->cmap_handle, "logging.fileline", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING, "logging.fileline value is not valid");
+		} else {
+			fileline = i;
+		}
+		free(str);
+	}
+
+	timestamp = QDEVICE_LOG_DEFAULT_TIMESTAMP;
+	if (cmap_get_string(instance->cmap_handle, "logging.timestamp", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING, "logging.timestamp value is not valid");
+		} else {
+			timestamp = i;
+		}
+		free(str);
+	}
+
+	function_name = QDEVICE_LOG_DEFAULT_FUNCTION_NAME;
+	if (cmap_get_string(instance->cmap_handle, "logging.function_name", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			qdevice_log(LOG_WARNING, "logging.function_name value is not valid");
+		} else {
+			function_name = i;
+		}
+		free(str);
+	}
+
+	strcpy(log_format_syslog, "");
+
+	if (fileline) {
+		strcat(log_format_syslog, "%f:");
+
+		if (function_name) {
+			strcat(log_format_syslog, "%n:");
+		}
+
+		strcat(log_format_syslog, "%l ");
+	}
+
+	strcat(log_format_syslog, "%b");
+
+	strcpy(log_format_stderr, "");
+	if (timestamp) {
+		strcpy(log_format_stderr, "%t %7p ");
+	}
+	strcat(log_format_stderr, log_format_syslog);
+
+	/*
+	 * Finally reconfigure log system
+	 */
+	qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, to_stderr);
+	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, to_syslog);
+	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_FACILITY, syslog_facility);
+	qb_log_filter_ctl(QB_LOG_SYSLOG, QB_LOG_FILTER_CLEAR_ALL, QB_LOG_FILTER_FILE, "*", LOG_TRACE);
+	qb_log_filter_ctl(QB_LOG_SYSLOG, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*",
+	    (debug ? LOG_DEBUG : syslog_priority));
+	qb_log_filter_ctl(QB_LOG_STDERR, QB_LOG_FILTER_CLEAR_ALL, QB_LOG_FILTER_FILE, "*", LOG_TRACE);
+	qb_log_filter_ctl(QB_LOG_STDERR, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*",
+	    (debug ? LOG_DEBUG : logfile_priority));
+
+	qb_log_format_set(QB_LOG_STDERR, log_format_stderr);
+	qb_log_format_set(QB_LOG_SYSLOG, log_format_syslog);
+}
+
+void
+qdevice_log_init(struct qdevice_instance *instance)
+{
+	qb_log_init(QDEVICE_PROGRAM_NAME, QDEVICE_LOG_DEFAULT_SYSLOG_FACILITY,
+	    QDEVICE_LOG_DEFAULT_SYSLOG_PRIORITY);
+
+	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE);
+	qb_log_ctl(QB_LOG_STDOUT, QB_LOG_CONF_ENABLED, QB_FALSE);
+	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
+	qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_TRUE);
+
+	qb_log_filter_ctl(QB_LOG_STDERR, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*", LOG_INFO);
+	qb_log_filter_ctl(QB_LOG_SYSLOG, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*", LOG_INFO);
+	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_PRIORITY_BUMP, LOG_INFO - LOG_DEBUG);
+	qb_log_format_set(QB_LOG_STDERR, "%t %7p %b");
+
+	qdevice_log_configure(instance);
+}
+
+void
+qdevice_log_close(struct qdevice_instance *instance)
+{
+
+	qb_log_fini();
+}

qdevices/qdevice-log.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_LOG_H_
+#define _QDEVICE_LOG_H_
+
+#include <qb/qbdefs.h>
+#include <qb/qblog.h>
+#include <prerror.h>
+
+#include "qdevice-instance.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define qdevice_log	qb_log
+#define qdevice_log_nss(priority, str) qdevice_log(priority, "%s (%d): %s", \
+    str, PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
+
+extern void		qdevice_log_init(struct qdevice_instance *instance);
+
+extern void		qdevice_log_configure(struct qdevice_instance *instance);
+
+extern void		qdevice_log_close(struct qdevice_instance *instance);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_LOG_H_ */

qdevices/qdevice-model-net.c

@@ -0,0 +1,439 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <config.h>
+
+#include "qdevice-model.h"
+#include "qdevice-model-net.h"
+#include "qdevice-log.h"
+#include "qdevice-net-cast-vote-timer.h"
+#include "qdevice-net-instance.h"
+#include "qdevice-net-algorithm.h"
+#include "qdevice-net-poll.h"
+#include "qdevice-net-send.h"
+#include "qdevice-net-votequorum.h"
+#include "qnet-config.h"
+#include "nss-sock.h"
+
+int
+qdevice_model_net_init(struct qdevice_instance *instance)
+{
+
+	struct qdevice_net_instance *net_instance;
+
+	qdevice_log(LOG_DEBUG, "Initializing qdevice_net_instance");
+	if (qdevice_net_instance_init_from_cmap(instance) != 0) {
+		return (-1);
+	}
+
+	net_instance = instance->model_data;
+
+	qdevice_log(LOG_DEBUG, "Registering algorithms");
+	if (qdevice_net_algorithm_register_all() != 0) {
+		return (-1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Initializing NSS");
+	if (nss_sock_init_nss((net_instance->tls_supported != TLV_TLS_UNSUPPORTED ?
+	    (char *)QDEVICE_NET_NSS_DB_DIR : NULL)) != 0) {
+		qdevice_log_nss(LOG_ERR, "Can't init nss");
+		return (-1);
+	}
+
+	if (qdevice_net_cast_vote_timer_update(net_instance, TLV_VOTE_ASK_LATER) != 0) {
+		qdevice_log(LOG_ERR, "Can't update cast vote timer");
+		return (-1);
+	}
+
+	if (qdevice_net_algorithm_init(net_instance) != 0) {
+		qdevice_log(LOG_ERR, "Algorithm init failed");
+		return (-1);
+	}
+
+	return (0);
+}
+
+int
+qdevice_model_net_destroy(struct qdevice_instance *instance)
+{
+	struct qdevice_net_instance *net_instance;
+
+	net_instance = instance->model_data;
+
+	qdevice_log(LOG_DEBUG, "Destroying algorithm");
+	qdevice_net_algorithm_destroy(net_instance);
+
+	qdevice_log(LOG_DEBUG, "Destroying qdevice_net_instance");
+	qdevice_net_instance_destroy(net_instance);
+
+	qdevice_log(LOG_DEBUG, "Shutting down NSS");
+	SSL_ClearSessionCache();
+
+	if (NSS_Shutdown() != SECSuccess) {
+		qdevice_log_nss(LOG_WARNING, "Can't shutdown NSS");
+	}
+
+	PR_Cleanup();
+
+	free(net_instance);
+
+	return (0);
+}
+
+static int
+qdevice_model_net_timer_connect_timeout(void *data1, void *data2)
+{
+	struct qdevice_net_instance *instance;
+
+	instance = (struct qdevice_net_instance *)data1;
+
+	qdevice_log(LOG_ERR, "Connect timeout");
+
+	instance->schedule_disconnect = 1;
+
+	instance->connect_timer = NULL;
+	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER;
+
+	return (0);
+}
+
+int
+qdevice_model_net_run(struct qdevice_instance *instance)
+{
+	struct qdevice_net_instance *net_instance;
+	int try_connect;
+	int res;
+
+	net_instance = instance->model_data;
+
+	qdevice_log(LOG_DEBUG, "Executing qdevice-net");
+
+	try_connect = 1;
+	while (try_connect) {
+		net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT;
+		net_instance->socket = NULL;
+
+		net_instance->connect_timer = timer_list_add(&net_instance->main_timer_list,
+			net_instance->connect_timeout, qdevice_model_net_timer_connect_timeout,
+			(void *)net_instance, NULL);
+
+		if (net_instance->connect_timer == NULL) {
+			qdevice_log(LOG_CRIT, "Can't schedule connect timer");
+
+			try_connect = 0;
+			break ;
+		}
+
+		qdevice_log(LOG_DEBUG, "Trying connect to qnetd server %s:%u (timeout = %u)",
+		    net_instance->host_addr, net_instance->host_port, net_instance->connect_timeout);
+
+		res = nss_sock_non_blocking_client_init(net_instance->host_addr,
+		    net_instance->host_port, PR_AF_UNSPEC, &net_instance->non_blocking_client);
+		if (res == -1) {
+			qdevice_log_nss(LOG_ERR, "Can't initialize non blocking client connection");
+		}
+
+		res = nss_sock_non_blocking_client_try_next(&net_instance->non_blocking_client);
+		if (res == -1) {
+			qdevice_log_nss(LOG_ERR, "Can't connect to qnetd host");
+			nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client);
+		}
+
+		while (qdevice_net_poll(net_instance) == 0) {
+		};
+
+		if (net_instance->connect_timer != NULL) {
+			timer_list_delete(&net_instance->main_timer_list, net_instance->connect_timer);
+			net_instance->connect_timer = NULL;
+		}
+
+		if (net_instance->echo_request_timer != NULL) {
+			timer_list_delete(&net_instance->main_timer_list, net_instance->echo_request_timer);
+			net_instance->echo_request_timer = NULL;
+		}
+
+		try_connect = qdevice_net_disconnect_reason_try_reconnect(net_instance->disconnect_reason);
+
+		if (qdevice_net_algorithm_disconnected(net_instance,
+		    net_instance->disconnect_reason, &try_connect) != 0) {
+			qdevice_log(LOG_ERR, "Algorithm returned error, force exit");
+			return (-1);
+		}
+
+		if (net_instance->disconnect_reason ==
+		    QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED) {
+			try_connect = 0;
+		}
+
+		if (net_instance->socket != NULL) {
+			if (PR_Close(net_instance->socket) != PR_SUCCESS) {
+				qdevice_log_nss(LOG_WARNING, "Unable to close connection");
+			}
+		}
+
+		qdevice_net_instance_clean(net_instance);
+	}
+
+	return (0);
+}
+
+/*
+ * Called when cmap reload (or nodelist) was requested.
+ *
+ * nlist is node list
+ * config_version is valid only if config_version_set != 0
+ *
+ * Should return 0 if processing should continue or -1 to call exit
+ */
+int
+qdevice_model_net_config_node_list_changed(struct qdevice_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version)
+{
+	struct qdevice_net_instance *net_instance;
+	int send_node_list;
+	enum tlv_vote vote;
+
+	net_instance = instance->model_data;
+
+	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
+		/*
+		 * Nodelist changed, but connection to qnetd not initiated yet.
+		 */
+		send_node_list = 0;
+
+		if (net_instance->cast_vote_timer_vote == TLV_VOTE_ACK) {
+			vote = TLV_VOTE_NACK;
+		} else {
+			vote = TLV_VOTE_NO_CHANGE;
+		}
+	} else {
+		send_node_list = 1;
+		vote = TLV_VOTE_NO_CHANGE;
+	}
+
+	if (qdevice_net_algorithm_config_node_list_changed(net_instance, nlist, config_version_set,
+	    config_version, &send_node_list, &vote) != 0) {
+		qdevice_log(LOG_ERR, "Algorithm returned error, force exit");
+		return (-1);
+	} else {
+		qdevice_log(LOG_DEBUG, "Algorithm decided to %s node list and result vote is %s",
+		    (send_node_list ? "send" : "not send"), tlv_vote_to_str(vote));
+	}
+
+	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
+		qdevice_log(LOG_CRIT, "qdevice_model_net_config_node_list_changed fatal error. "
+				" Can't update cast vote timer vote");
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+		net_instance->schedule_disconnect = 1;
+	}
+
+	if (send_node_list) {
+		if (qdevice_net_send_config_node_list(net_instance, nlist, config_version_set,
+		    config_version, 0) != 0) {
+			net_instance->schedule_disconnect = 1;
+			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+		}
+	}
+
+	return (0);
+}
+
+/*
+ * Called when cmap reload (or nodelist) was requested, but it was not possible to
+ * get node list.
+ *
+ * Should return 0 if processing should continue or -1 to call exit
+ */
+int
+qdevice_model_net_get_config_node_list_failed(struct qdevice_instance *instance)
+{
+	struct qdevice_net_instance *net_instance;
+
+	net_instance = instance->model_data;
+
+	net_instance->schedule_disconnect = 1;
+	net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+	return (0);
+}
+
+int
+qdevice_model_net_votequorum_quorum_notify(struct qdevice_instance *instance,
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[])
+{
+	struct qdevice_net_instance *net_instance;
+	int send_node_list;
+	enum tlv_vote vote;
+
+	net_instance = instance->model_data;
+
+	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
+		/*
+		 * Nodelist changed, but connection to qnetd not initiated yet.
+		 */
+		send_node_list = 0;
+
+		if (net_instance->cast_vote_timer_vote == TLV_VOTE_ACK) {
+			vote = TLV_VOTE_NACK;
+		} else {
+			vote = TLV_VOTE_NO_CHANGE;
+		}
+	} else {
+		send_node_list = 1;
+		vote = TLV_VOTE_NO_CHANGE;
+	}
+
+	if (qdevice_net_algorithm_votequorum_quorum_notify(net_instance, quorate,
+	    node_list_entries, node_list, &send_node_list, &vote) != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
+
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTEQUORUM_QUORUM_NOTIFY_ERR;
+		net_instance->schedule_disconnect = 1;
+	} else {
+		qdevice_log(LOG_DEBUG, "Algorithm decided to %s list and result vote is %s",
+		    (send_node_list ? "send" : "not send"), tlv_vote_to_str(vote));
+	}
+
+	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
+		qdevice_log(LOG_CRIT, "qdevice_model_net_votequorum_quorum_notify fatal error. "
+				" Can't update cast vote timer vote");
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+		net_instance->schedule_disconnect = 1;
+	}
+
+	if (send_node_list) {
+		if (qdevice_net_send_quorum_node_list(net_instance,
+		    (quorate ? TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE),
+		    node_list_entries, node_list) != 0) {
+			/*
+			 * Fatal error -> schedule disconnect
+			 */
+			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+			net_instance->schedule_disconnect = 1;
+		}
+	}
+
+	return (0);
+}
+
+int
+qdevice_model_net_votequorum_node_list_notify(struct qdevice_instance *instance,
+    votequorum_ring_id_t votequorum_ring_id, uint32_t node_list_entries, uint32_t node_list[])
+{
+	struct qdevice_net_instance *net_instance;
+	struct tlv_ring_id tlv_rid;
+	enum tlv_vote vote;
+	int send_node_list;
+
+	net_instance = instance->model_data;
+
+	qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid, &votequorum_ring_id);
+
+	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
+		/*
+		 * Nodelist changed, but connection to qnetd not initiated yet.
+		 */
+		send_node_list = 0;
+
+		if (net_instance->cast_vote_timer_vote == TLV_VOTE_ACK) {
+			vote = TLV_VOTE_NACK;
+		} else {
+			vote = TLV_VOTE_NO_CHANGE;
+		}
+	} else {
+		send_node_list = 1;
+		vote = TLV_VOTE_NO_CHANGE;
+	}
+
+	if (qdevice_net_algorithm_votequorum_node_list_notify(net_instance, &tlv_rid,
+	    node_list_entries, node_list, &send_node_list, &vote) != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
+
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTEQUORUM_NODE_LIST_NOTIFY_ERR;
+		net_instance->schedule_disconnect = 1;
+	} else {
+		qdevice_log(LOG_DEBUG, "Algorithm decided to %s list and result vote is %s",
+		    (send_node_list ? "send" : "not send"), tlv_vote_to_str(vote));
+	}
+
+	if (send_node_list) {
+		if (qdevice_net_send_membership_node_list(net_instance, &tlv_rid,
+		    node_list_entries, node_list) != 0) {
+			/*
+			 * Fatal error -> schedule disconnect
+			 */
+			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+			net_instance->schedule_disconnect = 1;
+		}
+	}
+
+	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
+		qdevice_log(LOG_CRIT, "qdevice_model_net_votequorum_node_list_notify fatal error "
+		    "Can't update cast vote timer");
+		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+		net_instance->schedule_disconnect = 1;
+	}
+
+	return (0);
+}
+
+int
+qdevice_model_net_votequorum_expected_votes_notify(struct qdevice_instance *instance,
+    uint32_t expected_votes)
+{
+	struct qdevice_net_instance *net_instance;
+
+	net_instance = instance->model_data;
+
+	return (0);
+}
+
+static struct qdevice_model qdevice_model_net = {
+	.name					= "net",
+	.init					= qdevice_model_net_init,
+	.destroy				= qdevice_model_net_destroy,
+	.run					= qdevice_model_net_run,
+	.get_config_node_list_failed		= qdevice_model_net_get_config_node_list_failed,
+	.config_node_list_changed		= qdevice_model_net_config_node_list_changed,
+	.votequorum_quorum_notify		= qdevice_model_net_votequorum_quorum_notify,
+	.votequorum_node_list_notify		= qdevice_model_net_votequorum_node_list_notify,
+	.votequorum_expected_votes_notify	= qdevice_model_net_votequorum_expected_votes_notify,
+};
+
+int
+qdevice_model_net_register(void)
+{
+	return (qdevice_model_register(QDEVICE_MODEL_TYPE_NET, &qdevice_model_net));
+}

qdevices/qdevice-model-net.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_MODEL_NET_H_
+#define _QDEVICE_MODEL_NET_H_
+
+#include "qdevice-instance.h"
+#include "qdevice-model.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int	qdevice_model_net_init(struct qdevice_instance *instance);
+
+extern int	qdevice_model_net_destroy(struct qdevice_instance *instance);
+
+extern int	qdevice_model_net_run(struct qdevice_instance *instance);
+
+extern int	qdevice_model_net_get_config_node_list_failed(struct qdevice_instance *instance);
+
+extern int	qdevice_model_net_config_node_list_changed(struct qdevice_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version);
+
+extern int	qdevice_model_net_votequorum_quorum_notify(struct qdevice_instance *instance,
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[]);
+
+extern int	qdevice_model_net_votequorum_node_list_notify(struct qdevice_instance *instance,
+    votequorum_ring_id_t votequorum_ring_id, uint32_t node_list_entries, uint32_t node_list[]);
+
+extern int 	qdevice_model_net_votequorum_expected_votes_notify(struct qdevice_instance *instance,
+    uint32_t expected_votes);
+
+extern int	qdevice_model_net_register(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_MODEL_NET_H_ */

qdevices/qdevice-model-type.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_MODEL_TYPE_H_
+#define _QDEVICE_MODEL_TYPE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum qdevice_model_type {
+	QDEVICE_MODEL_TYPE_NET = 0,
+	QDEVICE_MODEL_TYPE_ARRAY_SIZE,
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_MODEL_TYPE_H_ */

qdevices/qdevice-model.c

@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qdevice-log.h"
+#include "qdevice-model.h"
+#include "qdevice-model-net.h"
+
+static struct qdevice_model *qdevice_model_array[QDEVICE_MODEL_TYPE_ARRAY_SIZE];
+
+int
+qdevice_model_init(struct qdevice_instance *instance)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_init unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->init(instance));
+}
+
+int
+qdevice_model_destroy(struct qdevice_instance *instance)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_destroy unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->destroy(instance));
+}
+
+int
+qdevice_model_run(struct qdevice_instance *instance)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_run unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->run(instance));
+}
+
+int
+qdevice_model_get_config_node_list_failed(struct qdevice_instance *instance)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_run unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->get_config_node_list_failed(instance));
+}
+
+int
+qdevice_model_config_node_list_changed(struct qdevice_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_run unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->
+	    config_node_list_changed(instance, nlist, config_version_set, config_version));
+}
+
+int
+qdevice_model_votequorum_quorum_notify(struct qdevice_instance *instance,
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[])
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_votequorum_quorum_notify unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->
+	    votequorum_quorum_notify(instance, quorate, node_list_entries, node_list));
+}
+
+int
+qdevice_model_votequorum_node_list_notify(struct qdevice_instance *instance,
+    votequorum_ring_id_t votequorum_ring_id, uint32_t node_list_entries, uint32_t node_list[])
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_votequorum_node_list_notify unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->
+	    votequorum_node_list_notify(instance, votequorum_ring_id, node_list_entries, node_list));
+}
+
+int
+qdevice_model_votequorum_expected_votes_notify(struct qdevice_instance *instance,
+    uint32_t expected_votes)
+{
+
+	if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE ||
+	    qdevice_model_array[instance->model_type] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_model_votequorum_expected_votes_notify unhandled model");
+		exit(1);
+	}
+
+	return (qdevice_model_array[instance->model_type]->
+	    votequorum_expected_votes_notify(instance, expected_votes));
+}
+
+int
+qdevice_model_register(enum qdevice_model_type model_type,
+    struct qdevice_model *model)
+{
+
+	if (model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE) {
+		return (-1);
+	}
+
+	if (qdevice_model_array[model_type] != NULL) {
+		return (-1);
+	}
+
+	qdevice_model_array[model_type] = model;
+
+	return (0);
+}
+
+void
+qdevice_model_register_all(void)
+{
+
+	if (qdevice_model_net_register() != 0) {
+		qdevice_log(LOG_CRIT, "Failed to register model 'net' ");
+		exit(1);
+	}
+}
+
+int
+qdevice_model_str_to_type(const char *str, enum qdevice_model_type *model_type)
+{
+	int i;
+
+	for (i = 0; i < QDEVICE_MODEL_TYPE_ARRAY_SIZE; i++) {
+		if (qdevice_model_array[i] != NULL &&
+		    strcmp(qdevice_model_array[i]->name, str) == 0) {
+			*model_type = i;
+
+			return (0);
+		}
+	}
+
+	return (-1);
+}

qdevices/qdevice-model.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_MODEL_H_
+#define _QDEVICE_MODEL_H_
+
+#include "qdevice-instance.h"
+#include "qdevice-model-type.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int	qdevice_model_init(struct qdevice_instance *instance);
+
+extern int	qdevice_model_destroy(struct qdevice_instance *instance);
+
+extern int	qdevice_model_run(struct qdevice_instance *instance);
+
+extern int	qdevice_model_get_config_node_list_failed(struct qdevice_instance *instance);
+
+extern int	qdevice_model_config_node_list_changed(struct qdevice_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version);
+
+extern int	qdevice_model_votequorum_quorum_notify(struct qdevice_instance *instance,
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[]);
+
+extern int	qdevice_model_votequorum_node_list_notify(struct qdevice_instance *instance,
+    votequorum_ring_id_t votequorum_ring_id, uint32_t node_list_entries, uint32_t node_list[]);
+
+extern int 	qdevice_model_votequorum_expected_votes_notify(struct qdevice_instance *instance,
+    uint32_t expected_votes);
+
+struct qdevice_model {
+	const char *name;
+	int (*init)(struct qdevice_instance *instance);
+	int (*destroy)(struct qdevice_instance *instance);
+	int (*run)(struct qdevice_instance *instance);
+	int (*get_config_node_list_failed)(struct qdevice_instance *instance);
+	int (*config_node_list_changed)(struct qdevice_instance *instance,
+	    const struct node_list *nlist, int config_version_set, uint64_t config_version);
+	int (*votequorum_quorum_notify)(struct qdevice_instance *instance,
+	    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[]);
+	int (*votequorum_node_list_notify)(struct qdevice_instance *instance,
+	    votequorum_ring_id_t votequorum_ring_id,uint32_t node_list_entries,
+	    uint32_t node_list[]);
+	int (*votequorum_expected_votes_notify)(struct qdevice_instance *instance,
+	    uint32_t expected_votes);
+};
+
+extern int		qdevice_model_register(
+    enum qdevice_model_type model_type, struct qdevice_model *model);
+
+extern void qdevice_model_register_all(void);
+
+extern int		qdevice_model_str_to_type(const char *str,
+    enum qdevice_model_type *model_type);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_MODEL_H_ */

qdevices/qdevice-net-algo-2nodelms.c

@@ -37,7 +37,7 @@
 #include <string.h>
 
 #include "qdevice-net-algo-2nodelms.h"
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
 #include "qdevice-net-send.h"
 #include "qdevice-net-cast-vote-timer.h"
 
@@ -56,10 +56,19 @@ qdevice_net_algo_2nodelms_connected(struct qdevice_net_instance *instance)
 	return (0);
 }
 
+int
+qdevice_net_algo_2nodelms_config_node_list_changed(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int *send_node_list, enum tlv_vote *vote)
+{
+
+	return (0);
+}
+
 int
 qdevice_net_algo_2nodelms_votequorum_node_list_notify(struct qdevice_net_instance *instance,
     const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-    enum tlv_vote *vote)
+    int *send_node_list, enum tlv_vote *vote)
 {
 
 	return (0);
@@ -67,7 +76,8 @@ qdevice_net_algo_2nodelms_votequorum_node_list_notify(struct qdevice_net_instanc
 
 int
 qdevice_net_algo_2nodelms_votequorum_quorum_notify(struct qdevice_net_instance *instance,
-    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list)
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+    enum tlv_vote *vote)
 {
 
 	return (0);
@@ -145,6 +155,7 @@ qdevice_net_algo_2nodelms_destroy(struct qdevice_net_instance *instance)
 static struct qdevice_net_algorithm qdevice_net_algo_2nodelms = {
 	.init					= qdevice_net_algo_2nodelms_init,
 	.connected				= qdevice_net_algo_2nodelms_connected,
+	.config_node_list_changed		= qdevice_net_algo_2nodelms_config_node_list_changed,
 	.votequorum_node_list_notify		= qdevice_net_algo_2nodelms_votequorum_node_list_notify,
 	.votequorum_quorum_notify		= qdevice_net_algo_2nodelms_votequorum_quorum_notify,
 	.config_node_list_reply_received	= qdevice_net_algo_2nodelms_config_node_list_reply_received,

qdevices/qdevice-net-algo-2nodelms.h

@@ -45,13 +45,17 @@ extern int	qdevice_net_algo_2nodelms_init(struct qdevice_net_instance *instance)
 
 extern int	qdevice_net_algo_2nodelms_connected(struct qdevice_net_instance *instance);
 
+extern int      qdevice_net_algo_2nodelms_config_node_list_changed(
+    struct qdevice_net_instance *instance, const struct node_list *nlist,
+    int config_version_set, uint64_t config_version, int *send_node_list, enum tlv_vote *vote);
+
 extern int	qdevice_net_algo_2nodelms_votequorum_node_list_notify(
     struct qdevice_net_instance *instance, const struct tlv_ring_id *ring_id,
-    uint32_t node_list_entries, uint32_t node_list[], enum tlv_vote *vote);
+    uint32_t node_list_entries, uint32_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_2nodelms_votequorum_quorum_notify(
     struct qdevice_net_instance *instance, uint32_t quorate, uint32_t node_list_entries,
-    votequorum_node_t node_list[], int *send_node_list);
+    votequorum_node_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_2nodelms_config_node_list_reply_received(
     struct qdevice_net_instance *instance, uint32_t seq_number, int initial, enum tlv_vote *vote);

qdevices/qdevice-net-algo-ffsplit.c

@@ -37,7 +37,7 @@
 #include <string.h>
 
 #include "qdevice-net-algo-ffsplit.h"
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
 #include "qdevice-net-send.h"
 #include "qdevice-net-cast-vote-timer.h"
 
@@ -56,10 +56,18 @@ qdevice_net_algo_ffsplit_connected(struct qdevice_net_instance *instance)
 	return (0);
 }
 
+int
+qdevice_net_algo_ffsplit_config_node_list_changed(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int *send_node_list, enum tlv_vote *vote)
+{
+	return (0);
+}
+
 int
 qdevice_net_algo_ffsplit_votequorum_node_list_notify(struct qdevice_net_instance *instance,
     const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-    enum tlv_vote *vote)
+    int *send_node_list, enum tlv_vote *vote)
 {
 
 	return (0);
@@ -67,7 +75,8 @@ qdevice_net_algo_ffsplit_votequorum_node_list_notify(struct qdevice_net_instance
 
 int
 qdevice_net_algo_ffsplit_votequorum_quorum_notify(struct qdevice_net_instance *instance,
-    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list)
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+    enum tlv_vote *vote)
 {
 
 	return (0);
@@ -145,6 +154,7 @@ qdevice_net_algo_ffsplit_destroy(struct qdevice_net_instance *instance)
 static struct qdevice_net_algorithm qdevice_net_algo_ffsplit = {
 	.init					= qdevice_net_algo_ffsplit_init,
 	.connected				= qdevice_net_algo_ffsplit_connected,
+	.config_node_list_changed		= qdevice_net_algo_ffsplit_config_node_list_changed,
 	.votequorum_node_list_notify		= qdevice_net_algo_ffsplit_votequorum_node_list_notify,
 	.votequorum_quorum_notify		= qdevice_net_algo_ffsplit_votequorum_quorum_notify,
 	.config_node_list_reply_received	= qdevice_net_algo_ffsplit_config_node_list_reply_received,

qdevices/qdevice-net-algo-ffsplit.h

@@ -45,13 +45,17 @@ extern int	qdevice_net_algo_ffsplit_init(struct qdevice_net_instance *instance);
 
 extern int	qdevice_net_algo_ffsplit_connected(struct qdevice_net_instance *instance);
 
+extern int      qdevice_net_algo_ffsplit_config_node_list_changed(
+    struct qdevice_net_instance *instance, const struct node_list *nlist,
+    int config_version_set, uint64_t config_version, int *send_node_list, enum tlv_vote *vote);
+
 extern int	qdevice_net_algo_ffsplit_votequorum_node_list_notify(
     struct qdevice_net_instance *instance, const struct tlv_ring_id *ring_id,
-    uint32_t node_list_entries, uint32_t node_list[], enum tlv_vote *vote);
+    uint32_t node_list_entries, uint32_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_ffsplit_votequorum_quorum_notify(
     struct qdevice_net_instance *instance, uint32_t quorate, uint32_t node_list_entries,
-    votequorum_node_t node_list[], int *send_node_list);
+    votequorum_node_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_ffsplit_config_node_list_reply_received(
     struct qdevice_net_instance *instance, uint32_t seq_number, int initial, enum tlv_vote *vote);

qdevices/qdevice-net-algo-lms.c

@@ -37,7 +37,7 @@
 #include <string.h>
 
 #include "qdevice-net-algo-lms.h"
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
 #include "qdevice-net-send.h"
 #include "qdevice-net-cast-vote-timer.h"
 
@@ -56,10 +56,19 @@ qdevice_net_algo_lms_connected(struct qdevice_net_instance *instance)
 	return (0);
 }
 
+int
+qdevice_net_algo_lms_config_node_list_changed(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int *send_node_list, enum tlv_vote *vote)
+{
+
+	return (0);
+}
+
 int
 qdevice_net_algo_lms_votequorum_node_list_notify(struct qdevice_net_instance *instance,
     const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-    enum tlv_vote *vote)
+    int *send_node_list, enum tlv_vote *vote)
 {
 
 	return (0);
@@ -67,7 +76,8 @@ qdevice_net_algo_lms_votequorum_node_list_notify(struct qdevice_net_instance *in
 
 int
 qdevice_net_algo_lms_votequorum_quorum_notify(struct qdevice_net_instance *instance,
-    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list)
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+    enum tlv_vote *vote)
 {
 
 	return (0);
@@ -145,6 +155,7 @@ qdevice_net_algo_lms_destroy(struct qdevice_net_instance *instance)
 static struct qdevice_net_algorithm qdevice_net_algo_lms = {
 	.init					= qdevice_net_algo_lms_init,
 	.connected				= qdevice_net_algo_lms_connected,
+	.config_node_list_changed		= qdevice_net_algo_lms_config_node_list_changed,
 	.votequorum_node_list_notify		= qdevice_net_algo_lms_votequorum_node_list_notify,
 	.votequorum_quorum_notify		= qdevice_net_algo_lms_votequorum_quorum_notify,
 	.config_node_list_reply_received	= qdevice_net_algo_lms_config_node_list_reply_received,

qdevices/qdevice-net-algo-lms.h

@@ -45,13 +45,17 @@ extern int	qdevice_net_algo_lms_init(struct qdevice_net_instance *instance);
 
 extern int	qdevice_net_algo_lms_connected(struct qdevice_net_instance *instance);
 
+extern int      qdevice_net_algo_lms_config_node_list_changed(
+    struct qdevice_net_instance *instance, const struct node_list *nlist,
+    int config_version_set, uint64_t config_version, int *send_node_list, enum tlv_vote *vote);
+
 extern int	qdevice_net_algo_lms_votequorum_node_list_notify(
     struct qdevice_net_instance *instance, const struct tlv_ring_id *ring_id,
-    uint32_t node_list_entries, uint32_t node_list[], enum tlv_vote *vote);
+    uint32_t node_list_entries, uint32_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_lms_votequorum_quorum_notify(
     struct qdevice_net_instance *instance, uint32_t quorate, uint32_t node_list_entries,
-    votequorum_node_t node_list[], int *send_node_list);
+    votequorum_node_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_lms_config_node_list_reply_received(
     struct qdevice_net_instance *instance, uint32_t seq_number, int initial, enum tlv_vote *vote);

qdevices/qdevice-net-algo-test.c

@@ -37,7 +37,7 @@
 #include <string.h>
 
 #include "qdevice-net-algo-test.h"
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
 #include "qdevice-net-send.h"
 #include "qdevice-net-cast-vote-timer.h"
 
@@ -52,7 +52,7 @@ qdevice_net_algo_test_init(struct qdevice_net_instance *instance)
 {
 
 	instance->algorithm_data = NULL;
-	qnetd_log(LOG_INFO, "algo-test: Initialized");
+	qdevice_log(LOG_INFO, "algo-test: Initialized");
 
 	return (0);
 }
@@ -66,7 +66,32 @@ int
 qdevice_net_algo_test_connected(struct qdevice_net_instance *instance)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Connected");
+	qdevice_log(LOG_INFO, "algo-test: Connected");
+
+	return (0);
+}
+
+/*
+ * Called after config node list changed.
+ *
+ * Callback can override send_node_list and vote.
+ * Depending on net_instance->state, they are set acordingly:
+ * If net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS
+ *   send_node_list = 0
+ *   if cast_vote_timer_vote != TLV_VOTE_ACK
+ *     vote = TLV_VOTE_NO_CHANGE
+ *   if cast_vote_timer_vote = TLV_VOTE_ACK
+ *     vote = TLV_VOTE_NACK.
+ * Otherwise send_node_list = 0 and vote = TLV_VOTE_NO_CHANGE
+ * If send_node_list is set to non zero, node list is send to qnetd
+ */
+int
+qdevice_net_algo_test_config_node_list_changed(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int *send_node_list, enum tlv_vote *vote)
+{
+
+	qdevice_log(LOG_INFO, "algo-test: Config node list changed");
 
 	return (0);
 }
@@ -76,24 +101,22 @@ qdevice_net_algo_test_connected(struct qdevice_net_instance *instance)
  *
  * Callback should return 0 on success or -1 on failure (-> disconnect client).
  *
- * Main responsibility for this function is to set vote variable (variable is set by caller to
- * TLV_VOTE_WAIT_FOR_REPLY).
- *
- * TLV_VOTE_ACK = do not send list and vote ACK
- * TLV_VOTE_NACK = do not send list and vote NACK
- * TLV_VOTE_NO_CHANGE = do not send list and no change in vote
- * TLV_VOTE_WAIT_FOR_REPLY = send list and wait with voting until reply is received
- * TLV_VOTE_ASK_LATER = send list but do not change vote
+ * If net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS
+ *   send_node_list = 0
+ *   if cast_vote_timer_vote != TLV_VOTE_ACK
+ *     vote = TLV_VOTE_NO_CHANGE
+ *   if cast_vote_timer_vote = TLV_VOTE_ACK
+ *     vote = TLV_VOTE_NACK.
+ * Otherwise send_node_list = 0 and vote = TLV_VOTE_NO_CHANGE
+ * If send_node_list is set to non zero, node list is send to qnetd
  */
 int
 qdevice_net_algo_test_votequorum_node_list_notify(struct qdevice_net_instance *instance,
     const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-    enum tlv_vote *vote)
+    int *send_node_list, enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Votequorum list notify");
-
-	*vote = TLV_VOTE_WAIT_FOR_REPLY;
+	qdevice_log(LOG_INFO, "algo-test: Votequorum list notify");
 
 	return (0);
 }
@@ -103,17 +126,25 @@ qdevice_net_algo_test_votequorum_node_list_notify(struct qdevice_net_instance *i
  *
  * Callback should return 0 on success or -1 on failure (-> disconnect client).
  *
- * Main responsibility is to set send_node_list (set by caller to 1). If send_node_list is set to
- * non zero, node list is send to qnetd
+ * Callback can override send_node_list and vote.
+ * Depending on net_instance->state, they are set acordingly:
+ * If net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS
+ *   send_node_list = 0
+ *   if cast_vote_timer_vote != TLV_VOTE_ACK
+ *     vote = TLV_VOTE_NO_CHANGE
+ *   if cast_vote_timer_vote = TLV_VOTE_ACK
+ *     vote = TLV_VOTE_NACK.
+ * Otherwise send_node_list = 0 and vote = TLV_VOTE_NO_CHANGE
+ *
+ * If send_node_list is set to non zero, node list is send to qnetd
  */
 int
 qdevice_net_algo_test_votequorum_quorum_notify(struct qdevice_net_instance *instance,
-    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list)
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+    enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Votequorum quorum notify");
-
-	*send_node_list = 1;
+	qdevice_log(LOG_INFO, "algo-test: Votequorum quorum notify");
 
 	return (0);
 }
@@ -129,7 +160,7 @@ qdevice_net_algo_test_config_node_list_reply_received(struct qdevice_net_instanc
     uint32_t seq_number, int initial, enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Config node list reply");
+	qdevice_log(LOG_INFO, "algo-test: Config node list reply");
 
 	return (0);
 }
@@ -149,7 +180,7 @@ qdevice_net_algo_test_membership_node_list_reply_received(struct qdevice_net_ins
     uint32_t seq_number, const struct tlv_ring_id *ring_id, enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Membership node list reply");
+	qdevice_log(LOG_INFO, "algo-test: Membership node list reply");
 
 	return (0);
 }
@@ -165,7 +196,7 @@ qdevice_net_algo_test_quorum_node_list_reply_received(struct qdevice_net_instanc
     uint32_t seq_number, enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Quorum node list reply");
+	qdevice_log(LOG_INFO, "algo-test: Quorum node list reply");
 
 	return (0);
 }
@@ -181,7 +212,7 @@ qdevice_net_algo_test_ask_for_vote_reply_received(struct qdevice_net_instance *i
     uint32_t seq_number, enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Ask for vote reply received");
+	qdevice_log(LOG_INFO, "algo-test: Ask for vote reply received");
 
 	return (0);
 }
@@ -197,7 +228,7 @@ qdevice_net_algo_test_vote_info_received(struct qdevice_net_instance *instance,
     uint32_t seq_number, enum tlv_vote *vote)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Vote info received");
+	qdevice_log(LOG_INFO, "algo-test: Vote info received");
 
 	return (0);
 }
@@ -213,7 +244,7 @@ qdevice_net_algo_test_echo_reply_received(struct qdevice_net_instance *instance,
     uint32_t seq_number, int is_expected_seq_number)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Echo reply received");
+	qdevice_log(LOG_INFO, "algo-test: Echo reply received");
 
 	return (is_expected_seq_number ? 0 : -1);
 }
@@ -229,7 +260,7 @@ int
 qdevice_net_algo_test_echo_reply_not_received(struct qdevice_net_instance *instance)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Echo reply not received");
+	qdevice_log(LOG_INFO, "algo-test: Echo reply not received");
 
 	return (-1);
 }
@@ -247,7 +278,7 @@ qdevice_net_algo_test_disconnected(struct qdevice_net_instance *instance,
     enum qdevice_net_disconnect_reason disconnect_reason, int *try_reconnect)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Disconnected");
+	qdevice_log(LOG_INFO, "algo-test: Disconnected");
 
 	return (0);
 }
@@ -259,12 +290,13 @@ void
 qdevice_net_algo_test_destroy(struct qdevice_net_instance *instance)
 {
 
-	qnetd_log(LOG_INFO, "algo-test: Destroy");
+	qdevice_log(LOG_INFO, "algo-test: Destroy");
 }
 
 static struct qdevice_net_algorithm qdevice_net_algo_test = {
 	.init					= qdevice_net_algo_test_init,
 	.connected				= qdevice_net_algo_test_connected,
+	.config_node_list_changed		= qdevice_net_algo_test_config_node_list_changed,
 	.votequorum_node_list_notify		= qdevice_net_algo_test_votequorum_node_list_notify,
 	.votequorum_quorum_notify		= qdevice_net_algo_test_votequorum_quorum_notify,
 	.config_node_list_reply_received	= qdevice_net_algo_test_config_node_list_reply_received,

qdevices/qdevice-net-algo-test.h

@@ -45,16 +45,21 @@ extern int	qdevice_net_algo_test_init(struct qdevice_net_instance *instance);
 
 extern int	qdevice_net_algo_test_connected(struct qdevice_net_instance *instance);
 
+extern int      qdevice_net_algo_test_config_node_list_changed(
+    struct qdevice_net_instance *instance, const struct node_list *nlist,
+    int config_version_set, uint64_t config_version, int *send_node_list, enum tlv_vote *vote);
+
 extern int	qdevice_net_algo_test_votequorum_node_list_notify(
     struct qdevice_net_instance *instance, const struct tlv_ring_id *ring_id,
-    uint32_t node_list_entries, uint32_t node_list[], enum tlv_vote *vote);
+    uint32_t node_list_entries, uint32_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_test_votequorum_quorum_notify(
     struct qdevice_net_instance *instance, uint32_t quorate, uint32_t node_list_entries,
-    votequorum_node_t node_list[], int *send_node_list);
+    votequorum_node_t node_list[], int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_test_config_node_list_reply_received(
-    struct qdevice_net_instance *instance, uint32_t seq_number, int initial, enum tlv_vote *vote);
+    struct qdevice_net_instance *instance, uint32_t seq_number, int initial,
+    enum tlv_vote *vote);
 
 extern int	qdevice_net_algo_test_membership_node_list_reply_received(
     struct qdevice_net_instance *instance, uint32_t seq_number, const struct tlv_ring_id *ring_id,

qdevices/qdevice-net-algorithm.c

@@ -36,7 +36,7 @@
 
 #include "qnet-config.h"
 #include "qdevice-net-algorithm.h"
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
 
 #include "qdevice-net-algo-test.h"
 #include "qdevice-net-algo-ffsplit.h"
@@ -51,7 +51,7 @@ qdevice_net_algorithm_init(struct qdevice_net_instance *instance)
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_init unhandled decision algorithm");
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_init unhandled decision algorithm");
 		exit(1);
 	}
 
@@ -64,44 +64,63 @@ qdevice_net_algorithm_connected(struct qdevice_net_instance *instance)
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_connected unhandled decision algorithm");
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_connected unhandled decision algorithm");
 		exit(1);
 	}
 
 	return (qdevice_net_algorithm_array[instance->decision_algorithm]->connected(instance));
 }
 
+int
+qdevice_net_algorithm_config_node_list_changed(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int *send_node_list, enum tlv_vote *vote)
+{
+
+	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
+	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_connected unhandled decision algorithm");
+		exit(1);
+	}
+
+	return (qdevice_net_algorithm_array[instance->decision_algorithm]->
+	    config_node_list_changed(instance, nlist, config_version_set, config_version,
+	    send_node_list, vote));
+}
+
 int
 qdevice_net_algorithm_votequorum_node_list_notify(struct qdevice_net_instance *instance,
     const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-    enum tlv_vote *vote)
+    int *send_node_list, enum tlv_vote *vote)
 {
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_votequorum_node_list_notify "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_votequorum_node_list_notify "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
 
 	return (qdevice_net_algorithm_array[instance->decision_algorithm]->votequorum_node_list_notify(
-	    instance, ring_id, node_list_entries, node_list, vote));
+	    instance, ring_id, node_list_entries, node_list, send_node_list, vote));
 }
 
 int
 qdevice_net_algorithm_votequorum_quorum_notify(struct qdevice_net_instance *instance,
-    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list)
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+    enum tlv_vote *vote)
 {
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_votequorum_quorum_notify "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_votequorum_quorum_notify "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
 
-	return (qdevice_net_algorithm_array[instance->decision_algorithm]->votequorum_quorum_notify(
-	    instance, quorate, node_list_entries, node_list, send_node_list));
+	return (qdevice_net_algorithm_array[instance->decision_algorithm]->
+	    votequorum_quorum_notify(instance, quorate, node_list_entries, node_list,
+	    send_node_list, vote));
 }
 
 int
@@ -111,7 +130,7 @@ qdevice_net_algorithm_config_node_list_reply_received(struct qdevice_net_instanc
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_config_node_list_reply_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_config_node_list_reply_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -127,7 +146,7 @@ qdevice_net_algorithm_membership_node_list_reply_received(struct qdevice_net_ins
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_membership_node_list_reply_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_membership_node_list_reply_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -143,7 +162,7 @@ qdevice_net_algorithm_quorum_node_list_reply_received(struct qdevice_net_instanc
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_quorum_node_list_reply_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_quorum_node_list_reply_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -159,7 +178,7 @@ qdevice_net_algorithm_ask_for_vote_reply_received(struct qdevice_net_instance *i
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_ask_for_vote_reply_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_ask_for_vote_reply_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -175,7 +194,7 @@ qdevice_net_algorithm_vote_info_received(struct qdevice_net_instance *instance,
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_vote_info_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_vote_info_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -191,7 +210,7 @@ qdevice_net_algorithm_echo_reply_received(struct qdevice_net_instance *instance,
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_echo_reply_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_echo_reply_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -206,7 +225,7 @@ qdevice_net_algorithm_echo_reply_not_received(struct qdevice_net_instance *insta
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_echo_reply_not_received "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_echo_reply_not_received "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -222,7 +241,7 @@ qdevice_net_algorithm_disconnected(struct qdevice_net_instance *instance,
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_disconnected "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_disconnected "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -237,7 +256,7 @@ qdevice_net_algorithm_destroy(struct qdevice_net_instance *instance)
 
 	if (instance->decision_algorithm >= QDEVICE_NET_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE ||
 	    qdevice_net_algorithm_array[instance->decision_algorithm] == NULL) {
-		qdevice_net_log(LOG_CRIT, "qdevice_net_algorithm_destroy "
+		qdevice_log(LOG_CRIT, "qdevice_net_algorithm_destroy "
 		    "unhandled decision algorithm");
 		exit(1);
 	}
@@ -264,27 +283,29 @@ qdevice_net_algorithm_register(enum tlv_decision_algorithm_type algorithm_number
 	return (0);
 }
 
-void
+int
 qdevice_net_algorithm_register_all(void)
 {
 
 	if (qdevice_net_algo_test_register() != 0) {
-		qdevice_net_log(LOG_CRIT, "Failed to register decision algorithm 'test' ");
-		exit(1);
+		qdevice_log(LOG_CRIT, "Failed to register decision algorithm 'test' ");
+		return (-1);
 	}
 
 	if (qdevice_net_algo_ffsplit_register() != 0) {
-		qdevice_net_log(LOG_CRIT, "Failed to register decision algorithm 'ffsplit' ");
-		exit(1);
+		qdevice_log(LOG_CRIT, "Failed to register decision algorithm 'ffsplit' ");
+		return (-1);
 	}
 
 	if (qdevice_net_algo_2nodelms_register() != 0) {
-		qdevice_net_log(LOG_CRIT, "Failed to register decision algorithm '2nodelms' ");
-		exit(1);
+		qdevice_log(LOG_CRIT, "Failed to register decision algorithm '2nodelms' ");
+		return (-1);
 	}
 
 	if (qdevice_net_algo_lms_register() != 0) {
-		qdevice_net_log(LOG_CRIT, "Failed to register decision algorithm 'lms' ");
-		exit(1);
+		qdevice_log(LOG_CRIT, "Failed to register decision algorithm 'lms' ");
+		return (-1);
 	}
+
+	return (0);
 }

qdevices/qdevice-net-algorithm.h

@@ -38,6 +38,7 @@
 #include <sys/types.h>
 #include <inttypes.h>
 
+#include "node-list.h"
 #include "qdevice-net-instance.h"
 
 #ifdef __cplusplus
@@ -48,12 +49,17 @@ extern int	qdevice_net_algorithm_init(struct qdevice_net_instance *instance);
 
 extern int	qdevice_net_algorithm_connected(struct qdevice_net_instance *instance);
 
+extern int	qdevice_net_algorithm_config_node_list_changed(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int *send_node_list, enum tlv_vote *vote);
+
 extern int	qdevice_net_algorithm_votequorum_node_list_notify(struct qdevice_net_instance *instance,
     const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-    enum tlv_vote *vote);
+    int *send_node_list, enum tlv_vote *vote);
 
 extern int	qdevice_net_algorithm_votequorum_quorum_notify(struct qdevice_net_instance *instance,
-    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list);
+    uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+    enum tlv_vote *vote);
 
 extern int	qdevice_net_algorithm_config_node_list_reply_received(struct qdevice_net_instance *instance,
     uint32_t seq_number, int initial, enum tlv_vote *vote);
@@ -83,11 +89,15 @@ extern void	qdevice_net_algorithm_destroy(struct qdevice_net_instance *instance)
 struct qdevice_net_algorithm {
 	int (*init)(struct qdevice_net_instance *instance);
 	int (*connected)(struct qdevice_net_instance *instance);
+	int (*config_node_list_changed)(struct qdevice_net_instance *instance,
+	    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+	    int *send_node_list, enum tlv_vote *vote);
 	int (*votequorum_node_list_notify)(struct qdevice_net_instance *instance,
 	    const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[],
-	    enum tlv_vote *vote);
+	    int *send_node_list, enum tlv_vote *vote);
 	int (*votequorum_quorum_notify)(struct qdevice_net_instance *instance, uint32_t quorate,
-	    uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list);
+	    uint32_t node_list_entries, votequorum_node_t node_list[], int *send_node_list,
+	    enum tlv_vote *vote);
 	int (*config_node_list_reply_received)(struct qdevice_net_instance *instance,
 	    uint32_t seq_number, int initial, enum tlv_vote *vote);
 	int (*membership_node_list_reply_received)(struct qdevice_net_instance *instance,
@@ -109,7 +119,7 @@ struct qdevice_net_algorithm {
 extern int		qdevice_net_algorithm_register(
 	enum tlv_decision_algorithm_type algorithm_number, struct qdevice_net_algorithm *algorithm);
 
-extern void qdevice_net_algorithm_register_all(void);
+extern int		qdevice_net_algorithm_register_all(void);
 
 #ifdef __cplusplus
 }

qdevices/qdevice-net-cast-vote-timer.c

@@ -32,17 +32,15 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <err.h>
-
 #include "qnet-config.h"
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
 #include "qdevice-net-cast-vote-timer.h"
+#include "qdevice-votequorum.h"
 
 static int
 qdevice_net_cast_vote_timer_callback(void *data1, void *data2)
 {
 	struct qdevice_net_instance *instance;
-	cs_error_t res;
 	int cast_vote;
 
 	instance = (struct qdevice_net_instance *)data1;
@@ -58,27 +56,17 @@ qdevice_net_cast_vote_timer_callback(void *data1, void *data2)
 	case TLV_VOTE_WAIT_FOR_REPLY:
 	case TLV_VOTE_NO_CHANGE:
 	default:
-		errx(1, "qdevice_net_timer_cast_vote: Unhandled cast_vote_timer_vote %u\n",
+		qdevice_log(LOG_CRIT, "qdevice_net_timer_cast_vote: Unhandled cast_vote_timer_vote %u\n",
 		    instance->cast_vote_timer_vote);
+		exit(1);
 		break;
 	}
 
-	res = votequorum_qdevice_poll(instance->votequorum_handle,
-	    QDEVICE_NET_VOTEQUORUM_DEVICE_NAME,	cast_vote,
-	    instance->last_received_votequorum_ring_id);
-
-	if (res != CS_OK && res != CS_ERR_TRY_AGAIN) {
-		if (res == CS_ERR_MESSAGE_ERROR) {
-			qdevice_net_log(LOG_INFO, "votequorum_qdevice_poll called with old ring id,"
-			    " rescheduling timer");
-		} else {
-			qdevice_net_log(LOG_CRIT, "Can't call votequorum_qdevice_poll. Error %u",
-			    res);
-
-			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
-			instance->schedule_disconnect = 1;
-			return (0);
-		}
+	if (qdevice_votequorum_poll(instance->qdevice_instance_ptr, cast_vote) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+		instance->schedule_disconnect = 1;
+		instance->cast_vote_timer = NULL;
+		return (0);
 	}
 
 	/*
@@ -106,8 +94,9 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t
 
 		break;
 	default:
-		errx(1, "qdevice_net_cast_vote_timer_update_vote: Unhandled vote parameter %u\n",
+		qdevice_log(LOG_CRIT, "qdevice_net_cast_vote_timer_update_vote: Unhandled vote parameter %u\n",
 		    vote);
+		exit(1);
 		break;
 	}
 
@@ -120,12 +109,12 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t
 			    qdevice_net_cast_vote_timer_callback, (void *)instance, NULL);
 
 			if (instance->cast_vote_timer == NULL) {
-				qdevice_net_log(LOG_ERR, "Can't schedule sending of "
+				qdevice_log(LOG_ERR, "Can't schedule sending of "
 				    "votequorum poll");
 
 				return (-1);
 			} else {
-				qdevice_net_log(LOG_DEBUG, "Cast vote timer is now scheduled every "
+				qdevice_log(LOG_DEBUG, "Cast vote timer is now scheduled every "
 				    "%"PRIu32"ms.", instance->cast_vote_timer_interval);
 			}
 		}
@@ -137,9 +126,9 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t
 		if (instance->cast_vote_timer != NULL) {
 			timer_list_delete(&instance->main_timer_list, instance->cast_vote_timer);
 			instance->cast_vote_timer = NULL;
-			qdevice_net_log(LOG_DEBUG, "Cast vote timer is now stopped.");
+			qdevice_log(LOG_DEBUG, "Cast vote timer is now stopped.");
 		} else {
-			qdevice_net_log(LOG_DEBUG, "Cast vote timer remains stopped.");
+			qdevice_log(LOG_DEBUG, "Cast vote timer remains stopped.");
 		}
 	}
 

qdevices/qdevice-net-disconnect-reason.h

@@ -85,6 +85,9 @@ enum qdevice_net_disconnect_reason {
 	/* Can't dispatch cmap or votequroum. This cannot be overwritten and always means end of qdevice-net */
 	QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED,
 
+	/* It was not possible to establish connection with qnetd */
+	QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER,
+
 	QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR,
 	QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTEQUORUM_QUORUM_NOTIFY_ERR,
 	QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTEQUORUM_NODE_LIST_NOTIFY_ERR,
@@ -95,11 +98,13 @@ enum qdevice_net_disconnect_reason {
 	QDEVICE_NET_DISCONNECT_REASON_ALGO_ECHO_REPLY_NOT_RECEIVED_ERR,
 };
 
-#define qdevice_net_disconnect_reason_try_reconnect(reason)	(	\
-    reason == QDEVICE_NET_DISCONNECT_REASON_MSG_DECODE_ERROR ||		\
-    reason == QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION ||	\
-    reason == QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE ||	\
-    reason == QDEVICE_NET_DISCONNECT_REASON_CANT_SEND_MESSAGE)
+#define qdevice_net_disconnect_reason_try_reconnect(reason)	(		\
+    reason == QDEVICE_NET_DISCONNECT_REASON_MSG_DECODE_ERROR ||			\
+    reason == QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION ||		\
+    reason == QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE ||		\
+    reason == QDEVICE_NET_DISCONNECT_REASON_CANT_SEND_MESSAGE ||		\
+    reason == QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER ||	\
+    reason == QDEVICE_NET_DISCONNECT_REASON_ALGO_ECHO_REPLY_NOT_RECEIVED_ERR)
 
 #ifdef __cplusplus
 }

qdevices/qdevice-net-instance.c

@@ -32,17 +32,26 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qdevice-log.h"
 #include "qdevice-net-instance.h"
+#include "qnet-config.h"
+#include "utils.h"
+
+/*
+ * Needed for creating nspr handle from unix fd
+ */
+#include <private/pprio.h>
 
 int
 qdevice_net_instance_init(struct qdevice_net_instance *instance, size_t initial_receive_size,
     size_t initial_send_size, size_t min_send_size, size_t max_send_buffers,
     size_t max_receive_size,
-    enum tlv_tls_supported tls_supported, uint32_t node_id,
+    enum tlv_tls_supported tls_supported,
     enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
     uint32_t sync_heartbeat_interval, uint32_t cast_vote_timer_interval,
     const char *host_addr, uint16_t host_port, const char *cluster_name,
-    const struct tlv_tie_breaker *tie_breaker, uint32_t connect_timeout)
+    const struct tlv_tie_breaker *tie_breaker, uint32_t connect_timeout,
+    int cmap_fd, int votequorum_fd)
 {
 
 	memset(instance, 0, sizeof(*instance));
@@ -51,7 +60,6 @@ qdevice_net_instance_init(struct qdevice_net_instance *instance, size_t initial_
 	instance->initial_send_size = initial_send_size;
 	instance->min_send_size = min_send_size;
 	instance->max_receive_size = max_receive_size;
-	instance->node_id = node_id;
 	instance->decision_algorithm = decision_algorithm;
 	instance->heartbeat_interval = heartbeat_interval;
 	instance->sync_heartbeat_interval = sync_heartbeat_interval;
@@ -73,10 +81,18 @@ qdevice_net_instance_init(struct qdevice_net_instance *instance, size_t initial_
 
 	timer_list_init(&instance->main_timer_list);
 
-	node_list_init(&instance->last_sent_config_node_list);
-
 	instance->tls_supported = tls_supported;
 
+	if ((instance->cmap_poll_fd = PR_CreateSocketPollFd(cmap_fd)) == NULL) {
+		qdevice_log_nss(LOG_CRIT, "Can't create NSPR cmap poll fd");
+		return (-1);
+	}
+
+	if ((instance->votequorum_poll_fd = PR_CreateSocketPollFd(votequorum_fd)) == NULL) {
+		qdevice_log_nss(LOG_CRIT, "Can't create NSPR votequorum poll fd");
+		return (-1);
+	}
+
 	return (0);
 }
 
@@ -94,15 +110,8 @@ qdevice_net_instance_clean(struct qdevice_net_instance *instance)
 	instance->echo_request_expected_msg_seq_num = instance->echo_reply_received_msg_seq_num;
 	instance->using_tls = 0;
 
-	node_list_free(&instance->last_sent_config_node_list);
-
-	timer_list_free(&instance->main_timer_list);
-	instance->cast_vote_timer = NULL;
-	instance->echo_request_timer = NULL;
-
 	instance->schedule_disconnect = 0;
 	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNDEFINED;
-	instance->cmap_reload_in_progress = 0;
 }
 
 int
@@ -113,12 +122,195 @@ qdevice_net_instance_destroy(struct qdevice_net_instance *instance)
 
 	send_buffer_list_free(&instance->send_buffer_list);
 
-	node_list_free(&instance->last_sent_config_node_list);
-
 	timer_list_free(&instance->main_timer_list);
 
 	free((void *)instance->cluster_name);
 	free((void *)instance->host_addr);
 
+	if (PR_DestroySocketPollFd(instance->votequorum_poll_fd) != PR_SUCCESS) {
+		qdevice_log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
+	}
+
+	if (PR_DestroySocketPollFd(instance->cmap_poll_fd) != PR_SUCCESS) {
+		qdevice_log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
+	}
+
+	return (0);
+}
+
+int
+qdevice_net_instance_init_from_cmap(struct qdevice_instance *instance)
+{
+	char *str;
+	cmap_handle_t cmap_handle;
+	enum tlv_tls_supported tls_supported;
+	int i;
+	long int li;
+	enum tlv_decision_algorithm_type decision_algorithm;
+	struct tlv_tie_breaker tie_breaker;
+	uint32_t heartbeat_interval;
+	uint32_t sync_heartbeat_interval;
+	uint32_t cast_vote_timer_interval;
+	char *host_addr;
+	int host_port;
+	char *ep;
+	char *cluster_name;
+	uint32_t connect_timeout;
+	struct qdevice_net_instance *net_instance;
+
+	cmap_handle = instance->cmap_handle;
+
+	net_instance = malloc(sizeof(*net_instance));
+	if (net_instance == NULL) {
+		qdevice_log(LOG_ERR, "Can't alloc qdevice_net_instance");
+		return (-1);
+	}
+
+	/*
+	 * Check tls
+	 */
+	if (cmap_get_string(cmap_handle, "quorum.device.net.tls", &str) == CS_OK) {
+		if ((i = utils_parse_bool_str(str)) == -1) {
+			free(str);
+			qdevice_log(LOG_ERR, "quorum.device.net.tls value is not valid.");
+			return (-1);
+		}
+
+		if (i == 1) {
+			tls_supported = TLV_TLS_SUPPORTED;
+		} else {
+			tls_supported = TLV_TLS_UNSUPPORTED;
+		}
+
+		free(str);
+	}
+
+	/*
+	 * Host
+	 */
+	if (cmap_get_string(cmap_handle, "quorum.device.net.host", &str) != CS_OK) {
+		qdevice_log(LOG_ERR, "Qdevice net daemon address is not defined (quorum.device.net.host)");
+		return (-1);
+	}
+	host_addr = str;
+
+	if (cmap_get_string(cmap_handle, "quorum.device.net.port", &str) == CS_OK) {
+		host_port = strtol(str, &ep, 10);
+
+
+		if (host_port <= 0 || host_port > ((uint16_t)~0) || *ep != '\0') {
+			qdevice_log(LOG_ERR, "quorum.device.net.port must be in range 0-65535");
+			return (-1);
+		}
+
+		free(str);
+	} else {
+		host_port = QNETD_DEFAULT_HOST_PORT;
+	}
+
+	/*
+	 * Cluster name
+	 */
+	if (cmap_get_string(cmap_handle, "totem.cluster_name", &str) != CS_OK) {
+		qdevice_log(LOG_ERR, "Cluster name (totem.cluster_name) has to be defined.");
+		return (-1);
+	}
+	cluster_name = str;
+
+	/*
+	 * Adjust qdevice timeouts to better suit qnetd
+	 */
+	cast_vote_timer_interval = instance->heartbeat_interval * 0.5;
+	heartbeat_interval = instance->heartbeat_interval * 0.8;
+
+	sync_heartbeat_interval = instance->sync_heartbeat_interval * 0.8;
+
+	/*
+	 * Choose decision algorithm
+	 */
+	if (cmap_get_string(cmap_handle, "quorum.device.net.algorithm", &str) != CS_OK) {
+		decision_algorithm = QDEVICE_NET_DEFAULT_ALGORITHM;
+	} else {
+		if (strcmp(str, "test") == 0) {
+			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_TEST;
+		} else if (strcmp(str, "ffsplit") == 0) {
+			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_FFSPLIT;
+		} else if (strcmp(str, "2nodelms") == 0) {
+			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_2NODELMS;
+		} else if (strcmp(str, "lms") == 0) {
+			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_LMS;
+		} else {
+			qdevice_log(LOG_ERR, "Unknown decision algorithm %s", str);
+			free(str);
+			return (-1);
+		}
+
+		free(str);
+	}
+
+	/*
+	 * Load tie_breaker mode
+	 */
+	memset(&tie_breaker, 0, sizeof(tie_breaker));
+
+	if (cmap_get_string(cmap_handle, "quorum.device.net.tie_breaker", &str) != CS_OK) {
+		tie_breaker.mode = QDEVICE_NET_DEFAULT_TIE_BREAKER_MODE;
+	} else {
+		if (strcmp(str, "lowest") == 0) {
+			tie_breaker.mode = TLV_TIE_BREAKER_MODE_LOWEST;
+		} else if (strcmp(str, "highest") == 0) {
+			tie_breaker.mode = TLV_TIE_BREAKER_MODE_HIGHEST;
+		} else {
+			li = strtol(str, &ep, 10);
+			if (li <= 0 || li > ((uint32_t)~0) || *ep != '\0') {
+				qdevice_log(LOG_ERR, "tie_breaker must be lowest|highest|valid_node_id");
+				free(str);
+				return (-1);
+			}
+
+			tie_breaker.mode = TLV_TIE_BREAKER_MODE_NODE_ID;
+			tie_breaker.node_id = li;
+		}
+
+		free(str);
+	}
+
+	/*
+	 * Get connect timeout
+	 */
+	if (cmap_get_string(cmap_handle, "quorum.device.net.connect_timeout", &str) != CS_OK) {
+		connect_timeout = heartbeat_interval;
+	} else {
+		li = strtol(str, &ep, 10);
+		if (li < QDEVICE_NET_MIN_CONNECT_TIMEOUT || li > QDEVICE_NET_MAX_CONNECT_TIMEOUT || *ep != '\0') {
+			qdevice_log(LOG_ERR, "connect_timeout must be valid number in range <%lu,%lu>",
+			    QDEVICE_NET_MIN_CONNECT_TIMEOUT, QDEVICE_NET_MAX_CONNECT_TIMEOUT);
+			free(str);
+			return (-1);
+		}
+
+		connect_timeout = li;
+
+		free(str);
+	}
+
+	/*
+	 * Really initialize instance
+	 */
+	if (qdevice_net_instance_init(net_instance,
+	    QDEVICE_NET_INITIAL_MSG_RECEIVE_SIZE, QDEVICE_NET_INITIAL_MSG_SEND_SIZE,
+	    QDEVICE_NET_MIN_MSG_SEND_SIZE, QDEVICE_NET_MAX_SEND_BUFFERS,
+	    QDEVICE_NET_MAX_MSG_RECEIVE_SIZE,
+	    tls_supported, decision_algorithm,
+	    heartbeat_interval, sync_heartbeat_interval, cast_vote_timer_interval,
+	    host_addr, host_port, cluster_name, &tie_breaker, connect_timeout,
+	    instance->cmap_poll_fd, instance->votequorum_poll_fd) == -1) {
+		qdevice_log(LOG_ERR, "Can't initialize qdevice-net instance");
+		return (-1);
+	}
+
+	net_instance->qdevice_instance_ptr = instance;
+	instance->model_data = net_instance;
+
 	return (0);
 }

qdevices/qdevice-net-instance.h

@@ -40,10 +40,9 @@
 #include <stdlib.h>
 #include <stdint.h>
 
-#include <prio.h>
+#include "nss-sock.h"
 
-#include <cmap.h>
-#include <votequorum.h>
+#include "qdevice-instance.h"
 
 #include "dynar.h"
 #include "node-list.h"
@@ -57,6 +56,7 @@ extern "C" {
 #endif
 
 enum qdevice_net_instance_state {
+	QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT,
 	QDEVICE_NET_INSTANCE_STATE_WAITING_PREINIT_REPLY,
 	QDEVICE_NET_INSTANCE_STATE_WAITING_STARTTLS_BEING_SENT,
 	QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY,
@@ -80,9 +80,8 @@ struct qdevice_net_instance {
 	uint32_t echo_reply_received_msg_seq_num;
 	enum tlv_tls_supported tls_supported;
 	int using_tls;
-	uint32_t node_id;
-	uint32_t heartbeat_interval;		/* Heartbeat interval during normal operation */
-	uint32_t sync_heartbeat_interval;	/* Heartbeat interval during corosync sync */
+	uint32_t heartbeat_interval;            /* Adjusted heartbeat interval during normal operation */
+	uint32_t sync_heartbeat_interval;       /* Adjusted heartbeat interval during corosync sync */
 	uint32_t cast_vote_timer_interval;	/* Timer for cast vote */
 	uint32_t connect_timeout;
 	struct timer_list_entry *cast_vote_timer;
@@ -94,34 +93,33 @@ struct qdevice_net_instance {
 	struct timer_list main_timer_list;
 	struct timer_list_entry *echo_request_timer;
 	int schedule_disconnect;
-	cmap_handle_t cmap_handle;
-	votequorum_handle_t votequorum_handle;
 	PRFileDesc *votequorum_poll_fd;
 	PRFileDesc *cmap_poll_fd;
-	votequorum_ring_id_t last_received_votequorum_ring_id;
 	struct tlv_ring_id last_sent_ring_id;
 	struct tlv_tie_breaker tie_breaker;
 	void *algorithm_data;
-	cmap_track_handle_t cmap_reload_track_handle;
-	cmap_track_handle_t cmap_nodelist_track_handle;
-	int cmap_reload_in_progress;
-	struct node_list last_sent_config_node_list;
 	enum qdevice_net_disconnect_reason disconnect_reason;
+	struct qdevice_instance *qdevice_instance_ptr;
+	struct nss_sock_non_blocking_client non_blocking_client;
+	struct timer_list_entry *connect_timer;
 };
 
 extern int		qdevice_net_instance_init(struct qdevice_net_instance *instance,
     size_t initial_receive_size, size_t initial_send_size, size_t min_send_size,
     size_t max_send_buffers, size_t max_receive_size,
-    enum tlv_tls_supported tls_supported, uint32_t node_id,
+    enum tlv_tls_supported tls_supported,
     enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
     uint32_t sync_heartbeat_interval, uint32_t cast_vote_timer_interval,
     const char *host_addr, uint16_t host_port, const char *cluster_name,
-    const struct tlv_tie_breaker *tie_breaker, uint32_t connect_timeout);
+    const struct tlv_tie_breaker *tie_breaker, uint32_t connect_timeout,
+    int cmap_fd, int votequorum_fd);
 
 extern void		qdevice_net_instance_clean(struct qdevice_net_instance *instance);
 
 extern int		qdevice_net_instance_destroy(struct qdevice_net_instance *instance);
 
+extern int		qdevice_net_instance_init_from_cmap(struct qdevice_instance *instance);
+
 #ifdef __cplusplus
 }
 #endif

qdevices/qdevice-net-msg-received.c

@@ -31,3 +31,889 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
+
+#include "qdevice-log.h"
+#include "qdevice-net-algorithm.h"
+#include "qdevice-net-cast-vote-timer.h"
+#include "qdevice-net-msg-received.h"
+#include "qdevice-net-send.h"
+#include "qdevice-net-votequorum.h"
+#include "msg.h"
+
+/*
+ * -1 - Incompatible tls combination
+ *  0 - Don't use TLS
+ *  1 - Use TLS
+ */
+static int
+qdevice_net_msg_received_check_tls_compatibility(enum tlv_tls_supported server_tls,
+    enum tlv_tls_supported client_tls)
+{
+	int res;
+
+	res = -1;
+
+	switch (server_tls) {
+	case TLV_TLS_UNSUPPORTED:
+		switch (client_tls) {
+		case TLV_TLS_UNSUPPORTED: res = 0; break;
+		case TLV_TLS_SUPPORTED: res = 0; break;
+		case TLV_TLS_REQUIRED: res = -1; break;
+		}
+		break;
+	case TLV_TLS_SUPPORTED:
+		switch (client_tls) {
+		case TLV_TLS_UNSUPPORTED: res = 0; break;
+		case TLV_TLS_SUPPORTED: res = 1; break;
+		case TLV_TLS_REQUIRED: res = 1; break;
+		}
+		break;
+	case TLV_TLS_REQUIRED:
+		switch (client_tls) {
+		case TLV_TLS_UNSUPPORTED: res = -1; break;
+		case TLV_TLS_SUPPORTED: res = 1; break;
+		case TLV_TLS_REQUIRED: res = 1; break;
+		}
+		break;
+	}
+
+	return (res);
+}
+
+static void
+qdevice_net_msg_received_log_msg_decode_error(int ret)
+{
+
+	switch (ret) {
+	case -1:
+		qdevice_log(LOG_WARNING, "Received message with option with invalid length");
+		break;
+	case -2:
+		qdevice_log(LOG_CRIT, "Can't allocate memory");
+		break;
+	case -3:
+		qdevice_log(LOG_WARNING, "Received inconsistent msg (tlv len > msg size)");
+		break;
+	case -4:
+		qdevice_log(LOG_ERR, "Received message with option with invalid value");
+		break;
+	default:
+		qdevice_log(LOG_ERR, "Unknown error occured when decoding message");
+		break;
+	}
+}
+
+static int
+qdevice_net_msg_received_unexpected_msg(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg, const char *msg_str)
+{
+
+	qdevice_log(LOG_ERR, "Received unexpected %s message. Disconnecting from server",
+	    msg_str);
+
+	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+
+	return (-1);
+}
+
+static int
+qdevice_net_msg_received_preinit(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "preinit"));
+}
+
+static int
+qdevice_net_msg_check_seq_number(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	if (!msg->seq_number_set || msg->seq_number != instance->last_msg_seq_num) {
+		qdevice_log(LOG_ERR, "Received message doesn't contain seq_number or "
+		    "it's not expected one.");
+
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_preinit_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+	int res;
+	struct send_buffer_list_entry *send_buffer;
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_PREINIT_REPLY) {
+		qdevice_log(LOG_ERR, "Received unexpected preinit reply message. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+
+		return (-1);
+	}
+
+	if (qdevice_net_msg_check_seq_number(instance, msg) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	/*
+	 * Check TLS support
+	 */
+	if (!msg->tls_supported_set || !msg->tls_client_cert_required_set) {
+		qdevice_log(LOG_ERR, "Required tls_supported or tls_client_cert_required "
+		    "option is unset");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	res = qdevice_net_msg_received_check_tls_compatibility(msg->tls_supported, instance->tls_supported);
+	if (res == -1) {
+		qdevice_log(LOG_ERR, "Incompatible tls configuration (server %u client %u)",
+		    msg->tls_supported, instance->tls_supported);
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_INCOMPATIBLE_TLS;
+
+		return (-1);
+	} else if (res == 1) {
+		/*
+		 * Start TLS
+		 */
+		send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
+		if (send_buffer == NULL) {
+			qdevice_log(LOG_ERR, "Can't allocate send list buffer for "
+			    "starttls msg");
+
+			instance->disconnect_reason =
+			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+			return (-1);
+		}
+
+		instance->last_msg_seq_num++;
+		if (msg_create_starttls(&send_buffer->buffer, 1,
+		    instance->last_msg_seq_num) == 0) {
+			qdevice_log(LOG_ERR, "Can't allocate send buffer for starttls msg");
+
+			instance->disconnect_reason =
+			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+			send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
+			return (-1);
+		}
+
+		send_buffer_list_put(&instance->send_buffer_list, send_buffer);
+
+		instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_STARTTLS_BEING_SENT;
+	} else if (res == 0) {
+		if (qdevice_net_send_init(instance) != 0) {
+			instance->disconnect_reason =
+			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+			return (-1);
+		}
+	}
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+	size_t zi;
+	int res;
+	struct send_buffer_list_entry *send_buffer;
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY) {
+		qdevice_log(LOG_ERR, "Received unexpected init reply message. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+
+		return (-1);
+	}
+
+	if (qdevice_net_msg_check_seq_number(instance, msg) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (!msg->reply_error_code_set) {
+		qdevice_log(LOG_ERR, "Received init reply message without error code."
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (msg->reply_error_code != TLV_REPLY_ERROR_CODE_NO_ERROR) {
+		qdevice_log(LOG_ERR, "Received init reply message with error code %"PRIu16". "
+		    "Disconnecting from server", msg->reply_error_code);
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_SENT_ERROR;
+		return (-1);
+	}
+
+	if (!msg->server_maximum_request_size_set || !msg->server_maximum_reply_size_set) {
+		qdevice_log(LOG_ERR, "Required maximum_request_size or maximum_reply_size "
+		    "option is unset");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (msg->supported_messages == NULL || msg->supported_options == NULL) {
+		qdevice_log(LOG_ERR, "Required supported messages or supported options "
+		    "option is unset");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (msg->supported_decision_algorithms == NULL) {
+		qdevice_log(LOG_ERR, "Required supported decision algorithms option is unset");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (msg->server_maximum_request_size < instance->min_send_size) {
+		qdevice_log(LOG_ERR,
+		    "Server accepts maximum %zu bytes message but this client minimum "
+		    "is %zu bytes.", msg->server_maximum_request_size, instance->min_send_size);
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_INCOMPATIBLE_MSG_SIZE;
+		return (-1);
+	}
+
+	if (msg->server_maximum_reply_size > instance->max_receive_size) {
+		qdevice_log(LOG_ERR,
+		    "Server may send message up to %zu bytes message but this client maximum "
+		    "is %zu bytes.", msg->server_maximum_reply_size, instance->max_receive_size);
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_INCOMPATIBLE_MSG_SIZE;
+		return (-1);
+	}
+
+	/*
+	 * Change buffer sizes
+	 */
+	dynar_set_max_size(&instance->receive_buffer, msg->server_maximum_reply_size);
+	send_buffer_list_set_max_buffer_size(&instance->send_buffer_list,
+	    msg->server_maximum_request_size);
+
+
+	/*
+	 * Check if server supports decision algorithm we need
+	 */
+	res = 0;
+
+	for (zi = 0; zi < msg->no_supported_decision_algorithms && !res; zi++) {
+		if (msg->supported_decision_algorithms[zi] == instance->decision_algorithm) {
+			res = 1;
+		}
+	}
+
+	if (!res) {
+		qdevice_log(LOG_ERR, "Server doesn't support required decision algorithm");
+
+		instance->disconnect_reason =
+		    QDEVICE_NET_DISCONNECT_REASON_SERVER_DOESNT_SUPPORT_REQUIRED_ALGORITHM;
+
+		return (-1);
+	}
+
+	/*
+	 * Send set options message
+	 */
+	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
+	if (send_buffer == NULL) {
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for set option msg");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+		return (-1);
+	}
+
+	instance->last_msg_seq_num++;
+
+	if (msg_create_set_option(&send_buffer->buffer, 1, instance->last_msg_seq_num,
+	    1, instance->heartbeat_interval, 1, &instance->tie_breaker) == 0) {
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for set option msg");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+		return (-1);
+	}
+
+	send_buffer_list_put(&instance->send_buffer_list, send_buffer);
+
+	instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_SET_OPTION_REPLY;
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_starttls(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "starttls"));
+}
+
+static int
+qdevice_net_msg_received_server_error(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	if (!msg->reply_error_code_set) {
+		qdevice_log(LOG_ERR, "Received server error without error code set. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+	} else {
+		qdevice_log(LOG_ERR, "Received server error %"PRIu16". "
+		    "Disconnecting from server", msg->reply_error_code);
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_SENT_ERROR;
+	}
+
+	return (-1);
+}
+
+static int
+qdevice_net_msg_received_set_option(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "set option"));
+}
+
+static int
+qdevice_net_timer_send_heartbeat(void *data1, void *data2)
+{
+	struct qdevice_net_instance *instance;
+
+	instance = (struct qdevice_net_instance *)data1;
+
+	if (instance->echo_reply_received_msg_seq_num !=
+	    instance->echo_request_expected_msg_seq_num) {
+		qdevice_log(LOG_ERR, "Server didn't send echo reply message on time");
+
+		if (qdevice_net_algorithm_echo_reply_not_received(instance) != 0) {
+			qdevice_log(LOG_DEBUG, "Algorithm decided to disconnect");
+
+			instance->schedule_disconnect = 1;
+			instance->disconnect_reason =
+			    QDEVICE_NET_DISCONNECT_REASON_ALGO_ECHO_REPLY_NOT_RECEIVED_ERR;
+
+			instance->echo_request_timer = NULL;
+			return (0);
+		} else {
+			qdevice_log(LOG_DEBUG, "Algorithm decided to continue send heartbeat");
+
+			return (-1);
+		}
+	}
+
+	if (qdevice_net_send_echo_request(instance) == -1) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+		instance->schedule_disconnect = 1;
+		instance->echo_request_timer = NULL;
+		return (0);
+	}
+
+	/*
+	 * Schedule this function callback again
+	 */
+	return (-1);
+}
+
+static int
+qdevice_net_msg_received_set_option_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+	struct tlv_ring_id tlv_rid;
+	enum tlv_quorate quorate;
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_SET_OPTION_REPLY) {
+		qdevice_log(LOG_ERR, "Received unexpected set option reply message. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+
+		return (-1);
+	}
+
+	if (qdevice_net_msg_check_seq_number(instance, msg) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (!msg->decision_algorithm_set || !msg->heartbeat_interval_set) {
+		qdevice_log(LOG_ERR, "Received set option reply message without "
+		    "required options. Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	if (msg->decision_algorithm != instance->decision_algorithm ||
+	    msg->heartbeat_interval != instance->heartbeat_interval) {
+		qdevice_log(LOG_ERR, "Server doesn't accept sent decision algorithm or "
+		    "heartbeat interval.");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+
+		return (-1);
+	}
+
+	/*
+	 * Finally fully connected so it's possible to remove connection timer
+	 */
+	if (instance->connect_timer != NULL) {
+		timer_list_delete(&instance->main_timer_list, instance->connect_timer);
+		instance->connect_timer = NULL;
+	}
+
+	/*
+	 * Server accepted heartbeat interval -> schedule regular sending of echo request
+	 */
+	if (instance->heartbeat_interval > 0) {
+		instance->echo_request_expected_msg_seq_num = 0;
+		instance->echo_reply_received_msg_seq_num = 0;
+
+		instance->echo_request_timer = timer_list_add(&instance->main_timer_list,
+		    instance->heartbeat_interval, qdevice_net_timer_send_heartbeat,
+		    (void *)instance, NULL);
+
+		if (instance->echo_request_timer == NULL) {
+			qdevice_log(LOG_ERR, "Can't schedule regular sending of heartbeat.");
+
+			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_HB_TIMER;
+
+			return (-1);
+		}
+	}
+
+	if (qdevice_net_algorithm_connected(instance) != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR;
+		return (-1);
+	}
+
+	/*
+	 * Now we can finally really send node list, votequorum node list and update timer
+	 */
+	if (qdevice_net_send_config_node_list(instance,
+	    &instance->qdevice_instance_ptr->config_node_list,
+	    instance->qdevice_instance_ptr->config_node_list_version_set,
+	    instance->qdevice_instance_ptr->config_node_list_version, 1) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+		return (-1);
+	}
+
+	qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid,
+	    &instance->qdevice_instance_ptr->vq_node_list_ring_id);
+
+	if (qdevice_net_send_membership_node_list(instance, &tlv_rid,
+	    instance->qdevice_instance_ptr->vq_node_list_entries,
+	    instance->qdevice_instance_ptr->vq_node_list) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+		return (-1);
+	}
+
+	quorate = (instance->qdevice_instance_ptr->vq_quorum_quorate ?
+	    TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE);
+
+	if (qdevice_net_send_quorum_node_list(instance,
+	    quorate,
+	    instance->qdevice_instance_ptr->vq_quorum_node_list_entries,
+	    instance->qdevice_instance_ptr->vq_quorum_node_list) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+		return (-1);
+	}
+
+	if (qdevice_net_cast_vote_timer_update(instance, TLV_VOTE_WAIT_FOR_REPLY) != 0) {
+		qdevice_log(LOG_CRIT, "qdevice_net_msg_received_set_option_reply fatal error. "
+		    " Can't update cast vote timer vote");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+	}
+
+	instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_echo_request(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "echo request"));
+}
+
+static int
+qdevice_net_msg_received_echo_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	if (!msg->seq_number_set) {
+		qdevice_log(LOG_ERR, "Received echo reply message doesn't contain seq_number.");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+		return (-1);
+	}
+
+	if (msg->seq_number != instance->echo_request_expected_msg_seq_num) {
+		qdevice_log(LOG_WARNING, "Received echo reply message seq_number is not expected one.");
+	}
+
+	if (qdevice_net_algorithm_echo_reply_received(instance, msg->seq_number,
+	    msg->seq_number == instance->echo_request_expected_msg_seq_num) != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_ECHO_REPLY_RECEIVED_ERR;
+		return (-1);
+	}
+
+	instance->echo_reply_received_msg_seq_num = msg->seq_number;
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_node_list(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "node list"));
+}
+
+static int
+qdevice_net_msg_received_node_list_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+	const char *str;
+	enum tlv_vote result_vote;
+	int res;
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
+		qdevice_log(LOG_ERR, "Received unexpected node list reply message. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+		return (-1);
+	}
+
+	if (!msg->vote_set || !msg->seq_number_set || !msg->node_list_type_set) {
+		qdevice_log(LOG_ERR, "Received node list reply message without "
+		    "required options. Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+		return (-1);
+	}
+
+	if (msg->node_list_type == TLV_NODE_LIST_TYPE_MEMBERSHIP && !msg->ring_id_set) {
+		qdevice_log(LOG_ERR, "Received node list reply message with type membership "
+		    "without ring id set. Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+		return (-1);
+	}
+
+	switch (msg->node_list_type) {
+	case TLV_NODE_LIST_TYPE_INITIAL_CONFIG: str = "initial config"; break;
+	case TLV_NODE_LIST_TYPE_CHANGED_CONFIG: str = "changed config"; break;
+	case TLV_NODE_LIST_TYPE_MEMBERSHIP: str ="membership"; break;
+	case TLV_NODE_LIST_TYPE_QUORUM: str ="quorum"; break;
+	default:
+		qdevice_log(LOG_CRIT, "qdevice_net_msg_received_node_list_reply fatal error. "
+		    "Unhandled node_list_type (debug output)");
+		exit(1);
+		break;
+	}
+	qdevice_log(LOG_DEBUG, "Received %s node list reply", str);
+	qdevice_log(LOG_DEBUG, "  seq = %"PRIu32, msg->seq_number);
+	qdevice_log(LOG_DEBUG, "  vote = %s", tlv_vote_to_str(msg->vote));
+	if (msg->ring_id_set) {
+		qdevice_log(LOG_DEBUG, "  ring id = (%"PRIx32".%"PRIx64")",
+		    msg->ring_id.node_id, msg->ring_id.seq);
+	}
+
+	/*
+	 * Call algorithm
+	 */
+	result_vote = msg->vote;
+
+	switch (msg->node_list_type) {
+	case TLV_NODE_LIST_TYPE_INITIAL_CONFIG:
+	case TLV_NODE_LIST_TYPE_CHANGED_CONFIG:
+		res = qdevice_net_algorithm_config_node_list_reply_received(instance,
+		    msg->seq_number, (msg->node_list_type == TLV_NODE_LIST_TYPE_INITIAL_CONFIG),
+		    &result_vote);
+		break;
+	case TLV_NODE_LIST_TYPE_MEMBERSHIP:
+		res = qdevice_net_algorithm_membership_node_list_reply_received(instance,
+		    msg->seq_number, &msg->ring_id, &result_vote);
+		break;
+	case TLV_NODE_LIST_TYPE_QUORUM:
+		res = qdevice_net_algorithm_quorum_node_list_reply_received(instance,
+		    msg->seq_number, &result_vote);
+		break;
+	default:
+		qdevice_log(LOG_CRIT, "qdevice_net_msg_received_node_list_reply fatal error. "
+		    "Unhandled node_list_type (algorithm call)");
+		exit(1);
+		break;
+	}
+
+	if (res != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_NODE_LIST_REPLY_ERR;
+		return (-1);
+	} else {
+		qdevice_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(msg->vote));
+	}
+
+	if (result_vote != TLV_VOTE_NO_CHANGE) {
+		if (msg->node_list_type == TLV_NODE_LIST_TYPE_MEMBERSHIP &&
+		    !tlv_ring_id_eq(&msg->ring_id, &instance->last_sent_ring_id)) {
+			qdevice_log(LOG_INFO, "Received membership node list reply with "
+			    "old ring id. Not updating timer");
+		} else {
+			if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
+				instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+				return (-1);
+			}
+		}
+	}
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_ask_for_vote(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "ask for vote"));
+}
+
+static int
+qdevice_net_msg_received_ask_for_vote_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+	enum tlv_vote result_vote;
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
+		qdevice_log(LOG_ERR, "Received unexpected ask for vote reply message. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+		return (-1);
+	}
+
+	if (!msg->vote_set || !msg->seq_number_set) {
+		qdevice_log(LOG_ERR, "Received node list reply message without "
+		    "required options. Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+		return (-1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Received ask for vote reply");
+	qdevice_log(LOG_DEBUG, "  seq = %"PRIu32, msg->seq_number);
+	qdevice_log(LOG_DEBUG, "  vote = %s", tlv_vote_to_str(msg->vote));
+
+	result_vote = msg->vote;
+
+	if (qdevice_net_algorithm_ask_for_vote_reply_received(instance, msg->seq_number,
+	    &result_vote) != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_ASK_FOR_VOTE_REPLY_ERR;
+		return (-1);
+	} else {
+		qdevice_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(msg->vote));
+	}
+
+	if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_vote_info(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+	struct send_buffer_list_entry *send_buffer;
+	enum tlv_vote result_vote;
+
+	if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
+		qdevice_log(LOG_ERR, "Received unexpected vote info message. "
+		    "Disconnecting from server");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+		return (-1);
+	}
+
+	if (!msg->vote_set || !msg->seq_number_set) {
+		qdevice_log(LOG_ERR, "Received node list reply message without "
+		    "required options. Disconnecting from server");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_REQUIRED_OPTION_MISSING;
+		return (-1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Received vote info");
+	qdevice_log(LOG_DEBUG, "  seq = %"PRIu32, msg->seq_number);
+	qdevice_log(LOG_DEBUG, "  vote = %s", tlv_vote_to_str(msg->vote));
+
+	result_vote = msg->vote;
+	if (qdevice_net_algorithm_vote_info_received(instance, msg->seq_number,
+	    &result_vote) != 0) {
+		qdevice_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTE_INFO_ERR;
+		return (-1);
+	} else {
+		qdevice_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(result_vote));
+	}
+
+	if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
+		return (-1);
+	}
+
+	/*
+	 * Create reply message
+	 */
+	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
+	if (send_buffer == NULL) {
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for "
+		    "vote info reply msg");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+		return (-1);
+	}
+
+	if (msg_create_vote_info_reply(&send_buffer->buffer, msg->seq_number) == 0) {
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for "
+		    "vote info reply list msg");
+
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+qdevice_net_msg_received_vote_info_reply(struct qdevice_net_instance *instance,
+    const struct msg_decoded *msg)
+{
+
+	return (qdevice_net_msg_received_unexpected_msg(instance, msg, "vote info reply"));
+}
+
+int
+qdevice_net_msg_received(struct qdevice_net_instance *instance)
+{
+	struct msg_decoded msg;
+	int res;
+	int ret_val;
+
+	msg_decoded_init(&msg);
+
+	res = msg_decode(&instance->receive_buffer, &msg);
+	if (res != 0) {
+		/*
+		 * Error occurred. Disconnect.
+		 */
+		qdevice_net_msg_received_log_msg_decode_error(res);
+		qdevice_log(LOG_ERR, "Disconnecting from server");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_MSG_DECODE_ERROR;
+
+		return (-1);
+	}
+
+	ret_val = 0;
+
+	switch (msg.type) {
+	case MSG_TYPE_PREINIT:
+		ret_val = qdevice_net_msg_received_preinit(instance, &msg);
+		break;
+	case MSG_TYPE_PREINIT_REPLY:
+		ret_val = qdevice_net_msg_received_preinit_reply(instance, &msg);
+		break;
+	case MSG_TYPE_STARTTLS:
+		ret_val = qdevice_net_msg_received_starttls(instance, &msg);
+		break;
+	case MSG_TYPE_SERVER_ERROR:
+		ret_val = qdevice_net_msg_received_server_error(instance, &msg);
+		break;
+	case MSG_TYPE_INIT_REPLY:
+		ret_val = qdevice_net_msg_received_init_reply(instance, &msg);
+		break;
+	case MSG_TYPE_SET_OPTION:
+		ret_val = qdevice_net_msg_received_set_option(instance, &msg);
+		break;
+	case MSG_TYPE_SET_OPTION_REPLY:
+		ret_val = qdevice_net_msg_received_set_option_reply(instance, &msg);
+		break;
+	case MSG_TYPE_ECHO_REQUEST:
+		ret_val = qdevice_net_msg_received_echo_request(instance, &msg);
+		break;
+	case MSG_TYPE_ECHO_REPLY:
+		ret_val = qdevice_net_msg_received_echo_reply(instance, &msg);
+		break;
+	case MSG_TYPE_NODE_LIST:
+		ret_val = qdevice_net_msg_received_node_list(instance, &msg);
+		break;
+	case MSG_TYPE_NODE_LIST_REPLY:
+		ret_val = qdevice_net_msg_received_node_list_reply(instance, &msg);
+		break;
+	case MSG_TYPE_ASK_FOR_VOTE:
+		ret_val = qdevice_net_msg_received_ask_for_vote(instance, &msg);
+		break;
+	case MSG_TYPE_ASK_FOR_VOTE_REPLY:
+		ret_val = qdevice_net_msg_received_ask_for_vote_reply(instance, &msg);
+		break;
+	case MSG_TYPE_VOTE_INFO:
+		ret_val = qdevice_net_msg_received_vote_info(instance, &msg);
+		break;
+	case MSG_TYPE_VOTE_INFO_REPLY:
+		ret_val = qdevice_net_msg_received_vote_info_reply(instance, &msg);
+		break;
+	default:
+		qdevice_log(LOG_ERR, "Received unsupported message %u. "
+		    "Disconnecting from server", msg.type);
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNEXPECTED_MSG;
+
+		ret_val = -1;
+		break;
+	}
+
+	msg_decoded_destroy(&msg);
+
+	return (ret_val);
+}

qdevices/qdevice-net-msg-received.h

@@ -35,10 +35,14 @@
 #ifndef _QDEVICE_NET_MSG_RECEIVED_H_
 #define _QDEVICE_NET_MSG_RECEIVED_H_
 
+#include "qdevice-net-instance.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+extern int		qdevice_net_msg_received(struct qdevice_net_instance *instance);
+
 
 #ifdef __cplusplus
 }

qdevices/qdevice-net-nss.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sslerr.h>
+#include <secerr.h>
+
+#include "qdevice-log.h"
+#include "qdevice-net-nss.h"
+
+SECStatus
+qdevice_net_nss_bad_cert_hook(void *arg, PRFileDesc *fd) {
+	if (PR_GetError() == SEC_ERROR_EXPIRED_CERTIFICATE ||
+	    PR_GetError() == SEC_ERROR_EXPIRED_ISSUER_CERTIFICATE ||
+	    PR_GetError() == SEC_ERROR_CRL_EXPIRED ||
+	    PR_GetError() == SEC_ERROR_KRL_EXPIRED ||
+	    PR_GetError() == SSL_ERROR_EXPIRED_CERT_ALERT) {
+		qdevice_log(LOG_WARNING, "Server certificate is expired.");
+
+		return (SECSuccess);
+        }
+
+	qdevice_log_nss(LOG_ERR, "Server certificate verification failure.");
+
+	return (SECFailure);
+}
+
+SECStatus
+qdevice_net_nss_get_client_auth_data(void *arg, PRFileDesc *sock, struct CERTDistNamesStr *caNames,
+    struct CERTCertificateStr **pRetCert, struct SECKEYPrivateKeyStr **pRetKey)
+{
+
+	qdevice_log(LOG_DEBUG, "Sending client auth data.");
+
+	return (NSS_GetClientAuthData(arg, sock, caNames, pRetCert, pRetKey));
+}

qdevices/qdevice-net-log.h → qdevices/qdevice-net-nss.h

@@ -32,28 +32,26 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef _QDEVICE_NET_LOG_H_
-#define _QDEVICE_NET_LOG_H_
+#ifndef _QDEVICE_NET_NSS_H_
+#define _QDEVICE_NET_NSS_H_
 
-#include <syslog.h>
-
-#include "qnetd-log.h"
+#include <nspr.h>
+#include <nss.h>
+#include <ssl.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define qdevice_net_log                 qnetd_log
-#define qdevice_net_log_nss             qnetd_log_nss
-#define qdevice_net_log_init            qnetd_log_init
-#define qdevice_net_log_close           qnetd_log_close
-#define qdevice_net_log_set_debug       qnetd_log_set_debug
+extern SECStatus		qdevice_net_nss_bad_cert_hook(void *arg, PRFileDesc *fd);
+
+extern SECStatus		qdevice_net_nss_get_client_auth_data(void *arg,
+    PRFileDesc *sock, struct CERTDistNamesStr *caNames,
+    struct CERTCertificateStr **pRetCert, struct SECKEYPrivateKeyStr **pRetKey);
 
-#define QDEVICE_NET_LOG_TARGET_STDERR           QNETD_LOG_TARGET_STDERR
-#define QDEVICE_NET_LOG_TARGET_SYSLOG           QNETD_LOG_TARGET_SYSLOG
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* _QDEVICE_NET_LOG_H_ */
+#endif /* _QDEVICE_NET_NSS_H_ */

qdevices/qdevice-net-poll.c

@@ -0,0 +1,241 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qdevice-cmap.h"
+#include "qdevice-net-poll.h"
+#include "qdevice-log.h"
+#include "qdevice-net-send.h"
+#include "qdevice-net-socket.h"
+#include "qdevice-votequorum.h"
+
+enum qdevice_net_poll_pfd {
+	QDEVICE_NET_POLL_VOTEQUORUM,
+	QDEVICE_NET_POLL_CMAP,
+	QDEVICE_NET_POLL_SOCKET,
+	QDEVICE_NET_POLL_MAX_PFDS
+};
+
+static void
+qdevice_net_poll_read_socket(struct qdevice_net_instance *instance)
+{
+
+	if (qdevice_net_socket_read(instance) == -1) {
+		instance->schedule_disconnect = 1;
+	}
+}
+
+static void
+qdevice_net_poll_read_votequorum(struct qdevice_net_instance *instance)
+{
+
+	if (qdevice_votequorum_dispatch(instance->qdevice_instance_ptr) == -1) {
+		instance->schedule_disconnect = 1;
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
+	}
+}
+
+static void
+qdevice_net_poll_read_cmap(struct qdevice_net_instance *instance)
+{
+
+	if (qdevice_cmap_dispatch(instance->qdevice_instance_ptr) == -1) {
+		instance->schedule_disconnect = 1;
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED;
+	}
+}
+
+static void
+qdevice_net_poll_write_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd)
+{
+	int res;
+
+	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
+		res = nss_sock_non_blocking_client_succeeded(pfd);
+		if (res == -1) {
+			/*
+			 * Connect failed -> try next
+			 */
+			res = nss_sock_non_blocking_client_try_next(&instance->non_blocking_client);
+			if (res == -1) {
+				qdevice_log_nss(LOG_ERR, "Can't connect to qnetd host.");
+				nss_sock_non_blocking_client_destroy(&instance->non_blocking_client);
+			}
+		} else if (res == 0) {
+			/*
+			 * Poll again
+			 */
+		} else if (res == 1) {
+			/*
+			 * Connect success
+			 */
+			instance->socket = instance->non_blocking_client.socket;
+			nss_sock_non_blocking_client_destroy(&instance->non_blocking_client);
+			instance->non_blocking_client.socket = NULL;
+
+			qdevice_log(LOG_DEBUG, "Sending preinit msg to qnetd");
+			if (qdevice_net_send_preinit(instance) != 0) {
+				instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+				instance->schedule_disconnect = 1;
+			}
+		} else {
+			qdevice_log(LOG_CRIT, "Unhandled nss_sock_non_blocking_client_succeeded");
+			exit(1);
+		}
+	} else {
+		if (qdevice_net_socket_write(instance) == -1) {
+			instance->schedule_disconnect = 1;
+		}
+	}
+}
+
+static void
+qdevice_net_poll_err_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd)
+{
+
+	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) {
+		/*
+		 * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work
+		 * only with poll, select works as expected!???).
+		 * So test if client is still valid and if pollout was not already called (ensured
+		 * by default because of order in PR_Poll).
+		 * If both applies it's possible to emulate pollout set by calling poll_write.
+		 */
+		if (!instance->non_blocking_client.destroyed) {
+			qdevice_net_poll_write_socket(instance, pfd);
+		}
+	} else {
+		qdevice_log(LOG_ERR, "POLL_ERR (%u) on main socket", pfd->out_flags);
+
+		instance->schedule_disconnect = 1;
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
+	}
+}
+
+int
+qdevice_net_poll(struct qdevice_net_instance *instance)
+{
+	PRPollDesc pfds[QDEVICE_NET_POLL_MAX_PFDS];
+	PRInt32 poll_res;
+	PRIntn no_pfds;
+	int i;
+
+	no_pfds = 0;
+
+	pfds[QDEVICE_NET_POLL_VOTEQUORUM].fd = instance->votequorum_poll_fd;
+	pfds[QDEVICE_NET_POLL_VOTEQUORUM].in_flags = PR_POLL_READ;
+	no_pfds++;
+
+	pfds[QDEVICE_NET_POLL_CMAP].fd = instance->cmap_poll_fd;
+	pfds[QDEVICE_NET_POLL_CMAP].in_flags = PR_POLL_READ;
+	no_pfds++;
+
+	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT &&
+	    !instance->non_blocking_client.destroyed) {
+		pfds[QDEVICE_NET_POLL_SOCKET].fd = instance->non_blocking_client.socket;
+		pfds[QDEVICE_NET_POLL_SOCKET].in_flags = PR_POLL_WRITE | PR_POLL_EXCEPT;
+		no_pfds++;
+	} else {
+		pfds[QDEVICE_NET_POLL_SOCKET].fd = instance->socket;
+		pfds[QDEVICE_NET_POLL_SOCKET].in_flags = PR_POLL_READ;
+		if (!send_buffer_list_empty(&instance->send_buffer_list)) {
+			pfds[QDEVICE_NET_POLL_SOCKET].in_flags |= PR_POLL_WRITE;
+		}
+		no_pfds++;
+	}
+
+	instance->schedule_disconnect = 0;
+
+	if ((poll_res = PR_Poll(pfds, no_pfds,
+	    timer_list_time_to_expire(&instance->main_timer_list))) > 0) {
+		for (i = 0; i < no_pfds; i++) {
+			if (pfds[i].out_flags & PR_POLL_READ) {
+				switch (i) {
+				case QDEVICE_NET_POLL_SOCKET:
+					qdevice_net_poll_read_socket(instance);
+					break;
+				case QDEVICE_NET_POLL_VOTEQUORUM:
+					qdevice_net_poll_read_votequorum(instance);
+					break;
+				case QDEVICE_NET_POLL_CMAP:
+					qdevice_net_poll_read_cmap(instance);
+					break;
+				default:
+					qdevice_log(LOG_CRIT, "Unhandled read on poll descriptor %u", i);
+					exit(1);
+					break;
+				}
+			}
+
+			if (!instance->schedule_disconnect && pfds[i].out_flags & PR_POLL_WRITE) {
+				switch (i) {
+				case QDEVICE_NET_POLL_SOCKET:
+					qdevice_net_poll_write_socket(instance, &pfds[i]);
+					break;
+				default:
+					qdevice_log(LOG_CRIT, "Unhandled write on poll descriptor %u", i);
+					exit(1);
+					break;
+				}
+			}
+
+			if (!instance->schedule_disconnect &&
+			    (pfds[i].out_flags & (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) &&
+			    !(pfds[i].out_flags & (PR_POLL_READ|PR_POLL_WRITE))) {
+				switch (i) {
+				case QDEVICE_NET_POLL_SOCKET:
+					qdevice_net_poll_err_socket(instance, &pfds[i]);
+					break;
+				default:
+					qdevice_log(LOG_CRIT, "Unhandled error on poll descriptor %u", i);
+					exit(1);
+					break;
+				}
+			}
+		}
+	}
+
+	if (!instance->schedule_disconnect) {
+		timer_list_expire(&instance->main_timer_list);
+	}
+
+	if (instance->schedule_disconnect) {
+		/*
+		 * Schedule disconnect can be set by this function, by some timer_list callback
+		 * or cmap/votequorum callbacks
+		 */
+		return (-1);
+	}
+
+	return (0);
+}

qdevices/qdevice-net-poll.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_NET_POLL_H_
+#define _QDEVICE_NET_POLL_H_
+
+
+#include "qdevice-net-instance.h"
+
+extern int		qdevice_net_poll(struct qdevice_net_instance *instance);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_NET_POLL_H_ */

qdevices/qdevice-net-send.c

@@ -32,8 +32,10 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "qdevice-net-log.h"
+#include "qdevice-log.h"
+#include "qdevice-log-debug.h"
 #include "qdevice-net-send.h"
+#include "qdevice-cmap.h"
 #include "qdevice-net-votequorum.h"
 #include "msg.h"
 
@@ -44,7 +46,7 @@ qdevice_net_send_echo_request(struct qdevice_net_instance *instance)
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_CRIT, "Can't allocate send list buffer for reply msg.");
+		qdevice_log(LOG_CRIT, "Can't allocate send list buffer for reply msg.");
 
 		return (-1);
 	}
@@ -53,7 +55,7 @@ qdevice_net_send_echo_request(struct qdevice_net_instance *instance)
 
 	if (msg_create_echo_request(&send_buffer->buffer, 1,
 	    instance->echo_request_expected_msg_seq_num) == -1) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for echo request msg");
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for echo request msg");
 
 		return (-1);
 	}
@@ -63,6 +65,33 @@ qdevice_net_send_echo_request(struct qdevice_net_instance *instance)
 	return (0);
 }
 
+int
+qdevice_net_send_preinit(struct qdevice_net_instance *instance)
+{
+	struct send_buffer_list_entry *send_buffer;
+
+	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
+	if (send_buffer == NULL) {
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for preinit msg");
+
+		return (-1);
+	}
+
+	if (msg_create_preinit(&send_buffer->buffer, instance->cluster_name, 1,
+	    instance->last_msg_seq_num) == 0) {
+		qdevice_log(LOG_ERR, "Can't allocate buffer");
+
+		send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
+		return (-1);
+	}
+
+	send_buffer_list_put(&instance->send_buffer_list, send_buffer);
+
+	instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_PREINIT_REPLY;
+
+	return (0);
+}
+
 int
 qdevice_net_send_init(struct qdevice_net_instance *instance)
 {
@@ -78,7 +107,7 @@ qdevice_net_send_init(struct qdevice_net_instance *instance)
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for init msg");
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for init msg");
 
 		return (-1);
 	}
@@ -86,9 +115,10 @@ qdevice_net_send_init(struct qdevice_net_instance *instance)
 	if (msg_create_init(&send_buffer->buffer, 1, instance->last_msg_seq_num,
 	    instance->decision_algorithm,
 	    supported_msgs, no_supported_msgs, supported_opts, no_supported_opts,
-	    instance->node_id) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for init msg");
+	    instance->qdevice_instance_ptr->node_id) == 0) {
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for init msg");
 
+		send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
 		return (-1);
 	}
 
@@ -106,19 +136,20 @@ qdevice_net_send_ask_for_vote(struct qdevice_net_instance *instance)
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for ask for vote msg");
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for ask for vote msg");
 
 		return (-1);
 	}
 
 	instance->last_msg_seq_num++;
 
-	qdevice_net_log(LOG_DEBUG, "Sending ask for vote seq = %"PRIu32,
+	qdevice_log(LOG_DEBUG, "Sending ask for vote seq = %"PRIu32,
 	    instance->last_msg_seq_num);
 
 	if (msg_create_ask_for_vote(&send_buffer->buffer, instance->last_msg_seq_num) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for ask for vote msg");
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for ask for vote msg");
 
+		send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
 		return (-1);
 	}
 
@@ -127,86 +158,36 @@ qdevice_net_send_ask_for_vote(struct qdevice_net_instance *instance)
 	return (0);
 }
 
-static
-void qdevice_net_log_debug_node_list(const struct node_list *nlist)
-{
-	struct node_list_entry *node_info;
-	size_t zi;
-
-	qdevice_net_log(LOG_DEBUG, "  Node list:");
-
-	zi = 0;
-
-	TAILQ_FOREACH(node_info, nlist, entries) {
-		qdevice_net_log(LOG_DEBUG, "    %zu node_id = %"PRIx32", "
-		    "data_center_id = %"PRIx32", node_state = %s",
-		    zi, node_info->node_id, node_info->data_center_id,
-		    tlv_node_state_to_str(node_info->node_state));
-		zi++;
-        }
-}
-
 int
-qdevice_net_send_config_node_list(struct qdevice_net_instance *instance, int initial, int force_send)
+qdevice_net_send_config_node_list(struct qdevice_net_instance *instance,
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int initial)
 {
-	struct node_list nlist;
 	struct send_buffer_list_entry *send_buffer;
-	uint64_t config_version;
-	int send_config_version;
-
-	if (qdevice_net_cmap_get_nodelist(instance->cmap_handle, &nlist) != 0) {
-		qdevice_net_log(LOG_ERR, "Can't get initial configuration node list.");
-
-		return (-1);
-	}
-
-	/*
-	 * Send only if list changed or force was used
-	 */
-	if (!force_send && node_list_eq(&instance->last_sent_config_node_list, &nlist)) {
-		return (0);
-	}
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for config "
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for config "
 		    "node list msg");
 
-		node_list_free(&nlist);
-
 		return (-1);
 	}
 
-	send_config_version = qdevice_net_cmap_get_config_version(instance->cmap_handle,
-	    &config_version);
-
 	instance->last_msg_seq_num++;
 
-	qdevice_net_log(LOG_DEBUG, "Sending config node list seq = %"PRIu32,
+	qdevice_log(LOG_DEBUG, "Sending config node list seq = %"PRIu32,
 	    instance->last_msg_seq_num);
-	qdevice_net_log_debug_node_list(&nlist);
-
-	node_list_free(&instance->last_sent_config_node_list);
-	if (node_list_clone(&instance->last_sent_config_node_list, &nlist) != 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate last sent config node list clone");
-
-		node_list_free(&nlist);
-
-		return (-1);
-	}
+	qdevice_log_debug_dump_node_list(nlist);
 
 	if (msg_create_node_list(&send_buffer->buffer, instance->last_msg_seq_num,
 	    (initial ? TLV_NODE_LIST_TYPE_INITIAL_CONFIG : TLV_NODE_LIST_TYPE_CHANGED_CONFIG),
-	    0, NULL, send_config_version, config_version, 0, TLV_QUORATE_INQUORATE, &nlist) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for config list msg");
-
-		node_list_free(&nlist);
+	    0, NULL, config_version_set, config_version, 0, TLV_QUORATE_INQUORATE, nlist) == 0) {
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for config list msg");
 
+		send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
 		return (-1);
 	}
 
-	node_list_free(&nlist);
-
 	send_buffer_list_put(&instance->send_buffer_list, send_buffer);
 
 	return (0);
@@ -225,7 +206,7 @@ qdevice_net_send_membership_node_list(struct qdevice_net_instance *instance,
 
 	for (i = 0; i < node_list_entries; i++) {
 		if (node_list_add(&nlist, node_list[i], 0, TLV_NODE_STATE_NOT_SET) == NULL) {
-			qdevice_net_log(LOG_ERR, "Can't allocate membership node list.");
+			qdevice_log(LOG_ERR, "Can't allocate membership node list.");
 
 			node_list_free(&nlist);
 
@@ -235,7 +216,7 @@ qdevice_net_send_membership_node_list(struct qdevice_net_instance *instance,
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for membership "
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for membership "
 		    "node list msg");
 
 		node_list_free(&nlist);
@@ -245,18 +226,19 @@ qdevice_net_send_membership_node_list(struct qdevice_net_instance *instance,
 
 	instance->last_msg_seq_num++;
 
-	qdevice_net_log(LOG_DEBUG, "Sending membership node list seq = %"PRIu32", "
+	qdevice_log(LOG_DEBUG, "Sending membership node list seq = %"PRIu32", "
 	    "ringid = (%"PRIx32".%"PRIx64").", instance->last_msg_seq_num,
 	    ring_id->node_id, ring_id->seq);
-	qdevice_net_log_debug_node_list(&nlist);
+	qdevice_log_debug_dump_node_list(&nlist);
 
 	if (msg_create_node_list(&send_buffer->buffer, instance->last_msg_seq_num,
 	    TLV_NODE_LIST_TYPE_MEMBERSHIP,
 	    1, ring_id, 0, 0, 0, 0, &nlist) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for membership list msg");
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for membership list msg");
 
 		node_list_free(&nlist);
 
+		send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
 		return (-1);
 	}
 
@@ -287,7 +269,7 @@ qdevice_net_send_quorum_node_list(struct qdevice_net_instance *instance,
 
 		if (node_list_add(&nlist, node_list[i].nodeid, 0,
 		    qdevice_net_votequorum_node_state_to_tlv(node_list[i].state)) == NULL) {
-			qdevice_net_log(LOG_ERR, "Can't allocate quorum node list.");
+			qdevice_log(LOG_ERR, "Can't allocate quorum node list.");
 
 			node_list_free(&nlist);
 
@@ -297,7 +279,7 @@ qdevice_net_send_quorum_node_list(struct qdevice_net_instance *instance,
 
 	send_buffer = send_buffer_list_get_new(&instance->send_buffer_list);
 	if (send_buffer == NULL) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send list buffer for quorum "
+		qdevice_log(LOG_ERR, "Can't allocate send list buffer for quorum "
 		    "node list msg");
 
 		node_list_free(&nlist);
@@ -307,17 +289,18 @@ qdevice_net_send_quorum_node_list(struct qdevice_net_instance *instance,
 
 	instance->last_msg_seq_num++;
 
-	qdevice_net_log(LOG_DEBUG, "Sending quorum node list seq = %"PRIu32", quorate = %u",
+	qdevice_log(LOG_DEBUG, "Sending quorum node list seq = %"PRIu32", quorate = %u",
 	    instance->last_msg_seq_num, quorate);
-	qdevice_net_log_debug_node_list(&nlist);
+	qdevice_log_debug_dump_node_list(&nlist);
 
 	if (msg_create_node_list(&send_buffer->buffer, instance->last_msg_seq_num,
 	    TLV_NODE_LIST_TYPE_QUORUM,
 	    0, NULL, 0, 0, 1, quorate, &nlist) == 0) {
-		qdevice_net_log(LOG_ERR, "Can't allocate send buffer for quorum list msg");
+		qdevice_log(LOG_ERR, "Can't allocate send buffer for quorum list msg");
 
 		node_list_free(&nlist);
 
+		send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer);
 		return (-1);
 	}
 

qdevices/qdevice-net-send.h

@@ -38,7 +38,6 @@
 #include <sys/types.h>
 
 #include "qdevice-net-instance.h"
-#include "qdevice-net-cmap.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -46,12 +45,15 @@ extern "C" {
 
 extern int		qdevice_net_send_echo_request(struct qdevice_net_instance *instance);
 
+extern int		qdevice_net_send_preinit(struct qdevice_net_instance *instance);
+
 extern int		qdevice_net_send_init(struct qdevice_net_instance *instance);
 
 extern int		qdevice_net_send_ask_for_vote(struct qdevice_net_instance *instance);
 
 extern int		qdevice_net_send_config_node_list(struct qdevice_net_instance *instance,
-    int initial, int force_send);
+    const struct node_list *nlist, int config_version_set, uint64_t config_version,
+    int initial);
 
 extern int		qdevice_net_send_membership_node_list(
     struct qdevice_net_instance *instance, const struct tlv_ring_id *ring_id,

qdevices/qdevice-net-socket.c

@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "msg.h"
+#include "msgio.h"
+#include "qnet-config.h"
+#include "qdevice-log.h"
+#include "qdevice-net-msg-received.h"
+#include "qdevice-net-nss.h"
+#include "qdevice-net-send.h"
+#include "qdevice-net-socket.h"
+
+/*
+ * -1 means end of connection (EOF) or some other unhandled error. 0 = success
+ */
+int
+qdevice_net_socket_read(struct qdevice_net_instance *instance)
+{
+	int res;
+	int ret_val;
+	int orig_skipping_msg;
+
+	orig_skipping_msg = instance->skipping_msg;
+
+	res = msgio_read(instance->socket, &instance->receive_buffer,
+	    &instance->msg_already_received_bytes, &instance->skipping_msg);
+
+	if (!orig_skipping_msg && instance->skipping_msg) {
+		qdevice_log(LOG_DEBUG, "msgio_read set skipping_msg");
+	}
+
+	ret_val = 0;
+
+	switch (res) {
+	case 0:
+		/*
+		 * Partial read
+		 */
+		break;
+	case -1:
+		qdevice_log(LOG_DEBUG, "Server closed connection");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
+		ret_val = -1;
+		break;
+	case -2:
+		qdevice_log(LOG_ERR, "Unhandled error when reading from server. "
+		    "Disconnecting from server");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
+		ret_val = -1;
+		break;
+	case -3:
+		qdevice_log(LOG_ERR, "Can't store message header from server. "
+		    "Disconnecting from server");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
+		ret_val = -1;
+		break;
+	case -4:
+		qdevice_log(LOG_ERR, "Can't store message from server. "
+		    "Disconnecting from server");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
+		ret_val = -1;
+		break;
+	case -5:
+		qdevice_log(LOG_WARNING, "Server sent unsupported msg type %u. "
+		    "Disconnecting from server", msg_get_type(&instance->receive_buffer));
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNSUPPORTED_MSG;
+		ret_val = -1;
+		break;
+	case -6:
+		qdevice_log(LOG_WARNING,
+		    "Server wants to send too long message %u bytes. Disconnecting from server",
+		    msg_get_len(&instance->receive_buffer));
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_READ_MESSAGE;
+		ret_val = -1;
+		break;
+	case 1:
+		/*
+		 * Full message received / skipped
+		 */
+		if (!instance->skipping_msg) {
+			if (qdevice_net_msg_received(instance) == -1) {
+				ret_val = -1;
+			}
+		} else {
+			qdevice_log(LOG_CRIT, "net_socket_read in skipping msg state");
+			exit(1);
+		}
+
+		instance->skipping_msg = 0;
+		instance->msg_already_received_bytes = 0;
+		dynar_clean(&instance->receive_buffer);
+		break;
+	default:
+		qdevice_log(LOG_CRIT, "qdevice_net_socket_read unhandled error %d", res);
+		exit(1);
+		break;
+	}
+
+	return (ret_val);
+}
+
+static int
+qdevice_net_socket_write_finished(struct qdevice_net_instance *instance)
+{
+	PRFileDesc *new_pr_fd;
+
+	if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_STARTTLS_BEING_SENT) {
+		/*
+		 * StartTLS sent to server. Begin with TLS handshake
+		 */
+		if ((new_pr_fd = nss_sock_start_ssl_as_client(instance->socket, QNETD_NSS_SERVER_CN,
+		    qdevice_net_nss_bad_cert_hook,
+		    qdevice_net_nss_get_client_auth_data,
+		    (void *)QDEVICE_NET_NSS_CLIENT_CERT_NICKNAME, 0, NULL)) == NULL) {
+			qdevice_log_nss(LOG_ERR, "Can't start TLS");
+			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_TLS;
+			return (-1);
+		}
+
+		/*
+		 * And send init msg
+		 */
+		if (qdevice_net_send_init(instance) != 0) {
+			instance->disconnect_reason =
+			    QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
+
+			return (-1);
+		}
+
+		instance->socket = new_pr_fd;
+	}
+
+	return (0);
+}
+
+int
+qdevice_net_socket_write(struct qdevice_net_instance *instance)
+{
+	int res;
+	struct send_buffer_list_entry *send_buffer;
+	enum msg_type sent_msg_type;
+
+	send_buffer = send_buffer_list_get_active(&instance->send_buffer_list);
+	if (send_buffer == NULL) {
+		qdevice_log(LOG_CRIT, "send_buffer_list_get_active returned NULL");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SEND_MESSAGE;
+
+		return (-1);
+	}
+
+	res = msgio_write(instance->socket, &send_buffer->buffer,
+	    &send_buffer->msg_already_sent_bytes);
+
+	if (res == 1) {
+		sent_msg_type = msg_get_type(&send_buffer->buffer);
+
+		send_buffer_list_delete(&instance->send_buffer_list, send_buffer);
+
+		if (sent_msg_type != MSG_TYPE_ECHO_REQUEST) {
+			if (qdevice_net_socket_write_finished(instance) == -1) {
+				return (-1);
+			}
+		}
+	}
+
+	if (res == -1) {
+		qdevice_log_nss(LOG_CRIT, "PR_Send returned 0");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION;
+		return (-1);
+	}
+
+	if (res == -2) {
+		qdevice_log_nss(LOG_ERR, "Unhandled error when sending message to server");
+		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SEND_MESSAGE;
+
+		return (-1);
+	}
+
+	return (0);
+}

qdevices/qdevice-net-socket.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_NET_SOCKET_H_
+#define _QDEVICE_NET_SOCKET_H_
+
+#include "qdevice-net-instance.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int		qdevice_net_socket_read(struct qdevice_net_instance *instance);
+
+extern int		qdevice_net_socket_write(struct qdevice_net_instance *instance);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_NET_SOCKET_H_ */

qdevices/qdevice-net-votequorum.c

@@ -32,21 +32,8 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <err.h>
-#include <poll.h>
-
-/*
- * Needed for creating nspr handle from unix fd
- */
-#include <private/pprio.h>
-
-#include "qnet-config.h"
 #include "qdevice-net-votequorum.h"
-#include "qdevice-net-send.h"
-#include "qdevice-net-log.h"
-#include "qdevice-net-cast-vote-timer.h"
-#include "qdevice-net-algorithm.h"
-#include "nss-sock.h"
+#include "qdevice-log.h"
 
 enum tlv_node_state
 qdevice_net_votequorum_node_state_to_tlv(uint32_t votequorum_node_state)
@@ -58,8 +45,9 @@ qdevice_net_votequorum_node_state_to_tlv(uint32_t votequorum_node_state)
 	case VOTEQUORUM_NODESTATE_DEAD: res = TLV_NODE_STATE_DEAD; break;
 	case VOTEQUORUM_NODESTATE_LEAVING: res = TLV_NODE_STATE_LEAVING; break;
 	default:
-		errx(1, "qdevice_net_convert_votequorum_to_tlv_node_state: Unhandled votequorum "
+		qdevice_log(LOG_ERR, "qdevice_net_votequorum_node_state_to_tlv: Unhandled votequorum "
 		    "node state %"PRIu32, votequorum_node_state);
+		exit(1);
 		break;
 	}
 
@@ -74,180 +62,3 @@ qdevice_net_votequorum_ring_id_to_tlv(struct tlv_ring_id *tlv_rid,
 	tlv_rid->node_id = votequorum_rid->nodeid;
 	tlv_rid->seq = votequorum_rid->seq;
 }
-
-static void
-qdevice_net_votequorum_quorum_notify_callback(votequorum_handle_t votequorum_handle,
-    uint64_t context, uint32_t quorate,
-    uint32_t node_list_entries, votequorum_node_t node_list[])
-{
-	struct qdevice_net_instance *instance;
-	uint32_t u32;
-	int algo_send_list;
-
-	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
-		errx(1, "Fatal error. Can't get votequorum context");
-	}
-
-	qdevice_net_log(LOG_DEBUG, "Votequorum quorum notify callback:");
-	qdevice_net_log(LOG_DEBUG, "  Quorate = %u", quorate);
-
-	qdevice_net_log(LOG_DEBUG, "  Node list (size = %"PRIu32"):", node_list_entries);
-	for (u32 = 0; u32 < node_list_entries; u32++) {
-		qdevice_net_log(LOG_DEBUG, "    %"PRIu32" nodeid = %"PRIx32", state = %"PRIu32,
-		    u32, node_list[u32].nodeid, node_list[u32].state);
-	}
-
-	algo_send_list = 1;
-	if (qdevice_net_algorithm_votequorum_quorum_notify(instance, quorate,
-	    node_list_entries, node_list, &algo_send_list) != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTEQUORUM_QUORUM_NOTIFY_ERR;
-		instance->schedule_disconnect = 1;
-	} else {
-		qdevice_net_log(LOG_DEBUG, "Algorithm decided to %s list",
-		    (algo_send_list ? "send" : "not send"));
-	}
-
-	if (algo_send_list) {
-		if (qdevice_net_send_quorum_node_list(instance,
-		    (quorate ? TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE),
-		    node_list_entries, node_list) != 0) {
-			/*
-			 * Fatal error -> schedule disconnect
-			 */
-			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-			instance->schedule_disconnect = 1;
-		}
-	}
-}
-
-static void
-qdevice_net_votequorum_node_list_notify_callback(votequorum_handle_t votequorum_handle,
-    uint64_t context, votequorum_ring_id_t votequorum_ring_id,
-    uint32_t node_list_entries, uint32_t node_list[])
-{
-	struct qdevice_net_instance *instance;
-	struct tlv_ring_id tlv_rid;
-	uint32_t u32;
-	enum tlv_vote result_vote;
-
-	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
-		qdevice_net_log(LOG_ERR, "Fatal error. Can't get votequorum context");
-		exit(1);
-	}
-
-	qdevice_net_log(LOG_DEBUG, "Votequorum nodelist notify callback:");
-	qdevice_net_log(LOG_DEBUG, "  Ring_id = (%"PRIx32".%"PRIx64")",
-	    votequorum_ring_id.nodeid, votequorum_ring_id.seq);
-
-	qdevice_net_log(LOG_DEBUG, "  Node list (size = %"PRIu32"):", node_list_entries);
-	for (u32 = 0; u32 < node_list_entries; u32++) {
-		qdevice_net_log(LOG_DEBUG, "    %"PRIu32" nodeid = %"PRIx32,
-		    u32, node_list[u32]);
-	}
-
-	qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid, &votequorum_ring_id);
-
-	result_vote = TLV_VOTE_WAIT_FOR_REPLY;
-	if (qdevice_net_algorithm_votequorum_node_list_notify(instance, &tlv_rid, node_list_entries,
-	    node_list, &result_vote) != 0) {
-		qdevice_net_log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
-
-		instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_VOTEQUORUM_NODE_LIST_NOTIFY_ERR;
-		instance->schedule_disconnect = 1;
-	} else {
-		qdevice_net_log(LOG_DEBUG, "Algorithm result vote is %s", tlv_vote_to_str(result_vote));
-	}
-
-	if (result_vote == TLV_VOTE_WAIT_FOR_REPLY || result_vote == TLV_VOTE_ASK_LATER) {
-		if (qdevice_net_send_membership_node_list(instance, &tlv_rid, node_list_entries, node_list) != 0) {
-			/*
-			 * Fatal error -> schedule disconnect
-			 */
-			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
-			instance->schedule_disconnect = 1;
-		}
-	}
-
-	memcpy(&instance->last_received_votequorum_ring_id, &votequorum_ring_id,
-	    sizeof(votequorum_ring_id));
-
-	if (result_vote != TLV_VOTE_NO_CHANGE && result_vote != TLV_VOTE_ASK_LATER) {
-		if (qdevice_net_cast_vote_timer_update(instance, result_vote) != 0) {
-			qdevice_net_log(LOG_CRIT, "qdevice_net_votequorum_node_list_notify_callback fatal error "
-			    "Can't update cast vote timer");
-			instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
-			instance->schedule_disconnect = 1;
-		}
-	}
-}
-
-void
-qdevice_net_votequorum_init(struct qdevice_net_instance *instance)
-{
-	votequorum_callbacks_t votequorum_callbacks;
-	votequorum_handle_t votequorum_handle;
-	cs_error_t res;
-	int no_retries;
-	int fd;
-
-	memset(&votequorum_callbacks, 0, sizeof(votequorum_callbacks));
-
-	votequorum_callbacks.votequorum_quorum_notify_fn =
-	    qdevice_net_votequorum_quorum_notify_callback;
-
-	votequorum_callbacks.votequorum_nodelist_notify_fn =
-	    qdevice_net_votequorum_node_list_notify_callback;
-
-	no_retries = 0;
-
-	while ((res = votequorum_initialize(&votequorum_handle,
-	    &votequorum_callbacks)) == CS_ERR_TRY_AGAIN &&
-	    no_retries++ < QDEVICE_NET_MAX_CS_TRY_AGAIN) {
-		poll(NULL, 0, 1000);
-	}
-
-        if (res != CS_OK) {
-		errx(1, "Failed to initialize the votequorum API. Error %s", cs_strerror(res));
-	}
-
-	if ((res = votequorum_qdevice_register(votequorum_handle,
-	    QDEVICE_NET_VOTEQUORUM_DEVICE_NAME)) != CS_OK) {
-		errx(1, "Can't register votequorum device. Error %s", cs_strerror(res));
-	}
-
-	if ((res = votequorum_context_set(votequorum_handle, (void *)instance)) != CS_OK) {
-		errx(1, "Can't set votequorum context. Error %s", cs_strerror(res));
-	}
-
-	instance->votequorum_handle = votequorum_handle;
-
-	votequorum_fd_get(votequorum_handle, &fd);
-	if ((instance->votequorum_poll_fd = PR_CreateSocketPollFd(fd)) == NULL) {
-		qdevice_net_log_nss(LOG_CRIT, "Can't create NSPR votequorum poll fd");
-		exit(1);
-	}
-}
-
-void
-qdevice_net_votequorum_destroy(struct qdevice_net_instance *instance)
-{
-	cs_error_t res;
-
-	res = votequorum_qdevice_unregister(instance->votequorum_handle,
-		QDEVICE_NET_VOTEQUORUM_DEVICE_NAME);
-
-        if (res != CS_OK) {
-                qdevice_net_log(LOG_WARNING, "Unable to unregister votequorum device. Error %s", cs_strerror(res));
-	}
-
-	res = votequorum_finalize(instance->votequorum_handle);
-	if (res != CS_OK) {
-		qdevice_net_log(LOG_WARNING, "Unable to finalize votequorum. Error %s", cs_strerror(res));
-	}
-
-	if (PR_DestroySocketPollFd(instance->votequorum_poll_fd) != PR_SUCCESS) {
-		qdevice_net_log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
-	}
-}

qdevices/qdevice-net-votequorum.h

@@ -50,10 +50,6 @@ extern enum tlv_node_state	qdevice_net_votequorum_node_state_to_tlv(
 extern void			qdevice_net_votequorum_ring_id_to_tlv(struct tlv_ring_id *tlv_rid,
     const votequorum_ring_id_t *votequorum_rid);
 
-extern void			qdevice_net_votequorum_init(struct qdevice_net_instance *instance);
-
-extern void			qdevice_net_votequorum_destroy(struct qdevice_net_instance *instance);
-
 #ifdef __cplusplus
 }
 #endif

qdevices/qdevice-votequorum.c

@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <poll.h>
+
+#include "qdevice-config.h"
+#include "qdevice-log.h"
+#include "qdevice-votequorum.h"
+#include "qdevice-model.h"
+
+static void
+qdevice_votequorum_quorum_notify_callback(votequorum_handle_t votequorum_handle,
+    uint64_t context, uint32_t quorate,
+    uint32_t node_list_entries, votequorum_node_t node_list[])
+{
+	struct qdevice_instance *instance;
+	uint32_t u32;
+
+	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context");
+		exit(1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Votequorum quorum notify callback:");
+	qdevice_log(LOG_DEBUG, "  Quorate = %u", quorate);
+
+	qdevice_log(LOG_DEBUG, "  Node list (size = %"PRIu32"):", node_list_entries);
+	for (u32 = 0; u32 < node_list_entries; u32++) {
+		qdevice_log(LOG_DEBUG, "    %"PRIu32" nodeid = %"PRIx32", state = %"PRIu32,
+		    u32, node_list[u32].nodeid, node_list[u32].state);
+	}
+
+	if (qdevice_model_votequorum_quorum_notify(instance, quorate, node_list_entries,
+	    node_list) != 0) {
+		qdevice_log(LOG_DEBUG, "qdevice_model_votequorum_quorum_notify returned error -> exit");
+		exit(2);
+	}
+
+	instance->vq_quorum_quorate = quorate;
+	instance->vq_quorum_node_list_entries = node_list_entries;
+
+	free(instance->vq_quorum_node_list);
+	instance->vq_quorum_node_list = malloc(sizeof(*node_list) * node_list_entries);
+	if (instance->vq_quorum_node_list == NULL) {
+		qdevice_log(LOG_CRIT, "Can't alloc votequorum node list memory");
+		exit(1);
+	}
+	memcpy(instance->vq_quorum_node_list, node_list, sizeof(*node_list) * node_list_entries);
+}
+
+static void
+qdevice_votequorum_node_list_notify_callback(votequorum_handle_t votequorum_handle,
+    uint64_t context, votequorum_ring_id_t votequorum_ring_id,
+    uint32_t node_list_entries, uint32_t node_list[])
+{
+	struct qdevice_instance *instance;
+	uint32_t u32;
+
+	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context");
+		exit(1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Votequorum nodelist notify callback:");
+	qdevice_log(LOG_DEBUG, "  Ring_id = (%"PRIx32".%"PRIx64")",
+	    votequorum_ring_id.nodeid, votequorum_ring_id.seq);
+
+	qdevice_log(LOG_DEBUG, "  Node list (size = %"PRIu32"):", node_list_entries);
+	for (u32 = 0; u32 < node_list_entries; u32++) {
+		qdevice_log(LOG_DEBUG, "    %"PRIu32" nodeid = %"PRIx32,
+		    u32, node_list[u32]);
+	}
+
+	if (qdevice_model_votequorum_node_list_notify(instance, votequorum_ring_id, node_list_entries,
+	    node_list) != 0) {
+		qdevice_log(LOG_DEBUG, "qdevice_votequorum_node_list_notify_callback returned error -> exit");
+		exit(2);
+	}
+
+	memcpy(&instance->vq_node_list_ring_id, &votequorum_ring_id, sizeof(votequorum_ring_id));
+	instance->vq_node_list_entries = node_list_entries;
+	free(instance->vq_node_list);
+	instance->vq_node_list = malloc(sizeof(*node_list) * node_list_entries);
+	if (instance->vq_node_list == NULL) {
+		qdevice_log(LOG_CRIT, "Can't alloc votequorum node list memory");
+		exit(1);
+	}
+	memcpy(instance->vq_node_list, node_list, sizeof(*node_list) * node_list_entries);
+}
+
+static void
+qdevice_votequorum_expected_votes_notify_callback(votequorum_handle_t votequorum_handle,
+    uint64_t context, uint32_t expected_votes)
+{
+	struct qdevice_instance *instance;
+
+	if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context");
+		exit(1);
+	}
+
+	qdevice_log(LOG_DEBUG, "Votequorum expected_votes notify callback:");
+	qdevice_log(LOG_DEBUG, "  Expected_votes: %"PRIu32, expected_votes);
+
+	if (qdevice_model_votequorum_expected_votes_notify(instance, expected_votes) != 0) {
+		qdevice_log(LOG_DEBUG, "qdevice_votequorum_expected_votes_notify_callback returned error -> exit");
+		exit(2);
+	}
+
+	instance->vq_expected_votes = expected_votes;
+}
+
+void
+qdevice_votequorum_init(struct qdevice_instance *instance)
+{
+	votequorum_callbacks_t votequorum_callbacks;
+	votequorum_handle_t votequorum_handle;
+	cs_error_t res;
+	int no_retries;
+	struct votequorum_info vq_info;
+
+	memset(&votequorum_callbacks, 0, sizeof(votequorum_callbacks));
+
+	votequorum_callbacks.votequorum_quorum_notify_fn =
+	    qdevice_votequorum_quorum_notify_callback;
+
+	votequorum_callbacks.votequorum_nodelist_notify_fn =
+	    qdevice_votequorum_node_list_notify_callback;
+
+	votequorum_callbacks.votequorum_expectedvotes_notify_fn =
+	    qdevice_votequorum_expected_votes_notify_callback;
+
+	no_retries = 0;
+
+	while ((res = votequorum_initialize(&votequorum_handle,
+	    &votequorum_callbacks)) == CS_ERR_TRY_AGAIN &&
+	    no_retries++ < QDEVICE_MAX_CS_TRY_AGAIN) {
+		(void)poll(NULL, 0, 1000);
+	}
+
+        if (res != CS_OK) {
+		qdevice_log(LOG_CRIT, "Failed to initialize the votequorum API. Error %s", cs_strerror(res));
+		exit(1);
+	}
+
+	if ((res = votequorum_qdevice_register(votequorum_handle,
+	    QDEVICE_VOTEQUORUM_DEVICE_NAME)) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Can't register votequorum device. Error %s", cs_strerror(res));
+		exit(1);
+	}
+
+	if ((res = votequorum_context_set(votequorum_handle, (void *)instance)) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Can't set votequorum context. Error %s", cs_strerror(res));
+		exit(1);
+	}
+
+	if ((res = votequorum_getinfo(votequorum_handle, VOTEQUORUM_QDEVICE_NODEID,
+	    &vq_info)) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Can't get votequorum information. Error %s", cs_strerror(res));
+		exit(1);
+	}
+	instance->vq_expected_votes = vq_info.node_expected_votes;
+
+	instance->votequorum_handle = votequorum_handle;
+
+	votequorum_fd_get(votequorum_handle, &instance->votequorum_poll_fd);
+
+	if ((res = votequorum_trackstart(instance->votequorum_handle, 0,
+	    CS_TRACK_CHANGES)) != CS_OK) {
+		qdevice_log(LOG_CRIT, "Can't start tracking votequorum changes. Error %s",
+		    cs_strerror(res));
+		exit(1);
+	}
+}
+
+void
+qdevice_votequorum_destroy(struct qdevice_instance *instance)
+{
+	cs_error_t res;
+
+	free(instance->vq_quorum_node_list); instance->vq_quorum_node_list = NULL;
+	free(instance->vq_node_list); instance->vq_node_list = NULL;
+
+	res = votequorum_qdevice_unregister(instance->votequorum_handle,
+		QDEVICE_VOTEQUORUM_DEVICE_NAME);
+
+        if (res != CS_OK) {
+                qdevice_log(LOG_WARNING, "Unable to unregister votequorum device. Error %s", cs_strerror(res));
+	}
+
+	res = votequorum_finalize(instance->votequorum_handle);
+	if (res != CS_OK) {
+		qdevice_log(LOG_WARNING, "Unable to finalize votequorum. Error %s", cs_strerror(res));
+	}
+}
+
+int
+qdevice_votequorum_dispatch(struct qdevice_instance *instance)
+{
+	cs_error_t res;
+
+	res = votequorum_dispatch(instance->votequorum_handle, CS_DISPATCH_ALL);
+
+	if (res != CS_OK && res != CS_ERR_TRY_AGAIN) {
+		qdevice_log(LOG_ERR, "Can't dispatch votequorum messages");
+
+		return (-1);
+	}
+
+	return (0);
+}
+
+int
+qdevice_votequorum_poll(struct qdevice_instance *instance, int cast_vote)
+{
+	cs_error_t res;
+
+	res = votequorum_qdevice_poll(instance->votequorum_handle,
+	    QDEVICE_VOTEQUORUM_DEVICE_NAME, cast_vote,
+	    instance->vq_node_list_ring_id);
+
+	if (res != CS_OK && res != CS_ERR_TRY_AGAIN) {
+		if (res == CS_ERR_MESSAGE_ERROR) {
+			qdevice_log(LOG_INFO, "qdevice_votequorum_poll called with old ring id");
+		} else {
+			qdevice_log(LOG_CRIT, "Can't call votequorum_qdevice_poll. Error %s",
+			    cs_strerror(res));
+
+			return (-1);
+		}
+	}
+
+	return (0);
+}

qdevices/qdevice-votequorum.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Jan Friesse (jfriesse@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _QDEVICE_VOTEQUORUM_H_
+#define _QDEVICE_VOTEQUORUM_H_
+
+#include <votequorum.h>
+
+#include "qdevice-instance.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern void			qdevice_votequorum_init(struct qdevice_instance *instance);
+
+extern void			qdevice_votequorum_destroy(struct qdevice_instance *instance);
+
+extern int			qdevice_votequorum_dispatch(struct qdevice_instance *instance);
+
+extern int			qdevice_votequorum_poll(struct qdevice_instance *instance,
+    int cast_vote);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _QDEVICE_VOTEQUORUM_H_ */

qdevices/qnet-config.h

@@ -74,7 +74,6 @@ extern "C" {
 
 #define QNETD_NSS_SERVER_CN			"Qnetd Server"
 #define QDEVICE_NET_NSS_CLIENT_CERT_NICKNAME	"Cluster Cert"
-#define QDEVICE_NET_VOTEQUORUM_DEVICE_NAME	"QdeviceNet"
 
 #define QDEVICE_NET_MAX_SEND_BUFFERS		10
 
@@ -83,17 +82,9 @@ extern "C" {
 #define QNETD_DEFAULT_TIE_BREAKER_MODE		TLV_TIE_BREAKER_MODE_LOWEST
 #define QDEVICE_NET_DEFAULT_TIE_BREAKER_MODE	QNETD_DEFAULT_TIE_BREAKER_MODE
 
-#define QDEVICE_NET_MAX_CS_TRY_AGAIN		10
-
-#define QDEVICE_NET_LOCK_FILE			LOCALSTATEDIR"/run/corosync-qdevice-net.pid"
-
-#define QDEVICE_NET_CONNECT_TIMEOUT		(1000*30)
 #define QDEVICE_NET_MIN_CONNECT_TIMEOUT		1L
 #define QDEVICE_NET_MAX_CONNECT_TIMEOUT		(1000*60*10L)
 
-#define QDEVICE_NET_PAUSE_BEFORE_RECONNECT	1000
-
-
 /*
  * Decision algorithms supported by qnetd
  */

qdevices/qnetd-client-msg-received.c

@@ -173,6 +173,8 @@ qnetd_client_msg_received_preinit(struct qnetd_instance *instance, struct qnetd_
 		qnetd_log(LOG_ERR, "Can't alloc preinit reply msg. "
 		    "Disconnecting client connection.");
 
+		send_buffer_list_discard_new(&client->send_buffer_list, send_buffer);
+
 		return (-1);
 	};
 
@@ -393,6 +395,8 @@ qnetd_client_msg_received_init(struct qnetd_instance *instance, struct qnetd_cli
 	    QNETD_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE) == -1) {
 		qnetd_log(LOG_ERR, "Can't alloc init reply msg. Disconnecting client connection.");
 
+		send_buffer_list_discard_new(&client->send_buffer_list, send_buffer);
+
 		return (-1);
 	}
 
@@ -479,6 +483,8 @@ qnetd_client_msg_received_set_option(struct qnetd_instance *instance, struct qne
 		qnetd_log(LOG_ERR, "Can't alloc set option reply msg. "
 		    "Disconnecting client connection.");
 
+		send_buffer_list_discard_new(&client->send_buffer_list, send_buffer);
+
 		return (-1);
 	}
 
@@ -529,6 +535,8 @@ qnetd_client_msg_received_echo_request(struct qnetd_instance *instance, struct q
 	if (msg_create_echo_reply(&send_buffer->buffer, msg_orig) == -1) {
 		qnetd_log(LOG_ERR, "Can't alloc echo reply msg. Disconnecting client connection.");
 
+		send_buffer_list_discard_new(&client->send_buffer_list, send_buffer);
+
 		return (-1);
 	}
 
@@ -722,6 +730,8 @@ qnetd_client_msg_received_node_list(struct qnetd_instance *instance, struct qnet
 		qnetd_log(LOG_ERR, "Can't alloc node list reply msg. "
 		    "Disconnecting client connection.");
 
+		send_buffer_list_discard_new(&client->send_buffer_list, send_buffer);
+
 		return (-1);
 	}
 
@@ -809,6 +819,8 @@ qnetd_client_msg_received_ask_for_vote(struct qnetd_instance *instance, struct q
 		qnetd_log(LOG_ERR, "Can't alloc ask for vote reply msg. "
 		    "Disconnecting client connection.");
 
+		send_buffer_list_discard_new(&client->send_buffer_list, send_buffer);
+
 		return (-1);
 	}
 

qdevices/qnetd-client-net.c

@@ -189,7 +189,7 @@ qnetd_client_net_accept(struct qnetd_instance *instance)
 		return (-1);
 	}
 
-	if (nss_sock_set_nonblocking(client_socket) != 0) {
+	if (nss_sock_set_non_blocking(client_socket) != 0) {
 		qnetd_log_nss(LOG_ERR, "Can't set client socket to non blocking mode");
 
 		PR_Close(client_socket);

qdevices/qnetd-client.h

@@ -56,9 +56,9 @@ struct qnetd_client {
 	struct dynar receive_buffer;
 	struct send_buffer_list send_buffer_list;
 	size_t msg_already_received_bytes;
-	int skipping_msg;	// When incorrect message was received skip it
-	int tls_started;	// Set after TLS started
-	int tls_peer_certificate_verified;	// Certificate is verified only once
+	int skipping_msg;	/* When incorrect message was received skip it */
+	int tls_started;	/* Set after TLS started */
+	int tls_peer_certificate_verified;	/* Certificate is verified only once */
 	int preinit_received;
 	int init_received;
 	char *cluster_name;

qdevices/qnetd-instance.c

@@ -93,7 +93,9 @@ qnetd_instance_client_disconnect(struct qnetd_instance *instance, struct qnetd_c
 
 	qnetd_log_debug_client_disconnect(client, server_going_down);
 
-	qnetd_algorithm_client_disconnect(client, server_going_down);
+	if (client->init_received) {
+		qnetd_algorithm_client_disconnect(client, server_going_down);
+	}
 
 	PR_Close(client->socket);
 	if (client->cluster != NULL) {

qdevices/qnetd-log-debug.c

@@ -128,9 +128,9 @@ void
 qnetd_log_debug_client_disconnect(struct qnetd_client *client, int server_going_down)
 {
 
-	qnetd_log(LOG_DEBUG, "Client %p (cluster %s, node_id %"PRIx32
-		") disconnect%s", client, client->cluster_name, client->node_id,
-		(server_going_down ? " (server is going down)" : ""));
+	qnetd_log(LOG_DEBUG, "Client %p (init_received %u, cluster %s, node_id %"PRIx32
+		") disconnect%s", client, client->init_received, client->cluster_name,
+		client->node_id, (server_going_down ? " (server is going down)" : ""));
 }
 
 void

qdevices/send-buffer-list.c

@@ -96,6 +96,13 @@ send_buffer_list_put(struct send_buffer_list *sblist, struct send_buffer_list_en
 	TAILQ_INSERT_TAIL(&sblist->list, sblist_entry, entries);
 }
 
+void
+send_buffer_list_discard_new(struct send_buffer_list *sblist, struct send_buffer_list_entry *sblist_entry)
+{
+
+	TAILQ_INSERT_HEAD(&sblist->free_list, sblist_entry, entries);
+}
+
 struct send_buffer_list_entry *
 send_buffer_list_get_active(const struct send_buffer_list *sblist)
 {

qdevices/send-buffer-list.h

@@ -68,6 +68,9 @@ extern struct send_buffer_list_entry	*send_buffer_list_get_new(struct send_buffe
 extern void				 send_buffer_list_put(struct send_buffer_list *sblist,
     struct send_buffer_list_entry *sblist_entry);
 
+extern void				 send_buffer_list_discard_new(
+    struct send_buffer_list *sblist, struct send_buffer_list_entry *sblist_entry);
+
 extern struct send_buffer_list_entry	*send_buffer_list_get_active(
     const struct send_buffer_list *sblist);
 

qdevices/tlv.h

@@ -146,8 +146,8 @@ struct tlv_tie_breaker {
 
 struct tlv_node_info {
 	uint32_t node_id;
-	uint32_t data_center_id;		// 0 - data center id was not set
-	enum tlv_node_state node_state;		// TLV_NODE_STATE_NOT_SET - state was not set
+	uint32_t data_center_id;		/* 0 - data center id was not set */
+	enum tlv_node_state node_state;		/* TLV_NODE_STATE_NOT_SET - state was not set */
 };
 
 struct tlv_iterator {