corosync-qdevice/qdevices/qdevice-net-instance.c

/*
 * Copyright (c) 2015-2019 Red Hat, Inc.
 *
 * All rights reserved.
 *
 * Author: Jan Friesse (jfriesse@redhat.com)
 *
 * This software licensed under BSD license, the text of which follows:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * - Neither the name of the Red Hat, Inc. nor the names of its
 *   contributors may be used to endorse or promote products derived from this
 *   software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <limits.h>

#include "qdevice-config.h"
#include "qdevice-log.h"
#include "qdevice-net-instance.h"
#include "qnet-config.h"
#include "utils.h"
#include "qdevice-net-poll-array-user-data.h"
#include "qdevice-ipc.h"

/*
 * Needed for creating nspr handle from unix fd
 */
#include <private/pprio.h>

int
qdevice_net_instance_init(struct qdevice_net_instance *instance,
    enum tlv_tls_supported tls_supported,
    enum tlv_decision_algorithm_type decision_algorithm, uint32_t heartbeat_interval,
    uint32_t sync_heartbeat_interval, uint32_t cast_vote_timer_interval,
    const char *host_addr, uint16_t host_port, const char *cluster_name,
    const struct tlv_tie_breaker *tie_breaker, uint32_t connect_timeout,
    int force_ip_version, int cmap_fd, int votequorum_fd, int local_socket_fd,
    const struct qdevice_advanced_settings *advanced_settings,
    int heuristics_pipe_cmd_send_fd, int heuristics_pipe_cmd_recv_fd,
    int heuristics_pipe_log_recv_fd)
{

	memset(instance, 0, sizeof(*instance));

	instance->advanced_settings = advanced_settings;
	instance->decision_algorithm = decision_algorithm;
	instance->heartbeat_interval = heartbeat_interval;
	instance->sync_heartbeat_interval = sync_heartbeat_interval;
	instance->cast_vote_timer_interval = cast_vote_timer_interval;
	instance->cast_vote_timer = NULL;
	instance->host_addr = host_addr;
	instance->host_port = host_port;
	instance->cluster_name = cluster_name;
	instance->connect_timeout = connect_timeout;
	instance->last_msg_seq_num = 1;
	instance->echo_request_expected_msg_seq_num = 1;
	instance->echo_reply_received_msg_seq_num = 1;
	instance->force_ip_version = force_ip_version;
	instance->last_echo_reply_received_time = ((time_t) -1);
	instance->connected_since_time = ((time_t) -1);

	memcpy(&instance->tie_breaker, tie_breaker, sizeof(*tie_breaker));

	dynar_init(&instance->receive_buffer, advanced_settings->net_initial_msg_receive_size);

	send_buffer_list_init(&instance->send_buffer_list, advanced_settings->net_max_send_buffers,
	    advanced_settings->net_initial_msg_send_size);

	timer_list_init(&instance->main_timer_list);

	pr_poll_array_init(&instance->poll_array, sizeof(struct qdevice_net_poll_array_user_data));

	instance->tls_supported = tls_supported;

	if ((instance->cmap_poll_fd = PR_CreateSocketPollFd(cmap_fd)) == NULL) {
		log_nss(LOG_CRIT, "Can't create NSPR cmap poll fd");
		return (-1);
	}

	if ((instance->votequorum_poll_fd = PR_CreateSocketPollFd(votequorum_fd)) == NULL) {
		log_nss(LOG_CRIT, "Can't create NSPR votequorum poll fd");
		return (-1);
	}

	if ((instance->ipc_socket_poll_fd = PR_CreateSocketPollFd(local_socket_fd)) == NULL) {
		log_nss(LOG_CRIT, "Can't create NSPR IPC socket poll fd");
		return (-1);
	}

	if ((instance->heuristics_pipe_cmd_send_poll_fd =
	    PR_CreateSocketPollFd(heuristics_pipe_cmd_send_fd)) == NULL) {
		log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe command send poll fd");
		return (-1);
	}

	if ((instance->heuristics_pipe_cmd_recv_poll_fd =
	    PR_CreateSocketPollFd(heuristics_pipe_cmd_recv_fd)) == NULL) {
		log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe command recv poll fd");
		return (-1);
	}

	if ((instance->heuristics_pipe_log_recv_poll_fd =
	    PR_CreateSocketPollFd(heuristics_pipe_log_recv_fd)) == NULL) {
		log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe log recv poll fd");
		return (-1);
	}

	return (0);
}

void
qdevice_net_instance_clean(struct qdevice_net_instance *instance)
{

	dynar_clean(&instance->receive_buffer);

	send_buffer_list_free(&instance->send_buffer_list);

	instance->skipping_msg = 0;
	instance->msg_already_received_bytes = 0;
	instance->echo_request_expected_msg_seq_num = instance->echo_reply_received_msg_seq_num;
	instance->using_tls = 0;
	instance->tls_client_cert_sent = 0;
	instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT;

	instance->schedule_disconnect = 0;
	instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_UNDEFINED;
	instance->last_echo_reply_received_time = ((time_t) -1);
	instance->connected_since_time = ((time_t) -1);
}

int
qdevice_net_instance_destroy(struct qdevice_net_instance *instance)
{
	struct unix_socket_client *ipc_client;
	const struct unix_socket_client_list *ipc_client_list;
	struct qdevice_ipc_user_data *qdevice_ipc_user_data;
	PRFileDesc *prfd;

	ipc_client_list = &instance->qdevice_instance_ptr->local_ipc.clients;

	TAILQ_FOREACH(ipc_client, ipc_client_list, entries) {
		qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data;
		prfd = (PRFileDesc *)qdevice_ipc_user_data->model_data;

		if (PR_DestroySocketPollFd(prfd) != PR_SUCCESS) {
			log_nss(LOG_WARNING, "Unable to destroy client IPC poll socket fd");
		}
	}

	dynar_destroy(&instance->receive_buffer);

	send_buffer_list_free(&instance->send_buffer_list);

	pr_poll_array_destroy(&instance->poll_array);

	timer_list_free(&instance->main_timer_list);

	free((void *)instance->cluster_name);
	free((void *)instance->host_addr);

	if (PR_DestroySocketPollFd(instance->votequorum_poll_fd) != PR_SUCCESS) {
		log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
	}

	if (PR_DestroySocketPollFd(instance->cmap_poll_fd) != PR_SUCCESS) {
		log_nss(LOG_WARNING, "Unable to close votequorum connection fd");
	}

	if (PR_DestroySocketPollFd(instance->ipc_socket_poll_fd) != PR_SUCCESS) {
		log_nss(LOG_WARNING, "Unable to close local socket poll fd");
	}

	if (PR_DestroySocketPollFd(instance->heuristics_pipe_cmd_send_poll_fd) != PR_SUCCESS) {
		log_nss(LOG_WARNING, "Unable to close heuristics pipe command send poll fd");
		return (-1);
	}

	if (PR_DestroySocketPollFd(instance->heuristics_pipe_cmd_recv_poll_fd) != PR_SUCCESS) {
		log_nss(LOG_WARNING, "Unable to close heuristics pipe command recv poll fd");
		return (-1);
	}

	if (PR_DestroySocketPollFd(instance->heuristics_pipe_log_recv_poll_fd) != PR_SUCCESS) {
		log_nss(LOG_WARNING, "Unable to close heuristics pipe log recv poll fd");
		return (-1);
	}

	return (0);
}

int
qdevice_net_instance_init_from_cmap(struct qdevice_instance *instance)
{
	char *str;
	cmap_handle_t cmap_handle;
	enum tlv_tls_supported tls_supported;
	int i;
	long long int lli;
	enum tlv_decision_algorithm_type decision_algorithm;
	struct tlv_tie_breaker tie_breaker;
	uint32_t heartbeat_interval;
	uint32_t sync_heartbeat_interval;
	uint32_t cast_vote_timer_interval;
	char *host_addr;
	int host_port;
	char *cluster_name;
	uint32_t connect_timeout;
	struct qdevice_net_instance *net_instance;
	int force_ip_version;

	cmap_handle = instance->cmap_handle;

	net_instance = malloc(sizeof(*net_instance));
	if (net_instance == NULL) {
		log(LOG_ERR, "Can't alloc qdevice_net_instance");
		return (-1);
	}

	/*
	 * Check tls
	 */
	tls_supported = QDEVICE_NET_DEFAULT_TLS_SUPPORTED;

	if (cmap_get_string(cmap_handle, "quorum.device.net.tls", &str) == CS_OK) {
		if ((i = utils_parse_bool_str(str)) == -1) {
			if (strcasecmp(str, "required") != 0) {
				free(str);
				log(LOG_ERR, "quorum.device.net.tls value is not valid.");

				goto error_free_instance;
			} else {
				tls_supported = TLV_TLS_REQUIRED;
			}
		} else {
			if (i == 1) {
				tls_supported = TLV_TLS_SUPPORTED;
			} else {
				tls_supported = TLV_TLS_UNSUPPORTED;
			}
		}

		free(str);
	}

	/*
	 * Host
	 */
	if (cmap_get_string(cmap_handle, "quorum.device.net.host", &str) != CS_OK) {
		log(LOG_ERR, "Qdevice net daemon address is not defined (quorum.device.net.host)");
		goto error_free_instance;
	}
	host_addr = str;

	if (cmap_get_string(cmap_handle, "quorum.device.net.port", &str) == CS_OK) {
		if (utils_strtonum(str, 1, UINT16_MAX, &lli) == -1) {
			log(LOG_ERR, "quorum.device.net.port must be in range 1-%u", UINT16_MAX);
			free(str);
			goto error_free_host_addr;
		}
		free(str);

		host_port = lli;
	} else {
		host_port = QNETD_DEFAULT_HOST_PORT;
	}

	/*
	 * Cluster name
	 */
	if (cmap_get_string(cmap_handle, "totem.cluster_name", &str) != CS_OK) {
		log(LOG_ERR, "Cluster name (totem.cluster_name) has to be defined.");
		goto error_free_host_addr;
	}
	cluster_name = str;

	/*
	 * Adjust qdevice timeouts to better suit qnetd
	 */
	cast_vote_timer_interval = instance->heartbeat_interval * 0.5;
	heartbeat_interval = instance->heartbeat_interval * 0.8;
	if (heartbeat_interval < instance->advanced_settings->net_heartbeat_interval_min) {
		log(LOG_WARNING, "Heartbeat interval too small %"PRIu32". Adjusting to %"PRIu32".",
		    heartbeat_interval, instance->advanced_settings->net_heartbeat_interval_min);
		heartbeat_interval = instance->advanced_settings->net_heartbeat_interval_min;
	}
	if (heartbeat_interval > instance->advanced_settings->net_heartbeat_interval_max) {
		log(LOG_WARNING, "Heartbeat interval too big %"PRIu32". Adjusting to %"PRIu32".",
		    heartbeat_interval, instance->advanced_settings->net_heartbeat_interval_max);
		heartbeat_interval = instance->advanced_settings->net_heartbeat_interval_max;
	}
	sync_heartbeat_interval = instance->sync_heartbeat_interval * 0.8;

	/*
	 * Choose decision algorithm
	 */
	if (cmap_get_string(cmap_handle, "quorum.device.net.algorithm", &str) != CS_OK) {
		decision_algorithm = QDEVICE_NET_DEFAULT_ALGORITHM;
	} else {
		if (strcmp(str, "test") == 0) {
			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_TEST;
		} else if (strcmp(str, "ffsplit") == 0) {
			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_FFSPLIT;
		} else if (strcmp(str, "2nodelms") == 0) {
			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_2NODELMS;
		} else if (strcmp(str, "lms") == 0) {
			decision_algorithm = TLV_DECISION_ALGORITHM_TYPE_LMS;
		} else {
			log(LOG_ERR, "Unknown decision algorithm %s", str);
			free(str);
			goto error_free_cluster_name;
		}

		free(str);
	}

	if (decision_algorithm == TLV_DECISION_ALGORITHM_TYPE_TEST &&
	    !instance->advanced_settings->net_test_algorithm_enabled) {
		log(LOG_ERR, "Test algorithm is not enabled. You can force enable it by "
		    "passing -S net_test_algorithm_enabled=on to %s command", QDEVICE_PROGRAM_NAME);

		goto error_free_cluster_name;
	}
	/*
	 * Load tie_breaker mode
	 */
	memset(&tie_breaker, 0, sizeof(tie_breaker));

	if (cmap_get_string(cmap_handle, "quorum.device.net.tie_breaker", &str) != CS_OK) {
		tie_breaker.mode = QDEVICE_NET_DEFAULT_TIE_BREAKER_MODE;
	} else {
		if (strcmp(str, "lowest") == 0) {
			tie_breaker.mode = TLV_TIE_BREAKER_MODE_LOWEST;
		} else if (strcmp(str, "highest") == 0) {
			tie_breaker.mode = TLV_TIE_BREAKER_MODE_HIGHEST;
		} else {
			if (utils_strtonum(str, 1, UINT32_MAX, &lli) == -1) {
				log(LOG_ERR, "tie_breaker must be lowest|highest|valid_node_id");
				free(str);
				goto error_free_cluster_name;
			}

			tie_breaker.mode = TLV_TIE_BREAKER_MODE_NODE_ID;
			tie_breaker.node_id = lli;
		}

		free(str);
	}

	/*
	 * Get connect timeout
	 */
	if (cmap_get_string(cmap_handle, "quorum.device.net.connect_timeout", &str) != CS_OK) {
		connect_timeout = heartbeat_interval;
	} else {
		if (utils_strtonum(str, instance->advanced_settings->net_min_connect_timeout,
		    instance->advanced_settings->net_max_connect_timeout, &lli) == -1) {
			log(LOG_ERR, "connect_timeout must be valid number in "
			    "range <%"PRIu32",%"PRIu32">",
			    instance->advanced_settings->net_min_connect_timeout,
			    instance->advanced_settings->net_max_connect_timeout);
			free(str);
			goto error_free_cluster_name;
		}

		connect_timeout = (uint32_t)lli;

		free(str);
	}

	if (cmap_get_string(cmap_handle, "quorum.device.net.force_ip_version", &str) != CS_OK) {
		force_ip_version = 0;
	} else {
		if ((utils_strtonum(str, 0, 6, &lli) == -1) ||
		    (lli != 0 && lli != 4 && lli != 6)) {
			log(LOG_ERR, "force_ip_version must be one of 0|4|6");
			free(str);
			goto error_free_cluster_name;
		}

		force_ip_version = lli;

		free(str);
	}

	/*
	 * Really initialize instance
	 */
	if (qdevice_net_instance_init(net_instance,
	    tls_supported, decision_algorithm,
	    heartbeat_interval, sync_heartbeat_interval, cast_vote_timer_interval,
	    host_addr, host_port, cluster_name, &tie_breaker, connect_timeout,
	    force_ip_version,
	    instance->cmap_poll_fd, instance->votequorum_poll_fd,
	    instance->local_ipc.socket, instance->advanced_settings,
	    instance->heuristics_instance.pipe_cmd_send,
	    instance->heuristics_instance.pipe_cmd_recv,
	    instance->heuristics_instance.pipe_log_recv) == -1) {
		log(LOG_ERR, "Can't initialize qdevice-net instance");
		goto error_free_instance;
	}

	net_instance->qdevice_instance_ptr = instance;
	instance->model_data = net_instance;

	return (0);

error_free_cluster_name:
	free(cluster_name);
error_free_host_addr:
	free(host_addr);
error_free_instance:
	free(net_instance);
	return (-1);
}