corosync-qdevice/qdevices/qdevice-net-heuristics.c

/*
 * Copyright (c) 2017-2020 Red Hat, Inc.
 *
 * All rights reserved.
 *
 * Author: Jan Friesse (jfriesse@redhat.com)
 *
 * This software licensed under BSD license, the text of which follows:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * - Neither the name of the Red Hat, Inc. nor the names of its
 *   contributors may be used to endorse or promote products derived from this
 *   software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "log.h"
#include "qdevice-net-algorithm.h"
#include "qdevice-net-cast-vote-timer.h"
#include "qdevice-net-heuristics.h"
#include "qdevice-net-send.h"
#include "qdevice-net-votequorum.h"

enum tlv_heuristics
qdevice_net_heuristics_exec_result_to_tlv(enum qdevice_heuristics_exec_result exec_result)
{
	enum tlv_heuristics res;

	switch (exec_result) {
	case QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED: res = TLV_HEURISTICS_UNDEFINED; break;
	case QDEVICE_HEURISTICS_EXEC_RESULT_PASS: res = TLV_HEURISTICS_PASS; break;
	case QDEVICE_HEURISTICS_EXEC_RESULT_FAIL: res = TLV_HEURISTICS_FAIL; break;
	default:
		log(LOG_ERR, "qdevice_net_heuristics_exec_result_to_tlv: Unhandled "
		    "heuristics exec result %s",
		    qdevice_heuristics_exec_result_to_str(exec_result));
		exit(EXIT_FAILURE);
		break;
	}

	return (res);
}

static int
qdevice_net_regular_heuristics_exec_result_callback(uint32_t seq_number,
    enum qdevice_heuristics_exec_result exec_result, void *user_data1, void *user_data2)
{
	struct qdevice_heuristics_instance *heuristics_instance;
	struct qdevice_instance *instance;
	struct qdevice_net_instance *net_instance;
	int send_msg;
	enum tlv_vote vote;
	enum tlv_heuristics heuristics;

	instance = (struct qdevice_instance *)user_data1;
	heuristics_instance = &instance->heuristics_instance;
	net_instance = instance->model_data;

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
		log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (0);
	}

	heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);

	if (exec_result == QDEVICE_HEURISTICS_EXEC_RESULT_DISABLED) {
		/*
		 * Can happen when user disables heuristics during runtime
		 */
		return (0);
	}

	if (net_instance->latest_heuristics_result != heuristics) {
		log(heuristics == TLV_HEURISTICS_PASS ? LOG_NOTICE : LOG_ERR,
		    "Heuristics result changed from %s to %s",
		    tlv_heuristics_to_str(net_instance->latest_heuristics_result),
		    tlv_heuristics_to_str(heuristics));

		if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS) {
			/*
			 * Not connected to qnetd
			 */
			send_msg = 0;
		} else {
			send_msg = 1;
		}

		vote = TLV_VOTE_NO_CHANGE;

		if (qdevice_net_algorithm_heuristics_change(net_instance, &heuristics, &send_msg,
		    &vote) == -1) {
			log(LOG_ERR, "Algorithm returned error. Disconnecting.");

			net_instance->disconnect_reason =
			    QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
			net_instance->schedule_disconnect = 1;

			return (0);
		} else {
			log(LOG_DEBUG, "Algorithm decided to %s message with heuristics result "
			    "%s and result vote is %s", (send_msg ? "send" : "not send"),
			    tlv_heuristics_to_str(heuristics), tlv_vote_to_str(vote));
		}

		if (send_msg) {
			if (heuristics == TLV_HEURISTICS_UNDEFINED) {
				log(LOG_ERR, "Inconsistent algorithm result. "
				    "It's not possible to send message with undefined heuristics. "
				    "Disconnecting.");

				net_instance->disconnect_reason =
				    QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_ERR;
				net_instance->schedule_disconnect = 1;

				return (0);
			}

			if (!net_instance->server_supports_heuristics) {
				log(LOG_ERR, "Server doesn't support heuristics. "
				    "Disconnecting.");

				net_instance->disconnect_reason =
				    QDEVICE_NET_DISCONNECT_REASON_SERVER_DOESNT_SUPPORT_REQUIRED_OPT;
				net_instance->schedule_disconnect = 1;

				return (0);
			}

			if (qdevice_net_send_heuristics_change(net_instance, heuristics) != 0) {
				net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
				net_instance->schedule_disconnect = 1;

				return (0);
			}
		}

		if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
			log(LOG_CRIT, "qdevice_net_heuristics_exec_result_callback "
			    "Can't update cast vote timer");

			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
			net_instance->schedule_disconnect = 1;

			return (0);
		}
	}

	net_instance->latest_regular_heuristics_result = heuristics;
	net_instance->latest_heuristics_result = heuristics;

	if (qdevice_net_heuristics_schedule_timer(net_instance) != 0) {
		return (0);
	}

	return (0);
}

static int
qdevice_net_connect_heuristics_exec_result_callback(uint32_t seq_number,
    enum qdevice_heuristics_exec_result exec_result, void *user_data1, void *user_data2)
{
	struct qdevice_heuristics_instance *heuristics_instance;
	struct qdevice_instance *instance;
	struct qdevice_net_instance *net_instance;
	enum tlv_vote vote;
	enum tlv_heuristics heuristics;
	int send_config_node_list;
	int send_membership_node_list;
	int send_quorum_node_list;
	struct tlv_ring_id tlv_rid;
	enum tlv_quorate quorate;

	instance = (struct qdevice_instance *)user_data1;
	heuristics_instance = &instance->heuristics_instance;
	net_instance = instance->model_data;

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_connect_heuristics_exec_result_callback, 0) != 0) {
		log(LOG_ERR, "Can't deactivate net connect heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (0);
	}

	if (net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY) {
		/*
		 * Not connected to qnetd -> heuristics will be called again on new connect
		 */
		log(LOG_DEBUG, "Received unexpected net connect heuristics in state %u",
		    net_instance->state);
		return (0);
	}

	heuristics = qdevice_net_heuristics_exec_result_to_tlv(exec_result);

	send_config_node_list = 1;
	send_membership_node_list = 1;
	send_quorum_node_list = 1;
	vote = TLV_VOTE_WAIT_FOR_REPLY;

	if (qdevice_net_algorithm_connected(net_instance, &heuristics, &send_config_node_list,
	    &send_membership_node_list, &send_quorum_node_list, &vote) != 0) {
		log(LOG_DEBUG, "Algorithm returned error. Disconnecting.");
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_ALGO_CONNECTED_ERR;
		return (0);
	} else {
		log(LOG_DEBUG, "Algorithm decided to %s config node list, %s membership "
		    "node list, %s quorum node list, heuristics is %s and result vote is %s",
		    (send_config_node_list ? "send" : "not send"),
		    (send_membership_node_list ? "send" : "not send"),
		    (send_quorum_node_list ? "send" : "not send"),
		    tlv_heuristics_to_str(heuristics),
		    tlv_vote_to_str(vote));
	}

	/*
	 * Inform qnetd about connection options
	 */
	if (net_instance->server_supports_keep_active_partition_tie_breaker) {
		if (qdevice_net_send_set_option(net_instance, 0, 0,
		    net_instance->server_supports_keep_active_partition_tie_breaker,
		    net_instance->keep_active_partition_tie_breaker) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	/*
	 * Now we can finally really send node list, votequorum node list and update timer
	 */
	if (send_config_node_list) {
		if (qdevice_net_send_config_node_list(net_instance,
		    &instance->config_node_list,
		    instance->config_node_list_version_set,
		    instance->config_node_list_version, 1) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	if (send_membership_node_list) {
		qdevice_net_votequorum_ring_id_to_tlv(&tlv_rid,
		    &instance->vq_node_list_ring_id);

		if (qdevice_net_send_membership_node_list(net_instance, &tlv_rid,
		    instance->vq_node_list_entries,
		    instance->vq_node_list,
		    heuristics) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	if (send_quorum_node_list) {
		quorate = (instance->vq_quorum_quorate ?
		    TLV_QUORATE_QUORATE : TLV_QUORATE_INQUORATE);

		if (qdevice_net_send_quorum_node_list(net_instance,
		    quorate,
		    instance->vq_quorum_node_list_entries,
		    instance->vq_quorum_node_list) != 0) {
			net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER;
			return (0);
		}
	}

	if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) {
		log(LOG_CRIT, "qdevice_net_connect_heuristics_exec_result_callback fatal error. "
		    " Can't update cast vote timer vote");
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER;
	}

	net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_VOTEQUORUM_CMAP_EVENTS;
	net_instance->connected_since_time = time(NULL);

	net_instance->latest_connect_heuristics_result = heuristics;
	net_instance->latest_heuristics_result = heuristics;

	return (0);
}

static int
qdevice_net_heuristics_timer_callback(void *data1, void *data2)
{
	struct qdevice_net_instance *net_instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	net_instance = (struct qdevice_net_instance *)data1;
	heuristics_instance = &net_instance->qdevice_instance_ptr->heuristics_instance;

	if (qdevice_heuristics_waiting_for_result(heuristics_instance)) {
		log(LOG_DEBUG, "Not executing regular heuristics because other heuristics is already running.");

		return (1);
	}

	net_instance->regular_heuristics_timer = NULL;

	log(LOG_DEBUG, "Executing regular heuristics.");

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_regular_heuristics_exec_result_callback, 1) != 0) {
		log(LOG_ERR, "Can't activate net regular heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (0);
	}

	if (qdevice_heuristics_exec(heuristics_instance,
	    net_instance->qdevice_instance_ptr->sync_in_progress) != 0) {
		log(LOG_ERR, "Can't execute regular heuristics.");

		net_instance->schedule_disconnect = 1;
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;

		return (0);
	}

	/*
	 * Do not schedule this callback again. It's going to be scheduled in the
	 * qdevice_net_heuristics_exec_result_callback
	 */
	return (0);
}

int
qdevice_net_heuristics_stop_timer(struct qdevice_net_instance *net_instance)
{
	struct qdevice_instance *instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	instance = net_instance->qdevice_instance_ptr;
	heuristics_instance = &instance->heuristics_instance;

	if (net_instance->regular_heuristics_timer != NULL) {
		log(LOG_DEBUG, "Regular heuristics timer stopped");

		timer_list_entry_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop),
		    net_instance->regular_heuristics_timer);
		net_instance->regular_heuristics_timer = NULL;

		if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
		    qdevice_net_regular_heuristics_exec_result_callback, 0) != 0) {
			log(LOG_ERR, "Can't deactivate net regular heuristics exec callback notifier");

			net_instance->disconnect_reason =
			    QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
			net_instance->schedule_disconnect = 1;
			return (-1);
		}
	}

	return (0);
}

int
qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance *net_instance)
{
	uint32_t interval;
	struct qdevice_instance *instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	instance = net_instance->qdevice_instance_ptr;
	heuristics_instance = &instance->heuristics_instance;

        if (heuristics_instance->mode != QDEVICE_HEURISTICS_MODE_ENABLED) {
		log(LOG_DEBUG, "Not scheduling heuristics timer because mode is not enabled");

		if (qdevice_net_heuristics_stop_timer(net_instance) != 0) {
			return (-1);
		}

		return (0);
        }

	if (net_instance->regular_heuristics_timer != NULL) {
		log(LOG_DEBUG, "Not scheduling heuristics timer because it is already scheduled");

		return (0);
	}

	interval = heuristics_instance->interval;

	log(LOG_DEBUG, "Scheduling next regular heuristics in %"PRIu32"ms", interval);

	net_instance->regular_heuristics_timer = timer_list_add(
		pr_poll_loop_get_timer_list(&instance->main_poll_loop),
		interval,
		qdevice_net_heuristics_timer_callback,
	        (void *)net_instance, NULL);

	if (net_instance->regular_heuristics_timer == NULL) {
		log(LOG_ERR, "Can't schedule regular heuristics.");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_HEURISTICS_TIMER;
		net_instance->schedule_disconnect = 1;
		return (-1);
	}

	return (0);
}

int
qdevice_net_heuristics_init(struct qdevice_net_instance *net_instance)
{

	if (qdevice_heuristics_result_notifier_list_add(
	    &net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
	    qdevice_net_regular_heuristics_exec_result_callback,
	    net_instance->qdevice_instance_ptr, NULL) == NULL) {
		log(LOG_ERR, "Can't add net regular heuristics exec callback into notifier");

		return (-1);
	}

	if (qdevice_heuristics_result_notifier_list_add(
	    &net_instance->qdevice_instance_ptr->heuristics_instance.exec_result_notifier_list,
	    qdevice_net_connect_heuristics_exec_result_callback,
	    net_instance->qdevice_instance_ptr, NULL) == NULL) {
		log(LOG_ERR, "Can't add net connect heuristics exec callback into notifier");

		return (-1);
	}

	return (0);
}

int
qdevice_net_heuristics_exec_after_connect(struct qdevice_net_instance *net_instance)
{
	struct qdevice_instance *instance;
	struct qdevice_heuristics_instance *heuristics_instance;

	instance = net_instance->qdevice_instance_ptr;
	heuristics_instance = &instance->heuristics_instance;

	log(LOG_DEBUG, "Executing after-connect heuristics.");

	if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list,
	    qdevice_net_connect_heuristics_exec_result_callback, 1) != 0) {
		log(LOG_ERR, "Can't activate net connect heuristics exec callback notifier");

		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ACTIVATE_HEURISTICS_RESULT_NOTIFIER;
		net_instance->schedule_disconnect = 1;

		return (-1);
	}

	if (qdevice_heuristics_exec(heuristics_instance,
	    instance->sync_in_progress) != 0) {
		log(LOG_ERR, "Can't execute connect heuristics.");

		net_instance->schedule_disconnect = 1;
		net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_START_HEURISTICS;

		return (-1);
	}

	return (0);
}