corosync/lib/quorum.c

/*
 * Copyright (c) 2008-2020 Red Hat, Inc.
 *
 * All rights reserved.
 *
 * Author: Christine Caulfield (ccaulfie@redhat.com)
 *
 * This software licensed under BSD license, the text of which follows:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 * - Neither the name of the MontaVista Software, Inc. nor the names of its
 *   contributors may be used to endorse or promote products derived from this
 *   software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Provides a quorum API using the corosync executive
 */

#include <config.h>

#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <errno.h>

#include <qb/qbipcc.h>
#include <corosync/corotypes.h>
#include <corosync/corodefs.h>
#include <corosync/hdb.h>

#include <corosync/quorum.h>
#include <corosync/ipc_quorum.h>

#include "util.h"

struct quorum_inst {
	qb_ipcc_connection_t *c;
	int finalize;
	const void *context;
	union {
		quorum_model_data_t model_data;
		quorum_model_v0_data_t model_v0_data;
		quorum_model_v1_data_t model_v1_data;
	};
};

static void quorum_inst_free (void *inst);

DECLARE_HDB_DATABASE(quorum_handle_t_db, quorum_inst_free);

cs_error_t quorum_initialize (
	quorum_handle_t *handle,
	quorum_callbacks_t *callbacks,
	uint32_t *quorum_type)
{
	quorum_model_v0_data_t model_v0_data;

	memset (&model_v0_data, 0, sizeof(quorum_model_v0_data_t));

	if (callbacks) {
		model_v0_data.quorum_notify_fn = callbacks->quorum_notify_fn;
	}

	return (quorum_model_initialize(handle, QUORUM_MODEL_V0,
	    (quorum_model_data_t *)&model_v0_data, quorum_type, NULL));
}

cs_error_t quorum_model_initialize (
	quorum_handle_t *handle,
	quorum_model_t model,
	quorum_model_data_t *model_data,
	uint32_t *quorum_type,
	void *context)
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;
	struct iovec iov;
	struct qb_ipc_request_header quorum_gettype_req;
	struct req_lib_quorum_model_gettype quorum_model_gettype_req;
	struct res_lib_quorum_gettype res_lib_quorum_gettype;
	struct res_lib_quorum_model_gettype res_lib_quorum_model_gettype;
	uint32_t local_quorum_type;

	if (model != QUORUM_MODEL_V0 && model != QUORUM_MODEL_V1) {
		error = CS_ERR_INVALID_PARAM;
		goto error_no_destroy;
	}

	error = hdb_error_to_cs(hdb_handle_create (&quorum_handle_t_db, sizeof (struct quorum_inst), handle));
	if (error != CS_OK) {
		goto error_no_destroy;
	}

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, *handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		goto error_destroy;
	}

	error = CS_OK;
	quorum_inst->finalize = 0;
	quorum_inst->c = qb_ipcc_connect ("quorum", IPC_REQUEST_SIZE);
	if (quorum_inst->c == NULL) {
		error = qb_to_cs_error(-errno);
		goto error_put_destroy;
	}

	switch (model) {
	case QUORUM_MODEL_V0:
		quorum_gettype_req.size = sizeof (quorum_gettype_req);
		quorum_gettype_req.id = MESSAGE_REQ_QUORUM_GETTYPE;

		iov.iov_base = (char *)&quorum_gettype_req;
		iov.iov_len = sizeof (quorum_gettype_req);

		error = qb_to_cs_error(qb_ipcc_sendv_recv (
			quorum_inst->c,
			&iov,
			1,
			&res_lib_quorum_gettype,
			sizeof(res_lib_quorum_gettype), -1));

		if (error != CS_OK) {
			goto error_put_destroy;
		}
		error = res_lib_quorum_gettype.header.error;
		local_quorum_type = res_lib_quorum_gettype.quorum_type;
		break;
	case QUORUM_MODEL_V1:
		quorum_model_gettype_req.header.size = sizeof (quorum_model_gettype_req);
		quorum_model_gettype_req.header.id = MESSAGE_REQ_QUORUM_MODEL_GETTYPE;
		quorum_model_gettype_req.model = model;

		iov.iov_base = (char *)&quorum_model_gettype_req;
		iov.iov_len = sizeof (quorum_model_gettype_req);

		error = qb_to_cs_error(qb_ipcc_sendv_recv (
			quorum_inst->c,
			&iov,
			1,
			&res_lib_quorum_model_gettype,
			sizeof(res_lib_quorum_model_gettype), -1));

		if (error != CS_OK) {
			goto error_put_destroy;
		}
		error = res_lib_quorum_model_gettype.header.error;
		local_quorum_type = res_lib_quorum_model_gettype.quorum_type;
		break;
	}

	if (quorum_type != NULL) {
		*quorum_type = local_quorum_type;
	}

	if (model_data != NULL) {
		switch (model) {
		case QUORUM_MODEL_V0:
			memcpy(&quorum_inst->model_v0_data, model_data, sizeof(quorum_model_v0_data_t));
			break;
		case QUORUM_MODEL_V1:
			memcpy(&quorum_inst->model_v1_data, model_data, sizeof(quorum_model_v1_data_t));
			break;
		}
	}

	quorum_inst->model_data.model = model;
	quorum_inst->context = context;

	(void)hdb_handle_put (&quorum_handle_t_db, *handle);

	return (error);

error_put_destroy:
	(void)hdb_handle_put (&quorum_handle_t_db, *handle);
error_destroy:
	(void)hdb_handle_destroy (&quorum_handle_t_db, *handle);
error_no_destroy:
	return (error);
}

static void quorum_inst_free (void *inst)
{
	struct quorum_inst *quorum_inst = (struct quorum_inst *)inst;
	qb_ipcc_disconnect(quorum_inst->c);
}

cs_error_t quorum_finalize (
	quorum_handle_t handle)
{
	struct quorum_inst *quorum_inst;
	cs_error_t error;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	/*
	 * Another thread has already started finalizing
	 */
	if (quorum_inst->finalize) {
		(void)hdb_handle_put (&quorum_handle_t_db, handle);
		return (CS_ERR_BAD_HANDLE);
	}

	quorum_inst->finalize = 1;

	(void)hdb_handle_destroy (&quorum_handle_t_db, handle);

	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (CS_OK);
}

cs_error_t quorum_getquorate (
	quorum_handle_t handle,
	int *quorate)
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;
	struct iovec iov;
	struct qb_ipc_request_header req;
	struct res_lib_quorum_getquorate res_lib_quorum_getquorate;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	req.size = sizeof (req);
	req.id = MESSAGE_REQ_QUORUM_GETQUORATE;

	iov.iov_base = (char *)&req;
	iov.iov_len = sizeof (req);

	error = qb_to_cs_error(qb_ipcc_sendv_recv (
		quorum_inst->c,
		&iov,
		1,
		&res_lib_quorum_getquorate,
		sizeof (struct res_lib_quorum_getquorate), CS_IPC_TIMEOUT_MS));

	if (error != CS_OK) {
		goto error_exit;
	}

	error = res_lib_quorum_getquorate.header.error;

	*quorate = res_lib_quorum_getquorate.quorate;

error_exit:
	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (error);
}

cs_error_t quorum_fd_get (
	quorum_handle_t handle,
	int *fd)
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	error = qb_to_cs_error(qb_ipcc_fd_get (quorum_inst->c, fd));

	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (error);
}


cs_error_t quorum_context_get (
	quorum_handle_t handle,
	const void **context)
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	*context = quorum_inst->context;

	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (CS_OK);
}

cs_error_t quorum_context_set (
	quorum_handle_t handle,
	const void *context)
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	quorum_inst->context = context;

	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (CS_OK);
}


cs_error_t quorum_trackstart (
	quorum_handle_t handle,
	unsigned int flags )
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;
	struct iovec iov;
	struct req_lib_quorum_trackstart req_lib_quorum_trackstart;
	struct qb_ipc_response_header res;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	req_lib_quorum_trackstart.header.size = sizeof (struct req_lib_quorum_trackstart);
	req_lib_quorum_trackstart.header.id = MESSAGE_REQ_QUORUM_TRACKSTART;
	req_lib_quorum_trackstart.track_flags = flags;

	iov.iov_base = (char *)&req_lib_quorum_trackstart;
	iov.iov_len = sizeof (struct req_lib_quorum_trackstart);

       error = qb_to_cs_error(qb_ipcc_sendv_recv (
		quorum_inst->c,
                &iov,
                1,
                &res,
                sizeof (res), CS_IPC_TIMEOUT_MS));

	if (error != CS_OK) {
		goto error_exit;
	}

	error = res.error;

error_exit:
	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (error);
}

cs_error_t quorum_trackstop (
	quorum_handle_t handle)
{
	cs_error_t error;
	struct quorum_inst *quorum_inst;
	struct iovec iov;
	struct qb_ipc_request_header req;
	struct qb_ipc_response_header res;

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	req.size = sizeof (req);
	req.id = MESSAGE_REQ_QUORUM_TRACKSTOP;

	iov.iov_base = (char *)&req;
	iov.iov_len = sizeof (req);

       error = qb_to_cs_error(qb_ipcc_sendv_recv (
		quorum_inst->c,
                &iov,
                1,
                &res,
                sizeof (res), CS_IPC_TIMEOUT_MS));

	if (error != CS_OK) {
		goto error_exit;
	}

	error = res.error;

error_exit:
	(void)hdb_handle_put (&quorum_handle_t_db, handle);

	return (error);
}

cs_error_t quorum_dispatch (
	quorum_handle_t handle,
	cs_dispatch_flags_t dispatch_types)
{
	int timeout = -1;
	cs_error_t error;
	int cont = 1; /* always continue do loop except when set to 0 */
	struct quorum_inst *quorum_inst;
	struct quorum_inst quorum_inst_copy;
	struct qb_ipc_response_header *dispatch_data;
	char dispatch_buf[IPC_DISPATCH_SIZE];
	struct res_lib_quorum_notification *res_lib_quorum_notification;
	struct res_lib_quorum_v1_quorum_notification *res_lib_quorum_v1_quorum_notification;
	struct res_lib_quorum_v1_nodelist_notification *res_lib_quorum_v1_nodelist_notification;
	struct quorum_ring_id ring_id;
	mar_uint32_t *joined_list;
	mar_uint32_t *left_list;

	if (dispatch_types != CS_DISPATCH_ONE &&
		dispatch_types != CS_DISPATCH_ALL &&
		dispatch_types != CS_DISPATCH_BLOCKING &&
		dispatch_types != CS_DISPATCH_ONE_NONBLOCKING) {

		return (CS_ERR_INVALID_PARAM);
	}

	error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle,
		(void *)&quorum_inst));
	if (error != CS_OK) {
		return (error);
	}

	/*
	 * Timeout instantly for CS_DISPATCH_ONE_NONBLOCKING or CS_DISPATCH_ALL and
	 * wait indefinately for CS_DISPATCH_ONE or CS_DISPATCH_BLOCKING
	 */
	if (dispatch_types == CS_DISPATCH_ALL || dispatch_types == CS_DISPATCH_ONE_NONBLOCKING) {
		timeout = 0;
	}

	dispatch_data = (struct qb_ipc_response_header *)dispatch_buf;
	do {
		error = qb_to_cs_error (qb_ipcc_event_recv (
			quorum_inst->c,
			dispatch_buf,
			IPC_DISPATCH_SIZE,
			timeout));
		if (error == CS_ERR_BAD_HANDLE) {
			error = CS_OK;
			goto error_put;
		}
		if (error == CS_ERR_TRY_AGAIN) {
			if (dispatch_types == CS_DISPATCH_ONE_NONBLOCKING) {
				/*
				 * Don't mask error
				 */
				goto error_put;
			}
			error = CS_OK;
			if (dispatch_types == CS_DISPATCH_ALL) {
				break; /* exit do while cont is 1 loop */
			} else {
				continue; /* next poll */
			}
		}
		if (error != CS_OK) {
			goto error_put;
		}

		/*
		 * Make copy of callbacks, message data, unlock instance, and call callback
		 * A risk of this dispatch method is that the callback routines may
		 * operate at the same time that quorum_finalize has been called in another thread.
		 */
		memcpy (&quorum_inst_copy, quorum_inst, sizeof(quorum_inst_copy));
		switch (quorum_inst_copy.model_data.model) {
		case QUORUM_MODEL_V0:
			/*
			 * Dispatch incoming message
			 */
			switch (dispatch_data->id) {
			case MESSAGE_RES_QUORUM_NOTIFICATION:
				if (quorum_inst_copy.model_v0_data.quorum_notify_fn == NULL) {
					break;
				}
				res_lib_quorum_notification = (struct res_lib_quorum_notification *)dispatch_data;

				quorum_inst_copy.model_v0_data.quorum_notify_fn ( handle,
					res_lib_quorum_notification->quorate,
					res_lib_quorum_notification->ring_seq,
					res_lib_quorum_notification->view_list_entries,
					res_lib_quorum_notification->view_list);
				break;
			default:
				error = CS_ERR_LIBRARY;
				goto error_put;
				break;
			} /* switch (dispatch_data->id) */
			break; /* case QUORUM_MODEL_V0 */
		case QUORUM_MODEL_V1:
			/*
			 * Dispatch incoming message
			 */
			switch (dispatch_data->id) {
			case MESSAGE_RES_QUORUM_V1_QUORUM_NOTIFICATION:
				if (quorum_inst_copy.model_v1_data.quorum_notify_fn == NULL) {
					break;
				}
				res_lib_quorum_v1_quorum_notification =
					(struct res_lib_quorum_v1_quorum_notification *)dispatch_data;

				ring_id.nodeid = res_lib_quorum_v1_quorum_notification->ring_id.nodeid;
				ring_id.seq = res_lib_quorum_v1_quorum_notification->ring_id.seq;

				quorum_inst_copy.model_v1_data.quorum_notify_fn ( handle,
					res_lib_quorum_v1_quorum_notification->quorate,
					ring_id,
					res_lib_quorum_v1_quorum_notification->view_list_entries,
					res_lib_quorum_v1_quorum_notification->view_list);
				break;
			case MESSAGE_RES_QUORUM_V1_NODELIST_NOTIFICATION:
				if (quorum_inst_copy.model_v1_data.nodelist_notify_fn == NULL) {
					break;
				}
				res_lib_quorum_v1_nodelist_notification =
					(struct res_lib_quorum_v1_nodelist_notification *)dispatch_data;

				ring_id.nodeid = res_lib_quorum_v1_nodelist_notification->ring_id.nodeid;
				ring_id.seq = res_lib_quorum_v1_nodelist_notification->ring_id.seq;

				joined_list = res_lib_quorum_v1_nodelist_notification->member_list +
				    res_lib_quorum_v1_nodelist_notification->member_list_entries;
				left_list = joined_list +
				    res_lib_quorum_v1_nodelist_notification->joined_list_entries;

				quorum_inst_copy.model_v1_data.nodelist_notify_fn ( handle,
					ring_id,
					res_lib_quorum_v1_nodelist_notification->member_list_entries,
					res_lib_quorum_v1_nodelist_notification->member_list,
					res_lib_quorum_v1_nodelist_notification->joined_list_entries,
					joined_list,
					res_lib_quorum_v1_nodelist_notification->left_list_entries,
					left_list);
				break;
			default:
				error = CS_ERR_LIBRARY;
				goto error_put;
				break;
			} /* switch (dispatch_data->id) */
			break; /* case QUORUM_MODEL_V1 */
		}
		if (quorum_inst->finalize) {
			/*
			 * If the finalize has been called then get out of the dispatch.
			 */
			error = CS_ERR_BAD_HANDLE;
			goto error_put;
		}

		/*
		 * Determine if more messages should be processed
		 */
		if (dispatch_types == CS_DISPATCH_ONE || dispatch_types == CS_DISPATCH_ONE_NONBLOCKING) {
			cont = 0;
		}
	} while (cont);

error_put:
	(void)hdb_handle_put (&quorum_handle_t_db, handle);
	return (error);
}