Use new APIs for handle reference counting.

(Logical change 1.21)


git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@52 fd59a12c-fef9-0310-b244-a6a79926bd2f

lib/ckpt.c

@@ -1,4 +1,3 @@
-
 /*
  * Copyright (c) 2002-2004 MontaVista Software, Inc.
  *
@@ -62,9 +61,9 @@ struct ckptInstance {
 	int fd;
 	struct queue inq;
 	SaCkptCallbacksT callbacks;
+	int finalize;
 	pthread_mutex_t mutex;
 };
-#define CKPTINSTANCE_MUTEX_OFFSET HANDLECONVERT_NOLOCKING
 
 struct ckptCheckpointInstance {
 	int fd;
@@ -72,9 +71,6 @@ struct ckptCheckpointInstance {
 	SaUint32T maxSectionIdSize;
 	pthread_mutex_t mutex;
 };
-//#define CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET offset_of(struct ckptCheckpointInstance, mutex)
-#define CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET_DEMO offset_of(struct ckptCheckpointInstance, mutex)
-#define CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET HANDLECONVERT_NOLOCKING
 
 struct ckptSectionIteratorInstance {
 	int fd;
@@ -83,37 +79,38 @@ struct ckptSectionIteratorInstance {
 	pthread_mutex_t mutex;
 };
 
-//#define CKPTSECTIONITERATORINSTANCE_MUTEX_OFFSET offset_of(struct ckptSectionIteratorInstance, mutex)
-#define CKPTSECTIONITERATORINSTANCE_MUTEX_OFFSET HANDLECONVERT_NOLOCKING
+void ckptHandleInstanceDestructor (void *instance);
+void checkpointHandleInstanceDestructor (void *instance);
+void ckptSectionIteratorHandleInstanceDestructor (void *instance);
 
 /*
  * All CKPT instances in this database
  */
 static struct saHandleDatabase ckptHandleDatabase = {
-    handleCount: 0,
-    handles: 0,
-    generation: 0,
-    mutex: PTHREAD_MUTEX_INITIALIZER
+	handleCount: 0,
+	handles: 0,
+	mutex: PTHREAD_MUTEX_INITIALIZER,
+	handleInstanceDestructor: ckptHandleInstanceDestructor
 };
 
 /*
  *  All Checkpoint instances in this database
  */
-static struct saHandleDatabase ckptCheckpointHandleDatabase = {
-    handleCount: 0,
-    handles: 0,
-    generation: 0,
-    mutex: PTHREAD_MUTEX_INITIALIZER
+static struct saHandleDatabase checkpointHandleDatabase = {
+	handleCount: 0,
+	handles: 0,
+	mutex: PTHREAD_MUTEX_INITIALIZER,
+	handleInstanceDestructor: checkpointHandleInstanceDestructor
 };
 
 /*
  * All section iterators in this database
  */
 static struct saHandleDatabase ckptSectionIteratorHandleDatabase = {
-    handleCount: 0,
-    handles: 0,
-    generation: 0,
-    mutex: PTHREAD_MUTEX_INITIALIZER
+	handleCount: 0,
+	handles: 0,
+	mutex: PTHREAD_MUTEX_INITIALIZER,
+	handleInstanceDestructor: ckptSectionIteratorHandleInstanceDestructor
 };
 
 /*
@@ -130,13 +127,44 @@ static struct saVersionDatabase ckptVersionDatabase = {
 };
 
 
-//static struct timeval zerousec = {
-//	0, 0
-//};
-
 /*
  * Implementation
  */
+void ckptHandleInstanceDestructor (void *instance)
+{
+struct ckptInstance *ckptInstance = (struct ckptInstance *)instance;
+
+	if (ckptInstance->fd != -1) {
+		shutdown (ckptInstance->fd, 0);
+		close (ckptInstance->fd);
+	}
+	if (ckptInstance->inq.items) {
+		free (ckptInstance->inq.items);
+	}
+}
+
+void checkpointHandleInstanceDestructor (void *instance)
+{
+	struct ckptCheckpointInstance *ckptCheckpointInstance = (struct ckptCheckpointInstance *)instance;
+
+	if (ckptCheckpointInstance->fd != -1) {
+		shutdown (ckptCheckpointInstance->fd, 0);
+
+		close (ckptCheckpointInstance->fd);
+	}
+}
+
+void ckptSectionIteratorHandleInstanceDestructor (void *instance)
+{
+	struct ckptSectionIteratorInstance *ckptSectionIteratorInstance = (struct ckptSectionIteratorInstance *)instance;
+
+	if (ckptSectionIteratorInstance->fd != -1) {
+		shutdown (ckptSectionIteratorInstance->fd, 0);
+
+		close (ckptSectionIteratorInstance->fd);
+	}
+}
+
 SaErrorT
 saCkptInitialize (
 	SaCkptHandleT *ckptHandle,
@@ -148,39 +176,50 @@ saCkptInitialize (
 
 	error = saVersionVerify (&ckptVersionDatabase, version);
 	if (error != SA_OK) {
-		goto error_nofree;
+		goto error_no_destroy;
+	}
+
+	error = saHandleCreate (&ckptHandleDatabase, sizeof (struct ckptInstance),
+		ckptHandle);
+	if (error != SA_OK) {
+		goto error_no_destroy;
 	}
 
-	error = saHandleCreate (&ckptHandleDatabase, (void *)&ckptInstance,
-		sizeof (struct ckptInstance), ckptHandle);
+	error = saHandleInstanceGet (&ckptHandleDatabase, *ckptHandle,
+		(void *)&ckptInstance);
 	if (error != SA_OK) {
-		goto error_nofree;
+		goto error_destroy;
 	}
 
+	ckptInstance->fd = -1;
+
 	/*
 	 * An inq is needed to store async messages while waiting for a 
 	 * sync response
 	 */
 	error = saQueueInit (&ckptInstance->inq, 512, sizeof (void *));
 	if (error != SA_OK) {
-		goto error_free;
+		goto error_put_destroy;
 	}
 
 	error = saServiceConnect (&ckptInstance->fd, MESSAGE_REQ_CKPT_CHECKPOINT_INIT);
 	if (error != SA_OK) {
-		goto error_free2;
+		goto error_put_destroy;
 	}
 
 	memcpy (&ckptInstance->callbacks, callbacks, sizeof (SaCkptCallbacksT));
 
 	pthread_mutex_init (&ckptInstance->mutex, NULL);
 
+	saHandleInstancePut (&ckptHandleDatabase, *ckptHandle);
+
 	return (SA_OK);
-error_free2:
-	free (ckptInstance->inq.items);
-error_free:
-	saHandleRemove (&ckptHandleDatabase, *ckptHandle);
-error_nofree:
+
+error_put_destroy:
+	saHandleInstancePut (&ckptHandleDatabase, *ckptHandle);
+error_destroy:
+	saHandleDestroy (&ckptHandleDatabase, *ckptHandle);
+error_no_destroy:
 	return (error);
 }
 
@@ -192,12 +231,15 @@ saCkptSelectionObjectGet (
 	struct ckptInstance *ckptInstance;
 	SaErrorT error;
 
-	error = saHandleConvert (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance, CKPTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
 	*selectionObject = ckptInstance->fd;
+
+	saHandleInstancePut (&ckptHandleDatabase, *ckptHandle);
+
 	return (SA_OK);
 }
 
@@ -218,7 +260,7 @@ saCkptDispatch (
 	int ignore_dispatch = 0;
 	int cont = 1; /* always continue do loop except when set to 0 */
 
-	error = saHandleConvert (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance, CKPTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
@@ -382,14 +424,33 @@ saCkptFinalize (
 	struct ckptInstance *ckptInstance;
 	SaErrorT error;
 
-	error = saHandleConvert (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance, CKPTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&ckptHandleDatabase, *ckptHandle,
+		(void *)&ckptInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
-	close (ckptInstance->fd);
-	free (ckptInstance->inq.items);
-	saHandleRemove (&ckptHandleDatabase, *ckptHandle);
+	pthread_mutex_lock (&ckptInstance->mutex);
+
+	/*
+	 * Another thread has already started finalizing
+	 */
+	if (ckptInstance->finalize) {
+		pthread_mutex_unlock (&ckptInstance->mutex);
+		saHandleInstancePut (&ckptHandleDatabase, *ckptHandle);
+		return (SA_ERR_BAD_HANDLE);
+	}
+
+	ckptInstance->finalize = 1;
+
+	saActivatePoll (ckptInstance->fd);
+
+	pthread_mutex_unlock (&ckptInstance->mutex);
+
+	saHandleInstancePut (&ckptHandleDatabase, *ckptHandle);
+
+    saHandleDestroy (&ckptHandleDatabase, *ckptHandle);
+
 	return (SA_OK);
 }
 
@@ -406,22 +467,28 @@ saCkptCheckpointOpen (
 	struct req_lib_ckpt_checkpointopen req_lib_ckpt_checkpointopen;
 	struct res_lib_ckpt_checkpointopen res_lib_ckpt_checkpointopen;
 
-	error = saHandleCreate (&ckptCheckpointHandleDatabase, (void *)&ckptCheckpointInstance,
+	error = saHandleCreate (&checkpointHandleDatabase,
 		sizeof (struct ckptCheckpointInstance), checkpointHandle);
 	if (error != SA_OK) {
-		goto error_nofree;
+		goto error_no_destroy;
+	}
+
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
+	if (error != SA_OK) {
+		goto error_destroy;
 	}
 
+	ckptCheckpointInstance->fd = -1;
+
 	ckptCheckpointInstance->maxSectionIdSize =
 		checkpointCreationAttributes->maxSectionIdSize;
 
 	error = saServiceConnect (&ckptCheckpointInstance->fd, MESSAGE_REQ_CKPT_CHECKPOINT_INIT);
 	if (error != SA_OK) {
-		goto error_free;
+		goto error_put_destroy;
 	}
 
-	pthread_mutex_init (&ckptCheckpointInstance->mutex, NULL);
-
 	req_lib_ckpt_checkpointopen.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_checkpointopen.header.size = sizeof (struct req_lib_ckpt_checkpointopen);
 	req_lib_ckpt_checkpointopen.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_CHECKPOINTOPEN;
@@ -435,28 +502,31 @@ saCkptCheckpointOpen (
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_checkpointopen,
 		sizeof (struct req_lib_ckpt_checkpointopen), MSG_NOSIGNAL);
 	if (error != SA_OK) {
-		goto error_close;
+		goto error_put_destroy;
 	}
 
 	error = saRecvRetry (ckptCheckpointInstance->fd, &res_lib_ckpt_checkpointopen,
 		sizeof (struct res_lib_ckpt_checkpointopen), MSG_WAITALL | MSG_NOSIGNAL);
 	if (error != SA_OK) {
-		goto error_close;
+		goto error_put_destroy;
 	}
 	
 	if (res_lib_ckpt_checkpointopen.error != SA_OK) {
 		error = res_lib_ckpt_checkpointopen.error;
-		goto error_close;
+		goto error_put_destroy;
 	}
 
 	pthread_mutex_init (&ckptCheckpointInstance->mutex, NULL);
 
-error_nofree:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error);
-error_close:
-	close (ckptCheckpointInstance->fd);
-error_free:
-	saHandleRemove (&ckptCheckpointHandleDatabase, *checkpointHandle);
+
+error_put_destroy:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+error_destroy:
+	saHandleDestroy (&checkpointHandleDatabase, *checkpointHandle);
+error_no_destroy:
 	return (error);
 }
 
@@ -472,7 +542,7 @@ saCkptCheckpointOpenAsync (
 	SaErrorT error;
 	struct req_lib_ckpt_checkpointopenasync req_lib_ckpt_checkpointopenasync;
 
-	error = saHandleConvert (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance, CKPTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&ckptHandleDatabase, *ckptHandle, (void *)&ckptInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
@@ -488,9 +558,15 @@ saCkptCheckpointOpenAsync (
 	
 	req_lib_ckpt_checkpointopenasync.checkpointOpenFlags = checkpointOpenFlags;
 
+	pthread_mutex_lock (&ckptInstance->mutex);
+
         error = saSendRetry (ckptInstance->fd, &req_lib_ckpt_checkpointopenasync,
 		sizeof (struct req_lib_ckpt_checkpointopenasync), MSG_NOSIGNAL);
 
+	pthread_mutex_unlock (&ckptInstance->mutex);
+
+	saHandleInstancePut (&ckptHandleDatabase, *ckptHandle);
+
 	return (error);
 }
 
@@ -501,14 +577,15 @@ saCkptCheckpointClose (
 	SaErrorT error;
 	struct ckptCheckpointInstance *ckptCheckpointInstance;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
 		goto error_exit;
 	}
 
-	close (ckptCheckpointInstance->fd);
-	error = saHandleRemove (&ckptCheckpointHandleDatabase, *checkpointHandle);
+    saHandleDestroy (&checkpointHandleDatabase, *checkpointHandle);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
 
 error_exit:
 	return (error);
@@ -558,7 +635,8 @@ saCkptCheckpointRetentionDurationSet (
 	struct ckptCheckpointInstance *ckptCheckpointInstance;
 	struct req_lib_ckpt_checkpointretentiondurationset req_lib_ckpt_checkpointretentiondurationset;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle, (void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
 		goto error_exit;
 	}
@@ -567,10 +645,13 @@ saCkptCheckpointRetentionDurationSet (
 	req_lib_ckpt_checkpointretentiondurationset.header.size = sizeof (struct req_lib_ckpt_checkpointretentiondurationset);
 	req_lib_ckpt_checkpointretentiondurationset.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_CHECKPOINTRETENTIONDURATIONSET;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_checkpointretentiondurationset, sizeof (struct req_lib_ckpt_checkpointretentiondurationset), MSG_NOSIGNAL);
-	if (error != SA_OK) {
-		goto error_exit;
-	}
+
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
 
 error_exit:
 	return (error);
@@ -585,7 +666,8 @@ saCkptActiveCheckpointSet (
 	struct req_lib_ckpt_activecheckpointset req_lib_ckpt_activecheckpointset;
 	struct res_lib_ckpt_activecheckpointset res_lib_ckpt_activecheckpointset;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle, (void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		 (void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
 		goto error_exit;
 	}
@@ -594,13 +676,21 @@ saCkptActiveCheckpointSet (
 	req_lib_ckpt_activecheckpointset.header.size = sizeof (struct req_lib_ckpt_activecheckpointset);
 	req_lib_ckpt_activecheckpointset.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_ACTIVECHECKPOINTSET;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_activecheckpointset,
 		sizeof (struct req_lib_ckpt_activecheckpointset), MSG_NOSIGNAL);
 	if (error != SA_OK) {
 		goto error_exit;
 	}
 
-	error = saRecvQueue (ckptCheckpointInstance->fd, &res_lib_ckpt_activecheckpointset, 0, MESSAGE_RES_CKPT_CHECKPOINT_ACTIVECHECKPOINTSET);
+	error = saRecvQueue (ckptCheckpointInstance->fd,
+		&res_lib_ckpt_activecheckpointset, 0,
+		MESSAGE_RES_CKPT_CHECKPOINT_ACTIVECHECKPOINTSET);
+
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
 
 error_exit:
 	return (error == SA_OK ? res_lib_ckpt_activecheckpointset.error : error);
@@ -616,16 +706,18 @@ saCkptCheckpointStatusGet (
 	struct req_lib_ckpt_checkpointstatusget req_lib_ckpt_checkpointstatusget;
 	struct res_lib_ckpt_checkpointstatusget res_lib_ckpt_checkpointstatusget;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
 	req_lib_ckpt_checkpointstatusget.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_checkpointstatusget.header.size = sizeof (struct req_lib_ckpt_checkpointstatusget);
 	req_lib_ckpt_checkpointstatusget.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_CHECKPOINTSTATUSGET;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_checkpointstatusget,
 		sizeof (struct req_lib_ckpt_checkpointstatusget), MSG_NOSIGNAL);
 	if (error != SA_OK) {
@@ -638,11 +730,14 @@ saCkptCheckpointStatusGet (
 		goto error_exit;
 	}
 
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
 	memcpy (checkpointStatus,
 		&res_lib_ckpt_checkpointstatusget.checkpointStatus,
 		sizeof (SaCkptCheckpointStatusT));
 
 error_exit:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
 	return (error);
 }
 
@@ -658,10 +753,10 @@ saCkptSectionCreate (
 	struct req_lib_ckpt_sectioncreate req_lib_ckpt_sectioncreate;
 	struct res_lib_ckpt_sectioncreate res_lib_ckpt_sectioncreate;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
 	req_lib_ckpt_sectioncreate.header.magic = MESSAGE_MAGIC;
@@ -677,6 +772,8 @@ saCkptSectionCreate (
 	req_lib_ckpt_sectioncreate.initialDataSize = initialDataSize;
 
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_sectioncreate,
 		sizeof (struct req_lib_ckpt_sectioncreate), MSG_NOSIGNAL);
 	if (error != SA_OK) {
@@ -698,9 +795,15 @@ saCkptSectionCreate (
 		goto error_exit;
 	}
 
-	error = saRecvQueue (ckptCheckpointInstance->fd, &res_lib_ckpt_sectioncreate, 0, MESSAGE_RES_CKPT_CHECKPOINT_SECTIONCREATE);
+	error = saRecvQueue (ckptCheckpointInstance->fd,
+		&res_lib_ckpt_sectioncreate, 0,
+		MESSAGE_RES_CKPT_CHECKPOINT_SECTIONCREATE);
+
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
 
 error_exit:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error == SA_OK ? res_lib_ckpt_sectioncreate.error : error);
 }
 
@@ -715,12 +818,14 @@ saCkptSectionDelete (
 	struct req_lib_ckpt_sectiondelete req_lib_ckpt_sectiondelete;
 	struct res_lib_ckpt_sectiondelete res_lib_ckpt_sectiondelete;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	req_lib_ckpt_sectiondelete.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_sectiondelete.header.size = sizeof (struct req_lib_ckpt_sectiondelete) + sectionId->idLen; 
 	req_lib_ckpt_sectiondelete.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_SECTIONDELETE;
@@ -740,9 +845,14 @@ saCkptSectionDelete (
 	if (error != SA_OK) {
 		goto error_exit;
 	}
-	error = saRecvQueue (ckptCheckpointInstance->fd, &res_lib_ckpt_sectiondelete, 0, MESSAGE_RES_CKPT_CHECKPOINT_SECTIONDELETE);
+	error = saRecvQueue (ckptCheckpointInstance->fd,
+		&res_lib_ckpt_sectiondelete, 0,
+		MESSAGE_RES_CKPT_CHECKPOINT_SECTIONDELETE);
+
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
 
 error_exit:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
 	return (error == SA_OK ? res_lib_ckpt_sectiondelete.error : error);
 }
 
@@ -757,8 +867,8 @@ saCkptSectionExpirationTimeSet (
 	struct req_lib_ckpt_sectionexpirationtimeset req_lib_ckpt_sectionexpirationtimeset;
 	struct res_lib_ckpt_sectionexpirationtimeset res_lib_ckpt_sectionexpirationtimeset;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
 		goto error_exit;
 	}
@@ -769,6 +879,8 @@ saCkptSectionExpirationTimeSet (
 	req_lib_ckpt_sectionexpirationtimeset.idLen = sectionId->idLen;
 	req_lib_ckpt_sectionexpirationtimeset.expirationTime = expirationTime;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_sectionexpirationtimeset,
 		sizeof (struct req_lib_ckpt_sectionexpirationtimeset), MSG_NOSIGNAL);
 	if (error != SA_OK) {
@@ -786,10 +898,14 @@ saCkptSectionExpirationTimeSet (
 		}
 	}
 
-	error = saRecvQueue (ckptCheckpointInstance->fd, &res_lib_ckpt_sectionexpirationtimeset,
+	error = saRecvQueue (ckptCheckpointInstance->fd,
+		&res_lib_ckpt_sectionexpirationtimeset,
 		0, MESSAGE_RES_CKPT_CHECKPOINT_SECTIONEXPIRATIONTIMESET);
 
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
 error_exit:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
 	return (error == SA_OK ? res_lib_ckpt_sectionexpirationtimeset.error : error);
 }
 
@@ -806,18 +922,29 @@ saCkptSectionIteratorInitialize (
 	struct req_lib_ckpt_sectioniteratorinitialize req_lib_ckpt_sectioniteratorinitialize;
 	struct res_lib_ckpt_sectioniteratorinitialize res_lib_ckpt_sectioniteratorinitialize;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		goto error_no_destroy;
 	}
 
 	error = saHandleCreate (&ckptSectionIteratorHandleDatabase,
-		(void *)&ckptSectionIteratorInstance,
 		sizeof (struct ckptSectionIteratorInstance), sectionIterator);
 	if (error != SA_OK) {
-		goto error_exit;
+		goto error_put_checkpoint_db;
 	}
+
+	error = saHandleInstanceGet (&ckptSectionIteratorHandleDatabase,
+		*sectionIterator,
+		(void *)&ckptSectionIteratorInstance);
+	if (error != SA_OK) {
+		goto error_destroy;
+	}
+
+	ckptSectionIteratorInstance->fd = -1;
+
+	pthread_mutex_init (&ckptSectionIteratorInstance->mutex, NULL);
+
 	/*
 	 * Setup section id list for iterator next
 	 */
@@ -829,11 +956,9 @@ saCkptSectionIteratorInitialize (
 	error = saServiceConnect (&ckptSectionIteratorInstance->fd,	
 		MESSAGE_REQ_CKPT_SECTIONITERATOR_INIT);
 	if (error != SA_OK) {
-		goto error_remove;
+		goto error_put_destroy;
 	}
 
-	pthread_mutex_init (&ckptSectionIteratorInstance->mutex, NULL);
-
 	req_lib_ckpt_sectioniteratorinitialize.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_sectioniteratorinitialize.header.size = sizeof (struct req_lib_ckpt_sectioniteratorinitialize); 
 	req_lib_ckpt_sectioniteratorinitialize.header.id = MESSAGE_REQ_CKPT_SECTIONITERATOR_SECTIONITERATORINITIALIZE;
@@ -842,23 +967,36 @@ saCkptSectionIteratorInitialize (
 	memcpy (&req_lib_ckpt_sectioniteratorinitialize.checkpointName,
 		&ckptCheckpointInstance->checkpointName, sizeof (SaNameT));
 
+	pthread_mutex_lock (&ckptSectionIteratorInstance->mutex);
+
 	error = saSendRetry (ckptSectionIteratorInstance->fd,
-		&req_lib_ckpt_sectioniteratorinitialize, sizeof (struct req_lib_ckpt_sectioniteratorinitialize), MSG_NOSIGNAL);
+		&req_lib_ckpt_sectioniteratorinitialize,
+		sizeof (struct req_lib_ckpt_sectioniteratorinitialize),
+		MSG_NOSIGNAL);
 
 	if (error != SA_OK) {
-		goto error_close;
+		goto error_put_destroy;
 	}
 
 	error = saRecvQueue (ckptSectionIteratorInstance->fd,
 		&res_lib_ckpt_sectioniteratorinitialize, 0,
 		MESSAGE_RES_CKPT_SECTIONITERATOR_SECTIONITERATORINITIALIZE);
 
-error_exit:
+	pthread_mutex_unlock (&ckptSectionIteratorInstance->mutex);
+
+	saHandleInstancePut (&ckptSectionIteratorHandleDatabase, *sectionIterator);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error == SA_OK ? res_lib_ckpt_sectioniteratorinitialize.error : error);
-error_close:
-	close (ckptSectionIteratorInstance->fd);
-error_remove:
-	saHandleRemove (&ckptSectionIteratorHandleDatabase, *sectionIterator);
+
+error_put_destroy:
+	saHandleInstancePut (&ckptSectionIteratorHandleDatabase, *sectionIterator);
+error_destroy:
+	saHandleDestroy (&ckptSectionIteratorHandleDatabase, *sectionIterator);
+error_put_checkpoint_db:
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+error_no_destroy:
 	return (error);
 }
 
@@ -878,9 +1016,8 @@ saCkptSectionIteratorNext (
 	struct res_lib_ckpt_sectioniteratornext res_lib_ckpt_sectioniteratornext;
 	struct iteratorSectionIdListEntry *iteratorSectionIdListEntry;
 
-	error = saHandleConvert (&ckptSectionIteratorHandleDatabase, *sectionIterator,
-		(void *)&ckptSectionIteratorInstance,
-		CKPTSECTIONITERATORINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&ckptSectionIteratorHandleDatabase,
+		*sectionIterator, (void *)&ckptSectionIteratorInstance);
 	if (error != SA_OK) {
 		goto error_exit;
 	}
@@ -891,25 +1028,27 @@ saCkptSectionIteratorNext (
 		ckptSectionIteratorInstance->maxSectionIdSize);
 	if (iteratorSectionIdListEntry == 0) {
 		error = SA_ERR_NO_MEMORY;
-		goto error_exit;
+		goto error_put_nounlock;
 	}
 
 	req_lib_ckpt_sectioniteratornext.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_sectioniteratornext.header.size = sizeof (struct req_lib_ckpt_sectioniteratornext); 
 	req_lib_ckpt_sectioniteratornext.header.id = MESSAGE_REQ_CKPT_SECTIONITERATOR_SECTIONITERATORNEXT;
 
+	pthread_mutex_lock (&ckptSectionIteratorInstance->mutex);
+
 	error = saSendRetry (ckptSectionIteratorInstance->fd,
 		&req_lib_ckpt_sectioniteratornext,
 		sizeof (struct req_lib_ckpt_sectioniteratornext), MSG_NOSIGNAL);
 
 	if (error != SA_OK) {
-		goto error_exit;
+		goto error_put;
 	}
 
 	error = saRecvRetry (ckptSectionIteratorInstance->fd, &res_lib_ckpt_sectioniteratornext,
 		sizeof (struct res_lib_ckpt_sectioniteratornext), MSG_WAITALL | MSG_NOSIGNAL);
 	if (error != SA_OK) {
-		goto error_exit;
+		goto error_put;
 	}
 
 	memcpy (sectionDescriptor,
@@ -934,6 +1073,10 @@ saCkptSectionIteratorNext (
 		list_add (&iteratorSectionIdListEntry->list, &ckptSectionIteratorInstance->sectionIdListHead);
 	}
 
+error_put:
+	pthread_mutex_unlock (&ckptSectionIteratorInstance->mutex);
+error_put_nounlock:
+	saHandleInstancePut (&ckptSectionIteratorHandleDatabase, *sectionIterator);
 error_exit:
 	return (error == SA_OK ? res_lib_ckpt_sectioniteratornext.error : error);
 }
@@ -948,15 +1091,12 @@ saCkptSectionIteratorFinalize (
 	struct list_head *sectionIdIteratorList;
 	struct list_head *sectionIdIteratorListNext;
 
-	error = saHandleConvert (&ckptSectionIteratorHandleDatabase, *sectionIterator,
-		(void *)&ckptSectionIteratorInstance,
-		CKPTSECTIONITERATORINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&ckptSectionIteratorHandleDatabase,
+		*sectionIterator, (void *)&ckptSectionIteratorInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		goto error_noput;
 	}
 
-	close (ckptSectionIteratorInstance->fd);
-
 	/*
 	 * iterate list of section ids for this iterator to free the allocated memory
 	 * be careful to cache next pointer because free removes memory from use
@@ -973,9 +1113,11 @@ saCkptSectionIteratorFinalize (
 		free (iteratorSectionIdListEntry);
 	}
 
-	saHandleRemove (&ckptSectionIteratorHandleDatabase, *sectionIterator);
+	saHandleInstancePut (&ckptSectionIteratorHandleDatabase, *sectionIterator);
 
-error_exit:
+    saHandleDestroy (&ckptSectionIteratorHandleDatabase, *sectionIterator);
+
+error_noput:
 	return (error);
 }
 
@@ -994,15 +1136,16 @@ saCkptCheckpointWrite (
 	struct iovec iov[3];
 	int iov_len = 0;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET_DEMO, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
 	req_lib_ckpt_sectionwrite.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_sectionwrite.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_SECTIONWRITE;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
 
 	for (i = 0; i < numberOfElements; i++) {
 
@@ -1049,7 +1192,11 @@ saCkptCheckpointWrite (
 	}
 
 error_exit:
+
 	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error == SA_OK ? res_lib_ckpt_sectionwrite.error : error);
 }
 
@@ -1065,10 +1212,10 @@ saCkptSectionOverwrite (
 	struct req_lib_ckpt_sectionoverwrite req_lib_ckpt_sectionoverwrite;
 	struct res_lib_ckpt_sectionoverwrite res_lib_ckpt_sectionoverwrite;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
 	req_lib_ckpt_sectionoverwrite.header.magic = MESSAGE_MAGIC;
@@ -1077,6 +1224,8 @@ saCkptSectionOverwrite (
 	req_lib_ckpt_sectionoverwrite.idLen = sectionId->idLen;
 	req_lib_ckpt_sectionoverwrite.dataSize = dataSize;
 	
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_sectionoverwrite,
 		sizeof (struct req_lib_ckpt_sectionoverwrite), MSG_NOSIGNAL);
 	if (error != SA_OK) {
@@ -1099,6 +1248,10 @@ saCkptSectionOverwrite (
 		&res_lib_ckpt_sectionoverwrite, 0, MESSAGE_RES_CKPT_CHECKPOINT_SECTIONOVERWRITE);
 
 error_exit:
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error == SA_OK ? res_lib_ckpt_sectionoverwrite.error : error);
 }
 
@@ -1117,15 +1270,17 @@ saCkptCheckpointRead (
 	int i;
 	struct iovec iov[3];
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET_DEMO, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
 	req_lib_ckpt_sectionread.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_sectionread.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_SECTIONREAD;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	for (i = 0; i < numberOfElements; i++) {
 		req_lib_ckpt_sectionread.header.size = sizeof (struct req_lib_ckpt_sectionread) +
 			ioVector[i].sectionId.idLen;
@@ -1179,6 +1334,9 @@ saCkptCheckpointRead (
 
 error_exit:
 	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error == SA_OK ? res_lib_ckpt_sectionread.error : error);
 }
 
@@ -1192,16 +1350,18 @@ saCkptCheckpointSynchronize (
 	struct req_lib_ckpt_checkpointsynchronize req_lib_ckpt_checkpointsynchronize;
 	struct res_lib_ckpt_checkpointsynchronize res_lib_ckpt_checkpointsynchronize;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
+		return (error);
 	}
 
 	req_lib_ckpt_checkpointsynchronize.header.magic = MESSAGE_MAGIC;
 	req_lib_ckpt_checkpointsynchronize.header.size = sizeof (struct req_lib_ckpt_checkpointsynchronize); 
 	req_lib_ckpt_checkpointsynchronize.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_CHECKPOINTSYNCHRONIZE;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
 	error = saSendRetry (ckptCheckpointInstance->fd, &req_lib_ckpt_checkpointsynchronize,
 		sizeof (struct req_lib_ckpt_checkpointsynchronize), MSG_NOSIGNAL);
 
@@ -1213,6 +1373,10 @@ saCkptCheckpointSynchronize (
 		0, MESSAGE_RES_CKPT_CHECKPOINT_CHECKPOINTSYNCHRONIZE);
 
 error_exit:
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error == SA_OK ? res_lib_ckpt_checkpointsynchronize.error : error);
 }
 
@@ -1227,15 +1391,11 @@ saCkptCheckpointSynchronizeAsync (
 	SaErrorT error;
 	struct req_lib_ckpt_checkpointsynchronizeasync req_lib_ckpt_checkpointsynchronizeasync;
 
-	error = saHandleConvert (&ckptCheckpointHandleDatabase, *checkpointHandle,
-		(void *)&ckptCheckpointInstance, CKPTCHECKPOINTINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&checkpointHandleDatabase, *checkpointHandle,
+		(void *)&ckptCheckpointInstance);
 	if (error != SA_OK) {
-		goto error_exit;
-	}
-	error = saHandleConvert (&ckptHandleDatabase, *ckptHandle,
-		(void *)&ckptInstance, CKPTINSTANCE_MUTEX_OFFSET, 0);
-	if (error != SA_OK) {
-		goto error_exit;
+
+		return (error);
 	}
 
 	req_lib_ckpt_checkpointsynchronizeasync.header.magic = MESSAGE_MAGIC;
@@ -1243,9 +1403,18 @@ saCkptCheckpointSynchronizeAsync (
 	req_lib_ckpt_checkpointsynchronizeasync.header.id = MESSAGE_REQ_CKPT_CHECKPOINT_CHECKPOINTOPENASYNC;
 	req_lib_ckpt_checkpointsynchronizeasync.invocation = invocation;
 
+	pthread_mutex_lock (&ckptCheckpointInstance->mutex);
+
+	pthread_mutex_lock (&ckptInstance->mutex);
+
 	error = saSendRetry (ckptInstance->fd, &req_lib_ckpt_checkpointsynchronizeasync,
 		sizeof (struct req_lib_ckpt_checkpointsynchronizeasync), MSG_NOSIGNAL);
 
-error_exit:
+	pthread_mutex_unlock (&ckptInstance->mutex);
+
+	pthread_mutex_unlock (&ckptCheckpointInstance->mutex);
+
+	saHandleInstancePut (&checkpointHandleDatabase, *checkpointHandle);
+
 	return (error);
 }

lib/clm.c

@@ -57,16 +57,17 @@ struct message_overlay {
 struct clmInstance {
 	int fd;
 	SaClmCallbacksT callbacks;
-	struct message_overlay message;
+	int finalize;
 	pthread_mutex_t mutex;
 };
-#define CLMINSTANCE_MUTEX_OFFSET offset_of(struct clmInstance, mutex)
+
+static void clmHandleInstanceDestructor (void *);
 
 static struct saHandleDatabase clmHandleDatabase = {
 	handleCount: 0,
 	handles: 0,
-	generation: 0,
-	mutex: PTHREAD_MUTEX_INITIALIZER
+	mutex: PTHREAD_MUTEX_INITIALIZER,
+	handleInstanceDestructor: clmHandleInstanceDestructor
 };
 
 /*
@@ -82,6 +83,16 @@ static struct saVersionDatabase clmVersionDatabase = {
 	clmVersionsSupported
 };
 
+void clmHandleInstanceDestructor (void *instance)
+{
+	struct clmInstance *clmInstance = (struct clmInstance *)instance;
+
+	if (clmInstance->fd != -1) {
+		shutdown (clmInstance->fd, 0);
+		close (clmInstance->fd);
+	}
+}
+
 
 SaErrorT
 saClmInitialize (
@@ -94,29 +105,41 @@ saClmInitialize (
 
 	error = saVersionVerify (&clmVersionDatabase, version);
 	if (error != SA_OK) {
-		goto error_nofree;
+		goto error_no_destroy;
 	}
 
-	error = saHandleCreate (&clmHandleDatabase, (void *)&clmInstance,
-		sizeof (struct clmInstance), clmHandle);
+	error = saHandleCreate (&clmHandleDatabase, sizeof (struct clmInstance),
+		clmHandle);
 	if (error != SA_OK) {
-		goto error_nofree;
+		goto error_no_destroy;
 	}
+
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle,
+		(void *)&clmInstance);
+	if (error != SA_OK) {
+		goto error_destroy;
+	}
+
+	clmInstance->fd = -1;
 	
 	error = saServiceConnect (&clmInstance->fd, MESSAGE_REQ_CLM_INIT);
 	if (error != SA_OK) {
-		goto error_free;
+		goto error_put_destroy;
 	}
 
 	memcpy (&clmInstance->callbacks, clmCallbacks, sizeof (SaClmCallbacksT));
 
 	pthread_mutex_init (&clmInstance->mutex, NULL);
 
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
+
 	return (SA_OK);
 
-error_free:
-	saHandleRemove (&clmHandleDatabase, *clmHandle);
-error_nofree:
+error_put_destroy:
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
+error_destroy:
+	saHandleDestroy (&clmHandleDatabase, *clmHandle);
+error_no_destroy:
 	return (error);
 }
 
@@ -128,14 +151,14 @@ saClmSelectionObjectGet (
 	struct clmInstance *clmInstance;
 	SaErrorT error;
 
-	error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle, (void *)&clmInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
 	*selectionObject = clmInstance->fd;
 
-	pthread_mutex_unlock (&clmInstance->mutex);
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
 	return (SA_OK);
 }
 
@@ -150,15 +173,17 @@ saClmDispatch (
 	int cont = 1; /* always continue do loop except when set to 0 */
 	int dispatch_avail;
 	int poll_fd;
-	int handle_verified = 0;
 	struct clmInstance *clmInstance;
 	struct res_clm_trackcallback *res_clm_trackcallback;
 	struct res_clm_nodegetcallback *res_clm_nodegetcallback;
 	SaClmCallbacksT callbacks;
-	unsigned int gen_first;
-	unsigned int gen_second;
 	struct message_overlay dispatch_data;
 
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle, (void *)&clmInstance);
+	if (error != SA_OK) {
+		return (error);
+	}
+
 	/*
 	 * Timeout instantly for SA_DISPATCH_ONE or SA_DISPATCH_ALL and
 	 * wait indefinately for SA_DISPATCH_BLOCKING
@@ -168,21 +193,8 @@ saClmDispatch (
 	}
 
 	do {
-		error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET, &gen_first);
-		if (error != SA_OK) {
-			return (handle_verified ? SA_OK : error);
-		}
-		handle_verified = 1;
-
 		poll_fd = clmInstance->fd;
 
-		/*
-		 * Unlock mutex for potentially long wait in select.  If fd
-		 * is closed by clmFinalize in select, select will return
-		 */
-
-		pthread_mutex_unlock (&clmInstance->mutex);
-
 		ufds.fd = poll_fd;
 		ufds.events = POLLIN;
 		ufds.revents = 0;
@@ -192,32 +204,32 @@ saClmDispatch (
 			goto error_nounlock;
 		}
 
-		dispatch_avail = ufds.revents & POLLIN;
-		if (dispatch_avail == 0 && dispatchFlags == SA_DISPATCH_ALL) {
-			break; /* exit do while cont is 1 loop */
-		}
-		if (dispatch_avail == 0) {
-			continue; /* retry select */
-		}
+		pthread_mutex_lock (&clmInstance->mutex);
+
 		/*
-		 * Re-verify amfHandle
+		 * Handle has been finalized in another thread
 		 */
-		error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET, &gen_second);
-		if (error != SA_OK) {
-			return (handle_verified ? SA_OK : error);
+		if (clmInstance->finalize == 1) {
+			error = SA_OK;
+			pthread_mutex_unlock (&clmInstance->mutex);
+			goto error_unlock;
 		}
 
-		/*
-		 * Handle has been removed and then reallocated
-		 */
-		if (gen_first != gen_second) {
-			return SA_OK;
+		dispatch_avail = ufds.revents & POLLIN;
+		if (dispatch_avail == 0 && dispatchFlags == SA_DISPATCH_ALL) {
+			pthread_mutex_unlock (&clmInstance->mutex);
+			break; /* exit do while cont is 1 loop */
+		} else
+		if (dispatch_avail == 0) {
+			pthread_mutex_unlock (&clmInstance->mutex);
+			continue; /* next poll */
 		}
 
 		/*
 		 * Read header
 		 */
-		error = saRecvRetry (clmInstance->fd, &clmInstance->message.header, sizeof (struct message_header), MSG_WAITALL | MSG_NOSIGNAL);
+		error = saRecvRetry (clmInstance->fd, &dispatch_data.header,
+			sizeof (struct message_header), MSG_WAITALL | MSG_NOSIGNAL);
 		if (error != SA_OK) {
 			goto error_unlock;
 		}
@@ -225,9 +237,9 @@ saClmDispatch (
 		/*
 		 * Read data payload
 		 */
-		if (clmInstance->message.header.size > sizeof (struct message_header)) {
-			error = saRecvRetry (clmInstance->fd, &clmInstance->message.data,
-				clmInstance->message.header.size - sizeof (struct message_header), MSG_WAITALL | MSG_NOSIGNAL);
+		if (dispatch_data.header.size > sizeof (struct message_header)) {
+			error = saRecvRetry (clmInstance->fd, &dispatch_data.data,
+				dispatch_data.header.size - sizeof (struct message_header), MSG_WAITALL | MSG_NOSIGNAL);
 			if (error != SA_OK) {
 				goto error_unlock;
 			}
@@ -235,18 +247,15 @@ saClmDispatch (
 		/*
 		 * Make copy of callbacks, message data, unlock instance, and call callback
 		 * A risk of this dispatch method is that the callback routines may
-		 * operate at the same time that amfFinalize has been called.
+		 * operate at the same time that clmFinalize has been called.
 		*/
 		memcpy (&callbacks, &clmInstance->callbacks, sizeof (SaClmCallbacksT));
-		memcpy (&dispatch_data, &clmInstance->message, sizeof (struct message_overlay));
-
 
 		pthread_mutex_unlock (&clmInstance->mutex);
-
 		/*
 		 * Dispatch incoming message
 		 */
-		switch (clmInstance->message.header.id) {
+		switch (dispatch_data.header.id) {
 
 		case MESSAGE_RES_CLM_TRACKCALLBACK:
 			res_clm_trackcallback = (struct res_clm_trackcallback *)&dispatch_data;
@@ -277,6 +286,7 @@ saClmDispatch (
 			goto error_nounlock;
 			break;
 		}
+
 		/*
 		 * Determine if more messages should be processed
 		 * */
@@ -291,10 +301,8 @@ saClmDispatch (
 		}
 	} while (cont);
 
-	return (error);
-
 error_unlock:
-	pthread_mutex_unlock (&clmInstance->mutex);
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
 error_nounlock:
 	return (error);
 }
@@ -306,20 +314,31 @@ saClmFinalize (
 	struct clmInstance *clmInstance;
 	SaErrorT error;
 
-	error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET | HANDLECONVERT_DONTUNLOCKDB, 0);
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle, (void *)&clmInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
-	shutdown (clmInstance->fd, 0);
-	close (clmInstance->fd);
-	free (clmInstance);
+       pthread_mutex_lock (&clmInstance->mutex);
+
+	/*
+	 * Another thread has already started finalizing
+	 */
+	if (clmInstance->finalize) {
+		pthread_mutex_unlock (&clmInstance->mutex);
+		saHandleInstancePut (&clmHandleDatabase, *clmHandle);
+		return (SA_ERR_BAD_HANDLE);
+	}
+
+	clmInstance->finalize = 1;
 
-	error = saHandleRemove (&clmHandleDatabase, *clmHandle);
+	saActivatePoll (clmInstance->fd);
 
 	pthread_mutex_unlock (&clmInstance->mutex);
 
-	saHandleUnlockDatabase (&clmHandleDatabase);
+	saHandleDestroy (&clmHandleDatabase, *clmHandle);
+
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
 
 	return (error);
 }
@@ -342,15 +361,19 @@ saClmClusterTrackStart (
 	req_trackstart.notificationBufferAddress = notificationBuffer;
 	req_trackstart.numberOfItems = numberOfItems;
 
-	error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle, (void *)&clmInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
+	pthread_mutex_lock (&clmInstance->mutex);
+
 	error = saSendRetry (clmInstance->fd, &req_trackstart, sizeof (struct req_clm_trackstart), MSG_NOSIGNAL);
 
 	pthread_mutex_unlock (&clmInstance->mutex);
 
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
+
 	return (error);
 }
 
@@ -366,15 +389,19 @@ saClmClusterTrackStop (
 	req_trackstop.header.size = sizeof (struct req_clm_trackstop);
 	req_trackstop.header.id = MESSAGE_REQ_CLM_TRACKSTOP;
 
-	error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle, (void *)&clmInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
+	pthread_mutex_lock (&clmInstance->mutex);
+
 	error = saSendRetry (clmInstance->fd, &req_trackstop, sizeof (struct req_clm_trackstop), MSG_NOSIGNAL);
 
 	pthread_mutex_unlock (&clmInstance->mutex);
 
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
+
 	return (error);
 }
 
@@ -466,14 +493,19 @@ saClmClusterNodeGetAsync (
 	memcpy (&req_clm_nodeget.nodeId, &nodeId, sizeof (SaClmNodeIdT));
 	req_clm_nodeget.clusterNodeAddress = clusterNode;
 
-	error = saHandleConvert (&clmHandleDatabase, *clmHandle, (void *)&clmInstance, CLMINSTANCE_MUTEX_OFFSET, 0);
+	error = saHandleInstanceGet (&clmHandleDatabase, *clmHandle, (void *)&clmInstance);
 	if (error != SA_OK) {
 		return (error);
 	}
 
-	error = saSendRetry (clmInstance->fd, &req_clm_nodeget, sizeof (struct req_clm_nodeget), MSG_NOSIGNAL);
+	pthread_mutex_lock (&clmInstance->mutex);
+
+	error = saSendRetry (clmInstance->fd, &req_clm_nodeget,
+		sizeof (struct req_clm_nodeget), MSG_NOSIGNAL);
 
 	pthread_mutex_unlock (&clmInstance->mutex);
 
+	saHandleInstancePut (&clmHandleDatabase, *clmHandle);
+
 	return (error);
 }

lib/util.c

@@ -45,6 +45,7 @@
 #include <net/if.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <assert.h>
 
 #include "../include/ais_types.h"
 #include "../include/ais_msg.h"
@@ -201,19 +202,19 @@ error_exit:
 
 SaErrorT
 saActivatePoll (int s) {
-	struct req_amf_activatepoll req_amf_activatepoll;
+	struct req_lib_activatepoll req_lib_activatepoll;
 	SaErrorT error;
 
 	/*
 	 * Send activate poll to tell nodeexec to activate poll
 	 * on this file descriptor
 	 */
-	req_amf_activatepoll.header.magic = MESSAGE_MAGIC;
-	req_amf_activatepoll.header.size = sizeof (req_amf_activatepoll);
-	req_amf_activatepoll.header.id = MESSAGE_REQ_AMF_ACTIVATEPOLL;
+	req_lib_activatepoll.header.magic = MESSAGE_MAGIC;
+	req_lib_activatepoll.header.size = sizeof (req_lib_activatepoll);
+	req_lib_activatepoll.header.id = MESSAGE_REQ_LIB_ACTIVATEPOLL;
 
-	error = saSendRetry (s, &req_amf_activatepoll,
-		sizeof (struct req_amf_activatepoll), MSG_NOSIGNAL);
+	error = saSendRetry (s, &req_lib_activatepoll,
+		sizeof (struct req_lib_activatepoll), MSG_NOSIGNAL);
 	return (error);
 }
 
@@ -325,24 +326,10 @@ retry_poll:
 	return (error);
 }
 
-SaErrorT
-saHandleVerify (
-	struct saHandleDatabase *handleDatabase,
-	unsigned int handle)
-{
-	if (handle > handleDatabase->handleCount) {
-		return (SA_ERR_BAD_HANDLE);
-	}
-	if (handleDatabase->handles[handle].valid == 0) {
-		return (SA_ERR_BAD_HANDLE);
-	}
-	return (SA_OK);
-}
 
 SaErrorT
 saHandleCreate (
 	struct saHandleDatabase *handleDatabase,
-	void **instanceOut,
 	int instanceSize,
 	int *handleOut)
 {
@@ -354,11 +341,12 @@ saHandleCreate (
 	pthread_mutex_lock (&handleDatabase->mutex);
 
 	for (handle = 0; handle < handleDatabase->handleCount; handle++) {
-		if (handleDatabase->handles[handle].valid == 0) {
+		if (handleDatabase->handles[handle].state == SA_HANDLE_STATE_EMPTY) {
 			found = 1;
 			break;
 		}
 	}
+
 	if (found == 0) {
 		handleDatabase->handleCount += 1;
 		newHandles = (struct saHandle *)realloc (handleDatabase->handles,
@@ -369,90 +357,90 @@ saHandleCreate (
 		}
 		handleDatabase->handles = newHandles;
 	}
+
 	instance = malloc (instanceSize);
 	if (instance == 0) {
 		return (SA_ERR_NO_MEMORY);
 	}
 	memset (instance, 0, instanceSize);
 
-	handleDatabase->handles[handle].valid = 1;
+	handleDatabase->handles[handle].state = SA_HANDLE_STATE_ACTIVE;
+
 	handleDatabase->handles[handle].instance = instance;
-	handleDatabase->handles[handle].generation = handleDatabase->generation++;
+
+	handleDatabase->handles[handle].refCount = 1;
 
 	*handleOut = handle;
-	*instanceOut = instance;
 
 	pthread_mutex_unlock (&handleDatabase->mutex);
+
 	return (SA_OK);
 }
 
+
 SaErrorT
-saHandleRemove (
+saHandleDestroy (
 	struct saHandleDatabase *handleDatabase,
 	unsigned int handle)
 {
-	free (handleDatabase->handles[handle].instance);
-	memset (&handleDatabase->handles[handle], 0, sizeof (struct saHandle));
+	pthread_mutex_lock (&handleDatabase->mutex);
+	handleDatabase->handles[handle].state = SA_HANDLE_STATE_PENDINGREMOVAL;
+	pthread_mutex_unlock (&handleDatabase->mutex);
+	saHandleInstancePut (handleDatabase, handle);
 
 	return (SA_OK);
 }
 
+
 SaErrorT
-saHandleConvert (
+saHandleInstanceGet (
 	struct saHandleDatabase *handleDatabase,
 	unsigned int handle,
-	void **instance,
-	int offsetToMutex,
-	unsigned int *generationOut)
+	void **instance)
 { 
-	SaErrorT error;
-	int unlockDatabase;
-	int locking;
-
-	unlockDatabase = (0 == (offsetToMutex & HANDLECONVERT_DONTUNLOCKDB));
-	locking = (0 == (offsetToMutex & HANDLECONVERT_NOLOCKING));
-	offsetToMutex &= 0x00ffffff; /* remove 8 bits of flags */
+	pthread_mutex_lock (&handleDatabase->mutex);
 
-	if (locking) {
-		pthread_mutex_lock (&handleDatabase->mutex);
+	if (handle > handleDatabase->handleCount) {
+		return (SA_ERR_BAD_HANDLE);
 	}
-
-	error = saHandleVerify (handleDatabase, handle);
-	if (error != SA_OK) {
-		if (locking) {
-			pthread_mutex_unlock (&handleDatabase->mutex);
-		}
-		return (error);
+	if (handleDatabase->handles[handle].state != SA_HANDLE_STATE_ACTIVE) {
+		return (SA_ERR_BAD_HANDLE);
 	}
 
 	*instance = handleDatabase->handles[handle].instance;
-	if (generationOut) {
-		*generationOut = handleDatabase->handles[handle].generation;
-	}
 
-	/*
-	 * This function exits holding the mutex in the instance instance
-	 * pointed to by offsetToMutex (if NOLOCKING isn't set)
-	 */
-	if (locking) {
-		pthread_mutex_lock ((pthread_mutex_t *)(*instance + offsetToMutex));
-		if (unlockDatabase) {
-			pthread_mutex_unlock (&handleDatabase->mutex);
-		}
-	}
+	handleDatabase->handles[handle].refCount += 1;
+
+	pthread_mutex_unlock (&handleDatabase->mutex);
 
 	return (SA_OK);
 }
 
+
 SaErrorT
-saHandleUnlockDatabase (
-	struct saHandleDatabase *handleDatabase)
+saHandleInstancePut (
+	struct saHandleDatabase *handleDatabase,
+	unsigned int handle)
 {
+	pthread_mutex_lock (&handleDatabase->mutex);
+	void *instance;
+
+	handleDatabase->handles[handle].refCount -= 1;
+	assert (handleDatabase->handles[handle].refCount >= 0);
+
+	if (handleDatabase->handles[handle].refCount == 0) {
+		instance = (handleDatabase->handles[handle].instance);
+		handleDatabase->handleInstanceDestructor (instance);
+		free (instance);
+		memset (&handleDatabase->handles[handle], 0, sizeof (struct saHandle));
+	}
+
 	pthread_mutex_unlock (&handleDatabase->mutex);
 
 	return (SA_OK);
 }
 
+
 SaErrorT
 saVersionVerify (
         struct saVersionDatabase *versionDatabase,

lib/util.h

@@ -40,19 +40,26 @@
 #include <sys/poll.h>
 #include "../include/ais_msg.h"
 
+enum SA_HANDLE_STATE {
+	SA_HANDLE_STATE_EMPTY,
+	SA_HANDLE_STATE_PENDINGREMOVAL,
+	SA_HANDLE_STATE_ACTIVE
+};
+
 struct saHandle {
-	int valid;
+	int state;
 	void *instance;
-	unsigned int generation;
+	int refCount;
 };
 
 struct saHandleDatabase {
 	unsigned int handleCount;
 	struct saHandle *handles;
-	unsigned int generation;
 	pthread_mutex_t mutex;
+	void (*handleInstanceDestructor) (void *);
 };
 
+
 struct saVersionDatabase {
 	int versionCount;
 	SaVersionT *versionsSupported;
@@ -116,35 +123,27 @@ saPollRetry (
 	unsigned int nfds,
 	int timeout);
 
-SaErrorT
-saHandleVerify (
-	struct saHandleDatabase *handleDatabase,
-	unsigned int handle); 
-
 SaErrorT
 saHandleCreate (
 	struct saHandleDatabase *handleDatabase,
-	void **instance,
 	int instanceSize,
 	int *handleOut);
 
 SaErrorT
-saHandleRemove (
+saHandleDestroy (
 	struct saHandleDatabase *handleDatabase,
 	unsigned int handle);
 
 SaErrorT
-saHandleConvert (
+saHandleInstanceGet (
 	struct saHandleDatabase *handleDatabase,
 	unsigned int handle,
-	void **instance,
-	int offsetToMutex,
-	unsigned int *generationOut);
-
+	void **instance);
 
 SaErrorT
-saHandleUnlockDatabase (
-	struct saHandleDatabase *handleDatabase);
+saHandleInstancePut (
+	struct saHandleDatabase *handleDatabase,
+	unsigned int handle);
 
 SaErrorT
 saVersionVerify (
@@ -180,7 +179,4 @@ saQueueItemRemove (struct queue *queue);
 
 #define offset_of(type,member) (int)(&(((type *)0)->member))
 
-#define HANDLECONVERT_NOLOCKING		0x80000000
-#define HANDLECONVERT_DONTUNLOCKDB	0x40000000
-
 #endif /* AIS_UTIL_H_DEFINED */