- Error escalation improved, SU failover recovery action added

- Most runtime attributes in the inf. model calculated in runtime from
  more fundamental information. (improves consistency)
- sg_assign_si can now recalculate workloads considering existing
  assignments
- Logging improvements, similar to what is required as notification in
  AMF spec.
- CLC-CLI INSTANTIATE now exits aisexec when it fails (should later be
  sent as an NTF alarm)
- CLC-CLI CLEANUP correctly handles already terminated processes
- testamf1.c printouts removed for normal operation
- Iterator functions for SI/CSI assignments 



git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1108 fd59a12c-fef9-0310-b244-a6a79926bd2f

conf/amf.conf

@@ -19,9 +19,9 @@ safAmfCluster = TEST_CLUSTER {
 			saAmfSGNumPrefStandbySUs=1
 			saAmfSGMaxStandbySIsperSUs=2
 			saAmfSGCompRestartProb=100000
-			saAmfSGCompRestartMax=3
+			saAmfSGCompRestartMax=1
 			saAmfSGSuRestartProb=20000	
-			saAmfSGSuRestartMax=6
+			saAmfSGSuRestartMax=1
 			saAmfSGAutoAdjustProb=5000
 			safSu = SERVICE_X_1 {
 				saAmfSUHostedByNode=seasc0036
@@ -40,8 +40,7 @@ safAmfCluster = TEST_CLUSTER {
 					saAmfCompCleanupCmd = /tmp/aistest/clc_cli_script
 					saAmfCompCleanupCmdArgv = cleanup
 					saAmfCompCsTypes {
-						A-1
-						A-2
+						A
 					}
 					saAmfCompCmdEnv {
 						var1=val1
@@ -49,7 +48,33 @@ safAmfCluster = TEST_CLUSTER {
 					}
 					saAmfCompRecoveryOnError=component_restart
 					safHealthcheckKey = key1 {
-						saAmfHealthcheckPeriod = 3000
+						saAmfHealthcheckPeriod = 5000
+						saAmfHealthcheckMaxDuration = 350
+					}
+				}
+				safComp = B {
+					saAmfCompCategory=sa_aware
+					saAmfCompCapability=x_active_or_y_standby
+					saAmfCompNumMaxActiveCsi=1
+					saAmfCompNumMaxStandbyCsi=1
+					saAmfCompDefaultClcCliTimeout = 500
+					saAmfCompDefaultCallbackTimeOut = 500
+					saAmfCompInstantiateCmd = /tmp/aistest/clc_cli_script
+					saAmfCompInstantiateCmdArgv= instantiate /tmp/aistest/testamf1
+					saAmfCompTerminateCmd = /tmp/aistest/clc_cli_script
+					saAmfCompTerminateCmdArgv = terminate
+					saAmfCompCleanupCmd = /tmp/aistest/clc_cli_script
+					saAmfCompCleanupCmdArgv = cleanup
+					saAmfCompCsTypes {
+						B
+					}
+					saAmfCompCmdEnv {
+						var1=val1
+						var2=val2
+					}
+					saAmfCompRecoveryOnError=component_restart
+					safHealthcheckKey = key1 {
+						saAmfHealthcheckPeriod = 5000
 						saAmfHealthcheckMaxDuration = 350
 					}
 				}
@@ -72,8 +97,7 @@ safAmfCluster = TEST_CLUSTER {
 					saAmfCompCleanupCmd = clc_cli_script
 					saAmfCompCleanupCmdArgv = cleanup
 					saAmfCompCsTypes {
-						A-1
-						A-2
+						A
 					}
 					saAmfCompCmdEnv {
 						COMP_BINARY_PATH=/tmp/aistest
@@ -81,7 +105,7 @@ safAmfCluster = TEST_CLUSTER {
 					}
 					saAmfCompRecoveryOnError=component_restart
 					safHealthcheckKey = key1 {
-						saAmfHealthcheckPeriod = 3000
+						saAmfHealthcheckPeriod = 5000
 						saAmfHealthcheckMaxDuration = 350
 					}
 					safHealthcheckKey = key2 {
@@ -89,50 +113,71 @@ safAmfCluster = TEST_CLUSTER {
 						saAmfHealthcheckMaxDuration = 350
 					}
 				}
+				safComp = B {
+					saAmfCompCategory=sa_aware
+					saAmfCompCapability=x_active_or_y_standby
+					saAmfCompNumMaxActiveCsi=1
+					saAmfCompNumMaxStandbyCsi=1
+					saAmfCompDefaultClcCliTimeout = 500
+					saAmfCompDefaultCallbackTimeOut = 500
+					saAmfCompInstantiateCmd = /tmp/aistest/clc_cli_script
+					saAmfCompInstantiateCmdArgv= instantiate /tmp/aistest/testamf1
+					saAmfCompTerminateCmd = /tmp/aistest/clc_cli_script
+					saAmfCompTerminateCmdArgv = terminate
+					saAmfCompCleanupCmd = /tmp/aistest/clc_cli_script
+					saAmfCompCleanupCmdArgv = cleanup
+					saAmfCompCsTypes {
+						B
+					}
+					saAmfCompCmdEnv {
+						var1=val1
+						var2=val2
+					}
+					saAmfCompRecoveryOnError=component_restart
+					safHealthcheckKey = key1 {
+						saAmfHealthcheckPeriod = 5000
+						saAmfHealthcheckMaxDuration = 350
+					}
+				}
 			}
 		}
 		safSi = WL1 {
 			saAmfSINumCSIs=2
 			safCsi = WL1-1 {
-				saAmfCSTypeName = A-1
+				saAmfCSTypeName = A
 			}
 			safCsi = WL1-2 {
-				saAmfCSTypeName = A-2
-				saAmfCsiDependencies {
-					saAmfCSIDependency = WL1-1
-				}
+				saAmfCSTypeName = B
 				safCSIAttr = attr1 {
 					val1
 					val2
 				}
 				safCSIAttr = good_health_limit {
-					10
+					100000
 				}
 			}
-			safRankedSu = SU_1 {
-				saAmfRank = 1
-			}
-			safRankedSu = SU_2 {
-				saAmfRank = 2
-			}
 		}
 		safSi = WL2 {
-			saAmfSINumCSIs=1
+			saAmfSINumCSIs=2
 			safCsi = WL2-1 {
-				saAmfCSTypeName = A-1
+				saAmfCSTypeName = A
 			}
 			safCsi = WL2-2 {
-				saAmfCSTypeName = A-2
-			}
-			safDepend = WL1 {
-				saAmfToleranceTime=10000
+				saAmfCSTypeName = B
+				safCSIAttr = attr1 {
+					val1
+					val2
+				}
+				safCSIAttr = good_health_limit {
+					100000
+				}
 			}
 		}
-		safCSType = A-1 {
+		safCSType = A {
 			safAmfCSAttrName = attr1
 			safAmfCSAttrName = good_health_limit
 		}
-		safCSType = A-2 {
+		safCSType = B {
 		}
 	}
 }

exec/amf.h

@@ -46,13 +46,6 @@
 #include "objdb.h"
 #include "timer.h"
 
-enum escalation_levels {
-	ESCALATION_LEVEL_NO_ESCALATION = 1,	/* execute component restart */
-	ESCALATION_LEVEL_ONE = 2,		/* escalate to service unit restart */
-	ESCALATION_LEVEL_TWO = 3,		/* escalate to service unit failover */
-	ESCALATION_LEVEL_THREE = 4		/* escalate to node failover */
-};
-
 enum clc_component_types {
 	clc_component_sa_aware = 0,			/* sa aware */
 	clc_component_proxied_pre = 1,			/* proxied, pre-instantiable */
@@ -60,6 +53,38 @@ enum clc_component_types {
 	clc_component_non_proxied_non_sa_aware = 3	/* non-proxied, non sa aware */
 };
 
+typedef enum {
+	SG_AC_Idle = 0,
+	SG_AC_DeactivatingDependantWorkload,
+	SG_AC_TerminatingSuspected,
+	SG_AC_ActivatingStandby,
+	SG_AC_AssigningStandbyToSpare,
+	SG_AC_ReparingComponent,
+	SG_AC_ReparingSu,
+	SG_AC_AssigningOnRequest,
+	SG_AC_InstantiatingServiceUnits,
+	SG_AC_RemovingAssignment,
+	SG_AC_AssigningActiveworkload,
+	SG_AC_AssigningAutoAdjust,
+	SG_AC_AssigningStandBy,
+	SG_AC_WaitingAfterOperationFailed
+} sg_avail_control_state_t;
+
+typedef enum {
+	SU_RC_ESCALATION_LEVEL_0 = 0,
+	SU_RC_ESCALATION_LEVEL_1,
+	SU_RC_ESCALATION_LEVEL_2,
+	SU_RC_RESTART_COMP_DEACTIVATING,
+	SU_RC_RESTART_COMP_RESTARTING,
+	SU_RC_RESTART_COMP_SETTING,
+	SU_RC_RESTART_COMP_ACTIVATING,
+	SU_RC_RESTART_SU_DEACTIVATING,
+	SU_RC_RESTART_SU_TERMINATING,
+	SU_RC_RESTART_SU_INSTANTIATING,
+	SU_RC_RESTART_SU_SETTING,
+	SU_RC_RESTART_SU_ACTIVATING
+} su_restart_control_state_t;
+
 struct amf_si_assignment;
 struct amf_csi_assignment;
 struct amf_healthcheck;
@@ -153,6 +178,7 @@ struct amf_sg {
 	char clccli_path[PATH_MAX];
 	char binary_path[PATH_MAX];
 	struct amf_sg *next;
+	sg_avail_control_state_t avail_state;
 };
 
 struct amf_su {
@@ -167,24 +193,24 @@ struct amf_su {
 	SaBoolT saAmfSUPreInstantiable;
 	SaAmfOperationalStateT saAmfSUOperState;
 	SaAmfAdminStateT saAmfSUAdminState;
-	SaAmfReadinessStateT saAmfSUReadinessState;
+//	SaAmfReadinessStateT saAmfSUReadinessState;
 	SaAmfPresenceStateT saAmfSUPresenceState;
-	SaNameT saAmfSUAssignedSIs;
+//	SaNameT saAmfSUAssignedSIs;
 	SaNameT saAmfSUHostedByNode;
-	SaUint32T saAmfSUNumCurrActiveSIs;
-	SaUint32T saAmfSUNumCurrStandbySIs;
+/*     SaUint32T saAmfSUNumCurrActiveSIs;  */
+/*     SaUint32T saAmfSUNumCurrStandbySIs; */
 	SaUint32T saAmfSURestartCount;
 
 	/* Relations */
 	struct amf_sg *sg;
 	struct amf_comp *comp_head;
-	struct amf_si_assignment *assigned_sis;
 
 	/* Implementation */
+	su_restart_control_state_t restart_control_state;
+	su_restart_control_state_t escalation_level_history_state;
 	char clccli_path[PATH_MAX];
 	char binary_path[PATH_MAX];
 	SaUint32T              su_failover_cnt; /* missing in SAF specs? */
-	enum escalation_levels escalation_level;
 	struct amf_su         *next;
 };
 
@@ -230,11 +256,11 @@ struct amf_comp {
 
 	/* Runtime Attributes */
 	SaAmfOperationalStateT saAmfCompOperState;
-	SaAmfReadinessStateT saAmfCompReadinessState;
+//	SaAmfReadinessStateT saAmfCompReadinessState;
 	SaAmfPresenceStateT saAmfCompPresenceState;
 	SaUint32T saAmfCompRestartCount;
-	SaUint32T saAmfCompNumCurrActiveCsi;
-	SaUint32T saAmfCompNumCurrStandbyCsi;
+/*     SaUint32T saAmfCompNumCurrActiveCsi;  */
+/*     SaUint32T saAmfCompNumCurrStandbyCsi; */
 	SaNameT saAmfCompAssignedCsi;
 	SaNameT saAmfCompCurrProxyName;
 	SaNameT saAmfCompCurrProxiedNames;
@@ -242,7 +268,6 @@ struct amf_comp {
 	/* Relations */
 	struct amf_comp *proxy_comp;
 	struct amf_su *su;
-	struct amf_csi_assignment *assigned_csis;
 
 	/* Implementation */
 	char clccli_path[PATH_MAX];
@@ -251,6 +276,11 @@ struct amf_comp {
 	void *conn;
 	enum clc_component_types comptype;
 	struct amf_healthcheck *healthcheck_head;
+
+	/**
+     * Flag that indicates of this component has a suspected error
+	 */
+	int error_suspected;
 };
 
 struct amf_healthcheck {
@@ -282,14 +312,14 @@ struct amf_si {
 
 	/* Runtime Attributes */
 	SaAmfAdminStateT saAmfSIAdminState;
-	SaAmfAssignmentStateT saAmfSIAssignmentState;
-	SaUint32T saAmfSINumCurrActiveAssignments;
-	SaUint32T saAmfSINumCurrStandbyAssignments;
+//	SaAmfAssignmentStateT saAmfSIAssignmentState;
+//	SaUint32T saAmfSINumCurrActiveAssignments;
+//	SaUint32T saAmfSINumCurrStandbyAssignments;
 
 	/* Relations */
 	struct amf_application   *application;
 	struct amf_csi           *csi_head;
-	struct amf_si_assignment *si_assignments;
+	struct amf_si_assignment *assigned_sis;
 	struct amf_si_dependency *depends_on;
 	struct amf_si_ranked_su  *ranked_sus;
 
@@ -329,10 +359,13 @@ struct amf_si_assignment {
 
 	/* Relations */
 	struct amf_si *si;
+	struct amf_su *su;
 
 	/* Implementation */
 	SaAmfHAStateT requested_ha_state;
 	struct amf_si_assignment *next;
+	void (*assumed_callback_fn) (
+		struct amf_si_assignment *si_assignment, int result);
 };
 
 struct amf_csi {
@@ -343,7 +376,7 @@ struct amf_csi {
 
 	/* Relations */
 	struct amf_si *si;
-	struct amf_csi_assignment *csi_assignments;
+	struct amf_csi_assignment *assigned_csis;
 	struct amf_csi_attribute *attributes_head;
 
 	/* Implementation */
@@ -371,8 +404,8 @@ struct amf_csi_assignment {
 
 	/* Implementation */
 	SaAmfHAStateT requested_ha_state;
-	struct amf_csi_assignment *comp_next;
-	struct amf_csi_assignment *csi_next;
+	struct amf_csi_assignment *next;
+	struct amf_si_assignment *si_assignment;
 };
 
 enum amf_response_interfaces {
@@ -420,6 +453,7 @@ extern const char *amf_op_state (int state);
 extern const char *amf_presence_state (int state);
 extern const char *amf_ha_state (int state);
 extern const char *amf_readiness_state (int state);
+extern const char *amf_assignment_state (int state);
 
 /*===========================================================================*/
 /* amfnode.c */
@@ -515,7 +549,14 @@ extern int amf_sg_serialize (
 extern struct amf_sg *amf_sg_deserialize (
 	char **buf, int *size, struct amf_cluster *cluster);
 
-/* Event methods */
+/**
+ * Request SG to start (instantiate all SUs)
+ * 
+ * @param sg
+ * @param node - NULL start all SUs in the SG
+ * @param node - !NULL start all SUs in the SG for the specified
+ *             node.
+ */
 extern void amf_sg_start (struct amf_sg *sg, struct amf_node *node);
 
 /**
@@ -528,7 +569,7 @@ extern void amf_sg_assign_si (struct amf_sg *sg, int dependency_level);
 extern void amf_sg_failover_node_req (
 	struct amf_sg *sg, struct amf_node *node);
 extern void amf_sg_failover_su_req (
-	struct amf_sg *sg, struct amf_node *node);
+	struct amf_sg *sg, struct amf_su *su, struct amf_node *node);
 extern void amf_sg_failover_comp_req (
 	struct amf_sg *sg, struct amf_node *node);
 extern void amf_sg_switchover_node_req (
@@ -553,18 +594,35 @@ extern void amf_sg_si_activated (
 /* General methods */
 extern void amf_su_init (void);
 extern struct amf_su *amf_su_create (void);
+extern char *amf_su_dn_make (struct amf_su *su, SaNameT *name);
 extern int amf_su_serialize (
 	struct amf_su *su, char **buf, int *offset);
 extern struct amf_su *amf_su_deserialize (
 	char **buf, int *size, struct amf_cluster *cluster);
 extern int amf_su_is_local (struct amf_su *su);
+extern struct amf_si_assignment *amf_su_get_next_si_assignment (
+	struct amf_su *su, const struct amf_si_assignment *si_assignment);
+extern void amf_su_foreach_si_assignment (
+	struct amf_su *su,
+	void (*foreach_fn)(struct amf_su *su,
+					   struct amf_si_assignment *si_assignment));
+extern int amf_su_get_saAmfSUNumCurrActiveSIs (struct amf_su *su);
+extern int amf_su_get_saAmfSUNumCurrStandbySIs (struct amf_su *su);
+extern SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (
+	struct amf_su *su);
 
 /* Event methods */
 extern void amf_su_instantiate (struct amf_su *su);
 extern void amf_su_assign_si (
 	struct amf_su *su, struct amf_si *si, SaAmfHAStateT ha_state);
 extern void amf_su_restart_req (struct amf_su *su);
+
+/**
+ * Request termination of all component in an SU
+ * @param su
+ */
 extern void amf_su_terminate (struct amf_su *su);
+
 extern struct amf_node *amf_su_get_node (struct amf_su *su);
 extern void amf_su_escalation_level_reset (struct amf_su *su);
 extern void amf_su_remove_assignment (struct amf_su *su);
@@ -598,14 +656,29 @@ extern int amf_comp_serialize (
 	struct amf_comp *comp, char **buf, int *offset);
 extern struct amf_comp *amf_comp_deserialize (
 	char **buf, int *size, struct amf_cluster *cluster);
+extern void amf_comp_foreach_csi_assignment (
+	struct amf_comp *component,
+	void (*foreach_fn)(struct amf_comp *component,
+					   struct amf_csi_assignment *csi_assignment));
+extern struct amf_csi_assignment *amf_comp_get_next_csi_assignment (
+	struct amf_comp *component, const struct amf_csi_assignment *csi_assignment);
+extern SaAmfReadinessStateT amf_comp_get_saAmfCompReadinessState (
+	struct amf_comp *comp);
 
 /* Event methods */
 extern void amf_comp_instantiate (struct amf_comp *comp);
 extern void amf_comp_terminate (struct amf_comp *comp);
+
+/**
+ * Request the component to assume a HA state
+ * @param comp
+ * @param csi_assignment
+ * @param requested_ha_state
+ */
 extern void amf_comp_hastate_set (
 	struct amf_comp *comp,
-	struct amf_csi_assignment *csi_assignment,
-	SaAmfHAStateT requested_ha_state);
+	struct amf_csi_assignment *csi_assignment);
+
 extern void amf_comp_restart (struct amf_comp *comp);
 extern void amf_comp_operational_state_set (
 	struct amf_comp *comp, SaAmfOperationalStateT opstate);
@@ -617,9 +690,26 @@ extern void amf_comp_healthcheck_tmo (
 	struct amf_comp *comp, struct amf_healthcheck *healthcheck);
 extern void amf_comp_cleanup_completed (struct amf_comp *comp);
 
+/**
+ * Count number of active CSI assignments
+ * @param component
+ * 
+ * @return int
+ */
+extern int amf_comp_get_saAmfCompNumCurrActiveCsi(struct amf_comp *component);
+
+/**
+ * Count number of standby CSI assignments
+ * @param component
+ * 
+ * @return int
+ */
+extern int amf_comp_get_saAmfCompNumCurrStandbyCsi(struct amf_comp *component);
+
 /*
  * Originates from library                                                                
  */
+
 extern SaAisErrorT amf_comp_healthcheck_start (
 	struct amf_comp *comp,
 	SaAmfHealthcheckKeyT *healthcheckKey,
@@ -655,24 +745,69 @@ extern int amf_si_serialize (
 extern struct amf_si *amf_si_deserialize (
 	char **buf, int *size, struct amf_cluster *cluster);
 
+/**
+ * Get number of active assignments for the specified SI
+ * @param si
+ * 
+ * @return int
+ */
+extern int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si);
+
+/**
+ * Get number of standby assignments for the specified SI
+ * @param si
+ * 
+ * @return int
+ */
+extern int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si);
+
+/**
+ * Get assignment state for the specified SI.
+ * @param si
+ * 
+ * @return SaAmfAssignmentStateT
+ */
+extern SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState (
+	struct amf_si *si);
+
 /* Event methods */
 
 /**
- * Activate all active assignments
+ * Activate all active assignments. Request component to change
+ * HA state to active.
  * @param si
  * @param activated_callback_fn
  */
 extern void amf_si_activate (
 	struct amf_si *si,
 	void (*activated_callback_fn)(struct amf_si *si, int result));
-extern void amf_si_deactivate (
-	struct amf_si *si,
-	struct amf_csi *csi,
-	void (*deactivated_callback_fn)(struct amf_si *si, int result));
-extern void amf_si_set_ha_state (
-	struct amf_si *si,
-	SaAmfHAStateT ha_state,
-	void (*set_ha_state_callback_fn)(struct amf_si *si, int result));
+
+/**
+ * Deactivate all active assignments. Request component to
+ * change HA state to quiesced.
+ * 
+ * @param si_assignment
+ * @param deactivated_callback_fn
+ * 
+ * @return int 1 - deactived immediately
+ * @return int 0 - asynchronous response through callback
+ */
+/***/
+extern int amf_si_deactivate (
+	struct amf_si_assignment *si_assignment,
+	void (*deactivated_callback_fn)(struct amf_si_assignment *si_assignment,
+		int result));
+
+/**
+ * Request SI (SU) to assume a HA state (request component)
+ * 
+ * @param si_assignment
+ * @param assumed_ha_state_callback_fn
+ */
+extern void amf_si_ha_state_assume (
+	struct amf_si_assignment *si_assignment,
+	void (*assumed_ha_state_callback_fn)(struct amf_si_assignment *si_assignment,
+		int result));
 
 /**
  * Component reports to SI that a workload assignment succeeded.
@@ -691,6 +826,13 @@ extern void amf_si_comp_set_ha_state_done (
 extern void amf_si_comp_set_ha_state_failed (
 	struct amf_si *si, struct amf_csi_assignment *csi_assignment);
 
+/**
+ * Request a CSI to delete all CSI assignments.
+ * 
+ * @param component
+ * @param csi
+ */
+extern void amf_csi_delete_assignments (struct amf_csi *csi, struct amf_su *su);
 
 /* General methods */
 extern struct amf_csi *amf_csi_create (void);

exec/amfcluster.c

@@ -119,10 +119,10 @@ static void timer_function_cluster_startup_tmo (void *_cluster)
 	struct amf_cluster *cluster = _cluster;
 	struct amf_application *app;
 
-	dprintf("1st Cluster start timer expired, starting applications");
+	log_printf(LOG_NOTICE, "AMF Cluster: starting applications.");
 
 	for (app = cluster->application_head; app != NULL; app = app->next) {
-		amf_application_start (app, this_amf_node);
+		amf_application_start (app, NULL);
 	}
 
 	/* wait a while before assigning workload */
@@ -153,13 +153,14 @@ void amf_cluster_application_started (
 {
 	ENTER ("application '%s' started", application->name.value);
 
-	if (cluster->timeout_handle) {
-		poll_timer_delete (aisexec_poll_handle, cluster->timeout_handle);
-	}
-
 	if (all_applications_started (cluster)) {
 		struct amf_application *app;
-		dprintf("All applications started, assigning workload.");
+
+		log_printf(LOG_NOTICE, "AMF Cluster: all applications started, assigning workload.");
+
+		if (cluster->timeout_handle) {
+			poll_timer_delete (aisexec_poll_handle, cluster->timeout_handle);
+		}
 
 		for (app = cluster->application_head; app != NULL; app = app->next) {
 			amf_application_assign_workload (app, this_amf_node);
@@ -184,6 +185,6 @@ struct amf_cluster *amf_cluster_create (void)
 void amf_cluster_application_workload_assigned (
 	struct amf_cluster *cluster, struct amf_application *app)
 {
-	ENTER ("'%s'", app->name.value);
+	log_printf(LOG_NOTICE, "AMF Cluster: all workload assigned.");
 }
 

exec/amfcomp.c

@@ -9,7 +9,8 @@
  * - Use DN in API and multicast messages
  * - (Re-)Introduction of event based multicast messages
  * - Refactoring of code into several AMF files
- *  Author: Anders Eriksson
+ *  Author: Anders Eriksson, Lars Holm
+ *  - Component/SU restart, SU failover
  *
  * All rights reserved.
  *
@@ -171,9 +172,6 @@ struct component_terminate_callback_data {
 static void comp_presence_state_set (
 	struct amf_comp *comp,
 	SaAmfPresenceStateT presence_state);
-static void comp_operational_state_set (
-	struct amf_comp *comp,
-	SaAmfOperationalStateT operational_state);
 static int clc_cli_instantiate (struct amf_comp *comp);
 static int clc_instantiate_callback (struct amf_comp *comp);
 static int clc_csi_set_callback (struct amf_comp *comp);
@@ -185,6 +183,9 @@ static int clc_cli_cleanup_local (struct amf_comp *comp);
 static void healthcheck_deactivate (struct amf_healthcheck *healthcheck_active);
 static void lib_healthcheck_request (struct amf_healthcheck *healthcheck);
 static void timer_function_healthcheck_tmo (void *_healthcheck);
+static void lib_csi_set_request (
+	struct amf_comp *comp,
+	struct amf_csi_assignment *csi_assignment);
 
 /*
  * Life cycle functions
@@ -309,6 +310,21 @@ static void invocation_destroy_by_data (void *data)
 	}
 }
 
+/**
+ * Set suspected error flag and report to SU.
+ * 
+ * @param comp
+ * @param recommended_recovery
+ */
+static void report_error_suspected (
+	struct amf_comp *comp,
+	SaAmfRecommendedRecoveryT recommended_recovery)
+{
+	comp->error_suspected = 1;
+	amf_su_comp_error_suspected (
+		comp->su, comp, recommended_recovery);
+}
+
 char *amf_comp_dn_make (struct amf_comp *comp, SaNameT *name)
 {
 	int	i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH,
@@ -350,6 +366,15 @@ static void *clc_command_run (void *context)
 	if (pid) {
 		xprintf ("waiting for pid %d to finish\n", pid);
 		waitpid (pid, &status, 0);
+		if (WEXITSTATUS(status) != 0) {
+			fprintf (stderr, "Error: CLC_CLI failed with exit status:"
+							 " %d - %s\n", WEXITSTATUS(status),
+						strerror (WEXITSTATUS(status)));
+			/*                                                              
+             * TODO: remove this and handle properly later...
+             */
+			openais_exit_error (AIS_DONE_FATAL_ERR);
+		}
 		xprintf ("process (%d) finished with %d\n", pid, status);
 		if (clc_command_run_data->completion_callback) {
 			clc_command_run_data->completion_callback (context);
@@ -496,30 +521,24 @@ static int clc_cli_terminate (struct amf_comp *comp)
 	return (0);
 }
 
+/**
+ * Request component to terminate itself
+ * @param comp
+ * 
+ * @return int
+ */
 static int lib_comp_terminate_request (struct amf_comp *comp)
 {
-	struct res_lib_amf_componentterminatecallback res_lib_amf_componentterminatecallback;
+	struct res_lib_amf_componentterminatecallback res_lib;
 	struct component_terminate_callback_data *component_terminate_callback_data;
 
 	ENTER("comp %s\n", getSaNameT (&comp->name));
 
-	if (comp->saAmfCompPresenceState != SA_AMF_PRESENCE_INSTANTIATED) {
-		dprintf ("component terminated but not instantiated %s - %d\n",
-			getSaNameT (&comp->name), comp->saAmfCompPresenceState);
-		assert (0);
-		return (0);
-	}
-
-	dprintf ("component name terminating %s\n", getSaNameT (&comp->name));
-	dprintf ("component presence state %d\n", comp->saAmfCompPresenceState);
-
-	res_lib_amf_componentterminatecallback.header.id = MESSAGE_RES_AMF_COMPONENTTERMINATECALLBACK;
-	res_lib_amf_componentterminatecallback.header.size = sizeof (struct res_lib_amf_componentterminatecallback);
-	res_lib_amf_componentterminatecallback.header.error = SA_AIS_OK;
+	res_lib.header.id = MESSAGE_RES_AMF_COMPONENTTERMINATECALLBACK;
+	res_lib.header.size = sizeof (struct res_lib_amf_componentterminatecallback);
+	res_lib.header.error = SA_AIS_OK;
 
-
-	memcpy (&res_lib_amf_componentterminatecallback.compName,
-		&comp->name, sizeof (SaNameT));
+	memcpy (&res_lib.compName, &comp->name, sizeof (SaNameT));
 
 	component_terminate_callback_data =
 		malloc (sizeof (struct component_terminate_callback_data));
@@ -528,16 +547,14 @@ static int lib_comp_terminate_request (struct amf_comp *comp)
 	}
 	component_terminate_callback_data->comp = comp;
 
-	res_lib_amf_componentterminatecallback.invocation =
+	res_lib.invocation =
 		invocation_create (
 		AMF_RESPONSE_COMPONENTTERMINATECALLBACK,
 		component_terminate_callback_data);
-	dprintf ("Creating invocation %llu", 
-	(unsigned long long)res_lib_amf_componentterminatecallback.invocation);
 				        
 	openais_conn_send_response (
 		openais_conn_partner_get (comp->conn),
-		&res_lib_amf_componentterminatecallback,
+		&res_lib,
 		sizeof (struct res_lib_amf_componentterminatecallback));
 
 	return (0);
@@ -754,49 +771,25 @@ static void comp_ha_state_set (	struct amf_comp *comp,
 	struct amf_csi_assignment *csi_assignment,
 	SaAmfHAStateT ha_state)
 {
-	/*                                                              
-     * Filter component restart from reporting to SI. TODO: how is
-     * this handled the right way?
-     */
-	if (csi_assignment->saAmfCSICompHAState !=
-		csi_assignment->requested_ha_state) {
-
-		/* set confirmed HA state */
-		csi_assignment->saAmfCSICompHAState = ha_state;
-		TRACE1 ("Setting comp '%s', SU '%s' CSI '%s', HA state: %s\n",
-				comp->name.value, comp->su->name.value,
-				csi_assignment->csi->name.value,
-				amf_ha_state (csi_assignment->saAmfCSICompHAState));
-		amf_si_comp_set_ha_state_done (csi_assignment->csi->si, csi_assignment);
-	}
+	/* set confirmed HA state */
+	csi_assignment->saAmfCSICompHAState = ha_state;
+	TRACE1 ("Setting comp '%s', SU '%s' CSI '%s', HA state: %s\n",
+		comp->name.value, comp->su->name.value,
+		csi_assignment->csi->name.value,
+		amf_ha_state (csi_assignment->saAmfCSICompHAState));
+	amf_si_comp_set_ha_state_done (csi_assignment->csi->si, csi_assignment);
 }
 
 static void comp_presence_state_set (struct amf_comp *comp,
 	SaAmfPresenceStateT presence_state)
 {
-	SaAmfPresenceStateT old = comp->saAmfCompPresenceState;
-
 	comp->saAmfCompPresenceState = presence_state;
 	TRACE1 ("Setting comp '%s', SU '%s' presence state: %s\n",
 		comp->name.value, comp->su->name.value,
 		amf_presence_state (comp->saAmfCompPresenceState));
 
-	/* TODO: ugly, should be done by SU... */
-	if (old != SA_AMF_PRESENCE_RESTARTING) {
-		amf_su_comp_state_changed (
-			comp->su, comp, SA_AMF_PRESENCE_STATE, presence_state);
-	}
-}
-
-static void comp_operational_state_set (struct amf_comp *comp,
-	SaAmfOperationalStateT oper_state)
-{
-	comp->saAmfCompOperState = oper_state;
-	TRACE1 ("Setting comp '%s', SU '%s' operational state: %s\n",
-		comp->name.value, comp->su->name.value,
-		amf_op_state (comp->saAmfCompOperState));
 	amf_su_comp_state_changed (
-		comp->su, comp, SA_AMF_OP_STATE, oper_state);
+		comp->su, comp, SA_AMF_PRESENCE_STATE, presence_state);
 }
 
 #if 0
@@ -836,16 +829,99 @@ static void lib_csi_remove_request (struct amf_comp *comp,
 }
 #endif
 
-static void comp_reassign_csis (struct amf_comp *comp)
+struct amf_csi_assignment *amf_comp_get_next_csi_assignment (
+	struct amf_comp *component, const struct amf_csi_assignment *csi_assignment)
 {
-	struct amf_csi_assignment *csi_assignment = comp->assigned_csis;
+	struct amf_si *si;
+	struct amf_csi *csi;
+	struct amf_csi_assignment *tmp_csi_assignment;
+	SaNameT dn;
 
-	ENTER ("'%s'", comp->name.value);
+	amf_comp_dn_make (component, &dn);
 
-	for (; csi_assignment; csi_assignment = csi_assignment->comp_next) {
-		amf_comp_hastate_set (comp, csi_assignment,
-			csi_assignment->requested_ha_state);
+	if (csi_assignment == NULL) {
+		si = component->su->sg->application->si_head;
+		csi = si->csi_head;
+		tmp_csi_assignment = csi->assigned_csis;
+	} else {
+		tmp_csi_assignment = csi_assignment->next;
+		if (tmp_csi_assignment == NULL) {
+			csi = csi_assignment->csi->next;
+			if (csi == NULL) {
+				si = csi_assignment->csi->si->next;
+				if (si == NULL) {
+					return NULL;
+				} else {
+					csi = si->csi_head;
+					tmp_csi_assignment = csi->assigned_csis;
+				}
+			} else {
+				si = csi->si;
+				tmp_csi_assignment = csi->assigned_csis;
+			}
+		} else {	
+			csi = tmp_csi_assignment->csi;
+			si = csi->si;
+		}
 	}
+
+	for (; si != NULL; si = si->next) {
+		if (tmp_csi_assignment == NULL && csi == NULL && si != NULL) {
+			csi = si->csi_head;
+			tmp_csi_assignment = csi->assigned_csis;
+		}
+
+		for (; csi != NULL; csi = csi->next) {
+			if (tmp_csi_assignment == NULL && csi != NULL) {
+				tmp_csi_assignment = csi->assigned_csis;
+			}
+
+			for (; tmp_csi_assignment != NULL;
+				tmp_csi_assignment = tmp_csi_assignment->next) {
+
+				if (name_match (&tmp_csi_assignment->name, &dn)) {
+					return tmp_csi_assignment;
+				}
+			}
+		}
+	}
+
+	return NULL;
+}
+
+void amf_comp_foreach_csi_assignment (
+	struct amf_comp *component,
+	void (*foreach_fn)(struct amf_comp *component,
+					   struct amf_csi_assignment *csi_assignment))
+{
+	struct amf_csi_assignment *csi_assignment;
+
+	assert (foreach_fn != NULL);
+	csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
+	while (csi_assignment != NULL) {
+		foreach_fn (component, csi_assignment);
+		csi_assignment = amf_comp_get_next_csi_assignment (
+			component, csi_assignment);
+	}
+}
+
+static struct amf_csi_assignment *csi_assignment_find_in (
+	struct amf_comp *component, SaNameT *csi_name)
+{
+	struct amf_csi_assignment *csi_assignment;
+	SaNameT dn;
+
+	csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
+	while (csi_assignment != NULL) {
+		amf_csi_dn_make (csi_assignment->csi, &dn);
+		if (name_match (csi_name, &dn)) {
+			return csi_assignment;
+		}
+		csi_assignment = amf_comp_get_next_csi_assignment (
+			component, csi_assignment);
+	}
+
+	return NULL;
 }
 
 static void healthcheck_deactivate (
@@ -926,37 +1002,30 @@ static void timer_function_healthcheck_tmo (
 
 static void lib_healthcheck_request (struct amf_healthcheck *healthcheck)
 {
-	struct res_lib_amf_healthcheckcallback res_lib_amf_healthcheckcallback;
-
-	res_lib_amf_healthcheckcallback.header.id =
-		MESSAGE_RES_AMF_HEALTHCHECKCALLBACK;
-	res_lib_amf_healthcheckcallback.header.size =
-		sizeof (struct res_lib_amf_healthcheckcallback);
-	res_lib_amf_healthcheckcallback.header.error = SA_AIS_OK;
+	struct res_lib_amf_healthcheckcallback res_lib;
 
-	res_lib_amf_healthcheckcallback.invocation =
+	res_lib.header.id =	MESSAGE_RES_AMF_HEALTHCHECKCALLBACK;
+	res_lib.header.size = sizeof (struct res_lib_amf_healthcheckcallback);
+	res_lib.header.error = SA_AIS_OK;
+	res_lib.invocation =
 		invocation_create (AMF_RESPONSE_HEALTHCHECKCALLBACK, healthcheck);
 
-	amf_comp_dn_make (healthcheck->comp,
-					  &res_lib_amf_healthcheckcallback.compName);
-	memcpy (&res_lib_amf_healthcheckcallback.key,
-		&healthcheck->safHealthcheckKey,
+	amf_comp_dn_make (healthcheck->comp, &res_lib.compName);
+	memcpy (&res_lib.key, &healthcheck->safHealthcheckKey,
 		sizeof (SaAmfHealthcheckKeyT));
 
 	TRACE8 ("sending healthcheck request to component %s",
-			res_lib_amf_healthcheckcallback.compName.value);
+			res_lib.compName.value);
 	openais_conn_send_response (
 		openais_conn_partner_get (healthcheck->comp->conn),
-		&res_lib_amf_healthcheckcallback,
-		sizeof (struct res_lib_amf_healthcheckcallback));
+		&res_lib, sizeof (struct res_lib_amf_healthcheckcallback));
 }
 
 static void lib_csi_set_request (
 	struct amf_comp *comp,
-	struct amf_csi_assignment *csi_assignment,
-	SaAmfHAStateT requested_ha_state)
+	struct amf_csi_assignment *csi_assignment)
 {
-	struct res_lib_amf_csisetcallback* res_lib_amf_csisetcallback;     
+	struct res_lib_amf_csisetcallback* res_lib;     
 	void*  p;
 	struct amf_csi_attribute *attribute;
 	size_t char_length_of_csi_attrs=0;
@@ -966,16 +1035,18 @@ static void lib_csi_set_request (
 	char* csi_attribute_buf;
 	unsigned int byte_offset;
 
-	csi_assignment->requested_ha_state = requested_ha_state;
+	if (!amf_su_is_local (comp->su))
+		return;
+
 	csi = csi_assignment->csi;
 
-	dprintf("Assigning CSI '%s' state %s to comp '%s'\n",
-		getSaNameT (&csi->name), amf_ha_state (requested_ha_state),
+	ENTER ("Assigning CSI '%s' state %s to comp '%s'\n",
+		getSaNameT (&csi->name), amf_ha_state (csi_assignment->requested_ha_state),
 			comp->name.value);
 
 	for (attribute = csi->attributes_head;
-		 attribute != NULL;
-		 attribute = attribute->next) {
+		attribute != NULL;
+		attribute = attribute->next) {
 		for (i = 0; attribute->value[i] != NULL; i++) {
 			num_of_csi_attrs++;
 			char_length_of_csi_attrs += strlen(attribute->name);
@@ -983,23 +1054,23 @@ static void lib_csi_set_request (
 			char_length_of_csi_attrs += 2;
 		}
 	}
-	p = malloc(sizeof(struct res_lib_amf_csisetcallback)+
+	p = malloc(sizeof(struct res_lib_amf_csisetcallback) +
 			   char_length_of_csi_attrs);
 	if (p == NULL) {
 		openais_exit_error (AIS_DONE_OUT_OF_MEMORY);
 	}
 
-	res_lib_amf_csisetcallback = (struct res_lib_amf_csisetcallback*)p;
+	res_lib = (struct res_lib_amf_csisetcallback*)p;
 
 	/* Address of the buffer containing the Csi name value pair  */
-	csi_attribute_buf = res_lib_amf_csisetcallback->csi_attr_buf;
+	csi_attribute_buf = res_lib->csi_attr_buf;
 
 	/* Byteoffset start at the zero byte  */
 	byte_offset = 0;
 
 	for (attribute = csi->attributes_head;
-		 attribute != NULL;
-		 attribute = attribute->next) {
+		attribute != NULL;
+		attribute = attribute->next) {
 
 		for (i = 0; attribute->value[i] != NULL; i++) {
 			strcpy(&csi_attribute_buf[byte_offset], (char*)attribute->name);
@@ -1009,51 +1080,50 @@ static void lib_csi_set_request (
 		}
 	}
 
-	res_lib_amf_csisetcallback->number = num_of_csi_attrs;
-	res_lib_amf_csisetcallback->csiFlags = SA_AMF_CSI_ADD_ONE;  
+	res_lib->number = num_of_csi_attrs;
+	res_lib->csiFlags = SA_AMF_CSI_ADD_ONE;  
 
-	switch (requested_ha_state) {
+	switch (csi_assignment->requested_ha_state) {
 		case SA_AMF_HA_ACTIVE: {
-				res_lib_amf_csisetcallback->csiStateDescriptor.activeDescriptor.activeCompName.length = 0;
-				res_lib_amf_csisetcallback->csiStateDescriptor.activeDescriptor.transitionDescriptor =
-					SA_AMF_CSI_NEW_ASSIGN; 
-				break;
-			}
+			res_lib->csiStateDescriptor.activeDescriptor.activeCompName.length = 0;
+			res_lib->csiStateDescriptor.activeDescriptor.transitionDescriptor =
+				SA_AMF_CSI_NEW_ASSIGN; 
+			break;
+		}
 		case SA_AMF_HA_STANDBY: {
-				res_lib_amf_csisetcallback->csiStateDescriptor.standbyDescriptor.activeCompName.length = 0; 
-				res_lib_amf_csisetcallback->csiStateDescriptor.standbyDescriptor.standbyRank =  1;
-				break;
-			}
+			res_lib->csiStateDescriptor.standbyDescriptor.activeCompName.length = 0; 
+			res_lib->csiStateDescriptor.standbyDescriptor.standbyRank =  1;
+			break;
+		}
 		case SA_AMF_HA_QUIESCED: {
-				/*TODO*/
-				break;
-			}
+			/*TODO*/
+			break;
+		}
 		case SA_AMF_HA_QUIESCING: {
-				/*TODO*/
-				break;
-			}
+			/*TODO*/
+			break;
+		}
 		default: {
-				assert(SA_AMF_HA_ACTIVE||SA_AMF_HA_STANDBY||SA_AMF_HA_QUIESCING||SA_AMF_HA_QUIESCED);         
-				break;
-			}
+			assert(SA_AMF_HA_ACTIVE||SA_AMF_HA_STANDBY||SA_AMF_HA_QUIESCING||SA_AMF_HA_QUIESCED);         
+			break;
+		}
 	}
 
-	res_lib_amf_csisetcallback->header.id = MESSAGE_RES_AMF_CSISETCALLBACK;
-	res_lib_amf_csisetcallback->header.size = 
+	res_lib->header.id = MESSAGE_RES_AMF_CSISETCALLBACK;
+	res_lib->header.size = 
 		sizeof (struct res_lib_amf_csisetcallback) +
 		char_length_of_csi_attrs;
-	res_lib_amf_csisetcallback->header.error = SA_AIS_OK;
+	res_lib->header.error = SA_AIS_OK;
 
-	amf_comp_dn_make (comp, &res_lib_amf_csisetcallback->compName);
-	amf_csi_dn_make (csi, &res_lib_amf_csisetcallback->csiName);
+	amf_comp_dn_make (comp, &res_lib->compName);
+	amf_csi_dn_make (csi, &res_lib->csiName);
 
-	res_lib_amf_csisetcallback->haState = requested_ha_state;
-	res_lib_amf_csisetcallback->invocation =
+	res_lib->haState = csi_assignment->requested_ha_state;
+	res_lib->invocation =
 		invocation_create (AMF_RESPONSE_CSISETCALLBACK, csi_assignment);
 
-	openais_conn_send_response (openais_conn_partner_get (comp->conn),
-		res_lib_amf_csisetcallback,
-		res_lib_amf_csisetcallback->header.size);
+	openais_conn_send_response (
+		openais_conn_partner_get (comp->conn), res_lib,	res_lib->header.size);
 
 	free(p);
 }
@@ -1064,11 +1134,8 @@ SaAisErrorT amf_comp_register (struct amf_comp *comp)
 
 	if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING) {
 		comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATED);
-		if (comp->saAmfCompReadinessState == SA_AMF_READINESS_IN_SERVICE) {
-			comp_reassign_csis (comp);
-		}
 	} else if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_INSTANTIATING) {
-		comp_operational_state_set (comp, SA_AMF_OPERATIONAL_ENABLED);
+		amf_comp_operational_state_set (comp, SA_AMF_OPERATIONAL_ENABLED);
 		comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATED);
 	}
 	else {
@@ -1093,7 +1160,7 @@ void amf_comp_error_report (
 	}
 
 	/* report to SU and let it handle the problem */
-	amf_su_comp_error_suspected (comp->su, comp, recommendedRecovery);
+	report_error_suspected (comp, recommendedRecovery);
 }
 
 /**
@@ -1107,8 +1174,14 @@ void amf_comp_healthcheck_tmo (
 	TRACE2("Exec healthcheck tmo for '%s'", &comp->name.value);
 
 	/* report to SU and let it handle the problem */
-	amf_su_comp_error_suspected (
-		comp->su, comp, healthcheck->recommendedRecovery);
+	report_error_suspected (comp, healthcheck->recommendedRecovery);
+}
+
+static void clear_ha_state (
+	struct amf_comp *comp, struct amf_csi_assignment *csi_assignment)
+{
+	ENTER ("");
+	csi_assignment->saAmfCSICompHAState = 0;
 }
 
 /**
@@ -1119,12 +1192,16 @@ void amf_comp_cleanup_completed (struct amf_comp *comp)
 {
 	TRACE2("Exec CLC cleanup completed for '%s'", &comp->name.value);
 
+    /* Set all CSI's confirmed HA state to unknown  */
+	amf_comp_foreach_csi_assignment (comp, clear_ha_state);
+
+	/* clear error suspected flag, component is terminated now */
+	comp->error_suspected = 0;
+
 	if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING) {
 		amf_comp_instantiate (comp);
 	} else {
 		comp_presence_state_set (comp, SA_AMF_PRESENCE_UNINSTANTIATED);
-		amf_su_comp_state_changed (comp->su, comp, SA_AMF_PRESENCE_STATE,
-								   SA_AMF_PRESENCE_UNINSTANTIATED);
 	}
 }
 
@@ -1257,26 +1334,9 @@ void amf_comp_instantiate (struct amf_comp *comp)
 void amf_comp_readiness_state_set (struct amf_comp *comp,
 	SaAmfReadinessStateT state)
 {
-#if 0
-	/*
-	 * Set component readiness state appropriately
-	 * if unit in service and component is enabled, it is in service
-	 * otherwise it is out of service page 50 B.02.01
-	 */
-	if (comp->su->saAmfSUReadinessState == SA_AMF_READINESS_IN_SERVICE &&
-		comp->saAmfCompOperState == SA_AMF_OPERATIONAL_ENABLED) {
-		comp->saAmfCompReadinessState = SA_AMF_READINESS_IN_SERVICE;
-	} else if (comp->su->saAmfSUReadinessState == SA_AMF_READINESS_STOPPING && 
-		comp->saAmfCompOperState == SA_AMF_OPERATIONAL_ENABLED) {
-		comp->saAmfCompReadinessState = SA_AMF_READINESS_STOPPING;
-	} else {
-		comp->saAmfCompReadinessState = SA_AMF_READINESS_OUT_OF_SERVICE;
-	}
-#endif
-
-	comp->saAmfCompReadinessState = state;
+//	comp->saAmfCompReadinessState = state;
 	TRACE1 ("Setting comp '%s' readiness state: %s\n",
-		comp->name.value, amf_readiness_state (comp->saAmfCompReadinessState));
+		comp->name.value, amf_readiness_state (state));
 }
 
 /**
@@ -1389,8 +1449,8 @@ struct amf_comp *amf_comp_response_2 (
 				csi_assignment->comp->name.value, error);
 			comp = csi_assignment->comp;
 			if (error == SA_AIS_OK) {
-				comp_ha_state_set (comp, csi_assignment,
-								   csi_assignment->requested_ha_state);
+				comp_ha_state_set (
+					comp, csi_assignment, csi_assignment->requested_ha_state);
 			} else if (error == SA_AIS_ERR_FAILED_OPERATION) {
 				amf_si_comp_set_ha_state_failed (csi_assignment->csi->si,
 					csi_assignment);
@@ -1416,10 +1476,14 @@ struct amf_comp *amf_comp_response_2 (
 			}
 			break;
 		}
-#if 0
-		case AMF_RESPONSE_COMPONENTTERMINATECALLBACK:
+	    case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: {
+			struct component_terminate_callback_data *callback_data = data;
+			dprintf ("Lib comp '%s' terminate callback response, error: %d",
+					 callback_data->comp->name.value, error);
+			comp_presence_state_set (callback_data->comp,
+				SA_AMF_PRESENCE_UNINSTANTIATED);
 			break;
-#endif
+		}
 		default:
 			assert (0);
 			break;
@@ -1435,16 +1499,22 @@ struct amf_comp *amf_comp_response_2 (
  * @param requested_ha_state
  */
 void amf_comp_hastate_set (
-	struct amf_comp *comp,
-	struct amf_csi_assignment *csi_assignment,
-	SaAmfHAStateT requested_ha_state)
+	struct amf_comp *component,
+	struct amf_csi_assignment *csi_assignment)
 {
-	assert (comp != NULL && csi_assignment != NULL);
+	assert (component != NULL && csi_assignment != NULL);
 
-	if (!amf_su_is_local (comp->su))
-		return;
+	ENTER ("'%s'", csi_assignment->csi->name.value);
 
-	lib_csi_set_request(comp, csi_assignment, requested_ha_state);
+	if (!component->error_suspected) {
+		lib_csi_set_request(component, csi_assignment);
+	} else {
+		if (csi_assignment->requested_ha_state == SA_AMF_HA_QUIESCED) {
+			csi_assignment->saAmfCSICompHAState = csi_assignment->requested_ha_state;
+		} else {
+			assert (0);
+		}
+	}
 }
 
 /**
@@ -1458,7 +1528,11 @@ void amf_comp_terminate (struct amf_comp *comp)
 	comp_presence_state_set (comp, SA_AMF_PRESENCE_TERMINATING);
 
 	if (amf_su_is_local (comp->su)) {
-		clc_interfaces[comp->comptype]->terminate (comp);
+		if (comp->error_suspected) {
+			clc_interfaces[comp->comptype]->cleanup (comp);
+		} else {
+			clc_interfaces[comp->comptype]->terminate (comp);
+		}
 	}
 }
 
@@ -1469,8 +1543,9 @@ void amf_comp_terminate (struct amf_comp *comp)
 void amf_comp_restart (struct amf_comp *comp)
 {
 	dprintf ("comp restart '%s'\n", getSaNameT (&comp->name));
-	amf_comp_healthcheck_stop (comp, NULL);
 	comp_presence_state_set (comp, SA_AMF_PRESENCE_RESTARTING);
+	comp->saAmfCompRestartCount += 1;
+	amf_comp_healthcheck_stop (comp, NULL);
 
 	if (amf_su_is_local (comp->su)) {
 		clc_interfaces[comp->comptype]->cleanup (comp);
@@ -1489,20 +1564,16 @@ SaAisErrorT amf_comp_hastate_get (
 	struct amf_comp *comp, SaNameT *csi_name, SaAmfHAStateT *ha_state)
 {
 	struct amf_csi_assignment *assignment;
-	SaNameT name;
 
 	assert (comp != NULL && csi_name != NULL && ha_state != NULL);
 
 	dprintf ("comp ha state get from comp '%s' CSI '%s'\n",
 			 getSaNameT (&comp->name), csi_name->value);
 
-	for (assignment = comp->assigned_csis;
-		  assignment != NULL; assignment = assignment->comp_next) {
-		amf_csi_dn_make (assignment->csi, &name);
-		if (name_match (csi_name, &name)) {
-			*ha_state = assignment->saAmfCSICompHAState;
-			return SA_AIS_OK;
-		}
+	assignment = csi_assignment_find_in (comp, csi_name);
+	if (assignment != NULL) {
+		*ha_state = assignment->saAmfCSICompHAState;
+		return SA_AIS_OK;
 	}
 
 	return SA_AIS_ERR_INVALID_PARAM;
@@ -1560,3 +1631,60 @@ void amf_comp_init (void)
 	log_init ("AMF");
 }
 
+void amf_comp_operational_state_set (struct amf_comp *comp,
+	SaAmfOperationalStateT oper_state)
+{
+	comp->saAmfCompOperState = oper_state;
+	TRACE1 ("Setting comp '%s', SU '%s' operational state: %s\n",
+		comp->name.value, comp->su->name.value,
+		amf_op_state (comp->saAmfCompOperState));
+	amf_su_comp_state_changed (
+		comp->su, comp, SA_AMF_OP_STATE, oper_state);
+}
+
+int amf_comp_get_saAmfCompNumCurrActiveCsi(struct amf_comp *component)
+{
+	int cnt = 0;
+	struct amf_csi_assignment *csi_assignment;
+
+	csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
+	while (csi_assignment != NULL) {
+		if (csi_assignment->saAmfCSICompHAState == SA_AMF_HA_ACTIVE) {
+			cnt++;
+		}
+		csi_assignment = amf_comp_get_next_csi_assignment (
+			component, csi_assignment);
+	}
+
+	return cnt;
+}
+
+int amf_comp_get_saAmfCompNumCurrStandbyCsi(struct amf_comp *component)
+{
+	int cnt = 0;
+	struct amf_csi_assignment *csi_assignment;
+
+	csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
+	while (csi_assignment != NULL) {
+		if (csi_assignment->saAmfCSICompHAState == SA_AMF_HA_STANDBY) {
+			cnt++;
+		}
+		csi_assignment = amf_comp_get_next_csi_assignment (
+			component, csi_assignment);
+	}
+
+	return cnt;
+}
+
+SaAmfReadinessStateT amf_comp_get_saAmfCompReadinessState (
+	struct amf_comp *component)
+{
+	if (component->saAmfCompOperState == SA_AMF_OPERATIONAL_ENABLED) {
+		return amf_su_get_saAmfSUReadinessState (component->su);
+
+	} else if (component->saAmfCompOperState == SA_AMF_OPERATIONAL_DISABLED) {
+		return SA_AMF_READINESS_OUT_OF_SERVICE;
+	}
+
+	assert (0);
+}

exec/amfsg.c

@@ -9,7 +9,8 @@
  * - Use DN in API and multicast messages
  * - (Re-)Introduction of event based multicast messages
  * - Refactoring of code into several AMF files
- *  Author: Anders Eriksson
+ *  Author: Anders Eriksson, Lars Holm
+ *  - Component/SU restart, SU failover
  *
  * All rights reserved.
  *
@@ -146,6 +147,235 @@
 #include "main.h"
 #include "util.h"
 
+/**
+ * Delete all SI assignments and all CSI assignments
+ * by requesting all contained components.
+ * @param su
+ */
+static void delete_si_assignments (struct amf_su *su)
+{
+	struct amf_csi *csi;
+	struct amf_si *si;
+	struct amf_si_assignment *si_assignment;
+	
+	ENTER ("'%s'", su->name.value);
+
+	for (si = su->sg->application->si_head; si != NULL; si = si->next) {
+		if (!name_match (&si->saAmfSIProtectedbySG, &su->sg->name)) {
+			continue;
+		}
+
+		for (csi = si->csi_head; csi != NULL; csi = csi->next) {
+			amf_csi_delete_assignments (csi, su);
+		}
+
+		/*                                                              
+		 * TODO: this only works for n+m where each SI list has only two
+		 * assignments, one active and one standby.
+		 * TODO: use DN instead
+		 */
+		if (si->assigned_sis->su == su) {
+			si_assignment = si->assigned_sis;
+			si->assigned_sis = si_assignment->next;
+			dprintf ("first");
+		} else {
+			si_assignment = si->assigned_sis->next;
+			si->assigned_sis->next = NULL;
+			dprintf ("second");
+		}
+		dprintf ("%p, %d, %d",
+				 si_assignment, si_assignment->name.length,
+				 si->assigned_sis->name.length);
+		assert (si_assignment != NULL);
+		free (si_assignment);
+	}
+}
+
+static int all_si_has_hastate (struct amf_su *su, SaAmfHAStateT hastate)
+{
+	struct amf_si_assignment *si_assignment;
+	int all_confirmed = 1;
+
+	si_assignment = amf_su_get_next_si_assignment (su, NULL);
+	while (si_assignment != NULL) {
+		if (si_assignment->saAmfSISUHAState != hastate) {
+			all_confirmed = 0;
+			break;
+		}
+		si_assignment =	amf_su_get_next_si_assignment (su, si_assignment);
+	}
+
+	return all_confirmed;
+}
+
+/**
+ * Callback function used by SI when an SI has been deactivated.
+ * @param si_assignment
+ * @param result
+ */
+static void failover_su_si_deactivated_cbfn (
+	struct amf_si_assignment *si_assignment, int result)
+{
+	ENTER ("'%s', %d", si_assignment->si->name.value, result);
+
+	/*
+	 * If all SI assignments for the SU are quiesced, goto next
+	 * state (TerminatingSuspected).
+	 */
+
+	if (all_si_has_hastate (si_assignment->su, SA_AMF_HA_QUIESCED)) {
+		si_assignment->su->sg->avail_state = SG_AC_TerminatingSuspected;
+
+		/*                                                              
+		 * Terminate suspected SU(s)
+		 */
+		amf_su_terminate (si_assignment->su);
+	}
+}
+
+static int su_instantiated_count (struct amf_sg *sg)
+{
+	int cnt = 0;
+	struct amf_su *su;
+
+	for (su = sg->su_head; su != NULL; su = su->next) {
+		if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED)
+			cnt++;
+	}
+
+	return cnt;
+}
+
+static void standby_su_activated_cbfn (
+	struct amf_si_assignment *si_assignment, int result)
+{
+	struct amf_su *su;
+
+	ENTER ("'%s', %d", si_assignment->si->name.value, result);
+
+	/*                                                              
+	 * TODO: create SI assignment to spare and assign them
+	 */
+	si_assignment->su->sg->avail_state = SG_AC_AssigningStandbyToSpare;
+
+	si_assignment->su->sg->avail_state = SG_AC_ReparingSu;
+
+	if (all_si_has_hastate (si_assignment->su, SA_AMF_HA_ACTIVE)) {
+		for (su = si_assignment->su->sg->su_head; su != NULL; su = su->next) {
+			if ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_UNINSTANTIATED) &&
+				(su_instantiated_count (si_assignment->su->sg) <
+				si_assignment->su->sg->saAmfSGNumPrefInserviceSUs)) {
+
+				amf_su_instantiate (su);
+			}
+		}
+	}
+}
+
+static void assign_si_assumed_cbfn (
+	struct amf_si_assignment *si_assignment, int result)
+{
+	struct amf_si_assignment *tmp_si_assignment;
+	struct amf_si *si;
+	struct amf_sg *sg = si_assignment->su->sg;
+	int si_assignment_cnt = 0;
+	int confirmed_assignments = 0;
+
+	ENTER ("'%s', %d", si_assignment->si->name.value, result);
+
+	/*                                                              
+	 * Report to application when all SIs that this SG protects
+	 * has been assigned or go back to idle state if not cluster
+	 * start.
+	 */
+	for (si = sg->application->si_head;	si != NULL; si = si->next) {
+		if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
+
+			for (tmp_si_assignment = si->assigned_sis;
+				tmp_si_assignment != NULL;
+				tmp_si_assignment = tmp_si_assignment->next) {
+
+				si_assignment_cnt++;
+				if (tmp_si_assignment->requested_ha_state ==
+					tmp_si_assignment->saAmfSISUHAState) {
+
+					confirmed_assignments++;
+				}
+			}
+		}
+	}
+
+	assert (confirmed_assignments != 0);
+
+	switch (sg->avail_state) {
+		case SG_AC_AssigningOnRequest:
+			if (si_assignment_cnt == confirmed_assignments) {
+				sg->avail_state = SG_AC_Idle;
+				amf_application_sg_assigned (sg->application, sg);
+			} else {
+				dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments);
+			}
+			break;
+		case SG_AC_AssigningStandBy:
+			{
+				SaNameT dn;
+				if (si_assignment_cnt == confirmed_assignments) {
+					sg->avail_state = SG_AC_Idle;
+					amf_su_dn_make (si_assignment->su, &dn);
+					sg->avail_state = SG_AC_Idle;
+					log_printf (
+						LOG_NOTICE, "'%s' failover recovery action finished",
+						dn.value);
+				}
+				break;
+			}
+		default:
+			dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt,
+					 confirmed_assignments);
+			amf_runtime_attributes_print (amf_cluster);
+			assert (0);
+	}
+}
+
+/**
+ * Find an SU assigned with standby workload and activate it.
+ * @param su
+ */
+static void standby_su_activate (struct amf_su *su)
+{
+	struct amf_si_assignment *su_si_assignment;
+	struct amf_si_assignment *si_assignment;
+
+	ENTER ("Old SU '%s'", su->name.value);
+
+	su->sg->avail_state = SG_AC_ActivatingStandby;
+
+	/*                                                              
+	 * For each (active) SI assignment on the old SU, find a standby
+	 * SI assignment and activate it.
+	 */
+	su_si_assignment = amf_su_get_next_si_assignment (su, NULL);
+	while (su_si_assignment != NULL) {
+		for (si_assignment = su_si_assignment->si->assigned_sis;
+			  si_assignment != NULL;
+			  si_assignment = si_assignment->next) {
+
+			if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
+
+				si_assignment->requested_ha_state = SA_AMF_HA_ACTIVE;
+				amf_si_ha_state_assume (
+					si_assignment, standby_su_activated_cbfn);
+				break; /* one standby is enough */
+			}
+		}
+		su_si_assignment = amf_su_get_next_si_assignment (su, su_si_assignment);
+	}
+
+	delete_si_assignments (su);
+
+	LEAVE ("");
+}
+
 static inline int div_round (int a, int b)
 {
 	int res;
@@ -156,153 +386,156 @@ static inline int div_round (int a, int b)
 	return res;
 }
 
-static int all_su_instantiated(struct amf_sg *sg)
+static int all_su_has_presence_state(
+	struct amf_sg *sg, SaAmfPresenceStateT state)
 {
 	struct amf_su   *su;
-	int all_instantiated = 1;
+	int all_set = 1;
 
 	for (su = sg->su_head; su != NULL; su = su->next) {
-		if (su->saAmfSUPresenceState != SA_AMF_PRESENCE_INSTANTIATED) {
-			all_instantiated = 0;
+		if (su->saAmfSUPresenceState != state) {
+			all_set = 0;
 			break;
 		}
 	}
 
-	return all_instantiated;
+	return all_set;
 }
 
-static int application_si_count_get (struct amf_application *app)
+/**
+ * Get number of SIs protected by the specified SG.
+ * @param sg
+ * 
+ * @return int
+ */
+static int sg_si_count_get (struct amf_sg *sg)
 {
 	struct amf_si *si;
-	int answer = 0;
+	int cnt = 0;
 
-	for (si = app->si_head; si != NULL; si = si->next) {
-		answer += 1;
+	for (si = sg->application->si_head; si != NULL; si = si->next) {
+		if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
+			cnt += 1;
+		}
 	}
-	return (answer);
+	return (cnt);
 }
 
-static void sg_assign_nm_active (struct amf_sg *sg, int su_units_assign)
+static void sg_assign_nm_active (struct amf_sg *sg, int su_active_assign)
 {
-	struct amf_su *unit;
+	struct amf_su *su;
 	struct amf_si *si;
 	int assigned = 0;
 	int assign_per_su = 0;
 	int total_assigned = 0;
+	int si_cnt;
 
-	assign_per_su = application_si_count_get (sg->application);
-	assign_per_su = div_round (assign_per_su, su_units_assign);
+	ENTER ("'%s'", sg->name.value);
+
+	si_cnt = sg_si_count_get (sg);
+	assign_per_su = div_round (si_cnt, su_active_assign);
 	if (assign_per_su > sg->saAmfSGMaxActiveSIsperSUs) {
 		assign_per_su = sg->saAmfSGMaxActiveSIsperSUs;
 	}
 
 	si = sg->application->si_head;
-	unit = sg->su_head;
-	while (unit != NULL) {
-		if (unit->saAmfSUReadinessState != SA_AMF_READINESS_IN_SERVICE ||
-			unit->saAmfSUNumCurrActiveSIs == sg->saAmfSGMaxActiveSIsperSUs ||
-			unit->saAmfSUNumCurrStandbySIs > 0) {
-
-			unit = unit->next;
+	su = sg->su_head;
+	while (su != NULL) {
+		if (amf_su_get_saAmfSUReadinessState (su) !=
+			   SA_AMF_READINESS_IN_SERVICE ||
+			amf_su_get_saAmfSUNumCurrActiveSIs (su) == 
+			  sg->saAmfSGMaxActiveSIsperSUs ||
+			amf_su_get_saAmfSUNumCurrStandbySIs (su) > 0) {
+
+			su = su->next;
 			continue; /* Not in service */
 		}
 
 		assigned = 0;
 		while (si != NULL &&
 			assigned < assign_per_su &&
-			total_assigned < application_si_count_get (sg->application)) {
+			total_assigned < si_cnt) {
 
-			assigned += 1;
-			total_assigned += 1;
-			amf_su_assign_si (unit, si, SA_AMF_HA_ACTIVE);
+			if (amf_si_get_saAmfSINumCurrActiveAssignments (si) == 0) {
+				assigned += 1;
+				total_assigned += 1;
+				amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE);
+			}
 			si = si->next;
 		}
-		unit = unit->next;
+		su = su->next;
 	}
 	
 	if (total_assigned == 0) {
-		dprintf ("Error: No SIs assigned!");
+		dprintf ("Info: No SIs assigned!");
 	}
 }
 
-static void sg_assign_nm_standby (struct amf_sg *sg, int units_assign_standby)
+static void sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign)
 {
-	struct amf_su *unit;
+	struct amf_su *su;
 	struct amf_si *si;
 	int assigned = 0;
 	int assign_per_su = 0;
 	int total_assigned = 0;
+	int si_cnt;
 
-	if (units_assign_standby == 0) {
+	ENTER ("'%s'", sg->name.value);
+
+	if (su_standby_assign == 0) {
 		return;
 	}
-	assign_per_su = application_si_count_get (sg->application);
-	assign_per_su = div_round (assign_per_su, units_assign_standby);
+	si_cnt = sg_si_count_get (sg);
+	assign_per_su = div_round (si_cnt, su_standby_assign);
 	if (assign_per_su > sg->saAmfSGMaxStandbySIsperSUs) {
 		assign_per_su = sg->saAmfSGMaxStandbySIsperSUs;
 	}
 
 	si = sg->application->si_head;
-	unit = sg->su_head;
-	while (unit != NULL) {
-		if (unit->saAmfSUReadinessState != SA_AMF_READINESS_IN_SERVICE ||
-			unit->saAmfSUNumCurrActiveSIs > 0 ||
-			unit->saAmfSUNumCurrStandbySIs == sg->saAmfSGMaxStandbySIsperSUs) {
-
-			unit = unit->next;
+	su = sg->su_head;
+	while (su != NULL) {
+		if (amf_su_get_saAmfSUReadinessState (su) !=
+			   SA_AMF_READINESS_IN_SERVICE ||
+			amf_su_get_saAmfSUNumCurrActiveSIs (su) > 0 ||
+			amf_su_get_saAmfSUNumCurrStandbySIs (su) ==
+			   sg->saAmfSGMaxStandbySIsperSUs) {
+
+			su = su->next;
 			continue; /* Not available for assignment */
 		}
 
 		assigned = 0;
 		while (si != NULL && assigned < assign_per_su) {
-			assigned += 1;
-			total_assigned += 1;
-			amf_su_assign_si (unit, si, SA_AMF_HA_STANDBY);
+
+			if (amf_si_get_saAmfSINumCurrStandbyAssignments (si) == 0) {
+				assigned += 1;
+				total_assigned += 1;
+				amf_su_assign_si (su, si, SA_AMF_HA_STANDBY);
+			}
 			si = si->next;
 		}
-		unit = unit->next;
+		su = su->next;
 	}
 	if (total_assigned == 0) {
-		dprintf ("Error: No SIs assigned!");
+		dprintf ("Info: No SIs assigned!");
 	}
 }
-#if 0
-static void assign_nm_spare (struct amf_sg *sg)
-{
-	struct amf_su *unit;
-
-	for (unit = sg->su_head; unit != NULL; unit = unit->next) {
-		if (unit->saAmfSUReadinessState == SA_AMF_READINESS_IN_SERVICE &&
-			(unit->requested_ha_state != SA_AMF_HA_ACTIVE &&
-			unit->requested_ha_state != SA_AMF_HA_STANDBY)) {
-
-			dprintf ("Assigning to SU %s with SPARE\n",
-				getSaNameT (&unit->name));
-		}
-	}
-}
-#endif
 
 static int su_inservice_count_get (struct amf_sg *sg)
 {
-	struct amf_su *unit;
+	struct amf_su *su;
 	int answer = 0;
 
-	for (unit = sg->su_head; unit != NULL; unit = unit->next) {
-		if (unit->saAmfSUReadinessState == SA_AMF_READINESS_IN_SERVICE) {
+	for (su = sg->su_head; su != NULL; su = su->next) {
+		if (amf_su_get_saAmfSUReadinessState (su) ==
+			SA_AMF_READINESS_IN_SERVICE) {
+
 			answer += 1;
 		}
 	}
 	return (answer);
 }
 
-static void si_activated_callback (struct amf_si *si, int result)
-{
-	/*                                                              
-     * TODO: not implemented yet...
-     */
-}
-
 /**
  * TODO: dependency_level not used, hard coded
  * @param sg
@@ -322,21 +555,25 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level)
 
 	ENTER ("'%s'", sg->name.value);
 
+	if (sg->avail_state == SG_AC_Idle) {
+		sg->avail_state = SG_AC_AssigningOnRequest;
+	}
+
 	/**
-     * Phase 1: Calculate assignments and create all runtime objects in
-     * information model. Do not do the actual assignment, done in
-     * phase 2.
-     */
+	 * Phase 1: Calculate assignments and create all runtime objects in
+	 * information model. Do not do the actual assignment, done in
+	 * phase 2.
+	 */
 
 	/**
 	 * Calculate number of SUs to assign to active or standby state
 	 */
 	inservice_count = (float)su_inservice_count_get (sg);
 
-	active_sus_needed = div_round (application_si_count_get (sg->application),
+	active_sus_needed = div_round (sg_si_count_get (sg),
 		sg->saAmfSGMaxActiveSIsperSUs);
 
-	standby_sus_needed = div_round (application_si_count_get (sg->application),
+	standby_sus_needed = div_round (sg_si_count_get (sg),
 		sg->saAmfSGMaxStandbySIsperSUs);
 
 	units_for_active = inservice_count - sg->saAmfSGNumPrefStandbySUs;
@@ -349,12 +586,14 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level)
 		units_for_standby = 0;
 	}
 
-	ii_spare = inservice_count - sg->saAmfSGNumPrefActiveSUs - sg->saAmfSGNumPrefStandbySUs;
+	ii_spare = inservice_count - sg->saAmfSGNumPrefActiveSUs -
+		sg->saAmfSGNumPrefStandbySUs;
+
 	if (ii_spare < 0) {
 		ii_spare = 0;
 	}
 
-    /**
+	/**
 	 * Determine number of active and standby service units
 	 * to assign based upon reduction procedure
 	 */
@@ -366,7 +605,8 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level)
 		su_spare_assign = 0;
 	} else
 	if ((inservice_count - active_sus_needed - standby_sus_needed) < 0) {
-		dprintf ("assignment V - partial assignment with reduction of standby units\n");
+		dprintf ("assignment V - partial assignment with reduction "
+				 "of standby units\n");
 
 		su_active_assign = active_sus_needed;
 		if (standby_sus_needed > units_for_standby) {
@@ -376,15 +616,19 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level)
 		}
 		su_spare_assign = 0;
 	} else
-	if ((sg->saAmfSGMaxStandbySIsperSUs * units_for_standby) <= application_si_count_get (sg->application)) {
+	if ((sg->saAmfSGMaxStandbySIsperSUs * units_for_standby) <=
+		sg_si_count_get (sg)) {
+
 		dprintf ("IV: full assignment with reduction of active service units\n");
 		su_active_assign = inservice_count - standby_sus_needed;
 		su_standby_assign = standby_sus_needed;
 		su_spare_assign = 0;
 	} else 
-	if ((sg->saAmfSGMaxActiveSIsperSUs * units_for_active) <= application_si_count_get (sg->application)) {
+	if ((sg->saAmfSGMaxActiveSIsperSUs * units_for_active) <=
+		sg_si_count_get (sg)) {
 
-		dprintf ("III: full assignment with reduction of standby service units\n");
+		dprintf ("III: full assignment with reduction of standby "
+				 "service units\n");
 		su_active_assign = sg->saAmfSGNumPrefActiveSUs;
 		su_standby_assign = units_for_standby;
 		su_spare_assign = 0;
@@ -403,20 +647,33 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level)
 		su_spare_assign = ii_spare;
 	}
 
-	dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d) (assigning spares=%d)\n",
+	dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)"
+			 " (assigning spares=%d)\n",
 		inservice_count, su_active_assign, su_standby_assign, su_spare_assign);
 	sg_assign_nm_active (sg, su_active_assign);
 	sg_assign_nm_standby (sg, su_standby_assign);
 
+	sg->saAmfSGNumCurrAssignedSUs = inservice_count;
+
 	/**
-     * Phase 2: do the actual assignment to the component
-     */
+	 * Phase 2: do the actual assignment to the component
+	 * TODO: first do active, then standby
+	 */
 	{
 		struct amf_si *si;
+		struct amf_si_assignment *si_assignment;
 
 		for (si = sg->application->si_head; si != NULL; si = si->next) {
 			if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
-				amf_si_activate (si, si_activated_callback);
+				for (si_assignment = si->assigned_sis; si_assignment != NULL;
+					  si_assignment = si_assignment->next) {
+
+					if (si_assignment->requested_ha_state !=
+						si_assignment->saAmfSISUHAState) {
+						amf_si_ha_state_assume (
+							si_assignment, assign_si_assumed_cbfn);
+					}
+				}
 			}
 		}
 	}
@@ -430,8 +687,13 @@ void amf_sg_start (struct amf_sg *sg, struct amf_node *node)
 
 	ENTER ("'%s'", sg->name.value);
 
-	for (su = sg->su_head; su != NULL; su = su->next) {
-		amf_su_instantiate (su);
+	sg->avail_state = SG_AC_InstantiatingServiceUnits;
+
+	if (node == NULL) {
+		/* Cluster start */
+		for (su = sg->su_head; su != NULL; su = su->next) {
+			amf_su_instantiate (su);
+		}
 	}
 }
 
@@ -442,11 +704,25 @@ void amf_sg_su_state_changed (
 
 	if (type == SA_AMF_PRESENCE_STATE) {
 		if (state == SA_AMF_PRESENCE_INSTANTIATED) {
-			/*
-			 * If all SU presence states are INSTANTIATED, report to SG.
-			 */
-			if (all_su_instantiated(su->sg)) {
-				amf_application_sg_started (sg->application, sg, this_amf_node);
+			if (all_su_has_presence_state(su->sg,
+				SA_AMF_PRESENCE_INSTANTIATED)) {
+
+				if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
+					su->sg->avail_state = SG_AC_Idle;
+					amf_application_sg_started (
+						sg->application, sg, this_amf_node);
+				} else if (sg->avail_state == SG_AC_ReparingSu) {
+					su->sg->avail_state = SG_AC_AssigningStandBy;
+					amf_sg_assign_si (sg, 0);
+				} else {
+					assert (0);
+				}
+			}
+		} else if (state == SA_AMF_PRESENCE_UNINSTANTIATED) {
+			if (sg->avail_state == SG_AC_TerminatingSuspected) {
+				standby_su_activate (su);
+			} else {
+				assert (0);
 			}
 		} else {
 			assert (0);
@@ -461,9 +737,28 @@ void amf_sg_init (void)
 	log_init ("AMF");
 }
 
-void amf_sg_si_activated (struct amf_sg *sg, struct amf_si *si)
+void amf_sg_failover_su_req (
+	struct amf_sg *sg, struct amf_su *su, struct amf_node *node)
 {
+	struct amf_si_assignment *si_assignment;
+
 	ENTER ("");
-	amf_application_sg_assigned (sg->application, sg);
+
+	sg->avail_state = SG_AC_DeactivatingDependantWorkload;
+
+	/*                                                              
+	 * Deactivate workload for SU
+	 */
+	si_assignment = amf_su_get_next_si_assignment (su, NULL);
+	while (si_assignment != NULL) {
+
+		if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) {
+			si_assignment->requested_ha_state = SA_AMF_HA_QUIESCED;
+			amf_si_ha_state_assume (
+				si_assignment, failover_su_si_deactivated_cbfn);
+		}
+		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
+	}
 }
 
+

exec/amfsi.c

@@ -3,7 +3,8 @@
  * Copyright (c) 2006 Ericsson AB.
  *  Author: Hans Feldt
  * - Refactoring of code into several AMF files
- *  Author: Anders Eriksson
+ *  Author: Anders Eriksson, Lars Holm
+ *  - Component/SU restart, SU failover
  *
  * All rights reserved.
  *
@@ -112,17 +113,145 @@
 #include <stdio.h>
 #include "amf.h"
 #include "print.h"
+#include "util.h"
+#include "aispoll.h"
+#include "main.h"
+
+/**
+ * Check if any CSI assignment belonging to SU has the requested
+ * state.
+ * @param su
+ * @param hastate
+ * 
+ * @return int
+ */
+static int any_csi_has_hastate_in_su (struct amf_su *su, SaAmfHAStateT hastate)
+{
+	struct amf_comp *component;
+	struct amf_csi_assignment *csi_assignment;
+	int exist = 0;
+
+	for (component = su->comp_head; component != NULL;
+		component = component->next) {
+
+		csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
+		while (csi_assignment != NULL) {
+			if (csi_assignment->saAmfCSICompHAState == hastate) {
+				exist = 1;
+				goto done;
+			}
+			csi_assignment =
+				amf_comp_get_next_csi_assignment (component, csi_assignment);
+		}
+	}
+
+	done:
+	return exist;
+}
+
+/**
+ * Check if all CSI assignments belonging to a
+ * an SI assignemnt has the requested state.
+ * @param su
+ * @param hastate
+ * 
+ * @return int
+ */
+static int all_csi_has_hastate_for_si (
+	struct amf_si_assignment *si_assignment, SaAmfHAStateT hastate)
+{
+	struct amf_comp *component;
+	struct amf_csi_assignment *tmp_csi_assignment;
+	int all = 1;
+
+	for (component = si_assignment->su->comp_head; component != NULL;
+		component = component->next) {
+
+		tmp_csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
+		while (tmp_csi_assignment != NULL) {
+			if ((tmp_csi_assignment->si_assignment == si_assignment) &&
+				(tmp_csi_assignment->saAmfCSICompHAState != hastate)) {
+
+				all = 0;
+				goto done;
+			}
+			tmp_csi_assignment =
+				amf_comp_get_next_csi_assignment (component, tmp_csi_assignment);
+		}
+	}
+
+	done:
+	return all;
+}
+
+/**
+ * Implements table 6 in 3.3.2.4
+ * TODO: active & standby is not correct calculated acc. to
+ * table. This knowledge is e.g. used in assign_si_assumed_cbfn
+ * (sg.c)
+ * @param csi_assignment
+ */
+static void set_si_ha_state (struct amf_csi_assignment *csi_assignment)
+{
+	SaAmfHAStateT old_ha_state =
+		csi_assignment->si_assignment->saAmfSISUHAState;
+	SaAmfAssignmentStateT old_assigment_state =
+		amf_si_get_saAmfSIAssignmentState (csi_assignment->csi->si);
+
+	if (all_csi_has_hastate_for_si (
+		csi_assignment->si_assignment, SA_AMF_HA_ACTIVE)) {
+
+		csi_assignment->si_assignment->saAmfSISUHAState = SA_AMF_HA_ACTIVE;
+	}
+
+	if (all_csi_has_hastate_for_si (
+		csi_assignment->si_assignment, SA_AMF_HA_STANDBY)) {
+
+		csi_assignment->si_assignment->saAmfSISUHAState = SA_AMF_HA_STANDBY;
+	}
+
+	if (any_csi_has_hastate_in_su (
+		csi_assignment->comp->su, SA_AMF_HA_QUIESCING)) {
+
+		csi_assignment->si_assignment->saAmfSISUHAState = SA_AMF_HA_QUIESCING;
+	}
+	if (any_csi_has_hastate_in_su (
+		csi_assignment->comp->su, SA_AMF_HA_QUIESCED)) {
+
+		csi_assignment->si_assignment->saAmfSISUHAState = SA_AMF_HA_QUIESCED;
+	}
+
+	/* log changes to HA state */
+	if (old_ha_state != csi_assignment->si_assignment->saAmfSISUHAState) {
+		log_printf (LOG_NOTICE, "SU HA state changed to '%s' for:\n"
+			"\t\tSI '%s', SU '%s'",
+			amf_ha_state (csi_assignment->si_assignment->saAmfSISUHAState),
+			csi_assignment->si_assignment->si->name.value,
+			csi_assignment->si_assignment->name.value);
+	}
+
+	/* log changes to assignment state */
+	if (old_assigment_state !=
+		amf_si_get_saAmfSIAssignmentState (csi_assignment->csi->si)) {
+		log_printf (LOG_NOTICE, "SI Assignment state changed to '%s' for:\n"
+			"\t\tSI '%s', SU '%s'",
+			amf_assignment_state (
+			amf_si_get_saAmfSIAssignmentState (csi_assignment->csi->si)),
+			csi_assignment->si_assignment->si->name.value,
+			csi_assignment->si_assignment->name.value);
+	}
+}
 
 char *amf_csi_dn_make (struct amf_csi *csi, SaNameT *name)
 {
-	int	i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH,
+	int i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH,
 		"safCsi=%s,safSi=%s,safApp=%s",
 		csi->name.value, csi->si->name.value,
 		csi->si->application->name.value);
 	assert (i <= SA_MAX_NAME_LENGTH);
 	name->length = i;
 
-	return (char *)name->value;
+	return(char *)name->value;
 }
 
 void amf_si_init (void)
@@ -133,29 +262,22 @@ void amf_si_init (void)
 void amf_si_comp_set_ha_state_done (
 	struct amf_si *si, struct amf_csi_assignment *csi_assignment)
 {
-	struct amf_csi *csi;
-	int all_confirmed = 1;
+	ENTER ("'%s', '%s'", si->name.value, csi_assignment->csi->name.value);
 
-	ENTER ("");
+	set_si_ha_state (csi_assignment);
 
-	/**
-     * If all requested active/standby CSIs in SI are
-     * confirmed by the component, tell SG that the SI is
-     * activated.
-     */
-	for (csi = si->csi_head; csi != NULL; csi = csi->next) {
-		struct amf_csi_assignment *tmp;
-		for (tmp = csi->csi_assignments; tmp != NULL; tmp = tmp->csi_next) {
-			if (tmp->requested_ha_state != tmp->saAmfCSICompHAState) {
-				all_confirmed = 0;
-				goto done;
-			}
-		}
-	}
+	assert (csi_assignment->si_assignment->assumed_callback_fn != NULL);
+
+	/*                                                              
+	 * Report to caller when the requested SI assignment state is
+	 * confirmed.
+	 */
+	if (csi_assignment->si_assignment->requested_ha_state ==
+		csi_assignment->si_assignment->saAmfSISUHAState) {
 
-done:
-	if (all_confirmed) {
-		amf_sg_si_activated (csi_assignment->comp->su->sg, si);
+		csi_assignment->si_assignment->assumed_callback_fn (
+			csi_assignment->si_assignment, 0);
+		csi_assignment->si_assignment->assumed_callback_fn = NULL;
 	}
 }
 
@@ -170,15 +292,18 @@ void amf_si_activate (
 	for (csi = si->csi_head; csi != NULL; csi = csi->next) {
 		struct amf_csi_assignment *csi_assignment;
 
-		for (csi_assignment = csi->csi_assignments; csi_assignment != NULL;
-			csi_assignment = csi_assignment->csi_next) {
+		for (csi_assignment = csi->assigned_csis; csi_assignment != NULL;
+			csi_assignment = csi_assignment->next) {
+
+			csi_assignment->si_assignment->requested_ha_state =
+				SA_AMF_HA_ACTIVE;
 
 			/*                                                              
 			 * TODO: only active assignments should be set when dependency
 			 * levels are used.
 			 */
-			amf_comp_hastate_set (csi_assignment->comp, csi_assignment,
-				csi_assignment->requested_ha_state);
+			csi_assignment->requested_ha_state = SA_AMF_HA_ACTIVE;
+			amf_comp_hastate_set (csi_assignment->comp, csi_assignment);
 		}
 	}
 }
@@ -190,3 +315,149 @@ void amf_si_comp_set_ha_state_failed (
 	assert (0);
 }
 
+static void timer_function_ha_state_assumed (void *_si_assignment)
+{
+	struct amf_si_assignment *si_assignment = _si_assignment;
+
+	ENTER ("");
+	si_assignment->saAmfSISUHAState = si_assignment->requested_ha_state;
+	si_assignment->assumed_callback_fn (si_assignment, 0);
+}
+
+void amf_si_ha_state_assume (
+	struct amf_si_assignment *si_assignment,
+	void (*assumed_ha_state_callback_fn)(struct amf_si_assignment *si_assignment,
+	int result))
+{
+	struct amf_csi_assignment *csi_assignment;
+	struct amf_csi *csi;
+	int csi_assignment_cnt = 0;
+	int hastate_set_done_cnt = 0;
+
+	ENTER ("SI '%s' SU '%s' state %s", si_assignment->si->name.value,
+		si_assignment->su->name.value,
+		amf_ha_state (si_assignment->requested_ha_state));
+
+	si_assignment->assumed_callback_fn = assumed_ha_state_callback_fn;
+
+	for (csi = si_assignment->si->csi_head; csi != NULL; csi = csi->next) {
+		for (csi_assignment = csi->assigned_csis; csi_assignment != NULL;
+			csi_assignment = csi_assignment->next) {
+
+			/*                                                              
+			 * If the CSI assignment and the SI assignment belongs to the
+			 * same SU, we have a match and can request the component to
+			 * change HA state.
+			 */
+			if (name_match (&csi_assignment->comp->su->name,
+				&si_assignment->su->name) &&
+				(csi_assignment->saAmfCSICompHAState !=
+				si_assignment->requested_ha_state)) {
+
+				csi_assignment_cnt++;
+				csi_assignment->requested_ha_state =
+					si_assignment->requested_ha_state;
+				amf_comp_hastate_set (csi_assignment->comp, csi_assignment);
+
+				if (csi_assignment->saAmfCSICompHAState ==
+					csi_assignment->requested_ha_state) {
+
+					hastate_set_done_cnt++;
+				}
+			}
+		}
+	}
+
+	/*                                                              
+	 * If the SU has only one component which is the faulty one, we
+	 * will not get an asynchronous response from the component.
+	 * This response (amf_si_comp_set_ha_state_done) is used to do
+	 * the next state transition. The asynchronous response is
+	 * simulated using a timeout instead.
+	 */
+	if (csi_assignment_cnt == hastate_set_done_cnt) {
+		poll_timer_handle handle;
+		poll_timer_add (aisexec_poll_handle,
+			0,
+			si_assignment,
+			timer_function_ha_state_assumed,
+			&handle);
+	}
+}
+
+/**
+ * Get number of active assignments for the specified SI
+ * @param si
+ * 
+ * @return int
+ */
+int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si)
+{
+	int cnt = 0;
+	struct amf_si_assignment *si_assignment;
+
+	for (si_assignment = si->assigned_sis; si_assignment != NULL;
+		si_assignment = si_assignment->next) {
+
+		if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) {
+			cnt++;
+		}
+	}
+
+	return cnt;
+}
+
+int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si)
+{
+	int cnt = 0;
+	struct amf_si_assignment *si_assignment;
+
+	for (si_assignment = si->assigned_sis; si_assignment != NULL;
+		si_assignment = si_assignment->next) {
+
+		if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
+			cnt++;
+		}
+	}
+
+	return cnt;
+}
+
+SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState (struct amf_si *si)
+{
+	if ((amf_si_get_saAmfSINumCurrActiveAssignments (si) ==
+		si->saAmfSIPrefActiveAssignments) &&
+		(amf_si_get_saAmfSINumCurrStandbyAssignments (si) ==
+		si->saAmfSIPrefStandbyAssignments)) {
+
+		return SA_AMF_ASSIGNMENT_FULLY_ASSIGNED;
+	} else if (amf_si_get_saAmfSINumCurrActiveAssignments (si) == 0) {
+		return SA_AMF_ASSIGNMENT_UNASSIGNED;
+	} else {
+		return SA_AMF_ASSIGNMENT_PARTIALLY_ASSIGNED;
+	}
+}
+
+void amf_csi_delete_assignments (struct amf_csi *csi, struct amf_su *su)
+{
+	struct amf_csi_assignment *csi_assignment;
+
+	ENTER ("'%s'", su->name.value);
+
+	/*                                                              
+	* TODO: this only works for n+m where each CSI list has only
+	* two assignments, one active and one standby.
+	* TODO: use DN instead
+	*/
+	if (csi->assigned_csis->comp->su == su) {
+		csi_assignment = csi->assigned_csis;
+		csi->assigned_csis = csi_assignment->next;
+	} else {
+		csi_assignment = csi->assigned_csis->next;
+		csi->assigned_csis->next = NULL;
+		assert (csi_assignment != NULL && csi_assignment->comp->su == su);
+	}
+	assert (csi_assignment != NULL);
+	free (csi_assignment);
+}
+

exec/amfsu.c

@@ -9,7 +9,8 @@
  * - Use DN in API and multicast messages
  * - (Re-)Introduction of event based multicast messages
  * - Refactoring of code into several AMF files
- *  Author: Anders Eriksson
+ *  Author: Anders Eriksson, Lars Holm
+ *  - Component/SU restart, SU failover
  *
  * All rights reserved.
  *
@@ -141,21 +142,44 @@ static int presence_state_all_comps_in_su_are_set (struct amf_su *su,
 	return all_set;
 }
 
+/**
+ * This function only logs since the readiness state is runtime
+ * calculated.
+ * @param su
+ * @param amf_readiness_state
+ */
 static void su_readiness_state_set (struct amf_su *su,
 	SaAmfReadinessStateT readiness_state)
 {
-	su->saAmfSUReadinessState = readiness_state;
-	TRACE1 ("Setting SU '%s' readiness state: %s\n",
-		&su->name.value, amf_readiness_state(readiness_state));
+	log_printf (LOG_NOTICE, "Setting SU '%s' readiness state: %s\n",
+		su->name.value, amf_readiness_state (readiness_state));
+}
+
+static void clear_ha_state (
+	struct amf_su *su, struct amf_si_assignment *si_assignment)
+{
+	ENTER ("");
+	si_assignment->saAmfSISUHAState = 0;
 }
 
 static void su_presence_state_set (struct amf_su *su,
 	SaAmfPresenceStateT presence_state)
 {
+	/*                                                              
+     * Set all SI's confirmed HA state to unknown if uninstantiated
+     */
+	if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_UNINSTANTIATED) {
+		amf_su_foreach_si_assignment (su, clear_ha_state);
+	}
+
 	su->saAmfSUPresenceState = presence_state;
-	TRACE1 ("Setting SU '%s' presence state: %s\n",
+	log_printf (LOG_NOTICE, "Setting SU '%s' presence state: %s\n",
 		su->name.value, amf_presence_state (presence_state));
-	amf_sg_su_state_changed (su->sg, su, SA_AMF_PRESENCE_STATE, presence_state);
+
+	if (su->restart_control_state != SU_RC_RESTART_SU_SETTING) {
+		amf_sg_su_state_changed (
+			su->sg, su, SA_AMF_PRESENCE_STATE, presence_state);
+	}
 }
 
 static void su_operational_state_set (struct amf_su *su,
@@ -163,14 +187,8 @@ static void su_operational_state_set (struct amf_su *su,
 {
 	struct amf_comp* comp;
 
-	if (oper_state == su->saAmfSUOperState) {
-		log_printf (LOG_INFO,
-			"Not assigning service unit new operational state - same state\n");
-		return;
-	}
-
 	su->saAmfSUOperState = oper_state;
-	TRACE1 ("Setting SU '%s' operational state: %s\n",
+	log_printf (LOG_NOTICE, "Setting SU '%s' operational state: %s\n",
 		su->name.value, amf_op_state (oper_state));
 
 	if (oper_state == SA_AMF_OPERATIONAL_ENABLED) {
@@ -179,15 +197,16 @@ static void su_operational_state_set (struct amf_su *su,
 		for (comp = su->comp_head; comp; comp = comp->next) {
 			amf_comp_readiness_state_set (comp, SA_AMF_READINESS_IN_SERVICE);
 		}
-
-//		amf_sg_su_state_changed (su->sg, su, SA_AMF_OP_STATE, SA_AMF_OPERATIONAL_ENABLED);
 	} else if (oper_state == SA_AMF_OPERATIONAL_DISABLED) {
 		su_readiness_state_set (su, SA_AMF_READINESS_OUT_OF_SERVICE);
+		for (comp = su->comp_head; comp; comp = comp->next) {
+			amf_comp_readiness_state_set (comp, SA_AMF_READINESS_OUT_OF_SERVICE);
+		}
 	}
 }
 
 static void comp_assign_csi (struct amf_comp *comp, struct amf_csi *csi,
-	SaAmfHAStateT ha_state)
+	struct amf_si_assignment *si_assignment, SaAmfHAStateT ha_state)
 {
 	struct amf_csi_assignment *csi_assignment;
 
@@ -200,76 +219,52 @@ static void comp_assign_csi (struct amf_comp *comp, struct amf_csi *csi,
 		openais_exit_error (AIS_DONE_OUT_OF_MEMORY);
 	}
 
-	csi_assignment->comp_next = comp->assigned_csis;
-	comp->assigned_csis = csi_assignment;
-	csi_assignment->csi_next = csi->csi_assignments;
-	csi->csi_assignments = csi_assignment;
-	setSaNameT (&csi_assignment->name, (char*)comp->name.value);
-	csi_assignment->saAmfCSICompHAState = ha_state;
+	csi_assignment->next = csi->assigned_csis;
+	csi->assigned_csis = csi_assignment;
+	amf_comp_dn_make (comp, &csi_assignment->name);
 	csi_assignment->csi = csi;
 	csi_assignment->comp = comp;
 	csi_assignment->saAmfCSICompHAState = 0; /* undefined confirmed HA state */
 	csi_assignment->requested_ha_state = ha_state;
-
-	if (ha_state == SA_AMF_HA_ACTIVE)
-		comp->saAmfCompNumCurrActiveCsi++;
-	else if (ha_state == SA_AMF_HA_STANDBY)
-		comp->saAmfCompNumCurrStandbyCsi++;
-	else
-		assert (0);
+	csi_assignment->si_assignment = si_assignment;
 }
 
-static void su_cleanup (struct amf_su *su)
+static void su_restart (struct amf_su *su)
 {
 	struct amf_comp *comp;
+	SaNameT dn;
+
+	ENTER ("'%s'", su->name.value);
+
+	amf_su_dn_make (su, &dn);
+	log_printf (LOG_NOTICE, "Error detected for '%s', recovery "
+							"action:\n\t\tSU restart", dn.value);
+	
+	su->restart_control_state = SU_RC_RESTART_SU_DEACTIVATING;
+	su->restart_control_state = SU_RC_RESTART_SU_INSTANTIATING;
+	su->escalation_level_history_state =
+		SU_RC_ESCALATION_LEVEL_2;
+
+	su->saAmfSURestartCount += 1;
 
 	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 		amf_comp_restart (comp);
 	}
 }
 
-static void escalation_policy_cleanup (struct amf_comp *comp)
+static void comp_restart (struct amf_comp *comp)
 {
-//	escalation_timer_start (comp);
-
-	switch (comp->su->escalation_level) {
-	case ESCALATION_LEVEL_NO_ESCALATION:
-		comp->saAmfCompRestartCount += 1;
-		if (comp->saAmfCompRestartCount >= comp->su->sg->saAmfSGCompRestartMax) {
-			comp->su->escalation_level = ESCALATION_LEVEL_ONE;
-			escalation_policy_cleanup (comp);
-			comp->saAmfCompRestartCount = 0;
-			return;
-		}
-		dprintf ("Escalation level 0 - restart component\n");
-		dprintf ("Cleaning up and restarting component.\n");
-		amf_comp_restart (comp);
-		break;
-
-	case ESCALATION_LEVEL_ONE:
-		comp->su->saAmfSURestartCount += 1;
-		if (comp->su->saAmfSURestartCount >= comp->su->sg->saAmfSGSuRestartMax) {
-			comp->su->escalation_level = ESCALATION_LEVEL_TWO;
-			escalation_policy_cleanup (comp);
-			comp->saAmfCompRestartCount = 0;
-			comp->su->saAmfSURestartCount = 0;
-			return;
-		}
-		dprintf ("Escalation level 1 - restart unit\n");
-		dprintf ("Cleaning up and restarting unit.\n");
-		su_cleanup (comp->su);
-		break;
-
-	case ESCALATION_LEVEL_TWO:
-		dprintf ("Escalation level TWO\n");
-		su_cleanup (comp->su);
-//		unit_terminate_failover (comp);
-		break;
-
-	case ESCALATION_LEVEL_THREE:
-//TODO
-		break;
-	}
+	SaNameT dn;
+
+	ENTER ("'%s'", comp->name.value);
+	amf_comp_dn_make (comp, &dn);
+	log_printf (LOG_NOTICE, "Error detected for '%s', recovery "
+							"action:\n\t\tcomponent restart", dn.value);
+
+	comp->su->restart_control_state = SU_RC_RESTART_COMP_DEACTIVATING;
+	comp->su->restart_control_state = SU_RC_RESTART_COMP_RESTARTING;
+	comp->su->escalation_level_history_state = SU_RC_ESCALATION_LEVEL_1;
+	amf_comp_restart (comp);
 }
 
 void amf_su_instantiate (struct amf_su *su)
@@ -296,31 +291,13 @@ void amf_su_assign_si (struct amf_su *su, struct amf_si *si,
 	if (si_assignment == NULL) {
 		openais_exit_error (AIS_DONE_OUT_OF_MEMORY);
 	}
-	setSaNameT (&si_assignment->name, (char*)su->name.value);
+	amf_su_dn_make (su, &si_assignment->name);
 	si_assignment->saAmfSISUHAState = 0; /* undefined confirmed HA state */
 	si_assignment->requested_ha_state = ha_state;
-	si_assignment->next = su->assigned_sis;
-	su->assigned_sis = si_assignment;
+	si_assignment->next = si->assigned_sis;
+	si->assigned_sis = si_assignment;
 	si_assignment->si = si;
-	memcpy (&si_assignment->si->saAmfSIProtectedbySG,
-		&su->sg->name, sizeof (SaNameT));
-
-	if (ha_state == SA_AMF_HA_ACTIVE) {
-		si->saAmfSINumCurrActiveAssignments++;
-		su->saAmfSUNumCurrActiveSIs++;
-	} else if (ha_state == SA_AMF_HA_STANDBY) {
-		su->saAmfSUNumCurrStandbySIs++;
-		si->saAmfSINumCurrStandbyAssignments++;
-	} else
-		assert(0);
-
-	if ((si->saAmfSINumCurrActiveAssignments == si->saAmfSIPrefActiveAssignments) &&
-		(si->saAmfSINumCurrStandbyAssignments == si->saAmfSIPrefStandbyAssignments)) {
-		si->saAmfSIAssignmentState = SA_AMF_ASSIGNMENT_FULLY_ASSIGNED;
-	} else if ((si->saAmfSINumCurrActiveAssignments < si->saAmfSIPrefActiveAssignments) ||
-		(si->saAmfSINumCurrStandbyAssignments < si->saAmfSIPrefStandbyAssignments)) {
-		si->saAmfSIAssignmentState = SA_AMF_ASSIGNMENT_PARTIALLY_ASSIGNED;
-	}
+	si_assignment->su = su;
 
 	{
 		struct amf_csi *csi;
@@ -339,66 +316,162 @@ void amf_su_assign_si (struct amf_su *su, struct amf_si *si,
 				int no_of_assignments = 0;
 
 				for (csi = si->csi_head; csi != NULL; csi = csi->next) {
-					if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value, cs_type->length)) {
-						comp_assign_csi (comp, csi, ha_state);
+					if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value,
+								cs_type->length)) {
+						comp_assign_csi (comp, csi, si_assignment, ha_state);
 						no_of_assignments++;
 					}
 				}
 				if (no_of_assignments == 0) {
-					log_printf (LOG_WARNING, "\t   No CSIs of type %s configured?!!\n",
+					log_printf (
+						LOG_WARNING, "\t   No CSIs of type %s configured?!!\n",
 						getSaNameT (cs_type));
 				}
 			}
 			if (no_of_cs_types == 0) {
-				log_printf (LOG_LEVEL_ERROR, "\t   No CS types configured for comp %s ?!!\n",
+				log_printf (LOG_LEVEL_ERROR,
+					"\t   No CS types configured for comp %s ?!!\n",
 					getSaNameT (&comp->name));
 			}
 		}
 	}
 }
 
-/**
- * Used by a component to report a state change event
- * @param su
- * @param comp
- * @param type type of state
- * @param state new state
- */
-void amf_su_comp_state_changed (
-	struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state)
+static void si_ha_state_assumed_cbfn (
+	struct amf_si_assignment *si_assignment, int result)
 {
-	if (type == SA_AMF_PRESENCE_STATE) {
-		/*
-		 * If all comp presence states are INSTANTIATED, then SU should
-		 * be instantated.
-		 */
-		if (state == SA_AMF_PRESENCE_INSTANTIATED) {
+	struct amf_si_assignment *tmp_si_assignment;
+	struct amf_comp *comp;
+	struct amf_csi_assignment *csi_assignment;
+	int all_confirmed = 1;
+
+	ENTER ("");
+
+	tmp_si_assignment = amf_su_get_next_si_assignment(si_assignment->su, NULL);
+
+	while (tmp_si_assignment != NULL) {
+		for (comp = tmp_si_assignment->su->comp_head; comp != NULL;
+			  comp = comp->next) {
+
+			csi_assignment = amf_comp_get_next_csi_assignment(comp, NULL);
+			while (csi_assignment != NULL) {
+
+				if (csi_assignment->requested_ha_state != 
+					csi_assignment->saAmfCSICompHAState) {
+					all_confirmed = 0;
+				}
+				csi_assignment = amf_comp_get_next_csi_assignment(
+					comp, csi_assignment);
+			}
+		}
+		tmp_si_assignment = amf_su_get_next_si_assignment(
+			si_assignment->su, tmp_si_assignment);
+	}
+
+	if (all_confirmed) {
+		switch (si_assignment->su->restart_control_state) {
+			case SU_RC_RESTART_COMP_SETTING:
+				log_printf (LOG_NOTICE, "Component restart recovery finished");
+				break;
+			case SU_RC_RESTART_SU_SETTING:
+				log_printf (LOG_NOTICE, "SU restart recovery finished");
+				break;
+			default:
+				assert (0);
+		}
+		si_assignment->su->restart_control_state =
+			si_assignment->su->escalation_level_history_state;
+	}
+}
+
+static void reassign_sis(struct amf_su *su)
+{
+	struct amf_si_assignment *si_assignment;
+
+	ENTER ("");
+
+	si_assignment = amf_su_get_next_si_assignment(su, NULL);
+
+	while (si_assignment != NULL) {
+		si_assignment->saAmfSISUHAState = 0; /* unknown */
+		amf_si_ha_state_assume (si_assignment, si_ha_state_assumed_cbfn);
+		si_assignment = amf_su_get_next_si_assignment(su, si_assignment);
+	}
+}
+
+static void su_comp_presence_state_changed (
+	struct amf_su *su, struct amf_comp *comp, int state)
+{
+	ENTER ("'%s', '%s'", su->name.value, comp->name.value);
+
+	switch (state) {
+		case SA_AMF_PRESENCE_INSTANTIATED:
+			switch (su->restart_control_state) {
+				case SU_RC_ESCALATION_LEVEL_2:
+					/*                                                              
+                     * TODO: send to node
+                     */
+				case SU_RC_ESCALATION_LEVEL_0:
+					if (presence_state_all_comps_in_su_are_set (
+						comp->su, SA_AMF_PRESENCE_INSTANTIATED)) {
+
+						su_presence_state_set (
+							comp->su, SA_AMF_PRESENCE_INSTANTIATED);
+					}
+					break;
+				case SU_RC_RESTART_COMP_RESTARTING:
+					su->restart_control_state = SU_RC_RESTART_COMP_SETTING;
+					reassign_sis (comp->su);
+					break;
+				case SU_RC_RESTART_SU_INSTANTIATING:
+					if (presence_state_all_comps_in_su_are_set (
+						comp->su, SA_AMF_PRESENCE_INSTANTIATED)) {
+
+						su->restart_control_state = SU_RC_RESTART_SU_SETTING;
+						su_presence_state_set (
+							comp->su, SA_AMF_PRESENCE_INSTANTIATED);
+						reassign_sis (comp->su);
+					}
+					break;
+				default:
+					dprintf ("state %d", su->restart_control_state);
+					assert (0);
+			}
+			break;
+		case SA_AMF_PRESENCE_UNINSTANTIATED:
 			if (presence_state_all_comps_in_su_are_set (
-					comp->su, SA_AMF_PRESENCE_INSTANTIATED)) {
+				su, SA_AMF_PRESENCE_UNINSTANTIATED)) {
 
-				su_presence_state_set (comp->su, SA_AMF_PRESENCE_INSTANTIATED);
-			} else {
-				/*
-				 * This state occurs when there is more then
-				 * one SU
-				 */
-				return;
+				su_presence_state_set (comp->su,
+					SA_AMF_PRESENCE_UNINSTANTIATED);
 			}
-		} else if (state == SA_AMF_PRESENCE_INSTANTIATING) {
-		} else if (state == SA_AMF_PRESENCE_RESTARTING) {
-		} else {
+			break;
+		case SA_AMF_PRESENCE_INSTANTIATING:
+			break;
+		case SA_AMF_PRESENCE_RESTARTING:
+			break;
+		case SA_AMF_PRESENCE_TERMINATING:
+			break;
+		default:
 			assert (0);
-		}
-	} else if (type == SA_AMF_OP_STATE) {
-		/*
-		 * If all component op states are ENABLED, then SU op 
-		 * state should be ENABLED.
-		 */
-		if (state == SA_AMF_OPERATIONAL_ENABLED) {
+	}
+}
+
+static void su_comp_op_state_changed (
+	struct amf_su *su, struct amf_comp *comp, int state)
+{
+	ENTER ("'%s', '%s'", su->name.value, comp->name.value);
+
+	switch (state) {
+		case SA_AMF_OPERATIONAL_ENABLED:
+		{
 			struct amf_comp *comp_compare;
 			int all_set = 1;
-			for (comp_compare = comp->su->comp_head; comp_compare != NULL; comp_compare = comp_compare->next) {
-				if (comp_compare->saAmfCompOperState != SA_AMF_OPERATIONAL_ENABLED) {
+			for (comp_compare = comp->su->comp_head;
+				  comp_compare != NULL; comp_compare = comp_compare->next) {
+				if (comp_compare->saAmfCompOperState !=
+						SA_AMF_OPERATIONAL_ENABLED) {
+
 					all_set = 0;
 					break;
 				}
@@ -408,25 +481,35 @@ void amf_su_comp_state_changed (
 			} else {
 				su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_DISABLED);
 			}
-		} else {
-			assert (0);
+			break;
 		}
-	} else {
-		assert (0);
+		case SA_AMF_OPERATIONAL_DISABLED:
+			break;
+		default:
+			assert (0);
 	}
 }
 
 /**
- * Used by a component to report a change in HA state
+ * Used by a component to report a state change event
  * @param su
  * @param comp
- * @param csi_assignment
+ * @param type type of state
+ * @param state new state
  */
-void amf_su_comp_hastate_changed (
-	struct amf_su *su, struct amf_comp *comp,
-	struct amf_csi_assignment *csi_assignment)
+void amf_su_comp_state_changed (
+	struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state)
 {
-	ENTER("'%s' '%s'", comp->name.value, csi_assignment->csi->name.value);
+	switch (type) {
+		case SA_AMF_PRESENCE_STATE:
+			su_comp_presence_state_changed (su, comp, state);
+			break;
+		case SA_AMF_OP_STATE:
+			su_comp_op_state_changed (su, comp, state);
+			break;
+		default:
+			assert (0);
+	}
 }
 
 /**
@@ -455,7 +538,70 @@ void amf_su_comp_error_suspected (
 	struct amf_comp *comp,
 	SaAmfRecommendedRecoveryT recommended_recovery)
 {
-	escalation_policy_cleanup (comp);
+	ENTER ("Comp '%s', SU '%s'", comp->name.value, su->name.value);
+
+	/*                                                              
+     * Defer all new events. Workaround to be able to use gdb.
+     */
+	if (su->sg->avail_state != SG_AC_Idle) {
+		ENTER ("Comp '%s', SU '%s'", comp->name.value, su->name.value);
+		fprintf (stderr, "Warning Debug: event deferred!\n");
+		return;
+	}
+
+	switch (su->restart_control_state) {
+		case SU_RC_ESCALATION_LEVEL_0:
+			if (comp->saAmfCompRestartCount >= su->sg->saAmfSGCompRestartMax) {
+				su->restart_control_state = SU_RC_ESCALATION_LEVEL_1;
+				amf_su_comp_error_suspected (su, comp, recommended_recovery);
+			} else {
+				comp_restart (comp);
+			}
+			break;
+
+		case SU_RC_ESCALATION_LEVEL_1:
+			if (comp->saAmfCompRestartCount >= su->sg->saAmfSGCompRestartMax) {
+				if (su->saAmfSURestartCount >= su->sg->saAmfSGSuRestartMax) {
+					su->restart_control_state = SU_RC_ESCALATION_LEVEL_2;
+					amf_su_comp_error_suspected (su, comp, recommended_recovery);
+				} else {
+					su_restart (comp->su);
+				}
+			} else {
+				comp_restart (comp);
+			}
+			break;
+
+		case SU_RC_ESCALATION_LEVEL_2:
+			if (su->saAmfSURestartCount >= su->sg->saAmfSGSuRestartMax) {
+
+				/*                                                              
+                 * TODO: delegate to node
+                 */
+				struct amf_si_assignment *si_assignment =
+					amf_su_get_next_si_assignment (su, NULL);
+				if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) {
+					SaNameT dn;
+
+					su_operational_state_set (su, SA_AMF_OPERATIONAL_DISABLED);
+					amf_comp_operational_state_set (
+						comp, SA_AMF_OPERATIONAL_DISABLED);
+					amf_comp_dn_make (comp, &dn);
+					log_printf (LOG_NOTICE, "Error detected for '%s', recovery "
+											"action:\n\t\tSU failover", dn.value);
+					amf_sg_failover_su_req (comp->su->sg, comp->su, this_amf_node);
+					return;
+				} else {
+					su_restart (comp->su);
+				}
+			} else {
+				su_restart (comp->su);
+			}
+			break;
+
+		default:
+			assert (0);
+	}
 }
 
 void amf_su_init (void)
@@ -463,3 +609,142 @@ void amf_su_init (void)
 	log_init ("AMF");
 }
 
+void amf_su_terminate (struct amf_su *su)
+{
+	struct amf_comp *comp;
+
+	ENTER ("'%s'", su->name.value);
+
+	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
+		/*                                                              
+         * Terminate all components in SU abruptly
+         */
+		comp->error_suspected = 1;
+		amf_comp_terminate (comp);
+	}
+}
+
+char *amf_su_dn_make (struct amf_su *su, SaNameT *name)
+{
+	int	i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH,
+		"safSu=%s,safSg=%s,safApp=%s",
+		su->name.value,	su->sg->name.value, su->sg->application->name.value);
+	assert (i <= SA_MAX_NAME_LENGTH);
+	name->length = i;
+	return (char *)name->value;
+}
+
+struct amf_si_assignment *amf_su_get_next_si_assignment (
+	struct amf_su *su, const struct amf_si_assignment *si_assignment)
+{
+	struct amf_si *si;
+	struct amf_si_assignment *tmp_si_assignment;
+	SaNameT dn;
+
+	amf_su_dn_make (su, &dn);
+
+	if (si_assignment == NULL) {
+		si = su->sg->application->si_head;
+		tmp_si_assignment = si->assigned_sis;
+	} else {
+		tmp_si_assignment = si_assignment->next;
+		if (tmp_si_assignment == NULL) {
+			si = si_assignment->si->next;
+			if (si == NULL) {
+				return NULL;
+			} else {
+				tmp_si_assignment = si->assigned_sis;
+			}
+		} else {
+			si = tmp_si_assignment->si;
+		}
+	}
+
+	for (; si != NULL; si = si->next) {
+		if (tmp_si_assignment == NULL && si != NULL) {
+			tmp_si_assignment = si->assigned_sis;
+		}
+		for (; tmp_si_assignment != NULL;
+			tmp_si_assignment = tmp_si_assignment->next) {
+
+			if (name_match (&tmp_si_assignment->name, &dn)) {
+				return tmp_si_assignment;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+void amf_su_foreach_si_assignment (
+	struct amf_su *su,
+	void (*foreach_fn)(struct amf_su *su,
+					   struct amf_si_assignment *si_assignment))
+{
+	struct amf_si_assignment *si_assignment;
+
+	assert (foreach_fn != NULL);
+	si_assignment = amf_su_get_next_si_assignment (su, NULL);
+	while (si_assignment != NULL) {
+		foreach_fn (su, si_assignment);
+		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
+	}
+}
+
+
+int amf_su_get_saAmfSUNumCurrActiveSIs(struct amf_su *su)
+{
+	int cnt = 0;
+	struct amf_si_assignment *si_assignment;
+	
+	si_assignment = amf_su_get_next_si_assignment (su, NULL); 
+	while (si_assignment != NULL) {
+		if (su->sg->avail_state == SG_AC_AssigningOnRequest &&
+			si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) {
+			cnt++;
+		} else {
+			if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) {
+				cnt++;
+			}
+		}
+		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
+	}
+
+	return cnt;
+}
+
+
+int amf_su_get_saAmfSUNumCurrStandbySIs(struct amf_su *su)
+{
+	int cnt = 0;
+	struct amf_si_assignment *si_assignment;
+	
+	si_assignment = amf_su_get_next_si_assignment (su, NULL); 
+	while (si_assignment != NULL) {
+		if (su->sg->avail_state == SG_AC_AssigningOnRequest &&
+			si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) {
+			cnt++;
+		} else {
+			if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
+				cnt++;
+			}
+		}
+		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
+	}
+
+	return cnt;
+}
+
+SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (struct amf_su *su)
+{
+	if ((su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) &&
+		((su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) ||
+		 (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) {
+
+		return SA_AMF_READINESS_IN_SERVICE;
+	} else if (su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) {
+		return SA_AMF_READINESS_STOPPING;
+	} else {
+		return SA_AMF_READINESS_OUT_OF_SERVICE;
+	}
+}

exec/amfutil.c

@@ -267,6 +267,8 @@ struct amf_cluster *amf_config_read (char **error_string)
 	struct amf_csi           *csi = 0;
 	struct amf_csi_attribute *attribute = 0;
 	SaStringT                 env_var;
+	int                       su_cnt = 0;
+	int                       sg_cnt = 0;
 	int                       comp_env_var_cnt = 0;
 	int                       comp_cs_type_cnt = 0;
 	int                       csi_attr_cnt = 0;
@@ -350,6 +352,7 @@ struct amf_cluster *amf_config_read (char **error_string)
 				app->saAmfApplicationAdminState = SA_AMF_ADMIN_UNLOCKED;
 				setSaNameT (&app->name, trim_str (loc));
 				current_parse = AMF_APPLICATION;
+				sg_cnt = 0;
 			} else if (strstr_rs (line, "}")) {
 				if (cluster->saAmfClusterStartupTimeout == -1) {
 					error_reason = "saAmfClusterStartupTimeout missing";
@@ -413,12 +416,14 @@ struct amf_cluster *amf_config_read (char **error_string)
 			if ((loc = strstr_rs (line, "clccli_path=")) != 0) {
 				strcpy (app->clccli_path, loc);
 			} else if ((loc = strstr_rs (line, "safSg=")) != 0) {
+				sg_cnt++;
 				sg = calloc (1, sizeof (struct amf_sg));
 				sg->next = app->sg_head;
 				app->sg_head = sg;
 				sg->saAmfSGAdminState = SA_AMF_ADMIN_UNLOCKED;
 				sg->saAmfSGNumPrefActiveSUs = 1;
 				sg->saAmfSGNumPrefStandbySUs = 1;
+				sg->saAmfSGNumPrefInserviceSUs = ~0;
 				sg->saAmfSGCompRestartProb = -1;
 				sg->saAmfSGCompRestartMax = ~0;
 				sg->saAmfSGSuRestartProb = -1;
@@ -427,6 +432,7 @@ struct amf_cluster *amf_config_read (char **error_string)
 				sg->saAmfSGAutoRepair = SA_TRUE;
 				sg->application = app;
 				current_parse = AMF_SG;
+				su_cnt = 0;
 				setSaNameT (&sg->name, trim_str (loc));
 			} else if ((loc = strstr_rs (line, "safSi=")) != 0) {
 				si = calloc (1, sizeof (struct amf_si));
@@ -437,11 +443,24 @@ struct amf_cluster *amf_config_read (char **error_string)
 				si->saAmfSIPrefStandbyAssignments = 1;
 				setSaNameT (&si->name, trim_str (loc));
 				si->saAmfSIAdminState = SA_AMF_ADMIN_UNLOCKED;
-				si->saAmfSIAssignmentState = SA_AMF_ASSIGNMENT_UNASSIGNED;
 				current_parse = AMF_SI;
 			} else if ((loc = strstr_rs (line, "safCSType=")) != 0) {
 				current_parse = AMF_CS_TYPE;
 			} else if (strstr_rs (line, "}")) {
+				if (sg_cnt == 1) {
+					for (si = app->si_head; si != NULL; si = si->next) {
+						memcpy (&si->saAmfSIProtectedbySG, &sg->name,
+								sizeof (SaNameT));
+					}
+				} else {
+					for (si = app->si_head; si != NULL; si = si->next) {
+						if (si->saAmfSIProtectedbySG.length == 0) {
+							error_reason = "saAmfSIProtectedbySG not set in SI"
+								", needed when several SGs are specified.";
+							goto parse_error;
+						}
+					}
+				}
 				current_parse = AMF_CLUSTER;
 			} else {
 				goto parse_error;
@@ -489,13 +508,14 @@ struct amf_cluster *amf_config_read (char **error_string)
 				sg->saAmfSGAutoRepair = atoi (loc);
 			} else if ((loc = strstr_rs (line, "safSu=")) != 0) {
 				su = calloc (1, sizeof (struct amf_su));
+				su_cnt++;
 				su->next = sg->su_head;
 				sg->su_head = su;
 				su->sg = sg;
 				su->saAmfSUAdminState = SA_AMF_ADMIN_UNLOCKED;
 				su->saAmfSUOperState = SA_AMF_OPERATIONAL_DISABLED;
 				su->saAmfSUPresenceState = SA_AMF_PRESENCE_UNINSTANTIATED;
-				su->escalation_level = ESCALATION_LEVEL_NO_ESCALATION;
+				su->restart_control_state = SU_RC_ESCALATION_LEVEL_0;
 				su->saAmfSUFailover = 1;
 				setSaNameT (&su->name, trim_str (loc));
 				current_parse = AMF_SU;
@@ -528,6 +548,9 @@ struct amf_cluster *amf_config_read (char **error_string)
 					error_reason = "saAmfSGAutoRepair erroneous";
 					goto parse_error;
 				}
+				if (sg->saAmfSGNumPrefInserviceSUs == ~0) {
+					sg->saAmfSGNumPrefInserviceSUs = su_cnt;
+				}
 				current_parse = AMF_APPLICATION;
 			} else {
 				goto parse_error;
@@ -844,10 +867,12 @@ struct amf_cluster *amf_config_read (char **error_string)
 			} else if ((loc = strstr_rs (line, "saAmfCSIDependency=")) != 0) {
 				csi_dependencies_cnt++;
 				csi->saAmfCSIDependencies = realloc (csi->saAmfCSIDependencies,
-												 (csi_dependencies_cnt + 1) * sizeof(SaNameT));
+					(csi_dependencies_cnt + 1) * sizeof(SaNameT));
 				csi->saAmfCSIDependencies[csi_dependencies_cnt] = NULL;
-				csi->saAmfCSIDependencies[csi_dependencies_cnt - 1] = malloc (sizeof(SaNameT));
-				setSaNameT (csi->saAmfCSIDependencies[csi_dependencies_cnt - 1], loc);
+				csi->saAmfCSIDependencies[csi_dependencies_cnt - 1] =
+					malloc (sizeof(SaNameT));
+				setSaNameT (
+					csi->saAmfCSIDependencies[csi_dependencies_cnt - 1], loc);
 			} else {
 				goto parse_error;
 			}
@@ -858,9 +883,10 @@ struct amf_cluster *amf_config_read (char **error_string)
 				current_parse = AMF_CSI;
 			} else {
 				value = rm_beginning_ws (line);
-				attribute->value = realloc (attribute->value, 
-										   sizeof (SaStringT) * (csi_attr_cnt + 1));
-				attribute->value[csi_attr_cnt - 1] = malloc (strlen (value) + 1);
+				attribute->value = realloc (attribute->value,
+					sizeof (SaStringT) * (csi_attr_cnt + 1));
+				attribute->value[csi_attr_cnt - 1] =
+					malloc (strlen (value) + 1);
 				strcpy (attribute->value[csi_attr_cnt - 1], value);
 				attribute->value[csi_attr_cnt] = NULL;
 				csi_attr_cnt++;
@@ -891,6 +917,22 @@ parse_error:
 	return NULL;
 }
 
+static void print_csi_assignment (struct amf_comp *comp,
+	struct amf_csi_assignment *csi_assignment)
+{
+	log_printf (LOG_INFO, "            safCSI=%s\n", csi_assignment->csi->name.value);
+	log_printf (LOG_INFO, "              HA state: %s\n",
+		ha_state_text[csi_assignment->saAmfCSICompHAState]);
+}
+
+static void print_si_assignment (struct amf_su *su,
+	struct amf_si_assignment *si_assignment)
+{
+	log_printf (LOG_INFO, "          safSi=%s\n", si_assignment->si->name.value);
+	log_printf (LOG_INFO, "            HA state: %s\n",
+			ha_state_text[si_assignment->saAmfSISUHAState]);
+}
+
 void amf_runtime_attributes_print (struct amf_cluster *cluster)
 {
 	struct amf_node *node;
@@ -900,82 +942,95 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster)
 	struct amf_comp *comp;
 	struct amf_si *si;
 	struct amf_csi *csi;
-	struct amf_si_assignment *si_assignment;
-	struct amf_csi_assignment *csi_assignment;
 
-	dprintf("AMF runtime attributes:");
-	dprintf("===================================================");
-	dprintf("safCluster=%s", getSaNameT(&cluster->name));
-	dprintf("  admin state: %s\n", admin_state_text[cluster->saAmfClusterAdminState]);
+	log_printf (LOG_INFO, "AMF runtime attributes:");
+	log_printf (LOG_INFO, "===================================================");
+	log_printf (LOG_INFO, "safCluster=%s", getSaNameT(&cluster->name));
+	log_printf (LOG_INFO, "  admin state: %s\n",
+			admin_state_text[cluster->saAmfClusterAdminState]);
 	for (node = cluster->node_head; node != NULL; node = node->next) {
-		dprintf("  safNode=%s\n", getSaNameT (&node->name));
-		dprintf("    admin state: %s\n", admin_state_text[node->saAmfNodeAdminState]);
-		dprintf("    oper state:  %s\n", oper_state_text[node->saAmfNodeOperState]);
+		log_printf (LOG_INFO, "  safNode=%s\n", getSaNameT (&node->name));
+		log_printf (LOG_INFO, "    admin state: %s\n",
+				admin_state_text[node->saAmfNodeAdminState]);
+		log_printf (LOG_INFO, "    oper state:  %s\n",
+				oper_state_text[node->saAmfNodeOperState]);
 	}
 	for (app = cluster->application_head; app != NULL; app = app->next) {
-		dprintf("  safApp=%s\n", getSaNameT(&app->name));
-		dprintf("    admin state: %s\n", admin_state_text[app->saAmfApplicationAdminState]);
-		dprintf("    num_sg:      %d\n", app->saAmfApplicationCurrNumSG);
+		log_printf (LOG_INFO, "  safApp=%s\n", getSaNameT(&app->name));
+		log_printf (LOG_INFO, "    admin state: %s\n",
+				admin_state_text[app->saAmfApplicationAdminState]);
+		log_printf (LOG_INFO, "    num_sg:      %d\n", app->saAmfApplicationCurrNumSG);
 		for (sg = app->sg_head;	sg != NULL; sg = sg->next) {
-			dprintf("    safSG=%s\n", getSaNameT(&sg->name));
-			dprintf("      admin state:        %s\n", admin_state_text[sg->saAmfSGAdminState]);
-			dprintf("      assigned SUs        %d\n", sg->saAmfSGNumCurrAssignedSUs);
-			dprintf("      non inst. spare SUs %d\n", sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
-			dprintf("      inst. spare SUs     %d\n", sg->saAmfSGNumCurrInstantiatedSpareSUs);
+			log_printf (LOG_INFO, "    safSG=%s\n", getSaNameT(&sg->name));
+			log_printf (LOG_INFO, "      admin state:        %s\n",
+					admin_state_text[sg->saAmfSGAdminState]);
+			log_printf (LOG_INFO, "      assigned SUs        %d\n",
+					sg->saAmfSGNumCurrAssignedSUs);
+			log_printf (LOG_INFO, "      non inst. spare SUs %d\n",
+					sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
+			log_printf (LOG_INFO, "      inst. spare SUs     %d\n",
+					sg->saAmfSGNumCurrInstantiatedSpareSUs);
 			for (su = sg->su_head; su != NULL; su = su->next) {
-				dprintf("      safSU=%s\n", getSaNameT(&su->name));
-				dprintf("        oper state:      %s\n", oper_state_text[su->saAmfSUOperState]);
-				dprintf("        admin state:     %s\n", admin_state_text[su->saAmfSUAdminState]);
-				dprintf("        readiness state: %s\n", readiness_state_text[su->saAmfSUReadinessState]);
-				dprintf("        presence state:  %s\n", presence_state_text[su->saAmfSUPresenceState]);
-				dprintf("        hosted by node   %s\n", su->saAmfSUHostedByNode.value);
-				dprintf("        num active SIs   %d\n", su->saAmfSUNumCurrActiveSIs);
-				dprintf("        num standby SIs  %d\n", su->saAmfSUNumCurrStandbySIs);
-				dprintf("        restart count    %d\n", su->saAmfSURestartCount);
-				dprintf("        escalation level %d\n", su->escalation_level);
-				dprintf("        SU failover cnt  %d\n", su->su_failover_cnt);
-				dprintf("        assigned SIs:");
-				for (si_assignment = su->assigned_sis; si_assignment != NULL;
-					si_assignment = si_assignment->next) {
-					dprintf("          safSi=%s\n", si_assignment->si->name.value);
-					dprintf("            HA state: %s\n",
-						ha_state_text[si_assignment->saAmfSISUHAState]);
-				}
+				log_printf (LOG_INFO, "      safSU=%s\n", getSaNameT(&su->name));
+				log_printf (LOG_INFO, "        oper state:      %s\n",
+						oper_state_text[su->saAmfSUOperState]);
+				log_printf (LOG_INFO, "        admin state:     %s\n",
+						admin_state_text[su->saAmfSUAdminState]);
+				log_printf (LOG_INFO, "        readiness state: %s\n",
+					readiness_state_text[amf_su_get_saAmfSUReadinessState (su)]);
+				log_printf (LOG_INFO, "        presence state:  %s\n",
+						presence_state_text[su->saAmfSUPresenceState]);
+				log_printf (LOG_INFO, "        hosted by node   %s\n",
+						su->saAmfSUHostedByNode.value);
+				log_printf (LOG_INFO, "        num active SIs   %d\n",
+						amf_su_get_saAmfSUNumCurrActiveSIs (su));
+				log_printf (LOG_INFO, "        num standby SIs  %d\n",
+						amf_su_get_saAmfSUNumCurrStandbySIs (su));
+				log_printf (LOG_INFO, "        restart count    %d\n",
+						su->saAmfSURestartCount);
+				log_printf (LOG_INFO, "        restart control state %d\n",
+							su->restart_control_state);
+				log_printf (LOG_INFO, "        SU failover cnt  %d\n", su->su_failover_cnt);
+				log_printf (LOG_INFO, "        assigned SIs:");
+				amf_su_foreach_si_assignment (su, print_si_assignment);
+
 				for (comp = su->comp_head; comp != NULL; comp = comp->next) {
-					dprintf("        safComp=%s\n", getSaNameT(&comp->name));
-					dprintf("          oper state:      %s\n",
+					log_printf (LOG_INFO, "        safComp=%s\n", getSaNameT(&comp->name));
+					log_printf (LOG_INFO, "          oper state:      %s\n",
 						oper_state_text[comp->saAmfCompOperState]);
-					dprintf("          readiness state: %s\n",
-						readiness_state_text[comp->saAmfCompReadinessState]);
-					dprintf("          presence state:  %s\n",
+					log_printf (LOG_INFO, "          readiness state: %s\n",
+						readiness_state_text[amf_comp_get_saAmfCompReadinessState (comp)]);
+					log_printf (LOG_INFO, "          presence state:  %s\n",
 						presence_state_text[comp->saAmfCompPresenceState]);
-					dprintf("          num active CSIs  %d\n",
-						comp->saAmfCompNumCurrActiveCsi);
-					dprintf("          num standby CSIs %d\n",
-						comp->saAmfCompNumCurrStandbyCsi);
-					dprintf("          restart count    %d\n", comp->saAmfCompRestartCount);
-					dprintf("          assigned CSIs:");
-					for (csi_assignment = comp->assigned_csis; csi_assignment != NULL;
-						csi_assignment = csi_assignment->comp_next) {
-						dprintf("            safCSI=%s\n", csi_assignment->csi->name.value);
-						dprintf("              HA state: %s\n",
-							ha_state_text[csi_assignment->saAmfCSICompHAState]);
-					}
+					log_printf (LOG_INFO, "          num active CSIs  %d\n",
+						amf_comp_get_saAmfCompNumCurrActiveCsi (comp));
+					log_printf (LOG_INFO, "          num standby CSIs %d\n",
+						amf_comp_get_saAmfCompNumCurrStandbyCsi (comp));
+					log_printf (LOG_INFO, "          restart count    %d\n",
+							comp->saAmfCompRestartCount);
+					log_printf (LOG_INFO, "          assigned CSIs:");
+					amf_comp_foreach_csi_assignment (
+						comp, print_csi_assignment);
 				}
 			}
 		}
 		for (si = app->si_head; si != NULL; si = si->next) {
-			dprintf("    safSi=%s\n", getSaNameT(&si->name));
-			dprintf("      admin state:         %s\n", admin_state_text[si->saAmfSIAdminState]);
-			dprintf("      assignm. state:      %s\n", assignment_state_text[si->saAmfSIAssignmentState]);
-			dprintf("      active assignments:  %d\n", si->saAmfSINumCurrActiveAssignments);
-			dprintf("      standby assignments: %d\n", si->saAmfSINumCurrStandbyAssignments);
+			log_printf (LOG_INFO, "    safSi=%s\n", getSaNameT(&si->name));
+			log_printf (LOG_INFO, "      admin state:         %s\n",
+				admin_state_text[si->saAmfSIAdminState]);
+			log_printf (LOG_INFO, "      assignm. state:      %s\n",
+				assignment_state_text[
+					amf_si_get_saAmfSIAssignmentState (si)]);
+			log_printf (LOG_INFO, "      active assignments:  %d\n",
+					amf_si_get_saAmfSINumCurrActiveAssignments (si));
+			log_printf (LOG_INFO, "      standby assignments: %d\n",
+					amf_si_get_saAmfSINumCurrStandbyAssignments (si));
 			for (csi = si->csi_head; csi != NULL; csi = csi->next) {
-				dprintf("      safCsi=%s\n", getSaNameT(&csi->name));
+				log_printf (LOG_INFO, "      safCsi=%s\n", getSaNameT(&csi->name));
 			}
 		}
 	}
-	dprintf("===================================================");
+	log_printf (LOG_INFO, "===================================================");
 }
 
 /* to be removed... */
@@ -1036,3 +1091,8 @@ const char *amf_readiness_state (int state)
 	return readiness_state_text[state];
 }
 
+const char *amf_assignment_state (int state)
+{
+	return assignment_state_text[state];
+}
+

exec/openais-instantiate.c

@@ -45,6 +45,10 @@ struct timespec sleeptime = {
 	.tv_nsec = 10000 /* 10 msec */
 };
 
+#ifndef xprintf
+#define xprintf(...)
+#endif
+
 /*
  * The method by which the status is returned from execve
  * needs some performance enhancement
@@ -58,7 +62,8 @@ int main (int argc, char **argv, char **envp)
 
 	pid = fork();
 	if (pid == -1) {
-		printf ("openais-instantiate: could not fork process %s\n", strerror (errno));
+		fprintf (stderr, "openais-instantiate: could not fork process %s\n",
+				 strerror (errno));
 		return (errno);
 	}
 	if (pid) {
@@ -70,23 +75,24 @@ int main (int argc, char **argv, char **envp)
 			res = waitpid (pid, &status, WNOHANG);
 			if (res) {
 				if (WEXITSTATUS(status) == 0) {
-					printf ("openais-instantiate: component instantiated\n");
+					xprintf ("openais-instantiate: component instantiated\n");
 					return (0);
 				} else {
-					printf ("openais-instantiate: could not instantiate component (return code %d = %s)\n",
-						WEXITSTATUS(status),
+					fprintf (stderr, "openais-instantiate: could not execute "
+									 "program %s, (return code %d = %s)\n",
+						argv[1], WEXITSTATUS(status),
 						strerror (WEXITSTATUS(status)));
-					return (errno);
+					return (WEXITSTATUS(status));
 				}
 			}
 			nanosleep (&sleeptime, 0);
 			
 		}
-		printf ("openais-instantiate: component instantiated\n");
+		xprintf ("openais-instantiate: component instantiated\n");
 		return (0);
 	} else {
 
-printf ("childs pid %d\n", getpid());
+		xprintf ("childs pid %d\n", getpid());
 		/*
 		 * child process
 		 */

exec/util.h

@@ -59,6 +59,7 @@ enum e_ais_done {
 	AIS_DONE_OBJDB = -13,
 	AIS_DONE_INIT_SERVICES = -14,
 	AIS_DONE_OUT_OF_MEMORY = -15,
+	AIS_DONE_FATAL_ERR = -16
 };
 
 /*

test/clc_cli_script

@@ -33,7 +33,7 @@
 # It may be modified to do any action desired.
 
 instantiate() {
-	echo clc_cli_script instantiating component $SA_AMF_COMPONENT_NAME $1
+#	echo clc_cli_script instantiating component $SA_AMF_COMPONENT_NAME $1
 	./openais-instantiate $1
 	exit $?
 }
@@ -44,7 +44,8 @@ terminate() {
 	else
 		PIDFILEPATH=/tmp
 	fi
-	echo clc_cli_script terminating up component $SA_AMF_COMPONENT_NAME on PID `cat $PIDFILEPATH/openais_cleanup_$SA_AMF_COMPONENT_NAME`
+	echo clc_cli_script terminating component $SA_AMF_COMPONENT_NAME on PID \
+	`cat $PIDFILEPATH/openais_cleanup_$SA_AMF_COMPONENT_NAME`
 	kill -15 `cat $PIDFILEPATH/openais_cleanup_$SA_AMF_COMPONENT_NAME`
 	exit $?
 }
@@ -55,9 +56,15 @@ cleanup() {
 	else
 		PIDFILEPATH=/tmp
 	fi
-	echo clc_cli_script cleaning up component $SA_AMF_COMPONENT_NAME on PID `cat $PIDFILEPATH/openais_cleanup_$SA_AMF_COMPONENT_NAME`
+#	echo clc_cli_script cleaning up component $SA_AMF_COMPONENT_NAME on PID \
+#	`cat $PIDFILEPATH/openais_cleanup_$SA_AMF_COMPONENT_NAME`
 	kill -9 `cat $PIDFILEPATH/openais_cleanup_$SA_AMF_COMPONENT_NAME`
-	exit $?
+	STATUS=$?
+	if [ $STATUS == 1 ]; then
+		exit 0
+	else
+		exit $STATUS
+	fi
 }
 
 usage() {

test/testamf1.c

@@ -5,6 +5,9 @@
  *
  * Author: Steven Dake (sdake@mvista.com)
  *
+ * Copyright (c) 2006 Ericsson AB.
+ *  Author: Hans Feldt
+ *
  * This software licensed under BSD license, the text of which follows:
  * 
  * Redistribution and use in source and binary forms, with or without
@@ -135,11 +138,12 @@ void CSISetCallback (
 	int res;
 	switch (haState) {
 	case SA_AMF_HA_ACTIVE:
-		printf ("Component '%s' requested to enter hastate SA_AMF_ACTIVE for \n\tCSI '%s'\n",
-			compName->value, csiDescriptor->csiName.value);
+		printf ("%d: Component '%s' requested to enter hastate SA_AMF_ACTIVE"
+				" for \n\tCSI '%s'\n",
+			getpid(), compName->value, csiDescriptor->csiName.value);
 		res = saAmfResponse (handle, invocation, SA_AIS_OK);
 		if (res != SA_AIS_OK) {
-			fprintf (stderr, "saAmfResponse failed: %d\n", res);
+			fprintf (stderr, "%d: saAmfResponse failed: %d\n", getpid(), res);
 			exit (-1);
 		}
 
@@ -173,8 +177,9 @@ void CSISetCallback (
 		break;  
          
 	case SA_AMF_HA_STANDBY:
-		printf ("Component '%s' requested to enter hastate SA_AMF_STANDBY for \n\tCSI '%s'\n",
-			compName->value, csiDescriptor->csiName.value);
+		printf ("%d: Component '%s' requested to enter hastate SA_AMF_STANDBY "
+				"for \n\tCSI '%s'\n",
+			getpid(), compName->value, csiDescriptor->csiName.value);
 		res = saAmfResponse (handle, invocation, SA_AIS_OK);
 		
 		TR(TRU,csiDescriptor->csiAttr.number);
@@ -186,7 +191,6 @@ void CSISetCallback (
 		    TR(TRS,csiDescriptor->csiAttr.attr[i].attrName);
 		    TR(TRS,csiDescriptor->csiAttr.attr[i].attrValue);
 		} 
-		printf("%s:%d:%s:%d\n",__FILE__,__LINE__,__FUNCTION__,res);
 		TR(TRU,csiDescriptor->csiFlags);
 
 		printSaNameT((SaNameT*) &csiDescriptor->csiStateDescriptor.standbyDescriptor.activeCompName);
@@ -194,6 +198,10 @@ void CSISetCallback (
 
 		break;
 	case SA_AMF_HA_QUIESCED:
+		printf ("%d: Component '%s' requested to enter hastate SA_AMF_HA_QUIESCED "
+				"for \n\tCSI '%s'\n",
+			getpid(), compName->value, csiDescriptor->csiName.value);
+		res = saAmfResponse (handle, invocation, SA_AIS_OK);
 		break;
 	case SA_AMF_HA_QUIESCING:
 		break;
@@ -271,7 +279,7 @@ void write_pid (void) {
 	sprintf (filename,  "/var/run/openais_cleanup_%s", compNameGlobal.value);
 	fd = open (filename, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU);
 	if (fd == -1) {
-		printf("Failed using /var/run for pid file, using /tmp\n");
+		printf("%d: Failed using /var/run for pid file, using /tmp\n", getpid());
 		sprintf (filename,  "/tmp/openais_cleanup_%s", compNameGlobal.value);
 		fd = open (filename, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU);
 	}
@@ -295,18 +303,20 @@ int main (int argc, char **argv) {
 	}
 
 	/* test for correct value */
+#if 0
 	if (strstr (name, "safComp=A,safSu=SERVICE_X_") == NULL) {
 		fprintf(stderr, "SA_AMF_COMPONENT_NAME value wrong\n");
 		exit (-2);
 	}
+#endif
 
-	printf("Hello world from %s\n", name);
+	printf("%d: Hello world from %s\n", getpid(), name);
 
 	signal (SIGINT, sigintr_handler);
 #if defined(OPENAIS_BSD) || defined(OPENAIS_LINUX)
 	result = sched_setscheduler (0, SCHED_RR, &sched_param);
 	if (result == -1) {
-		printf ("couldn't set sched priority\n");
+		printf ("%d: couldn't set sched priority\n", getpid());
  	}
 #endif
 
@@ -327,18 +337,24 @@ int main (int argc, char **argv) {
 		&key0,
 		SA_AMF_HEALTHCHECK_AMF_INVOKED,
 		SA_AMF_COMPONENT_FAILOVER);
-    printf ("healthcheck start result %d (should be %d)\n", result, SA_AIS_OK);
+	if (result != SA_AIS_OK) {
+		printf ("Error: healthcheck start result %d\n", result);
+	}
 
     {
         SaNameT badname;
         strcpy ((char*)badname.value, "badname");
         badname.length = 7;
         result = saAmfComponentRegister (handle, &badname, NULL);
-        printf ("register result is %d (should be %d)\n", result, SA_AIS_ERR_INVALID_PARAM);
+		if (result != SA_AIS_ERR_INVALID_PARAM) {
+			printf ("Error: register result is %d\n", result);
+		}
     }
 
     result = saAmfComponentRegister (handle, &compNameGlobal, NULL);
-    printf ("register result is %d (should be %d)\n", result, SA_AIS_OK);
+	if (result != SA_AIS_OK) {
+		printf ("Error: register result is %d\n", result);
+	}
 
     /*
      * Test already started healthcheck
@@ -348,7 +364,9 @@ int main (int argc, char **argv) {
         &key0,
         SA_AMF_HEALTHCHECK_AMF_INVOKED,
         SA_AMF_COMPONENT_FAILOVER);
-    printf ("healthcheck start result %d (should be %d)\n", result, SA_AIS_ERR_EXIST);
+	if (result != SA_AIS_ERR_EXIST) {
+		printf ("Error: healthcheck start result %d\n", result);
+	}
 
 	do {
 		select (select_fd + 1, &read_fds, 0, 0, 0);