|
@@ -524,6 +524,45 @@ struct message_handlers {
|
|
|
int endian_conversion_needed);
|
|
int endian_conversion_needed);
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
|
|
+enum gather_state_from {
|
|
|
|
|
+ TOTEMSRP_GSFROM_CONSENSUS_TIMEOUT = 0,
|
|
|
|
|
+ TOTEMSRP_GSFROM_GATHER_MISSING1 = 1,
|
|
|
|
|
+ TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_OPERATIONAL_STATE = 2,
|
|
|
|
|
+ TOTEMSRP_GSFROM_THE_CONSENSUS_TIMEOUT_EXPIRED = 3,
|
|
|
|
|
+ TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_COMMIT_STATE = 4,
|
|
|
|
|
+ TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_RECOVERY_STATE = 5,
|
|
|
|
|
+ TOTEMSRP_GSFROM_FAILED_TO_RECEIVE = 6,
|
|
|
|
|
+ TOTEMSRP_GSFROM_FOREIGN_MESSAGE_IN_OPERATIONAL_STATE = 7,
|
|
|
|
|
+ TOTEMSRP_GSFROM_FOREIGN_MESSAGE_IN_GATHER_STATE = 8,
|
|
|
|
|
+ TOTEMSRP_GSFROM_MERGE_DURING_OPERATIONAL_STATE = 9,
|
|
|
|
|
+ TOTEMSRP_GSFROM_MERGE_DURING_GATHER_STATE = 10,
|
|
|
|
|
+ TOTEMSRP_GSFROM_MERGE_DURING_JOIN = 11,
|
|
|
|
|
+ TOTEMSRP_GSFROM_JOIN_DURING_OPERATIONAL_STATE = 12,
|
|
|
|
|
+ TOTEMSRP_GSFROM_JOIN_DURING_COMMIT_STATE = 13,
|
|
|
|
|
+ TOTEMSRP_GSFROM_JOIN_DURING_RECOVERY = 14,
|
|
|
|
|
+ TOTEMSRP_GSFROM_INTERFACE_CHANGE = 15,
|
|
|
|
|
+ TOTEMSRP_GSFROM_MAX = TOTEMSRP_GSFROM_INTERFACE_CHANGE,
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+const char* gather_state_from_desc [] = {
|
|
|
|
|
+ [TOTEMSRP_GSFROM_CONSENSUS_TIMEOUT] = "consensus timeout",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_GATHER_MISSING1] = "MISSING",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_OPERATIONAL_STATE] = "The token was lost in the OPERATIONAL state.",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_THE_CONSENSUS_TIMEOUT_EXPIRED] = "The consensus timeout expired.",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_COMMIT_STATE] = "The token was lost in the COMMIT state.",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_RECOVERY_STATE] = "The token was lost in the RECOVERY state.",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_FAILED_TO_RECEIVE] = "failed to receive",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_FOREIGN_MESSAGE_IN_OPERATIONAL_STATE] = "foreign message in operational state",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_FOREIGN_MESSAGE_IN_GATHER_STATE] = "foreign message in gather state",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_MERGE_DURING_OPERATIONAL_STATE] = "merge during operational state",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_MERGE_DURING_GATHER_STATE] = "merge during gather state",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_MERGE_DURING_JOIN] = "merge during join",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_JOIN_DURING_OPERATIONAL_STATE] = "join during operational state",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_JOIN_DURING_COMMIT_STATE] = "join during commit state",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_JOIN_DURING_RECOVERY] = "join during recovery",
|
|
|
|
|
+ [TOTEMSRP_GSFROM_INTERFACE_CHANGE] = "interface change",
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
/*
|
|
/*
|
|
|
* forward decls
|
|
* forward decls
|
|
|
*/
|
|
*/
|
|
@@ -586,7 +625,7 @@ static void memb_leave_message_send (struct totemsrp_instance *instance);
|
|
|
static void memb_ring_id_create_or_load (struct totemsrp_instance *, struct memb_ring_id *);
|
|
static void memb_ring_id_create_or_load (struct totemsrp_instance *, struct memb_ring_id *);
|
|
|
|
|
|
|
|
static void token_callbacks_execute (struct totemsrp_instance *instance, enum totem_callback_token_type type);
|
|
static void token_callbacks_execute (struct totemsrp_instance *instance, enum totem_callback_token_type type);
|
|
|
-static void memb_state_gather_enter (struct totemsrp_instance *instance, int gather_from);
|
|
|
|
|
|
|
+static void memb_state_gather_enter (struct totemsrp_instance *instance, enum gather_state_from gather_from);
|
|
|
static void messages_deliver_to_app (struct totemsrp_instance *instance, int skip, unsigned int end_point);
|
|
static void messages_deliver_to_app (struct totemsrp_instance *instance, int skip, unsigned int end_point);
|
|
|
static int orf_token_mcast (struct totemsrp_instance *instance, struct orf_token *oken,
|
|
static int orf_token_mcast (struct totemsrp_instance *instance, struct orf_token *oken,
|
|
|
int fcc_mcasts_allowed);
|
|
int fcc_mcasts_allowed);
|
|
@@ -617,6 +656,7 @@ static void timer_function_token_hold_retransmit_timeout (void *data);
|
|
|
static void timer_function_merge_detect_timeout (void *data);
|
|
static void timer_function_merge_detect_timeout (void *data);
|
|
|
static void *totemsrp_buffer_alloc (struct totemsrp_instance *instance);
|
|
static void *totemsrp_buffer_alloc (struct totemsrp_instance *instance);
|
|
|
static void totemsrp_buffer_release (struct totemsrp_instance *instance, void *ptr);
|
|
static void totemsrp_buffer_release (struct totemsrp_instance *instance, void *ptr);
|
|
|
|
|
+static const char* gsfrom_to_msg(enum gather_state_from gsfrom);
|
|
|
|
|
|
|
|
void main_deliver_fn (
|
|
void main_deliver_fn (
|
|
|
void *context,
|
|
void *context,
|
|
@@ -659,6 +699,16 @@ do { \
|
|
|
fmt ": %s (%d)\n", ##args, _error_ptr, err_num); \
|
|
fmt ": %s (%d)\n", ##args, _error_ptr, err_num); \
|
|
|
} while(0)
|
|
} while(0)
|
|
|
|
|
|
|
|
|
|
+static const char* gsfrom_to_msg(enum gather_state_from gsfrom)
|
|
|
|
|
+{
|
|
|
|
|
+ if (0 <= gsfrom && gsfrom <= TOTEMSRP_GSFROM_MAX) {
|
|
|
|
|
+ return gather_state_from_desc[gsfrom];
|
|
|
|
|
+ }
|
|
|
|
|
+ else {
|
|
|
|
|
+ return "UNKNOWN";
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
static void totemsrp_instance_initialize (struct totemsrp_instance *instance)
|
|
static void totemsrp_instance_initialize (struct totemsrp_instance *instance)
|
|
|
{
|
|
{
|
|
|
memset (instance, 0, sizeof (struct totemsrp_instance));
|
|
memset (instance, 0, sizeof (struct totemsrp_instance));
|
|
@@ -1601,7 +1651,7 @@ static void memb_state_consensus_timeout_expired (
|
|
|
|
|
|
|
|
memb_set_merge (no_consensus_list, no_consensus_list_entries,
|
|
memb_set_merge (no_consensus_list, no_consensus_list_entries,
|
|
|
instance->my_failed_list, &instance->my_failed_list_entries);
|
|
instance->my_failed_list, &instance->my_failed_list_entries);
|
|
|
- memb_state_gather_enter (instance, 0);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_CONSENSUS_TIMEOUT);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -1623,7 +1673,7 @@ static void timer_function_pause_timeout (void *data)
|
|
|
static void memb_recovery_state_token_loss (struct totemsrp_instance *instance)
|
|
static void memb_recovery_state_token_loss (struct totemsrp_instance *instance)
|
|
|
{
|
|
{
|
|
|
old_ring_state_restore (instance);
|
|
old_ring_state_restore (instance);
|
|
|
- memb_state_gather_enter (instance, 5);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_RECOVERY_STATE);
|
|
|
instance->stats.recovery_token_lost++;
|
|
instance->stats.recovery_token_lost++;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -1638,7 +1688,7 @@ static void timer_function_orf_token_timeout (void *data)
|
|
|
log_printf (instance->totemsrp_log_level_notice,
|
|
log_printf (instance->totemsrp_log_level_notice,
|
|
|
"A processor failed, forming new configuration.");
|
|
"A processor failed, forming new configuration.");
|
|
|
totemrrp_iface_check (instance->totemrrp_context);
|
|
totemrrp_iface_check (instance->totemrrp_context);
|
|
|
- memb_state_gather_enter (instance, 2);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_OPERATIONAL_STATE);
|
|
|
instance->stats.operational_token_lost++;
|
|
instance->stats.operational_token_lost++;
|
|
|
break;
|
|
break;
|
|
|
|
|
|
|
@@ -1646,14 +1696,14 @@ static void timer_function_orf_token_timeout (void *data)
|
|
|
log_printf (instance->totemsrp_log_level_debug,
|
|
log_printf (instance->totemsrp_log_level_debug,
|
|
|
"The consensus timeout expired.");
|
|
"The consensus timeout expired.");
|
|
|
memb_state_consensus_timeout_expired (instance);
|
|
memb_state_consensus_timeout_expired (instance);
|
|
|
- memb_state_gather_enter (instance, 3);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_THE_CONSENSUS_TIMEOUT_EXPIRED);
|
|
|
instance->stats.gather_token_lost++;
|
|
instance->stats.gather_token_lost++;
|
|
|
break;
|
|
break;
|
|
|
|
|
|
|
|
case MEMB_STATE_COMMIT:
|
|
case MEMB_STATE_COMMIT:
|
|
|
log_printf (instance->totemsrp_log_level_debug,
|
|
log_printf (instance->totemsrp_log_level_debug,
|
|
|
"The token was lost in the COMMIT state.");
|
|
"The token was lost in the COMMIT state.");
|
|
|
- memb_state_gather_enter (instance, 4);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_THE_TOKEN_WAS_LOST_IN_THE_COMMIT_STATE);
|
|
|
instance->stats.commit_token_lost++;
|
|
instance->stats.commit_token_lost++;
|
|
|
break;
|
|
break;
|
|
|
|
|
|
|
@@ -1986,7 +2036,7 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
|
|
|
|
|
|
|
|
static void memb_state_gather_enter (
|
|
static void memb_state_gather_enter (
|
|
|
struct totemsrp_instance *instance,
|
|
struct totemsrp_instance *instance,
|
|
|
- int gather_from)
|
|
|
|
|
|
|
+ enum gather_state_from gather_from)
|
|
|
{
|
|
{
|
|
|
instance->orf_token_discard = 1;
|
|
instance->orf_token_discard = 1;
|
|
|
|
|
|
|
@@ -2033,12 +2083,13 @@ static void memb_state_gather_enter (
|
|
|
memb_consensus_set (instance, &instance->my_id);
|
|
memb_consensus_set (instance, &instance->my_id);
|
|
|
|
|
|
|
|
log_printf (instance->totemsrp_log_level_debug,
|
|
log_printf (instance->totemsrp_log_level_debug,
|
|
|
- "entering GATHER state from %d.", gather_from);
|
|
|
|
|
|
|
+ "entering GATHER state from %d(%s).",
|
|
|
|
|
+ gather_from, gsfrom_to_msg(gather_from));
|
|
|
|
|
|
|
|
instance->memb_state = MEMB_STATE_GATHER;
|
|
instance->memb_state = MEMB_STATE_GATHER;
|
|
|
instance->stats.gather_entered++;
|
|
instance->stats.gather_entered++;
|
|
|
|
|
|
|
|
- if (gather_from == 3) {
|
|
|
|
|
|
|
+ if (TOTEMSRP_GSFROM_THE_CONSENSUS_TIMEOUT_EXPIRED == 3) {
|
|
|
/*
|
|
/*
|
|
|
* State 3 means gather, so we are continuously gathering.
|
|
* State 3 means gather, so we are continuously gathering.
|
|
|
*/
|
|
*/
|
|
@@ -3719,7 +3770,7 @@ printf ("token seq %d\n", token->seq);
|
|
|
instance->my_failed_list,
|
|
instance->my_failed_list,
|
|
|
&instance->my_failed_list_entries);
|
|
&instance->my_failed_list_entries);
|
|
|
|
|
|
|
|
- memb_state_gather_enter (instance, 6);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_FAILED_TO_RECEIVE);
|
|
|
} else {
|
|
} else {
|
|
|
instance->my_token_seq = token->token_seq;
|
|
instance->my_token_seq = token->token_seq;
|
|
|
token->token_seq += 1;
|
|
token->token_seq += 1;
|
|
@@ -3974,7 +4025,7 @@ static int message_handler_mcast (
|
|
|
memb_set_merge (
|
|
memb_set_merge (
|
|
|
&mcast_header.system_from, 1,
|
|
&mcast_header.system_from, 1,
|
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
|
- memb_state_gather_enter (instance, 7);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_FOREIGN_MESSAGE_IN_OPERATIONAL_STATE);
|
|
|
break;
|
|
break;
|
|
|
|
|
|
|
|
case MEMB_STATE_GATHER:
|
|
case MEMB_STATE_GATHER:
|
|
@@ -3986,7 +4037,7 @@ static int message_handler_mcast (
|
|
|
|
|
|
|
|
memb_set_merge (&mcast_header.system_from, 1,
|
|
memb_set_merge (&mcast_header.system_from, 1,
|
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
|
- memb_state_gather_enter (instance, 8);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_FOREIGN_MESSAGE_IN_GATHER_STATE);
|
|
|
return (0);
|
|
return (0);
|
|
|
}
|
|
}
|
|
|
break;
|
|
break;
|
|
@@ -4078,7 +4129,7 @@ static int message_handler_memb_merge_detect (
|
|
|
case MEMB_STATE_OPERATIONAL:
|
|
case MEMB_STATE_OPERATIONAL:
|
|
|
memb_set_merge (&memb_merge_detect.system_from, 1,
|
|
memb_set_merge (&memb_merge_detect.system_from, 1,
|
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
|
- memb_state_gather_enter (instance, 9);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_MERGE_DURING_OPERATIONAL_STATE);
|
|
|
break;
|
|
break;
|
|
|
|
|
|
|
|
case MEMB_STATE_GATHER:
|
|
case MEMB_STATE_GATHER:
|
|
@@ -4090,7 +4141,7 @@ static int message_handler_memb_merge_detect (
|
|
|
|
|
|
|
|
memb_set_merge (&memb_merge_detect.system_from, 1,
|
|
memb_set_merge (&memb_merge_detect.system_from, 1,
|
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
instance->my_proc_list, &instance->my_proc_list_entries);
|
|
|
- memb_state_gather_enter (instance, 10);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_MERGE_DURING_GATHER_STATE);
|
|
|
return (0);
|
|
return (0);
|
|
|
}
|
|
}
|
|
|
break;
|
|
break;
|
|
@@ -4217,7 +4268,7 @@ static void memb_join_process (
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
- memb_state_gather_enter (instance, 11);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_MERGE_DURING_JOIN);
|
|
|
gather_entered = 1;
|
|
gather_entered = 1;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -4225,7 +4276,7 @@ out:
|
|
|
if (gather_entered == 0 &&
|
|
if (gather_entered == 0 &&
|
|
|
instance->memb_state == MEMB_STATE_OPERATIONAL) {
|
|
instance->memb_state == MEMB_STATE_OPERATIONAL) {
|
|
|
|
|
|
|
|
- memb_state_gather_enter (instance, 12);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_JOIN_DURING_OPERATIONAL_STATE);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -4395,7 +4446,7 @@ static int message_handler_memb_join (
|
|
|
memb_join->ring_seq >= instance->my_ring_id.seq) {
|
|
memb_join->ring_seq >= instance->my_ring_id.seq) {
|
|
|
|
|
|
|
|
memb_join_process (instance, memb_join);
|
|
memb_join_process (instance, memb_join);
|
|
|
- memb_state_gather_enter (instance, 13);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_JOIN_DURING_COMMIT_STATE);
|
|
|
}
|
|
}
|
|
|
break;
|
|
break;
|
|
|
|
|
|
|
@@ -4409,7 +4460,7 @@ static int message_handler_memb_join (
|
|
|
|
|
|
|
|
memb_join_process (instance, memb_join);
|
|
memb_join_process (instance, memb_join);
|
|
|
memb_recovery_state_token_loss (instance);
|
|
memb_recovery_state_token_loss (instance);
|
|
|
- memb_state_gather_enter (instance, 14);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_JOIN_DURING_RECOVERY);
|
|
|
}
|
|
}
|
|
|
break;
|
|
break;
|
|
|
}
|
|
}
|
|
@@ -4598,7 +4649,7 @@ void main_iface_change_fn (
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if (instance->iface_changes >= instance->totem_config->interface_count) {
|
|
if (instance->iface_changes >= instance->totem_config->interface_count) {
|
|
|
- memb_state_gather_enter (instance, 15);
|
|
|
|
|
|
|
+ memb_state_gather_enter (instance, TOTEMSRP_GSFROM_INTERFACE_CHANGE);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|