Adding support for dynamic membership with UDPU transport

Add a new object called totem.interface.dynamic to allow creation/deletion
of new child objects using the corosync-objctl utility:

to add new member:
linux#  corosync-objctl -c totem.interface.dynamic.10-211-55-12

to delete an existing member:
linux#  corosync-objctl -d totem.interface.dynamic.10-211-55-12

Corosync will dynamically add these members to the configuration and start
communicating with those nodes.

Signed-off-by: Anton Jouline <anton.jouline@cbsinteractive.com>
Reviewed-by: Steven Dake <sdake@redhat.com>

TODO

@@ -20,6 +20,9 @@
 9. investigate if https://github.com/asalkeld/libqb/issues/1 is still an issue.
 10. allow a cluster name to autogenerate a mcastaddr
 11. ring status change via corosync-notifyd
+12. remove dashes in UDPU notifier and replace with dots once map code is in
+13. put addition/removal of members in totem.interface.member objects
+14. dynamic UDPU needs ring id settings for multiple rings
 
 --------------------------------------
  Current priority list for Needle 2.1

exec/main.c

@@ -680,6 +680,121 @@ static void corosync_totem_stats_updater (void *data)
 		&corosync_stats_timer_handle);
 }
 
+static void totem_dynamic_name_to_ip (char *dest,
+	size_t dest_size,
+	const void *src,
+	size_t src_len)
+{
+	char *p;
+	size_t len;
+
+	len = (src_len + 1 > dest_size) ? dest_size-1 : src_len;
+	memset(dest, 0, dest_size);
+	memcpy(dest, src, len);
+	for (p = dest; p != dest + len; p++) {
+		if (*p == '-') {
+			*p = '.';
+		}
+	}
+}
+
+static void totem_dynamic_create_notify_fn (
+	hdb_handle_t parent_object_handle,
+	hdb_handle_t object_handle,
+	const void *name_pt, size_t name_len,
+	void *priv_data_pt)
+{
+	struct totem_ip_address member;
+	int ring_no; 
+	char object_name[128];
+
+	totem_dynamic_name_to_ip (object_name,
+		sizeof object_name, name_pt, name_len);
+	log_printf (LOGSYS_LEVEL_DEBUG,
+		"adding dynamic member: %s\n", object_name);
+
+	/*
+	 * add new member
+	 */
+	if (totemip_parse (&member, object_name, 0) == 0) {
+		ring_no = 0;
+		totempg_member_add (&member, ring_no);
+	}
+}
+
+static void totem_dynamic_destroy_notify_fn(
+	hdb_handle_t parent_object_handle,
+	const void *name_pt, size_t name_len,
+	void *priv_data_pt)
+{
+	struct totem_ip_address member;
+	int ring_no; 
+	char object_name[128];
+
+	totem_dynamic_name_to_ip (object_name, sizeof object_name,
+		name_pt, name_len);
+	log_printf(LOGSYS_LEVEL_DEBUG,
+		"removing dynamic member: %s\n", object_name);
+
+	/*
+	 * remove member
+	 */
+	if (totemip_parse(&member, object_name, 0) == 0) {
+		ring_no = 0;
+		totempg_member_remove (&member, ring_no);
+	}
+}
+
+static void corosync_totem_dynamic_init (void)
+{
+	hdb_handle_t object_find_handle;
+	hdb_handle_t object_totem_handle;
+	hdb_handle_t object_interface_handle;
+	hdb_handle_t object_dynamic_handle;
+
+	if (objdb->object_find_create (OBJECT_PARENT_HANDLE,
+		"totem", strlen("totem"), &object_find_handle) != 0) {
+		log_printf(LOGSYS_LEVEL_ERROR,
+			"corosync_totem_dynamic_init:: FAILED to find totem!\n");
+		return;
+	}
+	if (objdb->object_find_next (object_find_handle,
+		&object_totem_handle) != 0) {
+		return;
+	}
+
+	if (objdb->object_find_create(object_totem_handle,
+		"interface", strlen("interface"), &object_find_handle) != 0) {
+
+		log_printf(LOGSYS_LEVEL_ERROR,
+			"corosync_totem_dynamic_init:: FAILED to find totem.interface!\n");
+		return;
+	}
+	if (objdb->object_find_next (object_find_handle,
+		&object_interface_handle) != 0) {
+
+		return;
+	}
+
+	/*
+	 * create new child object: dynamic
+	 */
+	if (objdb->object_create (object_interface_handle,
+		&object_dynamic_handle,
+		"dynamic", strlen("dynamic")) != 0) {
+
+		log_printf(LOGSYS_LEVEL_ERROR,
+			"unable to create object: \"totem.interface.dynamic\"\n");
+		return;
+	}
+
+	objdb->object_track_start (object_dynamic_handle,
+		OBJECT_TRACK_DEPTH_RECURSIVE,
+		NULL,
+		totem_dynamic_create_notify_fn,
+		totem_dynamic_destroy_notify_fn,
+		NULL, NULL);
+}
 static void corosync_totem_stats_init (void)
 {
 	totempg_stats_t * stats;
@@ -1126,6 +1241,7 @@ static void main_service_ready (void)
 	cs_ipcs_init();
 	corosync_totem_stats_init ();
 	corosync_fplay_control_init ();
+	corosync_totem_dynamic_init ();
 	if (minimum_sync_mode == CS_SYNC_V2) {
 		log_printf (LOGSYS_LEVEL_NOTICE, "Compatibility mode set to none.  Using V2 of the synchronization engine.\n");
 		sync_v2_init (

exec/totempg.c

@@ -1444,11 +1444,17 @@ void totempg_queue_level_register_callback (totem_queue_level_changed_fn fn)
 
 extern int totempg_member_add (
 	const struct totem_ip_address *member,
-	int ring_no);
+	int ring_no)
+{
+	return totemmrp_member_add (member, ring_no);
+}
 
 extern int totempg_member_remove (
 	const struct totem_ip_address *member,
-	int ring_no);
+	int ring_no)
+{
+	return totemmrp_member_remove (member, ring_no);
+}
 
 void totempg_threaded_mode_enable (void)
 {

exec/totemudpu.c

@@ -1677,6 +1677,8 @@ int totemudpu_member_add (
 	if (new_member == NULL) {
 		return (-1);
 	}
+	log_printf (LOGSYS_LEVEL_NOTICE, "adding new UDPU member {%s}\n", 
+		totemip_print(member));
 	list_init (&new_member->list);
 	list_add_tail (&new_member->list, &instance->member_list);
 	memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
@@ -1712,8 +1714,48 @@ int totemudpu_member_remove (
 	void *udpu_context,
 	const struct totem_ip_address *token_target)
 {
+	int found = 0;
+	struct list_head *list;
+	struct totemudpu_member *member;
+
 	struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
 
+	/*
+	 * Find the member to remove and close its socket
+	 */
+	for (list = instance->member_list.next;
+		list != &instance->member_list;
+		list = list->next) {
+
+		member = list_entry (list,
+			struct totemudpu_member,
+			list);
+
+		if (totemip_compare (token_target, &member->member)==0) {
+			log_printf(LOGSYS_LEVEL_NOTICE,
+				"removing UDPU member {%s}\n",
+				totemip_print(&member->member));
+
+			if (member->fd > 0) {
+				log_printf(LOGSYS_LEVEL_DEBUG,
+					"Closing socket to: {%s}\n",
+					totemip_print(&member->member));
+				qb_loop_poll_del (instance->totemudpu_poll_handle,
+					member->fd);
+				close (member->fd);
+			}
+			found = 1;
+			break;
+		}
+	}
+
+	/*
+	 * Delete the member from the list
+	 */
+	if (found) {
+		list_del (list);
+	}
+
 	instance = NULL;
 	return (0);
 }