amfsg.c 60 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260
  1. /** @file amfsg.c
  2. *
  3. * Copyright (c) 2002-2006 MontaVista Software, Inc.
  4. * Author: Steven Dake (sdake@mvista.com)
  5. *
  6. * Copyright (c) 2006 Ericsson AB.
  7. * Author: Hans Feldt, Anders Eriksson, Lars Holm
  8. * - Introduced AMF B.02 information model
  9. * - Use DN in API and multicast messages
  10. * - (Re-)Introduction of event based multicast messages
  11. * - Refactoring of code into several AMF files
  12. * - Component/SU restart, SU failover
  13. * - Constructors/destructors
  14. * - Serializers/deserializers
  15. *
  16. * All rights reserved.
  17. *
  18. *
  19. * This software licensed under BSD license, the text of which follows:
  20. *
  21. * Redistribution and use in source and binary forms, with or without
  22. * modification, are permitted provided that the following conditions are met:
  23. *
  24. * - Redistributions of source code must retain the above copyright notice,
  25. * this list of conditions and the following disclaimer.
  26. * - Redistributions in binary form must reproduce the above copyright notice,
  27. * this list of conditions and the following disclaimer in the documentation
  28. * and/or other materials provided with the distribution.
  29. * - Neither the name of the MontaVista Software, Inc. nor the names of its
  30. * contributors may be used to endorse or promote products derived from this
  31. * software without specific prior written permission.
  32. *
  33. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  34. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  35. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  36. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  37. * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  38. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  39. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  40. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  41. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  42. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  43. * THE POSSIBILITY OF SUCH DAMAGE.
  44. *
  45. * AMF Service Group Class Implementation
  46. *
  47. * This file contains functions for handling AMF-service groups(SGs). It can be
  48. * viewed as the implementation of the AMF Service Group class (called SG)
  49. * as described in SAI-Overview-B.02.01. The SA Forum specification
  50. * SAI-AIS-AMF-B.02.01 has been used as specification of the behaviour
  51. * and is referred to as 'the spec' below.
  52. *
  53. * The functions in this file are responsible for:
  54. * -on request start the service group by instantiating the contained SUs
  55. * -on request assign the service instances it protects to the in-service
  56. * service units it contains respecting as many as possible of the configured
  57. * requirements for the group
  58. * -create and delete an SI-assignment object for each relation between
  59. * an SI and an SU
  60. * -order each contained SU to create and delete CSI-assignments
  61. * -request the Service Instance class (SI) to execute the transfer of the
  62. * HA-state set/remove requests to each component involved
  63. * -fully control the execution of component failover and SU failover
  64. * -on request control the execution of the initial steps of node switchover
  65. * and node failover
  66. * -fully handle the auto adjust procedure
  67. *
  68. * Currently only the 'n+m' redundancy model is implemented. It is the
  69. * ambition to identify n+m specific variables and functions and add the suffix
  70. * '_nplusm' to them so that they can be easily recognized.
  71. *
  72. * When SG is requested to assign workload to all SUs or all SUs hosted on
  73. * a specific node, a procedure containing several steps is executed:
  74. * <1> An algorithm is executed which assigns SIs to SUs respecting the rules
  75. * that has been configured for SG. The algorithm also has to consider
  76. * if assignments between som SIs and SUs already exist. The scope of this
  77. * algorithm is to create SI-assignments and set up requested HA-state for
  78. * each assignment but not to transfer those HA-states to the components.
  79. * <2> All SI-assignments with a requested HA state == ACTIVE are transferred
  80. * to the components concerned before any STANDBY assignments are
  81. * transferred. All components have to acknowledge the setting of the
  82. * ACTIVE HA state before the transfer of any STANDBY assignment is
  83. * initiated.
  84. * <3> All active assignments can not be transferred at the same time to the
  85. * different components because the rules for dependencies between SI and
  86. * SI application wide and CSI and CSI within one SI, has to be respected.
  87. *
  88. * SG is fully responsible for step <1> but not fully responsible for handling
  89. * step <2> and <3>. However, SG uses an attribute called 'dependency level'
  90. * when requsted to assign workload. This parameter refers to an integer that
  91. * has been calculated initially for each SI. The 'dependency level' indicates
  92. * to which extent an SI depends on other SIs such that an SI that depends on
  93. * no other SI is on dependecy_level == 1, an SI that depends only on an SI on
  94. * dependency_level == 1 is on dependency-level == 2.
  95. * An SI that depends on several SIs gets a
  96. * dependency_level that is one unit higher than the SI with the highest
  97. * dependency_level it depends on. When SG is requested to assign the workload
  98. * on a certain dependency level, it requests all SI objects on that level to
  99. * activate (all) SI-assignments that during step <1> has been requested to
  100. * assume the active HA state.
  101. *
  102. * SG contains the following state machines:
  103. * - administrative state machine (ADSM) (NOT IN THIS RELEASE)
  104. * - availability control state machine (ACSM)
  105. *
  106. * The availability control state machine contains two states and one of them
  107. * is composite. Being a composite state means that it contains substates.
  108. * The states are:
  109. * - IDLE (non composite state)
  110. * - MANAGING_SG (composite state)
  111. * MANAGING_SG is entered at several different events which has in common
  112. * the need to set up or change the assignment of SIs to SUs. Only one such
  113. * event can be handled at the time. If new events occur while one event is
  114. * being handled then the new event is saved and will be handled after the
  115. * handling of the first event is ready (return to IDLE state has been done).
  116. * MANAGING_SG handles the following events:
  117. * - start (requests SG to order SU to instantiate all SUs in SG and waits
  118. * for SU to indicate presence state change reports from the SUs and
  119. * finally responds 'started' to the requester)
  120. * - assign (requests SG to assign SIs to SUs according to pre-configured
  121. * rules (if not already done) and transfer the HA state of
  122. * the SIs on the requested SI dependency level. Then SG waits for
  123. * confirmation that the HA state has been succesfully set and
  124. * finally responds 'assigned' to the reqeuster)
  125. * - auto_adjust (this event indicates that the auto-adjust probation timer has
  126. * expired and that SG should evaluate current assignments of
  127. * SIs to SUs and if needed remove current assignments and
  128. * create new according to what is specified in paragraph
  129. * 3.7.1.2)
  130. * - failover_comp (requests SG to failover a specific component according to
  131. * the procedure described in paragraph 3.12.1.3)
  132. * - failover_su (requests SG to failover a specific SU according to the
  133. * procedure described in paragraph 3.12.1.3 and 3.12.1.4)
  134. * - switchover_node (requests SG to execute the recovery actions described
  135. * in 3.12.1.3 and respond to the requester when recovery
  136. * is completed)
  137. * - failover_node (requests SG to execute the recovery actions described
  138. * in 3.12.1.3 and respond to the requester when recovery is
  139. * completed)
  140. *
  141. */
  142. #include <stdlib.h>
  143. #include <errno.h>
  144. #include "amf.h"
  145. #include "print.h"
  146. #include "main.h"
  147. #include "util.h"
  148. static int assign_si (struct amf_sg *sg, int dependency_level);
  149. static void acsm_enter_activating_standby (struct amf_sg *sg);
  150. static void delete_si_assignments_in_scope (struct amf_sg *sg);
  151. static void acsm_enter_repairing_su (struct amf_sg *sg);
  152. static void standby_su_activated_cbfn (
  153. struct amf_si_assignment *si_assignment, int result);
  154. static void dependent_si_deactivated_cbfn (
  155. struct amf_si_assignment *si_assignment, int result);
  156. static void acsm_enter_removing_standby_assignments (amf_sg_t *sg);
  157. static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg);
  158. static const char *sg_event_type_text[] = {
  159. "Unknown",
  160. "Failover SU",
  161. "Failover node",
  162. "Failover comp",
  163. "Switchover node",
  164. "Start",
  165. "Autoadjust",
  166. "Assign SI"
  167. };
  168. typedef struct sg_event {
  169. amf_sg_event_type_t event_type;
  170. amf_sg_t *sg;
  171. amf_su_t *su;
  172. amf_comp_t *comp;
  173. amf_node_t *node;
  174. } sg_event_t;
  175. static int is_cluster_start(amf_node_t *node_to_start)
  176. {
  177. return node_to_start == NULL;
  178. }
  179. static void sg_set_event (amf_sg_event_type_t sg_event_type,
  180. amf_sg_t *sg, amf_su_t *su, amf_comp_t *comp, amf_node_t * node,
  181. sg_event_t *sg_event)
  182. {
  183. sg_event->event_type = sg_event_type;
  184. sg_event->node = node;
  185. sg_event->su = su;
  186. sg_event->comp = comp;
  187. sg_event->sg = sg;
  188. }
  189. static void sg_defer_event (amf_sg_event_type_t event_type,
  190. sg_event_t *sg_event)
  191. {
  192. ENTER("Defered event = %d", event_type);
  193. amf_fifo_put (event_type, &sg_event->sg->deferred_events,
  194. sizeof (sg_event_t),
  195. sg_event);
  196. }
  197. static void sg_recall_deferred_events (amf_sg_t *sg)
  198. {
  199. sg_event_t sg_event;
  200. ENTER ("%s", sg->name.value);
  201. if (amf_fifo_get (&sg->deferred_events, &sg_event)) {
  202. switch (sg_event.event_type) {
  203. case SG_FAILOVER_SU_EV:
  204. amf_sg_failover_su_req (sg_event.sg,
  205. sg_event.su, sg_event.node);
  206. break;
  207. case SG_FAILOVER_NODE_EV:
  208. amf_sg_failover_node_req (sg_event.sg,
  209. sg_event.node);
  210. break;
  211. case SG_FAILOVER_COMP_EV:
  212. case SG_SWITCH_OVER_NODE_EV:
  213. case SG_START_EV:
  214. case SG_AUTO_ADJUST_EV:
  215. default:
  216. dprintf("event_type = %d", sg_event.event_type);
  217. break;
  218. }
  219. }
  220. }
  221. static void timer_function_sg_recall_deferred_events (void *data)
  222. {
  223. amf_sg_t *sg = (amf_sg_t*)data;
  224. ENTER ("");
  225. sg_recall_deferred_events (sg);
  226. }
  227. static void acsm_enter_idle (amf_sg_t *sg)
  228. {
  229. SaNameT dn;
  230. ENTER ("sg: %s state: %d", sg->name.value, sg->avail_state);
  231. sg->avail_state = SG_AC_Idle;
  232. if (sg->recovery_scope.event_type != 0) {
  233. switch (sg->recovery_scope.event_type) {
  234. case SG_FAILOVER_SU_EV:
  235. assert (sg->recovery_scope.sus[0] != NULL);
  236. amf_su_dn_make (sg->recovery_scope.sus[0], &dn);
  237. log_printf (
  238. LOG_NOTICE,
  239. "'%s' %s recovery action finished",
  240. dn.value,
  241. sg_event_type_text[sg->recovery_scope.event_type]);
  242. break;
  243. case SG_FAILOVER_NODE_EV:
  244. amf_node_sg_failed_over (
  245. sg->recovery_scope.node, sg);
  246. log_printf (
  247. LOG_NOTICE,
  248. "'%s for %s' recovery action finished",
  249. sg_event_type_text[sg->recovery_scope.event_type],
  250. sg->name.value);
  251. break;
  252. case SG_START_EV:
  253. amf_application_sg_started (sg->application,
  254. sg, this_amf_node);
  255. break;
  256. case SG_ASSIGN_SI_EV:
  257. log_printf (LOG_NOTICE, "All SI assigned");
  258. break;
  259. default:
  260. log_printf (
  261. LOG_NOTICE,
  262. "'%s' recovery action finished",
  263. sg_event_type_text[sg->recovery_scope.event_type]);
  264. break;
  265. }
  266. }
  267. if (sg->recovery_scope.sus != NULL) {
  268. free ((void *)sg->recovery_scope.sus);
  269. }
  270. if (sg->recovery_scope.sis != NULL) {
  271. free ((void *)sg->recovery_scope.sis);
  272. }
  273. memset (&sg->recovery_scope, 0, sizeof (struct sg_recovery_scope));
  274. sg->node_to_start = NULL;
  275. amf_call_function_asynchronous (
  276. timer_function_sg_recall_deferred_events, sg);
  277. }
  278. static int su_instantiated_count (struct amf_sg *sg)
  279. {
  280. int cnt = 0;
  281. struct amf_su *su;
  282. for (su = sg->su_head; su != NULL; su = su->next) {
  283. if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED)
  284. cnt++;
  285. }
  286. return cnt;
  287. }
  288. static int has_any_su_in_scope_active_workload (struct amf_sg *sg)
  289. {
  290. struct amf_su **sus= sg->recovery_scope.sus;
  291. struct amf_si_assignment *si_assignment;
  292. while (*sus != NULL) {
  293. si_assignment = amf_su_get_next_si_assignment (*sus, NULL);
  294. while (si_assignment != NULL) {
  295. if (si_assignment->saAmfSISUHAState !=
  296. SA_AMF_HA_ACTIVE) {
  297. break;
  298. }
  299. si_assignment = amf_su_get_next_si_assignment (
  300. *sus, si_assignment);
  301. }
  302. if (si_assignment != NULL) {
  303. break;
  304. }
  305. sus++;
  306. }
  307. return(*sus == NULL);
  308. }
  309. static int is_any_si_in_scope_assigned_standby (struct amf_sg *sg)
  310. {
  311. struct amf_si **sis= sg->recovery_scope.sis;
  312. struct amf_si_assignment *si_assignment;
  313. /*
  314. * Check if there is any si in the scope which has no active assignment
  315. * and at least one standby assignment.
  316. */
  317. while (*sis != NULL) {
  318. si_assignment = (*sis)->assigned_sis;
  319. while (si_assignment != NULL) {
  320. if (si_assignment->saAmfSISUHAState ==
  321. SA_AMF_HA_ACTIVE) {
  322. break;
  323. }
  324. si_assignment = si_assignment->next;
  325. }
  326. if (si_assignment == NULL) {
  327. /* There is no ACTIVE assignment ..*/
  328. si_assignment = (*sis)->assigned_sis;
  329. while (si_assignment != NULL) {
  330. if (si_assignment->saAmfSISUHAState ==
  331. SA_AMF_HA_STANDBY) {
  332. break;
  333. }
  334. si_assignment = si_assignment->next;
  335. }
  336. if (si_assignment != NULL) {
  337. /* .. and one STANDBY assignment*/
  338. break;
  339. }
  340. }
  341. sis++;
  342. }
  343. return(*sis != NULL);
  344. }
  345. static void acsm_enter_terminating_suspected (struct amf_sg *sg)
  346. {
  347. struct amf_su **sus= sg->recovery_scope.sus;
  348. ENTER("%s",sg->name.value);
  349. sg->avail_state = SG_AC_TerminatingSuspected;
  350. /*
  351. * Terminate suspected SU(s)
  352. */
  353. while (*sus != 0) {
  354. amf_su_terminate (*sus);
  355. sus++;
  356. }
  357. }
  358. static inline int su_presense_state_is_ored (amf_su_t *su,
  359. SaAmfPresenceStateT state1,SaAmfPresenceStateT state2,
  360. SaAmfPresenceStateT state3)
  361. {
  362. return(su->saAmfSUPresenceState == state1 || su->saAmfSUPresenceState ==
  363. state2 || su->saAmfSUPresenceState == state3) ? 1 : 0;
  364. }
  365. static inline int su_presense_state_is_not (amf_su_t *su,
  366. SaAmfPresenceStateT state1,SaAmfPresenceStateT state2,
  367. SaAmfPresenceStateT state3)
  368. {
  369. return(su->saAmfSUPresenceState != state1 && su->saAmfSUPresenceState !=
  370. state2 && su->saAmfSUPresenceState != state3) ? 1 : 0;
  371. }
  372. /**
  373. * Callback function used by SI when there is no dependent SI to
  374. * deactivate.
  375. * @param sg
  376. */
  377. static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg)
  378. {
  379. struct amf_su **sus = sg->recovery_scope.sus;
  380. ENTER("'%s'", sg->name.value);
  381. /*
  382. * Select next state depending on if some
  383. * SU in the scope is needs to be terminated.
  384. */
  385. while (*sus != NULL) {
  386. amf_su_t *su = *sus;
  387. ENTER("SU %s pr_state='%d'",su->name.value,
  388. su->saAmfSUPresenceState);
  389. if (su_presense_state_is_ored (su,
  390. SA_AMF_PRESENCE_UNINSTANTIATED,
  391. SA_AMF_PRESENCE_TERMINATION_FAILED,
  392. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  393. sus++;
  394. continue;
  395. }
  396. break;
  397. }
  398. if (*sus != NULL) {
  399. acsm_enter_terminating_suspected (sg);
  400. } else {
  401. delete_si_assignments_in_scope(sg);
  402. acsm_enter_removing_standby_assignments (sg);
  403. }
  404. }
  405. static void timer_function_dependent_si_deactivated2 (void *data)
  406. {
  407. amf_sg_t *sg = (amf_sg_t *)data;
  408. ENTER ("");
  409. dependent_si_deactivated_cbfn2 (sg);
  410. }
  411. static struct amf_si *si_get_dependent (struct amf_si *si)
  412. {
  413. struct amf_si *tmp_si = NULL;
  414. if (si->depends_on != NULL) {
  415. SaNameT res_arr[2];
  416. int is_match;
  417. if (si->depends_on->name.length < SA_MAX_NAME_LENGTH) {
  418. si->depends_on->name.value[si->depends_on->name.length] = '\0';
  419. }
  420. is_match = sa_amf_grep ((char*)si->depends_on->name.value,
  421. "safDepend=.*,safSi=(.*),safApp=.*",
  422. 2, res_arr);
  423. if (is_match) {
  424. tmp_si = amf_si_find (si->application, (char*)res_arr[1].value);
  425. } else {
  426. log_printf (LOG_LEVEL_ERROR, "distinguished name for "
  427. "amf_si_depedency failed\n");
  428. openais_exit_error (AIS_DONE_FATAL_ERR);
  429. }
  430. }
  431. return tmp_si;
  432. }
  433. static struct amf_si *amf_dependent_get_next (struct amf_si *si,
  434. struct amf_si *si_iter)
  435. {
  436. struct amf_si *tmp_si;
  437. struct amf_application *application;
  438. if (si_iter == NULL) {
  439. assert(amf_cluster != NULL);
  440. application = amf_cluster->application_head;
  441. assert(application != NULL);
  442. tmp_si = application->si_head;
  443. } else {
  444. tmp_si = si_iter->next;
  445. if (tmp_si == NULL) {
  446. application = si->application->next;
  447. if (application == NULL) {
  448. goto out;
  449. }
  450. }
  451. }
  452. for (; tmp_si != NULL; tmp_si = tmp_si->next) {
  453. struct amf_si *depends_on_si = si_get_dependent (tmp_si);
  454. while (depends_on_si != NULL) {
  455. if (depends_on_si == si) {
  456. goto out;
  457. }
  458. depends_on_si = depends_on_si->next;
  459. }
  460. }
  461. out:
  462. return tmp_si;
  463. }
  464. static void acsm_enter_deactivating_dependent_workload (amf_sg_t *sg)
  465. {
  466. struct amf_si **sis= sg->recovery_scope.sis;
  467. struct amf_si_assignment *si_assignment;
  468. int callback_pending = 0;
  469. sg->avail_state = SG_AC_DeactivatingDependantWorkload;
  470. ENTER("'%s'",sg->name.value);
  471. /*
  472. * For each SI in the recovery scope, find all active
  473. * assignments and request them to be deactivated.
  474. */
  475. while (*sis != NULL) {
  476. struct amf_si *dependent_si;
  477. struct amf_si *si = *sis;
  478. si_assignment = si->assigned_sis;
  479. dependent_si = amf_dependent_get_next (si, NULL);
  480. while (dependent_si != NULL) {
  481. si_assignment = dependent_si->assigned_sis;
  482. while (si_assignment != NULL) {
  483. if (si_assignment->saAmfSISUHAState ==
  484. SA_AMF_HA_ACTIVE) {
  485. si_assignment->requested_ha_state =
  486. SA_AMF_HA_QUIESCED;
  487. callback_pending = 1;
  488. amf_si_ha_state_assume (
  489. si_assignment,
  490. dependent_si_deactivated_cbfn);
  491. }
  492. si_assignment = si_assignment->next;
  493. }
  494. dependent_si = amf_dependent_get_next (si, dependent_si);
  495. }
  496. sis++;
  497. }
  498. if (callback_pending == 0) {
  499. static poll_timer_handle dependent_si_deactivated_handle;
  500. ENTER("");
  501. poll_timer_add (aisexec_poll_handle, 0, sg,
  502. timer_function_dependent_si_deactivated2,
  503. &dependent_si_deactivated_handle);
  504. }
  505. }
  506. /**
  507. * Enter function for state SG_AC_ActivatingStandby. It activates
  508. * one STANDBY assignment for each SI in the recovery scope.
  509. * @param sg
  510. */
  511. static void acsm_enter_activating_standby (struct amf_sg *sg)
  512. {
  513. struct amf_si **sis= sg->recovery_scope.sis;
  514. struct amf_si_assignment *si_assignment;
  515. int is_no_standby_activated = 1;
  516. ENTER("'%s'",sg->name.value);
  517. sg->avail_state = SG_AC_ActivatingStandby;
  518. /*
  519. * For each SI in the recovery scope, find one standby
  520. * SI assignment and activate it.
  521. */
  522. while (*sis != NULL) {
  523. si_assignment = (*sis)->assigned_sis;
  524. while (si_assignment != NULL) {
  525. if (si_assignment->saAmfSISUHAState ==
  526. SA_AMF_HA_STANDBY) {
  527. si_assignment->requested_ha_state =
  528. SA_AMF_HA_ACTIVE;
  529. amf_si_ha_state_assume (
  530. si_assignment, standby_su_activated_cbfn);
  531. is_no_standby_activated = 0;
  532. break;
  533. }
  534. si_assignment = si_assignment->next;
  535. }
  536. sis++;
  537. }
  538. if (is_no_standby_activated) {
  539. acsm_enter_assigning_standby_to_spare (sg);
  540. }
  541. }
  542. static void acsm_enter_repairing_su (struct amf_sg *sg)
  543. {
  544. struct amf_su **sus= sg->recovery_scope.sus;
  545. int is_any_su_instantiated = 0;
  546. ENTER("'%s'",sg->name.value);
  547. sg->avail_state = SG_AC_ReparingSu;
  548. /*
  549. * Instantiate SUs in current recovery scope until the configured
  550. * preference is fulfiled.
  551. */
  552. while (*sus != NULL) {
  553. if (su_instantiated_count ((*sus)->sg) <
  554. (*sus)->sg->saAmfSGNumPrefInserviceSUs) {
  555. struct amf_node *node =
  556. amf_node_find(&((*sus)->saAmfSUHostedByNode));
  557. if (node == NULL) {
  558. log_printf (LOG_LEVEL_ERROR,
  559. "no node to hosted on su found"
  560. "amf_si_depedency failed\n");
  561. openais_exit_error (AIS_DONE_FATAL_ERR);
  562. }
  563. if (node->saAmfNodeOperState == SA_AMF_OPERATIONAL_ENABLED) {
  564. /* node is synchronized */
  565. is_any_su_instantiated = 1;
  566. amf_su_instantiate ((*sus));
  567. }
  568. }
  569. sus++;
  570. }
  571. if (is_any_su_instantiated == 0) {
  572. acsm_enter_idle (sg);
  573. }
  574. }
  575. static inline void remove_all_suspected_sus (amf_sg_t *sg)
  576. {
  577. amf_su_t *su;
  578. ENTER("");
  579. for (su = sg->su_head; su != NULL; su =su->next) {
  580. amf_comp_t *component;
  581. for (component = su->comp_head; component != NULL;
  582. component = component->next) {
  583. component->error_suspected = 0;
  584. }
  585. }
  586. }
  587. static int is_all_si_assigned (amf_sg_t *sg)
  588. {
  589. struct amf_si_assignment *si_assignment;
  590. int si_assignment_cnt = 0;
  591. int confirmed_assignments = 0;
  592. amf_si_t *si;
  593. for (si = sg->application->si_head; si != NULL; si = si->next) {
  594. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  595. for (si_assignment = si->assigned_sis;
  596. si_assignment != NULL;
  597. si_assignment = si_assignment->next) {
  598. si_assignment_cnt++;
  599. if (si_assignment->requested_ha_state ==
  600. si_assignment->saAmfSISUHAState) {
  601. confirmed_assignments++;
  602. }
  603. }
  604. }
  605. }
  606. return (confirmed_assignments == si_assignment_cnt);
  607. }
  608. static void assign_si_assumed_cbfn (
  609. struct amf_si_assignment *si_assignment, int result)
  610. {
  611. struct amf_sg *sg = si_assignment->su->sg;
  612. int si_assignment_cnt = 0;
  613. int confirmed_assignments = 0;
  614. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  615. switch (sg->avail_state) {
  616. case SG_AC_AssigningOnRequest:
  617. if (is_all_si_assigned (sg)) {
  618. acsm_enter_idle (sg);
  619. amf_application_sg_assigned (sg->application, sg);
  620. } else {
  621. dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments);
  622. }
  623. break;
  624. case SG_AC_AssigningStandBy:
  625. {
  626. if (is_all_si_assigned(sg)) {
  627. acsm_enter_idle (sg);
  628. }
  629. break;
  630. }
  631. case SG_AC_AssigningStandbyToSpare:
  632. {
  633. if(is_all_si_assigned (sg)) {
  634. /*
  635. * All si_assignments has asumed
  636. * Prescense state SA_AMF_HA_STANDBY
  637. */
  638. switch (sg->recovery_scope.event_type) {
  639. case SG_FAILOVER_NODE_EV:
  640. acsm_enter_idle (sg);
  641. break;
  642. case SG_FAILOVER_SU_EV:
  643. if (sg->saAmfSGAutoRepair == SA_TRUE) {
  644. acsm_enter_repairing_su (sg);
  645. }
  646. break;
  647. default:
  648. assert (0);
  649. break;
  650. }
  651. } else {
  652. si_assignment->saAmfSISUHAState = SA_AMF_HA_STANDBY;
  653. }
  654. }
  655. break;
  656. default:
  657. dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt,
  658. confirmed_assignments);
  659. amf_runtime_attributes_print (amf_cluster);
  660. assert (0);
  661. break;
  662. }
  663. }
  664. /**
  665. * Inquire if SI is assigned to SU
  666. * @param si
  667. * @param su
  668. *
  669. * @return int
  670. */
  671. static int is_si_assigned_to_su (amf_si_t *si, amf_su_t *su)
  672. {
  673. amf_si_assignment_t *si_assignment = 0;
  674. int si_assignment_assigned_to_su = 0;
  675. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  676. si_assignment = si_assignment->next) {
  677. if (si_assignment->su == su) {
  678. si_assignment_assigned_to_su = 1;
  679. break;
  680. }
  681. }
  682. return si_assignment_assigned_to_su;
  683. }
  684. /**
  685. * Inquire if SU is a spare.
  686. * @param sg
  687. * @param su
  688. *
  689. * @return int
  690. */
  691. static int is_spare_su (amf_sg_t *sg, amf_su_t *su)
  692. {
  693. amf_si_t *si;
  694. int spare_su = 1;
  695. for (si = sg->application->si_head; si != NULL; si = si->next) {
  696. if(name_match(&sg->name, &si->saAmfSIProtectedbySG)) {
  697. if (is_si_assigned_to_su (si, su)) {
  698. spare_su = 0;
  699. break;
  700. }
  701. }
  702. }
  703. return (spare_su && su->saAmfSUPresenceState ==
  704. SA_AMF_PRESENCE_INSTANTIATED);
  705. }
  706. /**
  707. * Inqure if it is any spare SUs covered by SG
  708. * @param sg
  709. *
  710. * @return int
  711. */
  712. static int is_spare_sus (amf_sg_t *sg)
  713. {
  714. amf_su_t *su = NULL;
  715. int spare_sus = 0;
  716. for (su = sg->su_head; su != NULL; su = su->next) {
  717. if (is_spare_su(sg, su)) {
  718. spare_sus = 1;
  719. break;
  720. }
  721. }
  722. return spare_sus;
  723. }
  724. /**
  725. * Provide standby assignments for the spare SUs in SG
  726. * @param sg
  727. */
  728. static void assume_standby_si_assignment_for_spare_sus (amf_sg_t *sg)
  729. {
  730. ENTER("");
  731. assign_si (sg, 0);
  732. }
  733. /**
  734. * Enter the AssigningStandbyToSpare state.
  735. * @param sg
  736. */
  737. static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg)
  738. {
  739. ENTER("%s", sg->name.value);
  740. sg->avail_state = SG_AC_AssigningStandbyToSpare;
  741. if (is_spare_sus (sg)) {
  742. assume_standby_si_assignment_for_spare_sus (sg);
  743. } else {
  744. acsm_enter_repairing_su (sg);
  745. }
  746. }
  747. /**
  748. * Checks if the si pointed out is already in the scope.
  749. * @param sg
  750. * @param si
  751. */
  752. static int is_si_in_scope(struct amf_sg *sg, struct amf_si *si)
  753. {
  754. struct amf_si **tmp_sis= sg->recovery_scope.sis;
  755. while (*tmp_sis != NULL) {
  756. if (*tmp_sis == si) {
  757. break;
  758. }
  759. tmp_sis++;
  760. }
  761. return(*tmp_sis == si);
  762. }
  763. /**
  764. * Adds the si pointed out to the scope.
  765. * @param sg
  766. * @param si
  767. */
  768. static void add_si_to_scope ( struct amf_sg *sg, struct amf_si *si)
  769. {
  770. int number_of_si = 2; /* It shall be at least two */
  771. struct amf_si **tmp_sis= sg->recovery_scope.sis;
  772. ENTER ("'%s'", si->name.value);
  773. while (*tmp_sis != NULL) {
  774. number_of_si++;
  775. tmp_sis++;
  776. }
  777. sg->recovery_scope.sis = (struct amf_si **)
  778. realloc((void *)sg->recovery_scope.sis,
  779. sizeof (struct amf_si *)*number_of_si);
  780. assert (sg->recovery_scope.sis != NULL);
  781. tmp_sis= sg->recovery_scope.sis;
  782. while (*tmp_sis != NULL) {
  783. tmp_sis++;
  784. }
  785. *tmp_sis = si;
  786. *(++tmp_sis) = NULL;
  787. }
  788. /**
  789. * Adds the ssu pointed out to the scope.
  790. * @param sg
  791. * @param su
  792. */
  793. static void add_su_to_scope (struct amf_sg *sg, struct amf_su *su)
  794. {
  795. int number_of_su = 2; /* It shall be at least two */
  796. struct amf_su **tmp_sus= sg->recovery_scope.sus;
  797. ENTER ("'%s'", su->name.value);
  798. while (*tmp_sus != NULL) {
  799. number_of_su++;
  800. tmp_sus++;
  801. }
  802. sg->recovery_scope.sus = (struct amf_su **)
  803. realloc((void *)sg->recovery_scope.sus,
  804. sizeof (struct amf_su *)*number_of_su);
  805. assert (sg->recovery_scope.sus != NULL);
  806. tmp_sus= sg->recovery_scope.sus;
  807. while (*tmp_sus != NULL) {
  808. tmp_sus++;
  809. }
  810. *tmp_sus = su;
  811. *(++tmp_sus) = NULL;
  812. }
  813. /**
  814. * Set recovery scope for failover SU.
  815. * @param sg
  816. * @param su
  817. */
  818. static void set_scope_for_failover_su (struct amf_sg *sg, struct amf_su *su)
  819. {
  820. struct amf_si_assignment *si_assignment;
  821. struct amf_si **sis;
  822. struct amf_su **sus;
  823. SaNameT dn;
  824. sg->recovery_scope.event_type = SG_FAILOVER_SU_EV;
  825. sg->recovery_scope.node = NULL;
  826. sg->recovery_scope.comp = NULL;
  827. sg->recovery_scope.sus = (struct amf_su **)
  828. calloc (2, sizeof (struct amf_su *));
  829. sg->recovery_scope.sis = (struct amf_si **)
  830. calloc (1, sizeof (struct amf_si *));
  831. assert ((sg->recovery_scope.sus != NULL) &&
  832. (sg->recovery_scope.sis != NULL));
  833. sg->recovery_scope.sus[0] = su;
  834. amf_su_dn_make (sg->recovery_scope.sus[0], &dn);
  835. log_printf (
  836. LOG_NOTICE, "'%s' for %s recovery action started",
  837. sg_event_type_text[sg->recovery_scope.event_type],
  838. dn.value);
  839. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  840. while (si_assignment != NULL) {
  841. if (is_si_in_scope(sg, si_assignment->si) == 0) {
  842. add_si_to_scope(sg,si_assignment->si );
  843. }
  844. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  845. }
  846. sus = sg->recovery_scope.sus;
  847. dprintf("The following sus are within the scope:\n");
  848. while (*sus != NULL) {
  849. dprintf("%s\n", (*sus)->name.value);
  850. sus++;
  851. }
  852. sis= sg->recovery_scope.sis;
  853. dprintf("The following sis are within the scope:\n");
  854. while (*sis != NULL) {
  855. dprintf("%s\n", (*sis)->name.value);
  856. sis++;
  857. }
  858. }
  859. static void set_scope_for_failover_node (struct amf_sg *sg, struct amf_node *node)
  860. {
  861. struct amf_si_assignment *si_assignment;
  862. struct amf_si **sis;
  863. struct amf_su **sus;
  864. struct amf_su *su;
  865. ENTER ("'%s'", node->name.value);
  866. sg->recovery_scope.event_type = SG_FAILOVER_NODE_EV;
  867. sg->recovery_scope.node = node;
  868. sg->recovery_scope.comp = NULL;
  869. sg->recovery_scope.sus = (struct amf_su **)
  870. calloc (1, sizeof (struct amf_su *));
  871. sg->recovery_scope.sis = (struct amf_si **)
  872. calloc (1, sizeof (struct amf_si *));
  873. log_printf (
  874. LOG_NOTICE, "'%s' for node %s recovery action started",
  875. sg_event_type_text[sg->recovery_scope.event_type],
  876. node->name.value);
  877. assert ((sg->recovery_scope.sus != NULL) &&
  878. (sg->recovery_scope.sis != NULL));
  879. for (su = sg->su_head; su != NULL; su = su->next) {
  880. if (name_match (&node->name, &su->saAmfSUHostedByNode)) {
  881. add_su_to_scope (sg, su);
  882. }
  883. }
  884. sus = sg->recovery_scope.sus;
  885. while (*sus != 0) {
  886. su = *sus;
  887. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  888. while (si_assignment != NULL) {
  889. if (is_si_in_scope(sg, si_assignment->si) == 0) {
  890. add_si_to_scope(sg, si_assignment->si );
  891. }
  892. si_assignment = amf_su_get_next_si_assignment (
  893. su, si_assignment);
  894. }
  895. sus++;
  896. }
  897. sus = sg->recovery_scope.sus;
  898. dprintf("The following sus are within the scope:\n");
  899. while (*sus != NULL) {
  900. dprintf("%s\n", (*sus)->name.value);
  901. sus++;
  902. }
  903. sis = sg->recovery_scope.sis;
  904. dprintf("The following sis are within the scope:\n");
  905. while (*sis != NULL) {
  906. dprintf("%s\n", (*sis)->name.value);
  907. sis++;
  908. }
  909. }
  910. static void delete_si_assignment (amf_si_assignment_t *si_assignment)
  911. {
  912. amf_csi_t *csi;
  913. amf_si_assignment_t *si_assignment_tmp;
  914. amf_si_assignment_t **prev = &si_assignment->si->assigned_sis;
  915. for (csi = si_assignment->si->csi_head; csi != NULL; csi = csi->next) {
  916. amf_csi_delete_assignments (csi, si_assignment->su);
  917. }
  918. for (si_assignment_tmp = si_assignment->si->assigned_sis;
  919. si_assignment_tmp != NULL;
  920. si_assignment_tmp = si_assignment_tmp->next) {
  921. if (si_assignment_tmp == si_assignment) {
  922. amf_si_assignment_t *to_be_removed = si_assignment_tmp;
  923. *prev = si_assignment_tmp->next;
  924. dprintf ("SI assignment %s unlinked",
  925. to_be_removed->name.value);
  926. free (to_be_removed);
  927. } else {
  928. prev = &si_assignment_tmp->next;
  929. }
  930. }
  931. }
  932. /**
  933. * Delete all SI assignments and all CSI assignments
  934. * by requesting all contained components.
  935. * @param su
  936. */
  937. static void delete_si_assignments (struct amf_su *su)
  938. {
  939. struct amf_csi *csi;
  940. struct amf_si *si;
  941. struct amf_si_assignment *si_assignment;
  942. struct amf_si_assignment **prev;
  943. ENTER ("'%s'", su->name.value);
  944. for (si = su->sg->application->si_head; si != NULL; si = si->next) {
  945. prev = &si->assigned_sis;
  946. if (!name_match (&si->saAmfSIProtectedbySG, &su->sg->name)) {
  947. continue;
  948. }
  949. for (csi = si->csi_head; csi != NULL; csi = csi->next) {
  950. amf_csi_delete_assignments (csi, su);
  951. }
  952. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  953. si_assignment = si_assignment->next) {
  954. if (si_assignment->su == su) {
  955. struct amf_si_assignment *tmp = si_assignment;
  956. *prev = si_assignment->next;
  957. dprintf ("SI assignment %s unlinked", tmp->name.value);
  958. free (tmp);
  959. } else {
  960. prev = &si_assignment->next;
  961. }
  962. }
  963. }
  964. }
  965. /**
  966. * Delete all SI assignments and all CSI assignments in current
  967. * recovery scope.
  968. * @param sg
  969. */
  970. static void delete_si_assignments_in_scope (struct amf_sg *sg)
  971. {
  972. struct amf_su **sus= sg->recovery_scope.sus;
  973. while (*sus != NULL) {
  974. delete_si_assignments (*sus);
  975. sus++;
  976. }
  977. }
  978. /**
  979. * Given an SI, find and return the SU assigned as standby
  980. * @param si
  981. *
  982. * @return amf_su_t*
  983. */
  984. static amf_su_t *find_standby_su (amf_si_t *si)
  985. {
  986. amf_si_assignment_t *si_assignment;
  987. amf_su_t *standby_su = NULL;
  988. si_assignment = si->assigned_sis;
  989. while (si_assignment != NULL) {
  990. if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
  991. standby_su = si_assignment->su;
  992. break;
  993. }
  994. si_assignment = si_assignment->next;
  995. }
  996. return standby_su;
  997. }
  998. static int no_si_assignment_is_requested_to_be_removed (amf_sg_t *sg)
  999. {
  1000. amf_si_t *si;
  1001. int no_to_be_removed = 1;
  1002. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1003. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1004. amf_si_assignment_t *si_assignment = 0;
  1005. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  1006. si_assignment = si_assignment->next) {
  1007. if (si_assignment->requested_ha_state ==
  1008. USR_AMF_HA_STATE_REMOVED) {
  1009. no_to_be_removed = 0;
  1010. goto out;
  1011. }
  1012. }
  1013. }
  1014. }
  1015. out:
  1016. return no_to_be_removed;
  1017. }
  1018. static void removed_si_assignment_callback_fn (void *si_assignment_in)
  1019. {
  1020. amf_si_assignment_t *si_assignment = si_assignment_in;
  1021. ENTER("");
  1022. delete_si_assignment (si_assignment);
  1023. /*
  1024. * if all si assignments are remove then change state
  1025. */
  1026. if (no_si_assignment_is_requested_to_be_removed (si_assignment->su->sg)) {
  1027. acsm_enter_activating_standby (si_assignment->su->sg);
  1028. }
  1029. }
  1030. /**
  1031. *
  1032. * @param sg
  1033. *
  1034. * @return int, number of removed SI assignments
  1035. */
  1036. static int remove_standby_si_assignments (amf_sg_t *sg)
  1037. {
  1038. struct amf_si **sis = sg->recovery_scope.sis;
  1039. struct amf_si_assignment *si_assignment;
  1040. amf_su_t *standby_su;
  1041. int removed = 0;
  1042. ENTER("'%s'", sg->name.value);
  1043. /*
  1044. * For each SI in the recovery scope, find a standby
  1045. * SU, then remove all 'standby SI assignment' not in
  1046. * the recovery scope.
  1047. */
  1048. while (*sis != NULL) {
  1049. standby_su = find_standby_su (*sis);
  1050. if (standby_su != NULL) {
  1051. si_assignment = amf_su_get_next_si_assignment (standby_su, NULL);
  1052. while (si_assignment != NULL) {
  1053. amf_si_t **sia;
  1054. int in_recovery_scope;
  1055. for (sia = sg->recovery_scope.sis, in_recovery_scope = 0;
  1056. *sia != NULL; sia++) {
  1057. if (name_match (&si_assignment->si->name, &(*sia)->name)) {
  1058. in_recovery_scope = 1;
  1059. }
  1060. }
  1061. /*
  1062. * The si_assignment found with standby hastate is not in the
  1063. * recovery scope. The found si_assignment will then be
  1064. * requested to be removed once.
  1065. */
  1066. if (!in_recovery_scope &&
  1067. si_assignment->requested_ha_state !=
  1068. USR_AMF_HA_STATE_REMOVED) {
  1069. amf_si_assignment_remove (si_assignment,
  1070. removed_si_assignment_callback_fn);
  1071. removed++;
  1072. }
  1073. si_assignment = amf_su_get_next_si_assignment (standby_su,
  1074. si_assignment);
  1075. }
  1076. }
  1077. sis++;
  1078. }
  1079. return removed;
  1080. }
  1081. /**
  1082. * Entry function for state 'removing standby assignments'
  1083. * @param sg
  1084. */
  1085. static void acsm_enter_removing_standby_assignments (amf_sg_t *sg)
  1086. {
  1087. ENTER("SG: %s", sg->name.value);
  1088. sg->avail_state = SG_AC_RemovingStandbyAssignments;
  1089. if (sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) {
  1090. if (!remove_standby_si_assignments (sg)) {
  1091. acsm_enter_activating_standby (sg);
  1092. }
  1093. }
  1094. }
  1095. /**
  1096. * Callback function used by SI when an SI has been deactivated.
  1097. * @param si_assignment
  1098. * @param result
  1099. */
  1100. static void dependent_si_deactivated_cbfn (
  1101. struct amf_si_assignment *si_assignment, int result)
  1102. {
  1103. struct amf_sg *sg = si_assignment->su->sg;
  1104. struct amf_su **sus = sg->recovery_scope.sus;
  1105. struct amf_su *su;
  1106. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  1107. /*
  1108. * If all SI assignments for all SUs in the SG are not pending,
  1109. * goto next state (TerminatingSuspected).
  1110. */
  1111. for (su = sg->su_head ; su != NULL; su = su->next) {
  1112. struct amf_si_assignment *si_assignment;
  1113. si_assignment = amf_su_get_next_si_assignment(su, NULL);
  1114. while (si_assignment != NULL) {
  1115. if (si_assignment->saAmfSISUHAState !=
  1116. si_assignment->requested_ha_state) {
  1117. goto still_wating;
  1118. }
  1119. si_assignment = amf_su_get_next_si_assignment(su,
  1120. si_assignment);
  1121. }
  1122. }
  1123. still_wating:
  1124. if (su == NULL) {
  1125. sus = si_assignment->su->sg->recovery_scope.sus;
  1126. /*
  1127. * Select next state depending on if some
  1128. * SU in the scope is needs to be terminated.
  1129. */
  1130. while (*sus != NULL) {
  1131. if (su_presense_state_is_not (*sus,
  1132. SA_AMF_PRESENCE_UNINSTANTIATED,
  1133. SA_AMF_PRESENCE_TERMINATION_FAILED,
  1134. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  1135. break;
  1136. }
  1137. sus++;
  1138. }
  1139. if (*sus != NULL) {
  1140. acsm_enter_terminating_suspected (sg);
  1141. } else {
  1142. acsm_enter_removing_standby_assignments (sg);
  1143. }
  1144. }
  1145. LEAVE("");
  1146. }
  1147. static void standby_su_activated_cbfn (
  1148. struct amf_si_assignment *si_assignment, int result)
  1149. {
  1150. struct amf_su **sus = si_assignment->su->sg->recovery_scope.sus;
  1151. struct amf_si **sis = si_assignment->su->sg->recovery_scope.sis;
  1152. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  1153. /*
  1154. * If all SI assignments for all SIs in the scope are activated, goto next
  1155. * state.
  1156. */
  1157. while (*sis != NULL) {
  1158. if ((*sis)->assigned_sis != NULL &&
  1159. (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) {
  1160. break;
  1161. }
  1162. sis++;
  1163. }
  1164. if (*sis == NULL) {
  1165. acsm_enter_assigning_standby_to_spare ((*sus)->sg);
  1166. }
  1167. }
  1168. static inline int div_round (int a, int b)
  1169. {
  1170. int res;
  1171. assert (b != 0);
  1172. res = a / b;
  1173. if ((a % b) != 0)
  1174. res++;
  1175. return res;
  1176. }
  1177. static int no_su_has_presence_state (
  1178. struct amf_sg *sg, struct amf_node *node_to_start,
  1179. SaAmfPresenceStateT state)
  1180. {
  1181. struct amf_su *su;
  1182. int no_su_has_presence_state = 1;
  1183. for (su = sg->su_head; su != NULL; su = su->next) {
  1184. if (su->saAmfSUPresenceState == state) {
  1185. if (node_to_start == NULL) {
  1186. no_su_has_presence_state = 0;
  1187. break;
  1188. } else {
  1189. if (name_match(&node_to_start->name,
  1190. &su->saAmfSUHostedByNode)) {
  1191. no_su_has_presence_state = 0;
  1192. break;
  1193. }
  1194. }
  1195. }
  1196. }
  1197. return no_su_has_presence_state;
  1198. }
  1199. #if COMPILE_OUT
  1200. static int all_su_in_scope_has_presence_state (
  1201. struct amf_sg *sg, SaAmfPresenceStateT state)
  1202. {
  1203. struct amf_su **sus= sg->recovery_scope.sus;
  1204. while (*sus != NULL) {
  1205. if ((*sus)->saAmfSUPresenceState != state) {
  1206. break;
  1207. }
  1208. sus++;
  1209. }
  1210. return(*sus == NULL);
  1211. }
  1212. #endif
  1213. static int all_su_in_scope_has_either_two_presence_state (
  1214. amf_sg_t *sg,
  1215. SaAmfPresenceStateT state1,
  1216. SaAmfPresenceStateT state2)
  1217. {
  1218. struct amf_su **sus = sg->recovery_scope.sus;
  1219. while (*sus != NULL) {
  1220. if (!((*sus)->saAmfSUPresenceState == state1 ||
  1221. (*sus)->saAmfSUPresenceState == state2)) {
  1222. break;
  1223. }
  1224. sus++;
  1225. }
  1226. return (*sus == NULL);
  1227. }
  1228. static int all_su_in_scope_has_either_of_three_presence_state (amf_sg_t *sg,
  1229. SaAmfPresenceStateT state1, SaAmfPresenceStateT state2,
  1230. SaAmfPresenceStateT state3)
  1231. {
  1232. struct amf_su **sus = sg->recovery_scope.sus;
  1233. while (*sus != NULL) {
  1234. if (!((*sus)->saAmfSUPresenceState == state1 ||
  1235. (*sus)->saAmfSUPresenceState == state2 ||
  1236. (*sus)->saAmfSUPresenceState == state3)) {
  1237. break;
  1238. }
  1239. sus++;
  1240. }
  1241. return (*sus == NULL);
  1242. }
  1243. /**
  1244. * Get number of SIs protected by the specified SG.
  1245. * @param sg
  1246. *
  1247. * @return int
  1248. */
  1249. static int sg_si_count_get (struct amf_sg *sg)
  1250. {
  1251. struct amf_si *si;
  1252. int cnt = 0;
  1253. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1254. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1255. cnt += 1;
  1256. }
  1257. }
  1258. return(cnt);
  1259. }
  1260. static int amf_si_get_saAmfSINumReqActiveAssignments(struct amf_si *si)
  1261. {
  1262. struct amf_si_assignment *si_assignment = si->assigned_sis;
  1263. int number_of_req_active_assignments = 0;
  1264. for (; si_assignment != NULL; si_assignment = si_assignment->next) {
  1265. if (si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) {
  1266. number_of_req_active_assignments++;
  1267. }
  1268. }
  1269. return number_of_req_active_assignments;
  1270. }
  1271. static int amf_si_get_saAmfSINumReqStandbyAssignments(struct amf_si *si)
  1272. {
  1273. struct amf_si_assignment *si_assignment = si->assigned_sis;
  1274. int number_of_req_active_assignments = 0;
  1275. for (; si_assignment != NULL; si_assignment = si_assignment->next) {
  1276. if (si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) {
  1277. number_of_req_active_assignments++;
  1278. }
  1279. }
  1280. return number_of_req_active_assignments;
  1281. }
  1282. static int sg_assign_nm_active (struct amf_sg *sg, int su_active_assign)
  1283. {
  1284. struct amf_su *su;
  1285. struct amf_si *si;
  1286. int assigned = 0;
  1287. int assign_to_su = 0;
  1288. int total_assigned = 0;
  1289. int si_left;
  1290. int si_total;
  1291. int su_left_to_assign = su_active_assign;
  1292. ENTER("SG: %s", sg->name.value);
  1293. si_total = sg_si_count_get (sg);
  1294. si_left = si_total;
  1295. assign_to_su = div_round (si_left, su_active_assign);
  1296. if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) {
  1297. assign_to_su = sg->saAmfSGMaxActiveSIsperSUs;
  1298. }
  1299. su = sg->su_head;
  1300. while (su != NULL && su_left_to_assign > 0) {
  1301. if (amf_su_get_saAmfSUReadinessState (su) !=
  1302. SA_AMF_READINESS_IN_SERVICE ||
  1303. amf_su_get_saAmfSUNumCurrActiveSIs (su) ==
  1304. assign_to_su ||
  1305. amf_su_get_saAmfSUNumCurrStandbySIs (su) > 0) {
  1306. su = su->next;
  1307. continue; /* Not in service */
  1308. }
  1309. si = sg->application->si_head;
  1310. assigned = 0;
  1311. assign_to_su = div_round (si_left, su_left_to_assign);
  1312. if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) {
  1313. assign_to_su = sg->saAmfSGMaxActiveSIsperSUs;
  1314. }
  1315. while (si != NULL) {
  1316. if (name_match (&si->saAmfSIProtectedbySG, &sg->name) &&
  1317. assigned < assign_to_su &&
  1318. amf_si_get_saAmfSINumReqActiveAssignments(si) == 0) {
  1319. assigned += 1;
  1320. total_assigned += 1;
  1321. amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE);
  1322. }
  1323. si = si->next;
  1324. }
  1325. su = su->next;
  1326. su_left_to_assign -= 1;
  1327. si_left -= assigned;
  1328. dprintf (" su_left_to_assign =%d, si_left=%d\n",
  1329. su_left_to_assign, si_left);
  1330. }
  1331. assert (total_assigned <= si_total);
  1332. if (total_assigned == 0) {
  1333. dprintf ("Info: No SIs assigned");
  1334. }
  1335. return total_assigned;
  1336. }
  1337. static int sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign)
  1338. {
  1339. struct amf_su *su;
  1340. struct amf_si *si;
  1341. int assigned = 0;
  1342. int assign_to_su = 0;
  1343. int total_assigned = 0;
  1344. int si_left;
  1345. int si_total;
  1346. int su_left_to_assign = su_standby_assign;
  1347. ENTER ("'%s'", sg->name.value);
  1348. if (su_standby_assign == 0) {
  1349. return 0;
  1350. }
  1351. si_total = sg_si_count_get (sg);
  1352. si_left = si_total;
  1353. assign_to_su = div_round (si_left, su_standby_assign);
  1354. if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) {
  1355. assign_to_su = sg->saAmfSGMaxStandbySIsperSUs;
  1356. }
  1357. su = sg->su_head;
  1358. while (su != NULL && su_left_to_assign > 0) {
  1359. if (amf_su_get_saAmfSUReadinessState (su) !=
  1360. SA_AMF_READINESS_IN_SERVICE ||
  1361. amf_su_get_saAmfSUNumCurrActiveSIs (su) > 0 ||
  1362. amf_su_get_saAmfSUNumCurrStandbySIs (su) ==
  1363. assign_to_su) {
  1364. su = su->next;
  1365. continue; /* Not available for assignment */
  1366. }
  1367. si = sg->application->si_head;
  1368. assigned = 0;
  1369. assign_to_su = div_round (si_left, su_left_to_assign);
  1370. if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) {
  1371. assign_to_su = sg->saAmfSGMaxStandbySIsperSUs;
  1372. }
  1373. while (si != NULL) {
  1374. if (name_match (&si->saAmfSIProtectedbySG, &sg->name) &&
  1375. assigned < assign_to_su &&
  1376. amf_si_get_saAmfSINumReqStandbyAssignments (si) == 0) {
  1377. assigned += 1;
  1378. total_assigned += 1;
  1379. amf_su_assign_si (su, si, SA_AMF_HA_STANDBY);
  1380. }
  1381. si = si->next;
  1382. }
  1383. su_left_to_assign -= 1;
  1384. si_left -= assigned;
  1385. dprintf (" su_left_to_assign =%d, si_left=%d\n",
  1386. su_left_to_assign, si_left);
  1387. su = su->next;
  1388. }
  1389. assert (total_assigned <= si_total);
  1390. if (total_assigned == 0) {
  1391. dprintf ("Info: No SIs assigned!");
  1392. }
  1393. return total_assigned;
  1394. }
  1395. static int su_inservice_count_get (struct amf_sg *sg)
  1396. {
  1397. struct amf_su *su;
  1398. int answer = 0;
  1399. for (su = sg->su_head; su != NULL; su = su->next) {
  1400. if (amf_su_get_saAmfSUReadinessState (su) ==
  1401. SA_AMF_READINESS_IN_SERVICE) {
  1402. answer += 1;
  1403. }
  1404. }
  1405. return(answer);
  1406. }
  1407. /**
  1408. * TODO: dependency_level not used, hard coded
  1409. * @param sg
  1410. * @param dependency_level
  1411. */
  1412. static int assign_si (struct amf_sg *sg, int dependency_level)
  1413. {
  1414. int active_sus_needed = 0;
  1415. int standby_sus_needed = 0;
  1416. int inservice_count;
  1417. int su_active_assign;
  1418. int su_standby_assign;
  1419. int su_spare_assign;
  1420. int assigned = 0;
  1421. ENTER ("'%s'", sg->name.value);
  1422. /**
  1423. * Phase 1: Calculate assignments and create all runtime objects in
  1424. * information model. Do not do the actual assignment, done in
  1425. * phase 2.
  1426. */
  1427. /**
  1428. * Calculate number of SUs to assign to active or standby state
  1429. */
  1430. inservice_count = su_inservice_count_get (sg);
  1431. if (sg->saAmfSGNumPrefActiveSUs > 0) {
  1432. active_sus_needed = div_round (
  1433. sg_si_count_get (sg),
  1434. sg->saAmfSGMaxActiveSIsperSUs);
  1435. } else {
  1436. log_printf (LOG_LEVEL_ERROR, "ERROR: saAmfSGNumPrefActiveSUs == 0 !!");
  1437. openais_exit_error (AIS_DONE_FATAL_ERR);
  1438. }
  1439. if (sg->saAmfSGNumPrefStandbySUs > 0) {
  1440. standby_sus_needed = div_round (
  1441. sg_si_count_get (sg),
  1442. sg->saAmfSGMaxStandbySIsperSUs);
  1443. } else {
  1444. log_printf (LOG_LEVEL_ERROR, "ERROR: saAmfSGNumPrefStandbySUs == 0 !!");
  1445. openais_exit_error (AIS_DONE_FATAL_ERR);
  1446. }
  1447. dprintf ("(inservice=%d) (active_sus_needed=%d) (standby_sus_needed=%d)"
  1448. "\n",
  1449. inservice_count, active_sus_needed, standby_sus_needed);
  1450. /* Determine number of active and standby service units
  1451. * to assign based upon reduction procedure
  1452. */
  1453. if ((inservice_count < active_sus_needed)) {
  1454. dprintf ("assignment VI - partial assignment with SIs drop outs\n");
  1455. su_active_assign = inservice_count;
  1456. su_standby_assign = 0;
  1457. su_spare_assign = 0;
  1458. } else
  1459. if ((inservice_count < active_sus_needed + standby_sus_needed)) {
  1460. dprintf ("assignment V - partial assignment with reduction of"
  1461. " standby units\n");
  1462. su_active_assign = active_sus_needed;
  1463. su_standby_assign = inservice_count - active_sus_needed;
  1464. su_spare_assign = 0;
  1465. } else
  1466. if ((inservice_count < sg->saAmfSGNumPrefActiveSUs + standby_sus_needed)) {
  1467. dprintf ("IV: full assignment with reduction of active service"
  1468. " units\n");
  1469. su_active_assign = inservice_count - standby_sus_needed;
  1470. su_standby_assign = standby_sus_needed;
  1471. su_spare_assign = 0;
  1472. } else
  1473. if ((inservice_count <
  1474. sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) {
  1475. dprintf ("III: full assignment with reduction of standby service"
  1476. " units\n");
  1477. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1478. su_standby_assign = inservice_count - sg->saAmfSGNumPrefActiveSUs;
  1479. su_spare_assign = 0;
  1480. } else
  1481. if ((inservice_count ==
  1482. sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) {
  1483. if (sg->saAmfSGNumPrefInserviceSUs > inservice_count) {
  1484. dprintf ("II: full assignment with spare reduction\n");
  1485. } else {
  1486. dprintf ("II: full assignment without spares\n");
  1487. }
  1488. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1489. su_standby_assign = sg->saAmfSGNumPrefStandbySUs;
  1490. su_spare_assign = 0;
  1491. } else {
  1492. dprintf ("I: full assignment with spares\n");
  1493. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1494. su_standby_assign = sg->saAmfSGNumPrefStandbySUs;
  1495. su_spare_assign = inservice_count -
  1496. sg->saAmfSGNumPrefActiveSUs - sg->saAmfSGNumPrefStandbySUs;
  1497. }
  1498. dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)"
  1499. " (assigning spares=%d)\n",
  1500. inservice_count, su_active_assign, su_standby_assign, su_spare_assign);
  1501. if (inservice_count > 0) {
  1502. assigned = sg_assign_nm_active (sg, su_active_assign);
  1503. assigned += sg_assign_nm_standby (sg, su_standby_assign);
  1504. sg->saAmfSGNumCurrAssignedSUs = inservice_count;
  1505. /**
  1506. * Phase 2: do the actual assignment to the component
  1507. * TODO: first do active, then standby
  1508. */
  1509. {
  1510. struct amf_si *si;
  1511. struct amf_si_assignment *si_assignment;
  1512. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1513. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1514. for (si_assignment = si->assigned_sis;
  1515. si_assignment != NULL;
  1516. si_assignment = si_assignment->next) {
  1517. if (si_assignment->requested_ha_state !=
  1518. si_assignment->saAmfSISUHAState) {
  1519. amf_si_ha_state_assume (
  1520. si_assignment, assign_si_assumed_cbfn);
  1521. }
  1522. }
  1523. }
  1524. }
  1525. }
  1526. }
  1527. LEAVE ("'%s'", sg->name.value);
  1528. return assigned;
  1529. }
  1530. static void remove_si_in_scope (amf_sg_t *sg, amf_si_t *si)
  1531. {
  1532. int i;
  1533. int j;
  1534. amf_si_t **sis = sg->recovery_scope.sis;
  1535. amf_si_t **new_sis = amf_calloc (1, sizeof (amf_si_t*));
  1536. for (i = 0,j = 0; sis[i] != NULL; i++) {
  1537. if (sis[i] == si) {
  1538. continue;
  1539. }
  1540. new_sis[j] = sis[i];
  1541. new_sis = amf_realloc (new_sis, j + sizeof (amf_si_t *));
  1542. j++;
  1543. }
  1544. sg->recovery_scope.sis = new_sis;
  1545. }
  1546. static void remove_sis_for_term_failed_su_from_scope (amf_sg_t *sg,
  1547. amf_su_t *su)
  1548. {
  1549. amf_comp_t *component;
  1550. /*
  1551. * foreach component with presense state termiantion failed in su
  1552. */
  1553. for (component = su->comp_head; component != NULL;
  1554. component = component->next) {
  1555. amf_csi_assignment_t *csi_assignment;
  1556. if (component->saAmfCompPresenceState !=
  1557. SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
  1558. continue;
  1559. }
  1560. csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
  1561. while (csi_assignment != NULL) {
  1562. remove_si_in_scope (sg, csi_assignment->csi->si);
  1563. csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
  1564. }
  1565. }
  1566. }
  1567. int amf_sg_assign_si_req (struct amf_sg *sg, int dependency_level)
  1568. {
  1569. int posible_to_assign_si;
  1570. sg->recovery_scope.event_type = SG_ASSIGN_SI_EV;
  1571. sg->avail_state = SG_AC_AssigningOnRequest;
  1572. if ((posible_to_assign_si = assign_si (sg, dependency_level)) == 0) {
  1573. acsm_enter_idle (sg);
  1574. }
  1575. return posible_to_assign_si;
  1576. }
  1577. void amf_sg_failover_node_req (struct amf_sg *sg, struct amf_node *node)
  1578. {
  1579. ENTER("'%s, %s'",node->name.value, sg->name.value);
  1580. sg_event_t sg_event;
  1581. switch (sg->avail_state) {
  1582. case SG_AC_Idle:
  1583. set_scope_for_failover_node(sg, node);
  1584. if (has_any_su_in_scope_active_workload (sg)) {
  1585. acsm_enter_deactivating_dependent_workload (sg);
  1586. } else {
  1587. amf_su_t **sus = sg->recovery_scope.sus;
  1588. /*
  1589. * Select next state depending on if some
  1590. * SU in the scope needs to be terminated.
  1591. */
  1592. while (*sus != NULL) {
  1593. amf_su_t *su = *sus;
  1594. ENTER("SU %s pr_state='%d'",su->name.value,
  1595. su->saAmfSUPresenceState);
  1596. if (su_presense_state_is_ored (su,
  1597. SA_AMF_PRESENCE_UNINSTANTIATED,
  1598. SA_AMF_PRESENCE_TERMINATION_FAILED,
  1599. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  1600. sus++;
  1601. continue;
  1602. }
  1603. break;
  1604. }
  1605. if (*sus != NULL) {
  1606. acsm_enter_terminating_suspected (sg);
  1607. } else {
  1608. delete_si_assignments_in_scope (sg);
  1609. acsm_enter_idle (sg);
  1610. }
  1611. }
  1612. break;
  1613. case SG_AC_DeactivatingDependantWorkload:
  1614. case SG_AC_TerminatingSuspected:
  1615. case SG_AC_ActivatingStandby:
  1616. case SG_AC_AssigningStandbyToSpare:
  1617. case SG_AC_ReparingComponent:
  1618. case SG_AC_ReparingSu:
  1619. case SG_AC_AssigningOnRequest:
  1620. case SG_AC_InstantiatingServiceUnits:
  1621. case SG_AC_RemovingAssignment:
  1622. case SG_AC_AssigningActiveworkload:
  1623. case SG_AC_AssigningAutoAdjust:
  1624. case SG_AC_AssigningStandBy:
  1625. case SG_AC_WaitingAfterOperationFailed:
  1626. case SG_AC_RemovingStandbyAssignments:
  1627. sg_set_event (SG_FAILOVER_NODE_EV, sg, 0, 0, node, &sg_event);
  1628. sg_defer_event (SG_FAILOVER_NODE_EV, &sg_event);
  1629. break;
  1630. default:
  1631. assert (0);
  1632. break;
  1633. }
  1634. }
  1635. int amf_sg_start (struct amf_sg *sg, struct amf_node *node)
  1636. {
  1637. sg->recovery_scope.event_type = SG_START_EV;
  1638. ENTER ("'%s'", sg->name.value);
  1639. int instantiated_sus = 0;
  1640. switch (sg->avail_state) {
  1641. case SG_AC_InstantiatingServiceUnits:
  1642. case SG_AC_Idle: {
  1643. amf_su_t *su;
  1644. sg_avail_control_state_t old_avail_state = sg->avail_state;
  1645. ENTER ("'%s'", sg->name.value);
  1646. sg->node_to_start = node;
  1647. sg->avail_state = SG_AC_InstantiatingServiceUnits;
  1648. for (su = sg->su_head;
  1649. (su != NULL) &&
  1650. (instantiated_sus < sg->saAmfSGNumPrefInserviceSUs);
  1651. su = su->next) {
  1652. if (is_cluster_start (node)) {
  1653. amf_su_instantiate (su);
  1654. instantiated_sus++;
  1655. } else { /*is_not_cluster_start*/
  1656. /*
  1657. * Node start, match if SU is hosted on the
  1658. * specified node
  1659. */
  1660. if (name_match (&node->name,
  1661. &su->saAmfSUHostedByNode)) {
  1662. amf_su_instantiate (su);
  1663. instantiated_sus++;
  1664. }
  1665. }
  1666. }
  1667. if (instantiated_sus == 0) {
  1668. sg->avail_state = old_avail_state;
  1669. }
  1670. break;
  1671. }
  1672. case SG_AC_DeactivatingDependantWorkload:
  1673. case SG_AC_TerminatingSuspected:
  1674. case SG_AC_ActivatingStandby:
  1675. case SG_AC_AssigningStandbyToSpare:
  1676. case SG_AC_ReparingComponent:
  1677. case SG_AC_ReparingSu:
  1678. case SG_AC_AssigningOnRequest:
  1679. case SG_AC_RemovingAssignment:
  1680. case SG_AC_AssigningActiveworkload:
  1681. case SG_AC_AssigningAutoAdjust:
  1682. case SG_AC_AssigningStandBy:
  1683. case SG_AC_WaitingAfterOperationFailed:
  1684. case SG_AC_RemovingStandbyAssignments:
  1685. default:
  1686. assert (0);
  1687. break;
  1688. }
  1689. return instantiated_sus;
  1690. }
  1691. void amf_sg_su_state_changed (struct amf_sg *sg,
  1692. struct amf_su *su, SaAmfStateT type, int state)
  1693. {
  1694. ENTER ("'%s' SU '%s' state %s",
  1695. sg->name.value, su->name.value, amf_presence_state(state));
  1696. if (type == SA_AMF_PRESENCE_STATE) {
  1697. if (state == SA_AMF_PRESENCE_INSTANTIATED) {
  1698. if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
  1699. if (no_su_has_presence_state(sg, sg->node_to_start,
  1700. SA_AMF_PRESENCE_INSTANTIATING)) {
  1701. acsm_enter_idle (sg);
  1702. }
  1703. } else if (sg->avail_state == SG_AC_ReparingSu) {
  1704. if (all_su_in_scope_has_either_of_three_presence_state(
  1705. su->sg,
  1706. SA_AMF_PRESENCE_INSTANTIATED,
  1707. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1708. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1709. su->sg->avail_state = SG_AC_AssigningStandBy;
  1710. if (assign_si (sg, 0) == 0) {
  1711. acsm_enter_idle (sg);
  1712. }
  1713. } else {
  1714. dprintf ("avail-state: %u", sg->avail_state);
  1715. assert (0);
  1716. }
  1717. } else {
  1718. dprintf ("avail-state: %u", sg->avail_state);
  1719. assert (0);
  1720. }
  1721. } else if (state == SA_AMF_PRESENCE_UNINSTANTIATED) {
  1722. if (sg->avail_state == SG_AC_TerminatingSuspected) {
  1723. if (all_su_in_scope_has_either_two_presence_state (sg,
  1724. SA_AMF_PRESENCE_UNINSTANTIATED,
  1725. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  1726. delete_si_assignments_in_scope (sg);
  1727. if (is_any_si_in_scope_assigned_standby (sg)) {
  1728. remove_all_suspected_sus (sg);
  1729. acsm_enter_removing_standby_assignments (sg);
  1730. } else { /*is_no_si_in_scope_assigned_standby*/
  1731. remove_all_suspected_sus (sg);
  1732. acsm_enter_assigning_standby_to_spare (sg);
  1733. }
  1734. }
  1735. } else if (sg->avail_state == SG_AC_ReparingSu) {
  1736. if (all_su_in_scope_has_either_of_three_presence_state(
  1737. su->sg,
  1738. SA_AMF_PRESENCE_INSTANTIATED,
  1739. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1740. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1741. su->sg->avail_state = SG_AC_AssigningStandBy;
  1742. if (assign_si (sg, 0) == 0) {
  1743. acsm_enter_idle (sg);
  1744. }
  1745. } else {
  1746. dprintf("%d",sg->avail_state);
  1747. assert (0);
  1748. }
  1749. }
  1750. } else if (state == SA_AMF_PRESENCE_TERMINATION_FAILED) {
  1751. if (all_su_in_scope_has_either_two_presence_state (sg,
  1752. SA_AMF_PRESENCE_UNINSTANTIATED,
  1753. SA_AMF_PRESENCE_TERMINATION_FAILED) &&
  1754. is_any_si_in_scope_assigned_standby (sg)) {
  1755. remove_all_suspected_sus (sg);
  1756. acsm_enter_removing_standby_assignments (sg);
  1757. } else if (all_su_in_scope_has_either_two_presence_state (sg,
  1758. SA_AMF_PRESENCE_UNINSTANTIATED,
  1759. SA_AMF_PRESENCE_TERMINATION_FAILED) &&
  1760. !is_any_si_in_scope_assigned_standby (sg)) {
  1761. remove_all_suspected_sus (sg);
  1762. acsm_enter_assigning_standby_to_spare (sg);
  1763. } else {
  1764. remove_sis_for_term_failed_su_from_scope (sg, su);
  1765. }
  1766. } else if (state == SA_AMF_PRESENCE_INSTANTIATING) {
  1767. ; /* nop */
  1768. } else if (state == SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
  1769. if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
  1770. if (no_su_has_presence_state(sg, sg->node_to_start,
  1771. SA_AMF_PRESENCE_INSTANTIATING)) {
  1772. acsm_enter_idle (sg);
  1773. }
  1774. } else if (sg->avail_state == SG_AC_ReparingSu) {
  1775. if (all_su_in_scope_has_either_of_three_presence_state(
  1776. su->sg,
  1777. SA_AMF_PRESENCE_INSTANTIATED,
  1778. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1779. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1780. su->sg->avail_state = SG_AC_AssigningStandBy;
  1781. if (assign_si (sg, 0) == 0) {
  1782. acsm_enter_idle (sg);
  1783. }
  1784. }
  1785. } else {
  1786. /* TODO: Insert the assert (0) until solving defers in SU */
  1787. dprintf("sg->avail_state = %d, su instantiation state = %d",
  1788. sg->avail_state, state);
  1789. }
  1790. } else {
  1791. dprintf("sg->avail_state = %d, su instantiation state = %d",
  1792. sg->avail_state, state);
  1793. assert (0);
  1794. }
  1795. }
  1796. }
  1797. void amf_sg_init (void)
  1798. {
  1799. log_init ("AMF");
  1800. }
  1801. void amf_sg_failover_su_req (struct amf_sg *sg, struct amf_su *su,
  1802. struct amf_node *node)
  1803. {
  1804. ENTER ("%s", su->name.value);
  1805. sg_event_t sg_event;
  1806. switch (sg->avail_state) {
  1807. case SG_AC_Idle:
  1808. set_scope_for_failover_su (sg, su);
  1809. if (has_any_su_in_scope_active_workload (sg)) {
  1810. acsm_enter_deactivating_dependent_workload (sg);
  1811. } else {
  1812. acsm_enter_terminating_suspected (sg);
  1813. }
  1814. break;
  1815. case SG_AC_DeactivatingDependantWorkload:
  1816. case SG_AC_TerminatingSuspected:
  1817. case SG_AC_ActivatingStandby:
  1818. case SG_AC_AssigningStandbyToSpare:
  1819. case SG_AC_ReparingComponent:
  1820. case SG_AC_ReparingSu:
  1821. case SG_AC_AssigningOnRequest:
  1822. case SG_AC_InstantiatingServiceUnits:
  1823. case SG_AC_RemovingAssignment:
  1824. case SG_AC_AssigningActiveworkload:
  1825. case SG_AC_AssigningAutoAdjust:
  1826. case SG_AC_AssigningStandBy:
  1827. case SG_AC_WaitingAfterOperationFailed:
  1828. case SG_AC_RemovingStandbyAssignments:
  1829. sg_set_event (SG_FAILOVER_SU_EV, sg, su, 0, 0, &sg_event);
  1830. sg_defer_event (SG_FAILOVER_SU_EV, &sg_event);
  1831. break;
  1832. default:
  1833. assert (0);
  1834. break;
  1835. }
  1836. }
  1837. /**
  1838. * Constructor for SG objects. Adds SG to the list owned by
  1839. * the specified application. Always returns a valid SG
  1840. * object, out-of-memory problems are handled here. Default
  1841. * values are initialized.
  1842. * @param sg
  1843. * @param name
  1844. *
  1845. * @return struct amf_sg*
  1846. */
  1847. struct amf_sg *amf_sg_new (struct amf_application *app, char *name)
  1848. {
  1849. struct amf_sg *sg = amf_calloc (1, sizeof (struct amf_sg));
  1850. setSaNameT (&sg->name, name);
  1851. sg->saAmfSGAdminState = SA_AMF_ADMIN_UNLOCKED;
  1852. sg->saAmfSGNumPrefActiveSUs = 1;
  1853. sg->saAmfSGNumPrefStandbySUs = 1;
  1854. sg->saAmfSGNumPrefInserviceSUs = ~0;
  1855. sg->saAmfSGNumPrefAssignedSUs = ~0;
  1856. sg->saAmfSGCompRestartProb = -1;
  1857. sg->saAmfSGCompRestartMax = ~0;
  1858. sg->saAmfSGSuRestartProb = -1;
  1859. sg->saAmfSGSuRestartMax = ~0;
  1860. sg->saAmfSGAutoAdjustProb = -1;
  1861. sg->saAmfSGAutoRepair = SA_TRUE;
  1862. sg->application = app;
  1863. sg->next = app->sg_head;
  1864. app->sg_head = sg;
  1865. sg->deferred_events = NULL;
  1866. return sg;
  1867. }
  1868. void amf_sg_delete (struct amf_sg *sg)
  1869. {
  1870. struct amf_su *su;
  1871. for (su = sg->su_head; su != NULL;) {
  1872. struct amf_su *tmp = su;
  1873. su = su->next;
  1874. amf_su_delete (tmp);
  1875. }
  1876. free (sg);
  1877. }
  1878. void *amf_sg_serialize (struct amf_sg *sg, int *len)
  1879. {
  1880. char *buf = NULL;
  1881. int offset = 0, size = 0;
  1882. TRACE8 ("%s", sg->name.value);
  1883. buf = amf_serialize_SaNameT (buf, &size, &offset, &sg->name);
  1884. buf = amf_serialize_SaUint32T (buf, &size, &offset, sg->saAmfSGRedundancyModel);
  1885. buf = amf_serialize_SaUint32T (
  1886. buf, &size, &offset, sg->saAmfSGAutoAdjust);
  1887. buf = amf_serialize_SaUint32T (
  1888. buf, &size, &offset, sg->saAmfSGNumPrefActiveSUs);
  1889. buf = amf_serialize_SaUint32T (
  1890. buf, &size, &offset, sg->saAmfSGNumPrefStandbySUs);
  1891. buf = amf_serialize_SaUint32T (
  1892. buf, &size, &offset, sg->saAmfSGNumPrefInserviceSUs);
  1893. buf = amf_serialize_SaUint32T (
  1894. buf, &size, &offset, sg->saAmfSGNumPrefAssignedSUs);
  1895. buf = amf_serialize_SaUint32T (
  1896. buf, &size, &offset, sg->saAmfSGMaxActiveSIsperSUs);
  1897. buf = amf_serialize_SaUint32T (
  1898. buf, &size, &offset, sg->saAmfSGMaxStandbySIsperSUs);
  1899. buf = amf_serialize_SaUint32T (
  1900. buf, &size, &offset, sg->saAmfSGCompRestartProb);
  1901. buf = amf_serialize_SaUint32T (
  1902. buf, &size, &offset, sg->saAmfSGCompRestartMax);
  1903. buf = amf_serialize_SaUint32T (
  1904. buf, &size, &offset, sg->saAmfSGSuRestartProb);
  1905. buf = amf_serialize_SaUint32T (
  1906. buf, &size, &offset, sg->saAmfSGSuRestartMax);
  1907. buf = amf_serialize_SaUint32T (
  1908. buf, &size, &offset, sg->saAmfSGAutoAdjustProb);
  1909. buf = amf_serialize_SaUint32T (
  1910. buf, &size, &offset, sg->saAmfSGAutoRepair);
  1911. buf = amf_serialize_SaUint32T (
  1912. buf, &size, &offset, sg->saAmfSGAdminState);
  1913. buf = amf_serialize_SaUint32T (
  1914. buf, &size, &offset, sg->saAmfSGNumCurrAssignedSUs);
  1915. buf = amf_serialize_SaUint32T (
  1916. buf, &size, &offset, sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
  1917. buf = amf_serialize_SaUint32T (
  1918. buf, &size, &offset, sg->saAmfSGNumCurrInstantiatedSpareSUs);
  1919. buf = amf_serialize_SaStringT (
  1920. buf, &size, &offset, sg->clccli_path);
  1921. buf = amf_serialize_SaUint32T (
  1922. buf, &size, &offset, sg->avail_state);
  1923. buf = amf_serialize_SaUint32T (
  1924. buf, &size, &offset, sg->recovery_scope.event_type);
  1925. *len = offset;
  1926. return buf;
  1927. }
  1928. struct amf_sg *amf_sg_deserialize (struct amf_application *app, char *buf)
  1929. {
  1930. char *tmp = buf;
  1931. struct amf_sg *sg = amf_sg_new (app, "");
  1932. tmp = amf_deserialize_SaNameT (tmp, &sg->name);
  1933. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGRedundancyModel);
  1934. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoAdjust);
  1935. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefActiveSUs);
  1936. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefStandbySUs);
  1937. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefInserviceSUs);
  1938. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefAssignedSUs);
  1939. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGMaxActiveSIsperSUs);
  1940. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGMaxStandbySIsperSUs);
  1941. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGCompRestartProb);
  1942. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGCompRestartMax);
  1943. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGSuRestartProb);
  1944. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGSuRestartMax);
  1945. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoAdjustProb);
  1946. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoRepair);
  1947. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAdminState);
  1948. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrAssignedSUs);
  1949. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
  1950. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrInstantiatedSpareSUs);
  1951. tmp = amf_deserialize_SaStringT (tmp, &sg->clccli_path);
  1952. tmp = amf_deserialize_SaUint32T (tmp, &sg->avail_state);
  1953. tmp = amf_deserialize_SaUint32T (tmp, &sg->recovery_scope.event_type);
  1954. return sg;
  1955. }
  1956. struct amf_sg *amf_sg_find (struct amf_application *app, char *name)
  1957. {
  1958. struct amf_sg *sg;
  1959. for (sg = app->sg_head; sg != NULL; sg = sg->next) {
  1960. if (sg->name.length == strlen(name) &&
  1961. strncmp (name, (char*)sg->name.value, sg->name.length) == 0) {
  1962. break;
  1963. }
  1964. }
  1965. return sg;
  1966. }