4
0

amfsg.c 82 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854
  1. /** @file amfsg.c
  2. *
  3. * Copyright (c) 2002-2006 MontaVista Software, Inc.
  4. * Author: Steven Dake (sdake@redhat.com)
  5. *
  6. * Copyright (c) 2006 Ericsson AB.
  7. * Author: Hans Feldt, Anders Eriksson, Lars Holm
  8. * - Introduced AMF B.02 information model
  9. * - Use DN in API and multicast messages
  10. * - (Re-)Introduction of event based multicast messages
  11. * - Refactoring of code into several AMF files
  12. * - Component/SU restart, SU failover
  13. * - Constructors/destructors
  14. * - Serializers/deserializers
  15. *
  16. * All rights reserved.
  17. *
  18. *
  19. * This software licensed under BSD license, the text of which follows:
  20. *
  21. * Redistribution and use in source and binary forms, with or without
  22. * modification, are permitted provided that the following conditions are met:
  23. *
  24. * - Redistributions of source code must retain the above copyright notice,
  25. * this list of conditions and the following disclaimer.
  26. * - Redistributions in binary form must reproduce the above copyright notice,
  27. * this list of conditions and the following disclaimer in the documentation
  28. * and/or other materials provided with the distribution.
  29. * - Neither the name of the MontaVista Software, Inc. nor the names of its
  30. * contributors may be used to endorse or promote products derived from this
  31. * software without specific prior written permission.
  32. *
  33. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  34. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  35. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  36. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  37. * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  38. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  39. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  40. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  41. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  42. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  43. * THE POSSIBILITY OF SUCH DAMAGE.
  44. *
  45. * AMF Service Group Class Implementation
  46. *
  47. * This file contains functions for handling AMF-service groups(SGs). It can be
  48. * viewed as the implementation of the AMF Service Group class (called SG)
  49. * as described in SAI-Overview-B.02.01. The SA Forum specification
  50. * SAI-AIS-AMF-B.02.01 has been used as specification of the behaviour
  51. * and is referred to as 'the spec' below.
  52. *
  53. * The functions in this file are responsible for:
  54. * -on request start the service group by instantiating the contained SUs
  55. * -on request assign the service instances it protects to the in-service
  56. * service units it contains respecting as many as possible of the configured
  57. * requirements for the group
  58. * -create and delete an SI-assignment object for each relation between
  59. * an SI and an SU
  60. * -order each contained SU to create and delete CSI-assignments
  61. * -request the Service Instance class (SI) to execute the transfer of the
  62. * HA-state set/remove requests to each component involved
  63. * -fully control the execution of component failover and SU failover
  64. * -on request control the execution of the initial steps of node switchover
  65. * and node failover
  66. * -fully handle the auto adjust procedure
  67. *
  68. * Currently only the 'n+m' redundancy model is implemented. It is the
  69. * ambition to identify n+m specific variables and functions and add the suffix
  70. * '_nplusm' to them so that they can be easily recognized.
  71. *
  72. * When SG is requested to assign workload to all SUs or all SUs hosted on
  73. * a specific node, a procedure containing several steps is executed:
  74. * <1> An algorithm is executed which assigns SIs to SUs respecting the rules
  75. * that has been configured for SG. The algorithm also has to consider
  76. * if assignments between som SIs and SUs already exist. The scope of this
  77. * algorithm is to create SI-assignments and set up requested HA-state for
  78. * each assignment but not to transfer those HA-states to the components.
  79. * <2> All SI-assignments with a requested HA state == ACTIVE are transferred
  80. * to the components concerned before any STANDBY assignments are
  81. * transferred. All components have to acknowledge the setting of the
  82. * ACTIVE HA state before the transfer of any STANDBY assignment is
  83. * initiated.
  84. * <3> All active assignments can not be transferred at the same time to the
  85. * different components because the rules for dependencies between SI and
  86. * SI cluster wide and CSI and CSI within one SI, has to be respected.
  87. *
  88. * SG is fully responsible for step <1> but not fully responsible for handling
  89. * step <2> and <3>. However, SG uses an attribute called 'dependency level'
  90. * when requsted to assign workload. This parameter refers to an integer that
  91. * has been calculated initially for each SI. The 'dependency level' indicates
  92. * to which extent an SI depends on other SIs such that an SI that depends on
  93. * no other SI is on dependecy_level == 1, an SI that depends only on an SI on
  94. * dependency_level == 1 is on dependency-level == 2.
  95. * An SI that depends on several SIs gets a
  96. * dependency_level that is one unit higher than the SI with the highest
  97. * dependency_level it depends on. When SG is requested to assign the workload
  98. * on a certain dependency level, it requests all SI objects on that level to
  99. * activate (all) SI-assignments that during step <1> has been requested to
  100. * assume the active HA state.
  101. *
  102. * SG contains the following state machines:
  103. * - administrative state machine (ADSM) (NOT IN THIS RELEASE)
  104. * - availability control state machine (ACSM)
  105. *
  106. * The availability control state machine contains three states and one of them
  107. * is composite. Being a composite state means that it contains substates.
  108. * The states are:
  109. * - IDLE (non composite state)
  110. * - INSTANTIATING_SERVICE_UNITS
  111. * - MANAGING_SG (composite state)
  112. * MANAGING_SG is entered at several different events which has in common
  113. * the need to set up or change the assignment of SIs to SUs. Only one such
  114. * event can be handled at the time. If new events occur while one event is
  115. * being handled then the new event is saved and will be handled after the
  116. * handling of the first event is ready (return to IDLE state has been done).
  117. * MANAGING_SG handles the following events:
  118. * - start (requests SG to order SU to instantiate all SUs in SG and waits
  119. * for SU to indicate presence state change reports from the SUs and
  120. * finally responds 'started' to the requester)
  121. * - assign_si (requests SG to assign SIs to SUs according to pre-configured
  122. * rules (if not already done) and transfer the HA state of
  123. * the SIs on the requested SI dependency level. Then SG waits for
  124. * confirmation that the HA state has been succesfully set and
  125. * finally responds 'assigned' to the reqeuster)
  126. * - auto_adjust (this event indicates that the auto-adjust probation timer has
  127. * expired and that SG should evaluate current assignments of
  128. * SIs to SUs and if needed remove current assignments and
  129. * create new according to what is specified in paragraph
  130. * 3.7.1.2)
  131. * - failover_comp (requests SG to failover a specific component according to
  132. * the procedure described in paragraph 3.12.1.3)
  133. * - failover_su (requests SG to failover a specific SU according to the
  134. * procedure described in paragraph 3.12.1.3 and 3.12.1.4)
  135. * - switchover_node (requests SG to execute the recovery actions described
  136. * in 3.12.1.3 and respond to the requester when recovery
  137. * is completed)
  138. * - failover_node (requests SG to execute the recovery actions described
  139. * in 3.12.1.3 and respond to the requester when recovery is
  140. * completed)
  141. *
  142. * 1. SG Availability Control State Machine
  143. * ==========================================
  144. *
  145. * 1.1 State Transition Table
  146. *
  147. * State: Event: Action: New state:
  148. * ============================================================================
  149. * IDLE start A48,A28 INSTANTIATING_SUs
  150. * IDLE assign_si A48,A31 ASSIGNING_ON_REQ
  151. * IDLE failover_su A48,[C22]A10,A11 DEACTIVATING_DEP
  152. * IDLE failover_su A48,[!C22]A12 TERMINATING_SUSP
  153. * IDLE failover_node A48,[!C22]A12 TERMINATING_SUSP
  154. * IDLE failover_node A48,[C22]A10,A11 DEACTIVATING_DEP
  155. * IDLE failover_node A48,[C100]A34 IDLE
  156. * INSTANTIATING_SUs start A48,A28 INSTANTIATING_SUs
  157. * INSTANTIATING_SUs su_state_chg [C101]A26,A53 IDLE
  158. * INSTANTIATING_SUs su_state_chg [C102]A26,A53 IDLE
  159. * INSTANTIATING_SUs assign_si A31 ASSIGNING_ON_REQ
  160. * INSTANTIATING_SUs failover_su A52 INSTANTIATING_SUs
  161. * INSTANTIATING_SUs failover_node A52 INSTANTIATING_SUs
  162. * ASSIGNING_ON_REQ ha_state_assumed [C15]A54 IDLE
  163. * ASSIGNING_ON_REQ failover_su A52 ASSIGNING_ON_REQ
  164. * ASSIGNING_ON_REQ failover_node A52 ASSIGNING_ON_REQ
  165. * DEACTIVATING_DEP si_deactivated [C20] REMOVING_S-BY_ASS
  166. * DEACTIVATING_DEP si_deactivated [!C20]A12 TERMINATING_SUSP
  167. * TERMINATING_SUSP su_state_chg [C103]A24,A20 ASS_S-BY_TO_SPARE
  168. * TERMINATING_SUSP su_state_chg [C104]A24,A50 REMOVING_S-BY_ASS
  169. * TERMINATING_SUSP su_state_chg [C105]A16,A17 ACTIVATING_S-BY
  170. * TERMINATING_SUSP su_state_chg [C106]A20 ASS_S-BY_TO_SPARE
  171. * TERMINATING_SUSP su_state_chg [C108]A23 REPAIRING_SU
  172. * TERMINATING_SUSP su_state_chg [C109] IDLE
  173. * TERMINATING_SUSP failover_su A52 TERMINATING_SUSP
  174. * TERMINATING_SUSP failover_node A52 TERMINATING_SUSP
  175. * REMOVING_S-BY_ASS assignment_removed A51 REMOVING_S-BY_ASS
  176. * REMOVING_S-BY_ASS assignment_removed [C27]&[C24] ACTIVATING_S-BY
  177. * REMOVING_S-BY_ASS assignment_removed [C110]A20 ASS_S-BY_TO_SPARE
  178. * REMOVING_S-BY_ASS assignment_removed [C111]A23 REPAIRING_SU
  179. * REMOVING_S-BY_ASS assignment_removed [C112] IDLE
  180. * REMOVING_S-BY_ASS failover_su A52 REMOVING_S-BY_ASS
  181. * REMOVING_S-BY_ASS failover_node A52 REMOVING_S-BY_ASS
  182. * ACTIVATING_S-BY su_activated [C2]&[C11]A20 ASS_S-BY_TO_SPARE
  183. * ACTIVATING_S-BY su_activated [C113]A23 REPAIRING_SU
  184. * ACTIVATING_S-BY su_activated [C114] IDLE
  185. * ACTIVATING_S-BY failover_su A52 ACTIVATING_S-BY
  186. * ACTIVATING_S-BY failover_node A52 ACTIVATING_S-BY
  187. * ASS_S-BY_TO_SPARE ha_state_assumed [C115]A23 REPAIRING_SU
  188. * ASS_S-BY_TO_SPARE ha_state_assumed [C116] IDLE
  189. * ASS_S-BY_TO_SPARE failover_su A52 ASS_S-BY_TO_SPARE
  190. * ASS_S-BY_TO_SPARE failover_node A52 ASS_S-BY_TO_SPARE
  191. * REPAIRING_SU su_state_chg [C28]A36,A37,A31 ASSIGNING_WL
  192. * REPAIRING_SU su_state_chg [C28][C31] IDLE
  193. * REPAIRING_SU failover_su A52 REPAIRING_SU
  194. * REPAIRING_SU failover_node A52 REPAIRING_SU
  195. * ASSIGNING_WL ha_state_assumed [C15] IDLE
  196. * ASSIGNING_WL failover_su A52 ASSIGNING_WL
  197. * ASSIGNING_WL failover_node A52 ASSIGNING_WL
  198. *
  199. * 1.2 State Description
  200. * =====================
  201. * IDLE - SG is synchronized and idle. When IDLE state is set,
  202. * the oldest deferred event (if any) is recalled.
  203. * INSTANTIATING_SUs - INSTANTIATING_SERVICE_UNITS
  204. * SG has ordered all contained SUs to instantiate and
  205. * waits for the SUs to report a change of their
  206. * presence state. SG is also prepared to accept an
  207. * order to assign workload in this state.
  208. * ASSIGNING_ON_REQ - ASSIGNING_ON_REQUEST
  209. * SG has on request assigned workload to all service units
  210. * on the requested dependency level and waits for SIs to
  211. * indicate that the requested HA-state have been set to the
  212. * appropriate components.
  213. * TERMINATING_SUSP - TERMINATING_SUSPECTED
  214. * SG has cleaned up all components suspected to be
  215. * erroneous and waits for the concerned SUs to report a
  216. * change of their presence states.
  217. * REMOVING_S-BY_ASS - REMOVING_STANDBY_ASSIGNMENTS
  218. * This state is only applicable to the n-plus-m redundancy
  219. * model. In this state, SG has initiated the removal of
  220. * those assignments from standby SUs that do not match the
  221. * assignments of error suspected SUs. The reason for this
  222. * removal is a preparation for not violating the rule which
  223. * says an SU can not have both active and standby assign-
  224. * ments simultaneously in the n-plus-m redundancy model.
  225. * ACTIVATING_S-BY - ACTIVATING_STANDBY
  226. * SG has located all standby SI-assignments in the recovery
  227. * scope and ordered the corresponding SI to set the active
  228. * HA-state and waits for SI to indicate that the requested
  229. * HA-state has been set to the appropriate components.
  230. * ASS_S-BY_TO_SPARE - ASSIGNING_STANDBY_TO_SPARE
  231. * Current SG is configured with a spare SU. In this state,
  232. * SG has requested all SIs to assign the standby HA-state
  233. * to the spare SU and waits for the SIs to indicate that
  234. * the standby HA-state have been set.
  235. * REPAIRING_SU - REPAIRING_SU
  236. * In this state SG has initiated instantiation of all SUs
  237. * in current recovery scope until the configured preference
  238. * of number of instantiated SUs is fulfiled. SG then waits
  239. * for the concerned SUs to report a change of presence
  240. * state.
  241. * ASSIGNING_WL - ASSIGNING_WORKLOAD
  242. * In this state SG has initiated the assignment of workload
  243. * to all or a subset of its contained SUs and waits for the
  244. * concerned SIs to indicated that the requested HA-state
  245. * has been assumed.
  246. *
  247. * 1.3 Actions
  248. * ===========
  249. * A1 -
  250. * A2 -
  251. * A3 -
  252. * A4 -
  253. * A5 -
  254. * A6 -
  255. * A7 -
  256. * A8 -
  257. * A9 -
  258. * A10 - [foreach SI in the scope]&[foreach SI-assignment with
  259. * confirmed-HA-state == ACTIVE]/set requested-ha-state = QUIESCED
  260. * A11 - [foreach SI in the scope]/deactivate SI
  261. * A12 - [foreach suspected SU]/terminate all components
  262. * A13 -
  263. * A14 -
  264. * A15 -
  265. * A16 - [foreach SI in the scope]&[foreach SI-assignment with
  266. * confirmed-HA-state == STANDBY]/set requested-ha-state = ACTIVE
  267. * A17 - [foreach SI in the scope]/activate SI
  268. * A18 -
  269. * A19 -
  270. * A20 -
  271. * A21 -
  272. * A22 -
  273. * A23 -
  274. * A24 -
  275. * A25 -
  276. * A26 -
  277. * A27 -
  278. * A28 -
  279. * A29 -
  280. * A30 -
  281. * A31 -
  282. * A32 -
  283. * A33 -
  284. * A34 -
  285. * A35 -
  286. * A36 -
  287. * A37 -
  288. * A38 -
  289. * A39 -
  290. * A40 -
  291. * A41 -
  292. * A42 -
  293. * A43 -
  294. * A44 -
  295. * A45 -
  296. * A46 -
  297. * A47 -
  298. * A48 -
  299. * A49 -
  300. *
  301. * 1.4 Composite Guards
  302. * ====================
  303. * The meaning with these guards is just to save space in the state transition
  304. * table above.
  305. * C100 - [C7]&[!C22]&[C20]
  306. * C101 - [C12]&[C28]
  307. * C102 - [C13]&[C28]
  308. * C103 - [C6]&[C21]&[C11]
  309. * C104 - [C6]&[C25]&[C26]
  310. * C105 - [C6]&(!([C25]|[C26]))&[C24]
  311. * C106 - [C6]&(!([C25]|[C26]))&[!C24]&[C11]
  312. * C107 -
  313. * C108 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[!C30]
  314. * C109 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[C30]
  315. * C110 - [C27]&[!C24]&[C11]
  316. * C111 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[!C30]
  317. * C112 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[C30]
  318. * C113 - [C2]&[!C11]&[C9]&[C10]&[!C30]
  319. * C114 - [C2]&[!C11]&[C9]&[C10]&[C30]
  320. * C115 - [C9]&[C10]&[!C30]
  321. * C116 - [C9]&[C10]&[C30]
  322. *
  323. * 1.4 Guards
  324. * ==========
  325. * C1 -
  326. * C2 - all SI in the recovery scope
  327. * C3 -
  328. * C4 -
  329. * C5 -
  330. * C6 - all supected SUs or components have presence state == UNINSTANTIATED
  331. * or presence state == INSTANTIATION_FAILED
  332. * C7 - original event == failover node
  333. * C8 -
  334. * C9 - original event == failover su
  335. * C10 - SaAmfSGAutoRepair == true
  336. * C11 - spare SUs exist
  337. * C12 - original event == start(all SUs)
  338. * C13 - original event == start(node)
  339. * C14 -
  340. * C15 - all SI-assignments on current dependency-level have requested-ha-state
  341. == confirmed-ha-state or operation failed
  342. * C16 -
  343. * C17 -
  344. * C18 -
  345. * C19 -
  346. * C20 - all suspected SUs have presence state == UNINSTANTIATED or
  347. presence state == xx_FAILED
  348. * C21 - no SI in the scope has an SI-assignment with HA-state == STANDBY
  349. * C22 - the concerned entity has ACTIVE HA-state
  350. * C23 -
  351. * C24 - at least one SI-assignment related to an SI in the scope has HA-state
  352. == STANDBY
  353. * C25 - redundancy model == N plus M
  354. * C26 - at least one SU has STANDBY assignments for more SIs than those SIs
  355. that are within the recovery scope
  356. * C27 - no SI-assignment related to SI protected by current SG has requested
  357. HA-state == REMOVED
  358. * C28 - no SU has presence state == INSTANTIATING
  359. * C29 -
  360. * C30 - more SUs not needed or the node hosting the SU to repair is disabled.
  361. * C31 - no new additional assignments needed or possible
  362. */
  363. #include <stdlib.h>
  364. #include <errno.h>
  365. #include <assert.h>
  366. #include "amf.h"
  367. #include "logsys.h"
  368. #include "main.h"
  369. #include "util.h"
  370. LOGSYS_DECLARE_SUBSYS ("AMF", LOG_INFO);
  371. static int assign_si (struct amf_sg *sg, int dependency_level);
  372. static void acsm_enter_activating_standby (struct amf_sg *sg);
  373. static void delete_si_assignments_in_scope (struct amf_sg *sg);
  374. static void acsm_enter_repairing_su (struct amf_sg *sg);
  375. static void standby_su_activated_cbfn (
  376. struct amf_si_assignment *si_assignment, int result);
  377. static void dependent_si_deactivated_cbfn (
  378. struct amf_si_assignment *si_assignment, int result);
  379. static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg);
  380. static void assign_si_assumed_cbfn (
  381. struct amf_si_assignment *si_assignment, int result);
  382. static void acsm_enter_removing_standby_assignments (amf_sg_t *sg);
  383. static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg);
  384. static const char *sg_event_type_text[] = {
  385. "Unknown",
  386. "Failover SU",
  387. "Failover node",
  388. "Failover comp",
  389. "Switchover node",
  390. "Start",
  391. "Autoadjust",
  392. "Assign SI"
  393. };
  394. typedef struct sg_event {
  395. amf_sg_event_type_t event_type;
  396. amf_sg_t *sg;
  397. amf_su_t *su;
  398. amf_comp_t *comp;
  399. amf_node_t *node;
  400. } sg_event_t;
  401. /******************************************************************************
  402. * Internal (static) utility functions
  403. *****************************************************************************/
  404. static int is_cluster_start(amf_node_t *node_to_start)
  405. {
  406. return node_to_start == NULL;
  407. }
  408. static void sg_set_event (amf_sg_event_type_t sg_event_type,
  409. amf_sg_t *sg, amf_su_t *su, amf_comp_t *comp, amf_node_t * node,
  410. sg_event_t *sg_event)
  411. {
  412. sg_event->event_type = sg_event_type;
  413. sg_event->node = node;
  414. sg_event->su = su;
  415. sg_event->comp = comp;
  416. sg_event->sg = sg;
  417. }
  418. static void sg_defer_event (amf_sg_event_type_t event_type,
  419. sg_event_t *sg_event)
  420. {
  421. ENTER("Defered event = %d", event_type);
  422. amf_fifo_put (event_type, &sg_event->sg->deferred_events,
  423. sizeof (sg_event_t),
  424. sg_event);
  425. }
  426. static void sg_recall_deferred_events (amf_sg_t *sg)
  427. {
  428. sg_event_t sg_event;
  429. ENTER ("%s", sg->name.value);
  430. if (amf_fifo_get (&sg->deferred_events, &sg_event)) {
  431. switch (sg_event.event_type) {
  432. case SG_FAILOVER_SU_EV:
  433. amf_sg_failover_su_req (sg_event.sg,
  434. sg_event.su, sg_event.node);
  435. break;
  436. case SG_FAILOVER_NODE_EV:
  437. amf_sg_failover_node_req (sg_event.sg,
  438. sg_event.node);
  439. break;
  440. case SG_FAILOVER_COMP_EV:
  441. case SG_SWITCH_OVER_NODE_EV:
  442. case SG_START_EV:
  443. case SG_AUTO_ADJUST_EV:
  444. default:
  445. dprintf("event_type = %d", sg_event.event_type);
  446. break;
  447. }
  448. }
  449. }
  450. static void timer_function_sg_recall_deferred_events (void *data)
  451. {
  452. amf_sg_t *sg = (amf_sg_t*)data;
  453. ENTER ("");
  454. sg_recall_deferred_events (sg);
  455. }
  456. static void acsm_enter_idle (amf_sg_t *sg)
  457. {
  458. SaNameT dn;
  459. ENTER ("sg: %s state: %d", sg->name.value, sg->avail_state);
  460. sg->avail_state = SG_AC_Idle;
  461. if (sg->recovery_scope.event_type != 0) {
  462. switch (sg->recovery_scope.event_type) {
  463. case SG_FAILOVER_SU_EV:
  464. assert (sg->recovery_scope.sus[0] != NULL);
  465. amf_su_dn_make (sg->recovery_scope.sus[0], &dn);
  466. log_printf (
  467. LOG_NOTICE,
  468. "'%s' %s recovery action finished",
  469. dn.value,
  470. sg_event_type_text[sg->recovery_scope.event_type]);
  471. break;
  472. case SG_FAILOVER_NODE_EV:
  473. amf_node_sg_failed_over (
  474. sg->recovery_scope.node, sg);
  475. log_printf (
  476. LOG_NOTICE,
  477. "'%s for %s' recovery action finished",
  478. sg_event_type_text[sg->recovery_scope.event_type],
  479. sg->name.value);
  480. break;
  481. case SG_START_EV:
  482. amf_application_sg_started (sg->application,
  483. sg, this_amf_node);
  484. break;
  485. case SG_ASSIGN_SI_EV:
  486. log_printf (LOG_NOTICE, "All SI assigned");
  487. break;
  488. default:
  489. log_printf (
  490. LOG_NOTICE,
  491. "'%s' recovery action finished",
  492. sg_event_type_text[sg->recovery_scope.event_type]);
  493. break;
  494. }
  495. }
  496. if (sg->recovery_scope.sus != NULL) {
  497. free ((void *)sg->recovery_scope.sus);
  498. }
  499. if (sg->recovery_scope.sis != NULL) {
  500. free ((void *)sg->recovery_scope.sis);
  501. }
  502. memset (&sg->recovery_scope, 0, sizeof (struct sg_recovery_scope));
  503. sg->node_to_start = NULL;
  504. amf_call_function_asynchronous (
  505. timer_function_sg_recall_deferred_events, sg);
  506. }
  507. static int su_instantiated_count (struct amf_sg *sg)
  508. {
  509. int cnt = 0;
  510. struct amf_su *su;
  511. for (su = sg->su_head; su != NULL; su = su->next) {
  512. if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED)
  513. cnt++;
  514. }
  515. return cnt;
  516. }
  517. static int has_any_su_in_scope_active_workload (struct amf_sg *sg)
  518. {
  519. struct amf_su **sus= sg->recovery_scope.sus;
  520. struct amf_si_assignment *si_assignment;
  521. while (*sus != NULL) {
  522. si_assignment = amf_su_get_next_si_assignment (*sus, NULL);
  523. while (si_assignment != NULL) {
  524. if (si_assignment->saAmfSISUHAState !=
  525. SA_AMF_HA_ACTIVE) {
  526. break;
  527. }
  528. si_assignment = amf_su_get_next_si_assignment (
  529. *sus, si_assignment);
  530. }
  531. if (si_assignment != NULL) {
  532. break;
  533. }
  534. sus++;
  535. }
  536. return(*sus == NULL);
  537. }
  538. static int is_any_si_in_scope_assigned_standby (struct amf_sg *sg)
  539. {
  540. struct amf_si **sis= sg->recovery_scope.sis;
  541. struct amf_si_assignment *si_assignment;
  542. /*
  543. * Check if there is any si in the scope which has no
  544. * active assignment and at least one standby assignment.
  545. */
  546. while (*sis != NULL) {
  547. si_assignment = (*sis)->assigned_sis;
  548. while (si_assignment != NULL) {
  549. if (si_assignment->saAmfSISUHAState ==
  550. SA_AMF_HA_ACTIVE) {
  551. break;
  552. }
  553. si_assignment = si_assignment->next;
  554. }
  555. if (si_assignment == NULL) {
  556. /* There is no ACTIVE assignment ..*/
  557. si_assignment = (*sis)->assigned_sis;
  558. while (si_assignment != NULL) {
  559. if (si_assignment->saAmfSISUHAState ==
  560. SA_AMF_HA_STANDBY) {
  561. break;
  562. }
  563. si_assignment = si_assignment->next;
  564. }
  565. if (si_assignment != NULL) {
  566. /* .. and one STANDBY assignment*/
  567. break;
  568. }
  569. }
  570. sis++;
  571. }
  572. return(*sis != NULL);
  573. }
  574. static void acsm_enter_terminating_suspected (struct amf_sg *sg)
  575. {
  576. struct amf_su **sus= sg->recovery_scope.sus;
  577. ENTER("%s",sg->name.value);
  578. sg->avail_state = SG_AC_TerminatingSuspected;
  579. /*
  580. * Terminate suspected SU(s)
  581. */
  582. while (*sus != 0) {
  583. amf_su_terminate (*sus);
  584. sus++;
  585. }
  586. }
  587. static inline int su_presense_state_is_ored (amf_su_t *su,
  588. SaAmfPresenceStateT state1,SaAmfPresenceStateT state2,
  589. SaAmfPresenceStateT state3)
  590. {
  591. return(su->saAmfSUPresenceState == state1 || su->saAmfSUPresenceState ==
  592. state2 || su->saAmfSUPresenceState == state3) ? 1 : 0;
  593. }
  594. static inline int su_presense_state_is_not (amf_su_t *su,
  595. SaAmfPresenceStateT state1,SaAmfPresenceStateT state2,
  596. SaAmfPresenceStateT state3)
  597. {
  598. return(su->saAmfSUPresenceState != state1 && su->saAmfSUPresenceState !=
  599. state2 && su->saAmfSUPresenceState != state3) ? 1 : 0;
  600. }
  601. static void timer_function_dependent_si_deactivated2 (void *data)
  602. {
  603. amf_sg_t *sg = (amf_sg_t *)data;
  604. ENTER ("");
  605. dependent_si_deactivated_cbfn2 (sg);
  606. }
  607. static struct amf_si *si_get_dependent (struct amf_si *si)
  608. {
  609. struct amf_si *tmp_si = NULL;
  610. if (si->depends_on != NULL) {
  611. SaNameT res_arr[2];
  612. int is_match;
  613. if (si->depends_on->name.length < SA_MAX_NAME_LENGTH) {
  614. si->depends_on->name.value[si->depends_on->name.length] = '\0';
  615. }
  616. is_match = sa_amf_grep ((char*)si->depends_on->name.value,
  617. "safDepend=.*,safSi=(.*),safApp=.*",
  618. 2, res_arr);
  619. if (is_match) {
  620. tmp_si = amf_si_find (si->application, (char*)res_arr[1].value);
  621. } else {
  622. log_printf (LOG_LEVEL_ERROR, "distinguished name for "
  623. "amf_si_depedency failed\n");
  624. openais_exit_error (AIS_DONE_FATAL_ERR);
  625. }
  626. }
  627. return tmp_si;
  628. }
  629. static struct amf_si *amf_dependent_get_next (struct amf_si *si,
  630. struct amf_si *si_iter)
  631. {
  632. struct amf_si *tmp_si;
  633. struct amf_application *application;
  634. if (si_iter == NULL) {
  635. assert(amf_cluster != NULL);
  636. application = amf_cluster->application_head;
  637. assert(application != NULL);
  638. tmp_si = application->si_head;
  639. } else {
  640. tmp_si = si_iter->next;
  641. if (tmp_si == NULL) {
  642. application = si->application->next;
  643. if (application == NULL) {
  644. goto out;
  645. }
  646. }
  647. }
  648. for (; tmp_si != NULL; tmp_si = tmp_si->next) {
  649. struct amf_si *depends_on_si = si_get_dependent (tmp_si);
  650. while (depends_on_si != NULL) {
  651. if (depends_on_si == si) {
  652. goto out;
  653. }
  654. depends_on_si = depends_on_si->next;
  655. }
  656. }
  657. out:
  658. return tmp_si;
  659. }
  660. static void acsm_enter_deactivating_dependent_workload (amf_sg_t *sg)
  661. {
  662. struct amf_si **sis= sg->recovery_scope.sis;
  663. struct amf_si_assignment *si_assignment;
  664. int callback_pending = 0;
  665. sg->avail_state = SG_AC_DeactivatingDependantWorkload;
  666. ENTER("'%s'",sg->name.value);
  667. /*
  668. * For each SI in the recovery scope, find all active
  669. * assignments and request them to be deactivated.
  670. */
  671. while (*sis != NULL) {
  672. struct amf_si *dependent_si;
  673. struct amf_si *si = *sis;
  674. si_assignment = si->assigned_sis;
  675. dependent_si = amf_dependent_get_next (si, NULL);
  676. while (dependent_si != NULL) {
  677. si_assignment = dependent_si->assigned_sis;
  678. while (si_assignment != NULL) {
  679. if (si_assignment->saAmfSISUHAState ==
  680. SA_AMF_HA_ACTIVE) {
  681. si_assignment->requested_ha_state =
  682. SA_AMF_HA_QUIESCED;
  683. callback_pending = 1;
  684. amf_si_ha_state_assume (
  685. si_assignment,
  686. dependent_si_deactivated_cbfn);
  687. }
  688. si_assignment = si_assignment->next;
  689. }
  690. dependent_si = amf_dependent_get_next (si, dependent_si);
  691. }
  692. sis++;
  693. }
  694. if (callback_pending == 0) {
  695. static poll_timer_handle dependent_si_deactivated_handle;
  696. ENTER("");
  697. poll_timer_add (aisexec_poll_handle, 0, sg,
  698. timer_function_dependent_si_deactivated2,
  699. &dependent_si_deactivated_handle);
  700. }
  701. }
  702. /**
  703. * Enter function for state SG_AC_ActivatingStandby. It activates
  704. * one STANDBY assignment for each SI in the recovery scope.
  705. * @param sg
  706. */
  707. static void acsm_enter_activating_standby (struct amf_sg *sg)
  708. {
  709. struct amf_si **sis= sg->recovery_scope.sis;
  710. struct amf_si_assignment *si_assignment;
  711. int is_no_standby_activated = 1;
  712. ENTER("'%s'",sg->name.value);
  713. sg->avail_state = SG_AC_ActivatingStandby;
  714. /*
  715. * For each SI in the recovery scope, find one standby
  716. * SI assignment and activate it.
  717. */
  718. while (*sis != NULL) {
  719. si_assignment = (*sis)->assigned_sis;
  720. while (si_assignment != NULL) {
  721. if (si_assignment->saAmfSISUHAState ==
  722. SA_AMF_HA_STANDBY) {
  723. si_assignment->requested_ha_state =
  724. SA_AMF_HA_ACTIVE;
  725. amf_si_ha_state_assume (
  726. si_assignment, standby_su_activated_cbfn);
  727. is_no_standby_activated = 0;
  728. break;
  729. }
  730. si_assignment = si_assignment->next;
  731. }
  732. sis++;
  733. }
  734. if (is_no_standby_activated) {
  735. acsm_enter_assigning_standby_to_spare (sg);
  736. }
  737. }
  738. static void acsm_enter_repairing_su (struct amf_sg *sg)
  739. {
  740. struct amf_su **sus= sg->recovery_scope.sus;
  741. int is_any_su_instantiated = 0;
  742. const int PERFORMS_INSTANTIATING = 1;
  743. ENTER("'%s'",sg->name.value);
  744. sg->avail_state = SG_AC_ReparingSu;
  745. /*
  746. * Instantiate SUs in current recovery scope until the configured
  747. * preference is fulfiled.
  748. */
  749. while (*sus != NULL) {
  750. if (su_instantiated_count ((*sus)->sg) <
  751. (*sus)->sg->saAmfSGNumPrefInserviceSUs) {
  752. struct amf_node *node =
  753. amf_node_find(&((*sus)->saAmfSUHostedByNode));
  754. if (node == NULL) {
  755. log_printf (LOG_LEVEL_ERROR,
  756. "Su to recover not hosted on any node\n");
  757. openais_exit_error (AIS_DONE_FATAL_ERR);
  758. }
  759. if (node->saAmfNodeOperState == SA_AMF_OPERATIONAL_ENABLED) {
  760. /* node is synchronized */
  761. if (amf_su_instantiate ((*sus)) == PERFORMS_INSTANTIATING) {
  762. is_any_su_instantiated = 1;
  763. }
  764. }
  765. }
  766. sus++;
  767. }
  768. if (is_any_su_instantiated == 0) {
  769. acsm_enter_idle (sg);
  770. }
  771. }
  772. static inline void remove_all_suspected_sus (amf_sg_t *sg)
  773. {
  774. amf_su_t *su;
  775. ENTER("");
  776. for (su = sg->su_head; su != NULL; su =su->next) {
  777. amf_comp_t *component;
  778. for (component = su->comp_head; component != NULL;
  779. component = component->next) {
  780. amf_comp_error_suspected_clear (component);
  781. }
  782. }
  783. }
  784. static int is_all_si_assigned (amf_sg_t *sg)
  785. {
  786. struct amf_si_assignment *si_assignment;
  787. int si_assignment_cnt = 0;
  788. int confirmed_assignments = 0;
  789. amf_si_t *si;
  790. for (si = sg->application->si_head; si != NULL; si = si->next) {
  791. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  792. for (si_assignment = si->assigned_sis;
  793. si_assignment != NULL;
  794. si_assignment = si_assignment->next) {
  795. si_assignment_cnt++;
  796. if (si_assignment->requested_ha_state ==
  797. si_assignment->saAmfSISUHAState) {
  798. confirmed_assignments++;
  799. }
  800. }
  801. }
  802. }
  803. return (confirmed_assignments == si_assignment_cnt);
  804. }
  805. /**
  806. * Inquire if SI is assigned to SU
  807. * @param si
  808. * @param su
  809. *
  810. * @return int
  811. */
  812. static int is_si_assigned_to_su (amf_si_t *si, amf_su_t *su)
  813. {
  814. amf_si_assignment_t *si_assignment = 0;
  815. int si_assignment_assigned_to_su = 0;
  816. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  817. si_assignment = si_assignment->next) {
  818. if (si_assignment->su == su) {
  819. si_assignment_assigned_to_su = 1;
  820. break;
  821. }
  822. }
  823. return si_assignment_assigned_to_su;
  824. }
  825. /**
  826. * Inquire if SU is a spare.
  827. * @param sg
  828. * @param su
  829. *
  830. * @return int
  831. */
  832. static int is_spare_su (amf_sg_t *sg, amf_su_t *su)
  833. {
  834. amf_si_t *si;
  835. int spare_su = 1;
  836. for (si = sg->application->si_head; si != NULL; si = si->next) {
  837. if(name_match(&sg->name, &si->saAmfSIProtectedbySG)) {
  838. if (is_si_assigned_to_su (si, su)) {
  839. spare_su = 0;
  840. break;
  841. }
  842. }
  843. }
  844. return (spare_su && su->saAmfSUPresenceState ==
  845. SA_AMF_PRESENCE_INSTANTIATED);
  846. }
  847. /**
  848. * Inqure if it is any spare SUs covered by SG
  849. * @param sg
  850. *
  851. * @return int
  852. */
  853. static int is_spare_sus (amf_sg_t *sg)
  854. {
  855. amf_su_t *su = NULL;
  856. int spare_sus = 0;
  857. for (su = sg->su_head; su != NULL; su = su->next) {
  858. if (is_spare_su(sg, su)) {
  859. spare_sus = 1;
  860. break;
  861. }
  862. }
  863. return spare_sus;
  864. }
  865. /**
  866. * Provide standby assignments for the spare SUs in SG
  867. * @param sg
  868. */
  869. static void assume_standby_si_assignment_for_spare_sus (amf_sg_t *sg)
  870. {
  871. ENTER("");
  872. assign_si (sg, 0);
  873. }
  874. /**
  875. * Enter the AssigningStandbyToSpare state.
  876. * @param sg
  877. */
  878. static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg)
  879. {
  880. ENTER("%s", sg->name.value);
  881. sg->avail_state = SG_AC_AssigningStandbyToSpare;
  882. if (is_spare_sus (sg)) {
  883. assume_standby_si_assignment_for_spare_sus (sg);
  884. } else {
  885. switch (sg->recovery_scope.event_type) {
  886. case SG_FAILOVER_NODE_EV:
  887. acsm_enter_idle (sg);
  888. break;
  889. case SG_FAILOVER_SU_EV:
  890. acsm_enter_repairing_su (sg);
  891. break;
  892. default:
  893. dprintf("event_type %d",sg->recovery_scope.event_type);
  894. assert (0);
  895. break;
  896. }
  897. }
  898. }
  899. /**
  900. * Checks if the si pointed out is already in the scope.
  901. * @param sg
  902. * @param si
  903. */
  904. static int is_si_in_scope(struct amf_sg *sg, struct amf_si *si)
  905. {
  906. struct amf_si **tmp_sis= sg->recovery_scope.sis;
  907. while (*tmp_sis != NULL) {
  908. if (*tmp_sis == si) {
  909. break;
  910. }
  911. tmp_sis++;
  912. }
  913. return(*tmp_sis == si);
  914. }
  915. /**
  916. * Adds the si pointed out to the scope.
  917. * @param sg
  918. * @param si
  919. */
  920. static void add_si_to_scope ( struct amf_sg *sg, struct amf_si *si)
  921. {
  922. int number_of_si = 2; /* It shall be at least two */
  923. struct amf_si **tmp_sis= sg->recovery_scope.sis;
  924. ENTER ("'%s'", si->name.value);
  925. while (*tmp_sis != NULL) {
  926. number_of_si++;
  927. tmp_sis++;
  928. }
  929. sg->recovery_scope.sis = (struct amf_si **)
  930. realloc((void *)sg->recovery_scope.sis,
  931. sizeof (struct amf_si *)*number_of_si);
  932. assert (sg->recovery_scope.sis != NULL);
  933. tmp_sis= sg->recovery_scope.sis;
  934. while (*tmp_sis != NULL) {
  935. tmp_sis++;
  936. }
  937. *tmp_sis = si;
  938. *(++tmp_sis) = NULL;
  939. }
  940. /**
  941. * Adds the ssu pointed out to the scope.
  942. * @param sg
  943. * @param su
  944. */
  945. static void add_su_to_scope (struct amf_sg *sg, struct amf_su *su)
  946. {
  947. int number_of_su = 2; /* It shall be at least two */
  948. struct amf_su **tmp_sus= sg->recovery_scope.sus;
  949. ENTER ("'%s'", su->name.value);
  950. while (*tmp_sus != NULL) {
  951. number_of_su++;
  952. tmp_sus++;
  953. }
  954. sg->recovery_scope.sus = (struct amf_su **)
  955. realloc((void *)sg->recovery_scope.sus,
  956. sizeof (struct amf_su *)*number_of_su);
  957. assert (sg->recovery_scope.sus != NULL);
  958. tmp_sus= sg->recovery_scope.sus;
  959. while (*tmp_sus != NULL) {
  960. tmp_sus++;
  961. }
  962. *tmp_sus = su;
  963. *(++tmp_sus) = NULL;
  964. }
  965. /**
  966. * Set recovery scope for failover SU.
  967. * @param sg
  968. * @param su
  969. */
  970. static void set_scope_for_failover_su (struct amf_sg *sg, struct amf_su *su)
  971. {
  972. struct amf_si_assignment *si_assignment;
  973. struct amf_si **sis;
  974. struct amf_su **sus;
  975. SaNameT dn;
  976. sg->recovery_scope.event_type = SG_FAILOVER_SU_EV;
  977. sg->recovery_scope.node = NULL;
  978. sg->recovery_scope.comp = NULL;
  979. sg->recovery_scope.sus = (struct amf_su **)
  980. calloc (2, sizeof (struct amf_su *));
  981. sg->recovery_scope.sis = (struct amf_si **)
  982. calloc (1, sizeof (struct amf_si *));
  983. assert ((sg->recovery_scope.sus != NULL) &&
  984. (sg->recovery_scope.sis != NULL));
  985. sg->recovery_scope.sus[0] = su;
  986. amf_su_dn_make (sg->recovery_scope.sus[0], &dn);
  987. log_printf (
  988. LOG_NOTICE, "'%s' for %s recovery action started",
  989. sg_event_type_text[sg->recovery_scope.event_type],
  990. dn.value);
  991. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  992. while (si_assignment != NULL) {
  993. if (is_si_in_scope(sg, si_assignment->si) == 0) {
  994. add_si_to_scope(sg,si_assignment->si );
  995. }
  996. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  997. }
  998. sus = sg->recovery_scope.sus;
  999. dprintf("The following sus are within the scope:\n");
  1000. while (*sus != NULL) {
  1001. dprintf("%s\n", (*sus)->name.value);
  1002. sus++;
  1003. }
  1004. sis= sg->recovery_scope.sis;
  1005. dprintf("The following sis are within the scope:\n");
  1006. while (*sis != NULL) {
  1007. dprintf("%s\n", (*sis)->name.value);
  1008. sis++;
  1009. }
  1010. }
  1011. static void set_scope_for_failover_node (struct amf_sg *sg, struct amf_node *node)
  1012. {
  1013. struct amf_si_assignment *si_assignment;
  1014. struct amf_si **sis;
  1015. struct amf_su **sus;
  1016. struct amf_su *su;
  1017. ENTER ("'%s'", node->name.value);
  1018. sg->recovery_scope.event_type = SG_FAILOVER_NODE_EV;
  1019. sg->recovery_scope.node = node;
  1020. sg->recovery_scope.comp = NULL;
  1021. sg->recovery_scope.sus = (struct amf_su **)
  1022. calloc (1, sizeof (struct amf_su *));
  1023. sg->recovery_scope.sis = (struct amf_si **)
  1024. calloc (1, sizeof (struct amf_si *));
  1025. log_printf (
  1026. LOG_NOTICE, "'%s' for node %s recovery action started",
  1027. sg_event_type_text[sg->recovery_scope.event_type],
  1028. node->name.value);
  1029. assert ((sg->recovery_scope.sus != NULL) &&
  1030. (sg->recovery_scope.sis != NULL));
  1031. for (su = sg->su_head; su != NULL; su = su->next) {
  1032. if (name_match (&node->name, &su->saAmfSUHostedByNode)) {
  1033. add_su_to_scope (sg, su);
  1034. }
  1035. }
  1036. sus = sg->recovery_scope.sus;
  1037. while (*sus != 0) {
  1038. su = *sus;
  1039. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  1040. while (si_assignment != NULL) {
  1041. if (is_si_in_scope(sg, si_assignment->si) == 0) {
  1042. add_si_to_scope(sg, si_assignment->si );
  1043. }
  1044. si_assignment = amf_su_get_next_si_assignment (
  1045. su, si_assignment);
  1046. }
  1047. sus++;
  1048. }
  1049. sus = sg->recovery_scope.sus;
  1050. dprintf("The following sus are within the scope:\n");
  1051. while (*sus != NULL) {
  1052. dprintf("%s\n", (*sus)->name.value);
  1053. sus++;
  1054. }
  1055. sis = sg->recovery_scope.sis;
  1056. dprintf("The following sis are within the scope:\n");
  1057. while (*sis != NULL) {
  1058. dprintf("%s\n", (*sis)->name.value);
  1059. sis++;
  1060. }
  1061. }
  1062. static void delete_si_assignment (amf_si_assignment_t *si_assignment)
  1063. {
  1064. amf_csi_t *csi;
  1065. amf_si_assignment_t *si_assignment_tmp;
  1066. amf_si_assignment_t **prev = &si_assignment->si->assigned_sis;
  1067. for (csi = si_assignment->si->csi_head; csi != NULL; csi = csi->next) {
  1068. amf_csi_delete_assignments (csi, si_assignment->su);
  1069. }
  1070. for (si_assignment_tmp = si_assignment->si->assigned_sis;
  1071. si_assignment_tmp != NULL;
  1072. si_assignment_tmp = si_assignment_tmp->next) {
  1073. if (si_assignment_tmp == si_assignment) {
  1074. amf_si_assignment_t *to_be_removed = si_assignment_tmp;
  1075. *prev = si_assignment_tmp->next;
  1076. dprintf ("SI assignment %s unlinked",
  1077. to_be_removed->name.value);
  1078. free (to_be_removed);
  1079. } else {
  1080. prev = &si_assignment_tmp->next;
  1081. }
  1082. }
  1083. }
  1084. /**
  1085. * Delete all SI assignments and all CSI assignments
  1086. * by requesting all contained components.
  1087. * @param su
  1088. */
  1089. static void delete_si_assignments (struct amf_su *su)
  1090. {
  1091. struct amf_csi *csi;
  1092. struct amf_si *si;
  1093. struct amf_si_assignment *si_assignment;
  1094. struct amf_si_assignment **prev;
  1095. ENTER ("'%s'", su->name.value);
  1096. for (si = su->sg->application->si_head; si != NULL; si = si->next) {
  1097. prev = &si->assigned_sis;
  1098. if (!name_match (&si->saAmfSIProtectedbySG, &su->sg->name)) {
  1099. continue;
  1100. }
  1101. for (csi = si->csi_head; csi != NULL; csi = csi->next) {
  1102. amf_csi_delete_assignments (csi, su);
  1103. }
  1104. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  1105. si_assignment = si_assignment->next) {
  1106. if (si_assignment->su == su) {
  1107. struct amf_si_assignment *tmp = si_assignment;
  1108. *prev = si_assignment->next;
  1109. dprintf ("SI assignment %s unlinked", tmp->name.value);
  1110. free (tmp);
  1111. } else {
  1112. prev = &si_assignment->next;
  1113. }
  1114. }
  1115. }
  1116. }
  1117. /**
  1118. * Delete all SI assignments and all CSI assignments in current
  1119. * recovery scope.
  1120. * @param sg
  1121. */
  1122. static void delete_si_assignments_in_scope (struct amf_sg *sg)
  1123. {
  1124. struct amf_su **sus= sg->recovery_scope.sus;
  1125. while (*sus != NULL) {
  1126. delete_si_assignments (*sus);
  1127. sus++;
  1128. }
  1129. }
  1130. /**
  1131. * Given an SI, find and return the SU assigned as standby
  1132. * @param si
  1133. *
  1134. * @return amf_su_t*
  1135. */
  1136. static amf_su_t *find_standby_su (amf_si_t *si)
  1137. {
  1138. amf_si_assignment_t *si_assignment;
  1139. amf_su_t *standby_su = NULL;
  1140. si_assignment = si->assigned_sis;
  1141. while (si_assignment != NULL) {
  1142. if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
  1143. standby_su = si_assignment->su;
  1144. break;
  1145. }
  1146. si_assignment = si_assignment->next;
  1147. }
  1148. return standby_su;
  1149. }
  1150. static int no_si_assignment_is_requested_to_be_removed (amf_sg_t *sg)
  1151. {
  1152. amf_si_t *si;
  1153. int no_to_be_removed = 1;
  1154. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1155. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1156. amf_si_assignment_t *si_assignment = 0;
  1157. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  1158. si_assignment = si_assignment->next) {
  1159. if (si_assignment->requested_ha_state ==
  1160. USR_AMF_HA_STATE_REMOVED) {
  1161. no_to_be_removed = 0;
  1162. goto out;
  1163. }
  1164. }
  1165. }
  1166. }
  1167. out:
  1168. return no_to_be_removed;
  1169. }
  1170. static void removed_si_assignment_callback_fn (void *si_assignment_in)
  1171. {
  1172. amf_si_assignment_t *si_assignment = si_assignment_in;
  1173. ENTER("");
  1174. delete_si_assignment (si_assignment);
  1175. /*
  1176. * if all si assignments are remove then change state
  1177. */
  1178. if (no_si_assignment_is_requested_to_be_removed (si_assignment->su->sg)) {
  1179. acsm_enter_activating_standby (si_assignment->su->sg);
  1180. }
  1181. }
  1182. /**
  1183. *
  1184. * @param sg
  1185. *
  1186. * @return int, number of removed SI assignments
  1187. */
  1188. static int remove_standby_si_assignments (amf_sg_t *sg)
  1189. {
  1190. struct amf_si **sis = sg->recovery_scope.sis;
  1191. struct amf_si_assignment *si_assignment;
  1192. amf_su_t *standby_su;
  1193. int removed = 0;
  1194. ENTER("'%s'", sg->name.value);
  1195. /*
  1196. * For each SI in the recovery scope, find a standby
  1197. * SU, then remove all 'standby SI assignment' not in
  1198. * the recovery scope.
  1199. */
  1200. while (*sis != NULL) {
  1201. standby_su = find_standby_su (*sis);
  1202. if (standby_su != NULL) {
  1203. si_assignment = amf_su_get_next_si_assignment (standby_su, NULL);
  1204. while (si_assignment != NULL) {
  1205. amf_si_t **sia;
  1206. int in_recovery_scope;
  1207. for (sia = sg->recovery_scope.sis, in_recovery_scope = 0;
  1208. *sia != NULL; sia++) {
  1209. if (name_match (&si_assignment->si->name, &(*sia)->name)) {
  1210. in_recovery_scope = 1;
  1211. }
  1212. }
  1213. /*
  1214. * The si_assignment found with standby hastate is not in the
  1215. * recovery scope. The found si_assignment will then be
  1216. * requested to be removed once.
  1217. */
  1218. if (!in_recovery_scope &&
  1219. si_assignment->requested_ha_state !=
  1220. USR_AMF_HA_STATE_REMOVED) {
  1221. amf_si_assignment_remove (si_assignment,
  1222. removed_si_assignment_callback_fn);
  1223. removed++;
  1224. }
  1225. si_assignment = amf_su_get_next_si_assignment (standby_su,
  1226. si_assignment);
  1227. }
  1228. }
  1229. sis++;
  1230. }
  1231. return removed;
  1232. }
  1233. /**
  1234. * Entry function for state 'removing standby assignments'
  1235. * @param sg
  1236. */
  1237. static void acsm_enter_removing_standby_assignments (amf_sg_t *sg)
  1238. {
  1239. ENTER("SG: %s", sg->name.value);
  1240. sg->avail_state = SG_AC_RemovingStandbyAssignments;
  1241. if (sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) {
  1242. if (!remove_standby_si_assignments (sg)) {
  1243. acsm_enter_activating_standby (sg);
  1244. }
  1245. }
  1246. }
  1247. static inline int div_round (int a, int b)
  1248. {
  1249. int res;
  1250. assert (b != 0);
  1251. res = a / b;
  1252. if ((a % b) != 0)
  1253. res++;
  1254. return res;
  1255. }
  1256. static int no_su_has_presence_state (
  1257. struct amf_sg *sg, struct amf_node *node_to_start,
  1258. SaAmfPresenceStateT state)
  1259. {
  1260. struct amf_su *su;
  1261. int no_su_has_presence_state = 1;
  1262. for (su = sg->su_head; su != NULL; su = su->next) {
  1263. if (su->saAmfSUPresenceState == state) {
  1264. if (node_to_start == NULL) {
  1265. no_su_has_presence_state = 0;
  1266. break;
  1267. } else {
  1268. if (name_match(&node_to_start->name,
  1269. &su->saAmfSUHostedByNode)) {
  1270. no_su_has_presence_state = 0;
  1271. break;
  1272. }
  1273. }
  1274. }
  1275. }
  1276. return no_su_has_presence_state;
  1277. }
  1278. #if COMPILE_OUT
  1279. static int all_su_in_scope_has_presence_state (
  1280. struct amf_sg *sg, SaAmfPresenceStateT state)
  1281. {
  1282. struct amf_su **sus= sg->recovery_scope.sus;
  1283. while (*sus != NULL) {
  1284. if ((*sus)->saAmfSUPresenceState != state) {
  1285. break;
  1286. }
  1287. sus++;
  1288. }
  1289. return(*sus == NULL);
  1290. }
  1291. #endif
  1292. static int all_su_in_scope_has_either_two_presence_state (
  1293. amf_sg_t *sg,
  1294. SaAmfPresenceStateT state1,
  1295. SaAmfPresenceStateT state2)
  1296. {
  1297. struct amf_su **sus = sg->recovery_scope.sus;
  1298. while (*sus != NULL) {
  1299. if (!((*sus)->saAmfSUPresenceState == state1 ||
  1300. (*sus)->saAmfSUPresenceState == state2)) {
  1301. break;
  1302. }
  1303. sus++;
  1304. }
  1305. return (*sus == NULL);
  1306. }
  1307. static int all_su_in_scope_has_either_of_three_presence_state (amf_sg_t *sg,
  1308. SaAmfPresenceStateT state1, SaAmfPresenceStateT state2,
  1309. SaAmfPresenceStateT state3)
  1310. {
  1311. struct amf_su **sus = sg->recovery_scope.sus;
  1312. while (*sus != NULL) {
  1313. if (!((*sus)->saAmfSUPresenceState == state1 ||
  1314. (*sus)->saAmfSUPresenceState == state2 ||
  1315. (*sus)->saAmfSUPresenceState == state3)) {
  1316. break;
  1317. }
  1318. sus++;
  1319. }
  1320. return (*sus == NULL);
  1321. }
  1322. /**
  1323. * Get number of SIs protected by the specified SG.
  1324. * @param sg
  1325. *
  1326. * @return int
  1327. */
  1328. static int sg_si_count_get (struct amf_sg *sg)
  1329. {
  1330. struct amf_si *si;
  1331. int cnt = 0;
  1332. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1333. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1334. cnt += 1;
  1335. }
  1336. }
  1337. return(cnt);
  1338. }
  1339. static int amf_si_get_saAmfSINumReqActiveAssignments(struct amf_si *si)
  1340. {
  1341. struct amf_si_assignment *si_assignment = si->assigned_sis;
  1342. int number_of_req_active_assignments = 0;
  1343. for (; si_assignment != NULL; si_assignment = si_assignment->next) {
  1344. if (si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) {
  1345. number_of_req_active_assignments++;
  1346. }
  1347. }
  1348. return number_of_req_active_assignments;
  1349. }
  1350. static int amf_si_get_saAmfSINumReqStandbyAssignments(struct amf_si *si)
  1351. {
  1352. struct amf_si_assignment *si_assignment = si->assigned_sis;
  1353. int number_of_req_active_assignments = 0;
  1354. for (; si_assignment != NULL; si_assignment = si_assignment->next) {
  1355. if (si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) {
  1356. number_of_req_active_assignments++;
  1357. }
  1358. }
  1359. return number_of_req_active_assignments;
  1360. }
  1361. static int sg_assign_active_nplusm (struct amf_sg *sg, int su_active_assign)
  1362. {
  1363. struct amf_su *su;
  1364. struct amf_si *si;
  1365. int assigned = 0;
  1366. int assign_to_su = 0;
  1367. int total_assigned = 0;
  1368. int si_left;
  1369. int si_total;
  1370. int su_left_to_assign = su_active_assign;
  1371. ENTER("SG: %s", sg->name.value);
  1372. si_total = sg_si_count_get (sg);
  1373. si_left = si_total;
  1374. assign_to_su = div_round (si_left, su_active_assign);
  1375. if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) {
  1376. assign_to_su = sg->saAmfSGMaxActiveSIsperSUs;
  1377. }
  1378. su = sg->su_head;
  1379. while (su != NULL && su_left_to_assign > 0) {
  1380. if (amf_su_get_saAmfSUReadinessState (su) !=
  1381. SA_AMF_READINESS_IN_SERVICE ||
  1382. amf_su_get_saAmfSUNumCurrActiveSIs (su) == assign_to_su ||
  1383. amf_su_get_saAmfSUNumCurrStandbySIs (su) > 0) {
  1384. su = su->next;
  1385. continue; /* Not in service */
  1386. }
  1387. si = sg->application->si_head;
  1388. assigned = 0;
  1389. assign_to_su = div_round (si_left, su_left_to_assign);
  1390. if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) {
  1391. assign_to_su = sg->saAmfSGMaxActiveSIsperSUs;
  1392. }
  1393. while (si != NULL) {
  1394. if (name_match (&si->saAmfSIProtectedbySG, &sg->name) &&
  1395. assigned < assign_to_su &&
  1396. amf_si_get_saAmfSINumReqActiveAssignments(si) == 0) {
  1397. assigned += 1;
  1398. total_assigned += 1;
  1399. amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE);
  1400. }
  1401. si = si->next;
  1402. }
  1403. su = su->next;
  1404. su_left_to_assign -= 1;
  1405. si_left -= assigned;
  1406. dprintf (" su_left_to_assign =%d, si_left=%d\n",
  1407. su_left_to_assign, si_left);
  1408. }
  1409. assert (total_assigned <= si_total);
  1410. if (total_assigned == 0) {
  1411. dprintf ("Info: No SIs assigned");
  1412. }
  1413. return total_assigned;
  1414. }
  1415. static int sg_assign_standby_nplusm (struct amf_sg *sg, int su_standby_assign)
  1416. {
  1417. struct amf_su *su;
  1418. struct amf_si *si;
  1419. int assigned = 0;
  1420. int assign_to_su = 0;
  1421. int total_assigned = 0;
  1422. int si_left;
  1423. int si_total;
  1424. int su_left_to_assign = su_standby_assign;
  1425. ENTER ("'%s'", sg->name.value);
  1426. if (su_standby_assign == 0) {
  1427. return 0;
  1428. }
  1429. si_total = sg_si_count_get (sg);
  1430. si_left = si_total;
  1431. assign_to_su = div_round (si_left, su_standby_assign);
  1432. if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) {
  1433. assign_to_su = sg->saAmfSGMaxStandbySIsperSUs;
  1434. }
  1435. su = sg->su_head;
  1436. while (su != NULL && su_left_to_assign > 0) {
  1437. if (amf_su_get_saAmfSUReadinessState (su) !=
  1438. SA_AMF_READINESS_IN_SERVICE ||
  1439. amf_su_get_saAmfSUNumCurrActiveSIs (su) > 0 ||
  1440. amf_su_get_saAmfSUNumCurrStandbySIs (su) ==
  1441. assign_to_su) {
  1442. su = su->next;
  1443. continue; /* Not available for assignment */
  1444. }
  1445. si = sg->application->si_head;
  1446. assigned = 0;
  1447. assign_to_su = div_round (si_left, su_left_to_assign);
  1448. if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) {
  1449. assign_to_su = sg->saAmfSGMaxStandbySIsperSUs;
  1450. }
  1451. while (si != NULL) {
  1452. if (name_match (&si->saAmfSIProtectedbySG, &sg->name) &&
  1453. assigned < assign_to_su &&
  1454. amf_si_get_saAmfSINumReqStandbyAssignments (si) == 0) {
  1455. assigned += 1;
  1456. total_assigned += 1;
  1457. amf_su_assign_si (su, si, SA_AMF_HA_STANDBY);
  1458. }
  1459. si = si->next;
  1460. }
  1461. su_left_to_assign -= 1;
  1462. si_left -= assigned;
  1463. dprintf (" su_left_to_assign =%d, si_left=%d\n",
  1464. su_left_to_assign, si_left);
  1465. su = su->next;
  1466. }
  1467. assert (total_assigned <= si_total);
  1468. if (total_assigned == 0) {
  1469. dprintf ("Info: No SIs assigned!");
  1470. }
  1471. return total_assigned;
  1472. }
  1473. static int su_inservice_count_get (struct amf_sg *sg)
  1474. {
  1475. struct amf_su *su;
  1476. int answer = 0;
  1477. for (su = sg->su_head; su != NULL; su = su->next) {
  1478. if (amf_su_get_saAmfSUReadinessState (su) ==
  1479. SA_AMF_READINESS_IN_SERVICE) {
  1480. answer += 1;
  1481. }
  1482. }
  1483. return(answer);
  1484. }
  1485. static int su_active_out_of_service_count_get (amf_sg_t *sg)
  1486. {
  1487. int active_out_of_service_count = 0;
  1488. amf_su_t *su;
  1489. for (su = sg->su_head; su != NULL; su = su->next) {
  1490. amf_si_assignment_t *si_assignment;
  1491. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  1492. while (si_assignment != NULL) {
  1493. if ((si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) &&
  1494. (amf_su_get_saAmfSUReadinessState (su) ==
  1495. SA_AMF_READINESS_OUT_OF_SERVICE)) {
  1496. active_out_of_service_count += 1;
  1497. }
  1498. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  1499. }
  1500. }
  1501. return active_out_of_service_count;
  1502. }
  1503. static int su_standby_out_of_service_count_get (amf_sg_t *sg)
  1504. {
  1505. int active_out_of_service_count = 0;
  1506. amf_su_t *su;
  1507. for (su = sg->su_head; su != NULL; su = su->next) {
  1508. amf_si_assignment_t *si_assignment;
  1509. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  1510. while (si_assignment != NULL) {
  1511. if ((si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) &&
  1512. (amf_su_get_saAmfSUReadinessState (su) ==
  1513. SA_AMF_READINESS_OUT_OF_SERVICE)) {
  1514. active_out_of_service_count += 1;
  1515. }
  1516. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  1517. }
  1518. }
  1519. return active_out_of_service_count;
  1520. }
  1521. /**
  1522. * This function calculates the number of active and standby assignments that
  1523. * shall be done according to what is configured and the number of in-service
  1524. * SUs available. This function leaves possible existing assignments as they are
  1525. * but possibly adds new assignments. This function also initiates the transfer
  1526. * of the calculated assignments to the SUs that shall execute them.
  1527. *
  1528. * TODO: dependency_level not used, hard coded
  1529. * @param sg
  1530. * @param dependency_level
  1531. * @return - the sum of assignments initiated
  1532. */
  1533. static int assign_si (struct amf_sg *sg, int dependency_level)
  1534. {
  1535. int active_sus_needed = 0;
  1536. int standby_sus_needed = 0;
  1537. int inservice_count;
  1538. int su_active_assign;
  1539. int su_standby_assign;
  1540. int su_spare_assign;
  1541. int assigned = 0;
  1542. int active_out_of_service = 0;
  1543. int standby_out_of_service = 0;
  1544. ENTER ("'%s'", sg->name.value);
  1545. /**
  1546. * Phase 1: Calculate assignments and create all runtime objects in
  1547. * information model. Do not do the actual assignment, done in
  1548. * phase 2.
  1549. */
  1550. /**
  1551. * Calculate number of SUs to assign to active or standby state
  1552. */
  1553. inservice_count = su_inservice_count_get (sg);
  1554. active_out_of_service = su_active_out_of_service_count_get(sg);
  1555. standby_out_of_service = su_standby_out_of_service_count_get(sg);
  1556. if (sg->saAmfSGNumPrefActiveSUs > 0) {
  1557. active_sus_needed = div_round (
  1558. sg_si_count_get (sg),
  1559. sg->saAmfSGMaxActiveSIsperSUs);
  1560. } else {
  1561. log_printf (LOG_LEVEL_ERROR, "ERROR: saAmfSGNumPrefActiveSUs == 0 !!");
  1562. openais_exit_error (AIS_DONE_FATAL_ERR);
  1563. }
  1564. if (sg->saAmfSGNumPrefStandbySUs > 0) {
  1565. standby_sus_needed = div_round (
  1566. sg_si_count_get (sg),
  1567. sg->saAmfSGMaxStandbySIsperSUs);
  1568. } else {
  1569. log_printf (LOG_LEVEL_ERROR, "ERROR: saAmfSGNumPrefStandbySUs == 0 !!");
  1570. openais_exit_error (AIS_DONE_FATAL_ERR);
  1571. }
  1572. dprintf ("(inservice=%d) (active_sus_needed=%d) (standby_sus_needed=%d)"
  1573. "\n",
  1574. inservice_count, active_sus_needed, standby_sus_needed);
  1575. /* Determine number of active and standby service units
  1576. * to assign based upon reduction procedure
  1577. */
  1578. if ((inservice_count < active_sus_needed - active_out_of_service)) {
  1579. dprintf ("assignment VI - partial assignment with SIs drop outs\n");
  1580. su_active_assign = inservice_count;
  1581. su_standby_assign = 0;
  1582. su_spare_assign = 0;
  1583. } else
  1584. if ((inservice_count < active_sus_needed - active_out_of_service +
  1585. standby_sus_needed)) {
  1586. dprintf ("assignment V - partial assignment with reduction of"
  1587. " standby units\n");
  1588. su_active_assign = active_sus_needed;
  1589. su_standby_assign = inservice_count - active_sus_needed - active_out_of_service;
  1590. su_spare_assign = 0;
  1591. } else
  1592. if ((inservice_count < sg->saAmfSGNumPrefActiveSUs + standby_sus_needed)) {
  1593. dprintf ("IV: full assignment with reduction of active service"
  1594. " units\n");
  1595. su_active_assign = inservice_count - standby_sus_needed;
  1596. su_standby_assign = standby_sus_needed;
  1597. su_spare_assign = 0;
  1598. } else
  1599. if ((inservice_count <
  1600. sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) {
  1601. dprintf ("III: full assignment with reduction of standby service"
  1602. " units\n");
  1603. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1604. su_standby_assign = inservice_count - sg->saAmfSGNumPrefActiveSUs;
  1605. su_spare_assign = 0;
  1606. } else
  1607. if ((inservice_count ==
  1608. sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) {
  1609. if (sg->saAmfSGNumPrefInserviceSUs > inservice_count) {
  1610. dprintf ("II: full assignment with spare reduction\n");
  1611. } else {
  1612. dprintf ("II: full assignment without spares\n");
  1613. }
  1614. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1615. su_standby_assign = sg->saAmfSGNumPrefStandbySUs;
  1616. su_spare_assign = 0;
  1617. } else {
  1618. dprintf ("I: full assignment with spares\n");
  1619. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1620. su_standby_assign = sg->saAmfSGNumPrefStandbySUs;
  1621. su_spare_assign = inservice_count -
  1622. sg->saAmfSGNumPrefActiveSUs - sg->saAmfSGNumPrefStandbySUs;
  1623. }
  1624. dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)"
  1625. " (assigning spares=%d)\n",
  1626. inservice_count, su_active_assign, su_standby_assign, su_spare_assign);
  1627. if (inservice_count > 0) {
  1628. assigned = sg_assign_active_nplusm (sg, su_active_assign);
  1629. assigned += sg_assign_standby_nplusm (sg, su_standby_assign);
  1630. sg->saAmfSGNumCurrAssignedSUs = inservice_count;
  1631. /**
  1632. * Phase 2: do the actual assignment to the component
  1633. * TODO: first do active, then standby
  1634. */
  1635. {
  1636. struct amf_si *si;
  1637. struct amf_si_assignment *si_assignment;
  1638. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1639. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1640. for (si_assignment = si->assigned_sis;
  1641. si_assignment != NULL;
  1642. si_assignment = si_assignment->next) {
  1643. if (si_assignment->requested_ha_state !=
  1644. si_assignment->saAmfSISUHAState) {
  1645. amf_si_ha_state_assume (
  1646. si_assignment, assign_si_assumed_cbfn);
  1647. }
  1648. }
  1649. }
  1650. }
  1651. }
  1652. }
  1653. LEAVE ("'%s'", sg->name.value);
  1654. return assigned;
  1655. }
  1656. #ifdef COMPILE_OUT
  1657. static void remove_si_in_scope (amf_sg_t *sg, amf_si_t *si)
  1658. {
  1659. int i;
  1660. int j;
  1661. amf_si_t **sis = sg->recovery_scope.sis;
  1662. amf_si_t **new_sis = amf_calloc (1, sizeof (amf_si_t*));
  1663. for (i = 0,j = 0; sis[i] != NULL; i++) {
  1664. if (sis[i] == si) {
  1665. continue;
  1666. }
  1667. new_sis[j] = sis[i];
  1668. new_sis = amf_realloc (new_sis, j + sizeof (amf_si_t *));
  1669. j++;
  1670. }
  1671. sg->recovery_scope.sis = new_sis;
  1672. }
  1673. #endif
  1674. #ifdef COMPILE_OUT
  1675. static void remove_sis_for_term_failed_su_from_scope (amf_sg_t *sg,
  1676. amf_su_t *su)
  1677. {
  1678. amf_comp_t *component;
  1679. /*
  1680. * foreach component with presense state termiantion failed in su
  1681. */
  1682. for (component = su->comp_head; component != NULL;
  1683. component = component->next) {
  1684. amf_csi_assignment_t *csi_assignment;
  1685. if (component->saAmfCompPresenceState !=
  1686. SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
  1687. continue;
  1688. }
  1689. csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
  1690. while (csi_assignment != NULL) {
  1691. remove_si_in_scope (sg, csi_assignment->csi->si);
  1692. csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
  1693. }
  1694. }
  1695. }
  1696. #endif
  1697. /**
  1698. * This function returns 1 if the redundancy model is N plus M and at least one
  1699. * component of the specified SU has an active HA-state, else the function
  1700. * returns 0.
  1701. * @param sg
  1702. * @param su
  1703. * @return int
  1704. */
  1705. static int is_comp_in_active_ha_state_nplusm (
  1706. amf_sg_t *sg, amf_su_t *su)
  1707. {
  1708. amf_comp_t *component;
  1709. amf_csi_assignment_t *csi_assignment;
  1710. int comp_is_in_active_ha_state = 0;
  1711. if(sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) {
  1712. for (component = su->comp_head; component != NULL;
  1713. component = component->next) {
  1714. csi_assignment = amf_comp_get_next_csi_assignment(component, NULL);
  1715. while (csi_assignment != NULL) {
  1716. if (csi_assignment->saAmfCSICompHAState == SA_AMF_HA_ACTIVE) {
  1717. comp_is_in_active_ha_state = 1;
  1718. goto out;
  1719. }
  1720. csi_assignment = amf_comp_get_next_csi_assignment(component,
  1721. csi_assignment);
  1722. }
  1723. }
  1724. }
  1725. out:
  1726. return comp_is_in_active_ha_state;
  1727. }
  1728. /**
  1729. * This function handles a change of presence state reported by an SU contained
  1730. * in specified SG. The new presence state is INSTANTIATED.
  1731. * @param sg
  1732. * @param su
  1733. */
  1734. static void sg_su_state_changed_to_instantiated (struct amf_sg *sg, struct amf_su *su)
  1735. {
  1736. ENTER("%s %s",sg->name.value, su->name.value);
  1737. switch (sg->avail_state) {
  1738. case SG_AC_InstantiatingServiceUnits:
  1739. if (no_su_has_presence_state(sg, sg->node_to_start,
  1740. SA_AMF_PRESENCE_INSTANTIATING)) {
  1741. acsm_enter_idle (sg);
  1742. }
  1743. break;
  1744. case SG_AC_ReparingSu:
  1745. if (no_su_has_presence_state(sg, sg->node_to_start,
  1746. SA_AMF_PRESENCE_INSTANTIATING)) {
  1747. if (all_su_in_scope_has_either_of_three_presence_state(
  1748. su->sg,
  1749. SA_AMF_PRESENCE_INSTANTIATED,
  1750. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1751. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1752. su->sg->avail_state = SG_AC_AssigningWorkload;
  1753. if (assign_si (sg, 0) == 0) {
  1754. acsm_enter_idle (sg);
  1755. }
  1756. } else {
  1757. dprintf ("avail-state: %u", sg->avail_state);
  1758. assert (0);
  1759. }
  1760. }
  1761. break;
  1762. default:
  1763. dprintf ("avail-state: %u", sg->avail_state);
  1764. assert (0);
  1765. break;
  1766. }
  1767. }
  1768. /**
  1769. * This function handles a change of presence state reported by an SU contained
  1770. * in specified SG. The new presence state is UNINSTANTIATED.
  1771. * @param sg
  1772. * @param su
  1773. */
  1774. static void amf_sg_su_state_changed_to_uninstantiated (amf_sg_t *sg,
  1775. amf_su_t *su)
  1776. {
  1777. ENTER("%s %s",sg->name.value, su->name.value);
  1778. switch (sg->avail_state) {
  1779. case SG_AC_TerminatingSuspected:
  1780. if (no_su_has_presence_state(sg, sg->node_to_start,
  1781. SA_AMF_PRESENCE_TERMINATING)) {
  1782. if (all_su_in_scope_has_either_two_presence_state (sg,
  1783. SA_AMF_PRESENCE_UNINSTANTIATED,
  1784. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  1785. delete_si_assignments_in_scope (sg);
  1786. if (is_any_si_in_scope_assigned_standby (sg)) {
  1787. remove_all_suspected_sus (sg);
  1788. acsm_enter_removing_standby_assignments (sg);
  1789. } else { /*is_no_si_in_scope_assigned_standby*/
  1790. remove_all_suspected_sus (sg);
  1791. acsm_enter_assigning_standby_to_spare (sg);
  1792. }
  1793. }
  1794. }
  1795. break;
  1796. case SG_AC_ReparingSu:
  1797. if (no_su_has_presence_state(sg, sg->node_to_start,
  1798. SA_AMF_PRESENCE_TERMINATING)) {
  1799. if (all_su_in_scope_has_either_of_three_presence_state(
  1800. su->sg,
  1801. SA_AMF_PRESENCE_INSTANTIATED,
  1802. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1803. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1804. su->sg->avail_state = SG_AC_AssigningWorkload;
  1805. if (assign_si (sg, 0) == 0) {
  1806. acsm_enter_idle (sg);
  1807. }
  1808. }
  1809. }
  1810. break;
  1811. default:
  1812. log_printf (LOG_ERR, "sg avail_state = %d", sg->avail_state);
  1813. assert (0);
  1814. break;
  1815. }
  1816. }
  1817. /**
  1818. * This function handles a change of presence state reported by an SU contained
  1819. * in specified SG. The new presence state is TERMINATION_FAILED.
  1820. * @param sg
  1821. * @param su
  1822. */
  1823. static void amf_sg_su_state_changed_to_termination_failed (amf_sg_t *sg,
  1824. amf_su_t *su)
  1825. {
  1826. ENTER("%s %s",sg->name.value, su->name.value);
  1827. if (no_su_has_presence_state(sg, sg->node_to_start,
  1828. SA_AMF_PRESENCE_TERMINATING)) {
  1829. if (is_comp_in_active_ha_state_nplusm (sg, su)) {
  1830. acsm_enter_idle (sg);
  1831. goto out;
  1832. }
  1833. if (all_su_in_scope_has_either_two_presence_state (sg,
  1834. SA_AMF_PRESENCE_UNINSTANTIATED,
  1835. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  1836. delete_si_assignments_in_scope (sg);
  1837. if (is_any_si_in_scope_assigned_standby (sg)) {
  1838. remove_all_suspected_sus (sg);
  1839. acsm_enter_removing_standby_assignments (sg);
  1840. } else { /*is_no_si_in_scope_assigned_standby*/
  1841. remove_all_suspected_sus (sg);
  1842. acsm_enter_assigning_standby_to_spare (sg);
  1843. }
  1844. }
  1845. }
  1846. out:
  1847. return;
  1848. }
  1849. /**
  1850. * This function handles a change of presence state reported by an SU contained
  1851. * in specified SG. The new presence state is INSTANTIATION_FAILED.
  1852. * @param sg
  1853. * @param su
  1854. */
  1855. static void amf_sg_su_state_changed_to_instantiation_failed (amf_sg_t *sg,
  1856. amf_su_t *su)
  1857. {
  1858. ENTER("%s %s",sg->name.value, su->name.value);
  1859. switch (sg->avail_state) {
  1860. case SG_AC_InstantiatingServiceUnits:
  1861. if (no_su_has_presence_state(sg, sg->node_to_start,
  1862. SA_AMF_PRESENCE_INSTANTIATING)) {
  1863. acsm_enter_idle (sg);
  1864. }
  1865. break;
  1866. case SG_AC_ReparingSu:
  1867. if (no_su_has_presence_state(sg, sg->node_to_start,
  1868. SA_AMF_PRESENCE_INSTANTIATING)) {
  1869. if (all_su_in_scope_has_either_of_three_presence_state(
  1870. su->sg,
  1871. SA_AMF_PRESENCE_INSTANTIATED,
  1872. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1873. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1874. su->sg->avail_state = SG_AC_AssigningWorkload;
  1875. if (assign_si (sg, 0) == 0) {
  1876. acsm_enter_idle (sg);
  1877. }
  1878. }
  1879. }
  1880. break;
  1881. default:
  1882. /* TODO: Insert the assert (0) until solving defers in SU */
  1883. dprintf("sg->avail_state = %d", sg->avail_state);
  1884. break;
  1885. }
  1886. }
  1887. /******************************************************************************
  1888. * Event methods
  1889. *****************************************************************************/
  1890. /**
  1891. * This function starts all SUs in the SG or all SUs on a specified node.
  1892. * @param sg
  1893. * @param node - Node on which all SUs shall be started or
  1894. * NULL indicating that all SUs on all nodes shall be started.
  1895. * @return - No of SUs that has been attempted to start
  1896. */
  1897. int amf_sg_start (struct amf_sg *sg, struct amf_node *node)
  1898. {
  1899. sg->recovery_scope.event_type = SG_START_EV;
  1900. ENTER ("'%s'", sg->name.value);
  1901. int instantiated_sus = 0;
  1902. switch (sg->avail_state) {
  1903. case SG_AC_InstantiatingServiceUnits:
  1904. case SG_AC_Idle: {
  1905. amf_su_t *su;
  1906. sg_avail_control_state_t old_avail_state = sg->avail_state;
  1907. ENTER ("'%s'", sg->name.value);
  1908. sg->node_to_start = node;
  1909. sg->avail_state = SG_AC_InstantiatingServiceUnits;
  1910. for (su = sg->su_head;
  1911. (su != NULL) &&
  1912. (instantiated_sus < sg->saAmfSGNumPrefInserviceSUs);
  1913. su = su->next) {
  1914. if (is_cluster_start (node)) {
  1915. amf_su_instantiate (su);
  1916. instantiated_sus++;
  1917. } else { /*is_not_cluster_start*/
  1918. /*
  1919. * Node start, match if SU is hosted on the
  1920. * specified node
  1921. */
  1922. if (name_match (&node->name,
  1923. &su->saAmfSUHostedByNode)) {
  1924. amf_su_instantiate (su);
  1925. instantiated_sus++;
  1926. }
  1927. }
  1928. }
  1929. if (instantiated_sus == 0) {
  1930. sg->avail_state = old_avail_state;
  1931. }
  1932. break;
  1933. }
  1934. case SG_AC_DeactivatingDependantWorkload:
  1935. case SG_AC_TerminatingSuspected:
  1936. case SG_AC_ActivatingStandby:
  1937. case SG_AC_AssigningStandbyToSpare:
  1938. case SG_AC_ReparingComponent:
  1939. case SG_AC_ReparingSu:
  1940. case SG_AC_AssigningOnRequest:
  1941. case SG_AC_RemovingAssignment:
  1942. case SG_AC_AssigningActiveworkload:
  1943. case SG_AC_AssigningAutoAdjust:
  1944. case SG_AC_AssigningWorkload:
  1945. case SG_AC_WaitingAfterOperationFailed:
  1946. case SG_AC_RemovingStandbyAssignments:
  1947. default:
  1948. assert (0);
  1949. break;
  1950. }
  1951. return instantiated_sus;
  1952. }
  1953. /**
  1954. * This function initiates assignment of the subset of the workload which
  1955. * matches the specified workload dependency level, to all SUs contained in the
  1956. * SG according to the requirements specified in the configuration.
  1957. * @param sg -
  1958. * @param dependency_level - Dependency level to assign
  1959. * @return - No of SUs that has been attempted to start
  1960. */
  1961. int amf_sg_assign_si_req (struct amf_sg *sg, int dependency_level)
  1962. {
  1963. /* TODO: Introduce state control in this function
  1964. */
  1965. int posible_to_assign_si;
  1966. sg->recovery_scope.event_type = SG_ASSIGN_SI_EV;
  1967. sg->avail_state = SG_AC_AssigningOnRequest;
  1968. if ((posible_to_assign_si = assign_si (sg, dependency_level)) == 0) {
  1969. acsm_enter_idle (sg);
  1970. }
  1971. return posible_to_assign_si;
  1972. }
  1973. /**
  1974. * This function is called because an error has been detected and the analysis
  1975. * (done elsewhere) indicated that this error shall be recovered by a Node
  1976. * failover. This function initiates the recovery action 'Node failover'.
  1977. * @param sg
  1978. * @param su - SU to failover
  1979. * @param node -
  1980. */
  1981. void amf_sg_failover_node_req (struct amf_sg *sg, struct amf_node *node)
  1982. {
  1983. ENTER("'%s, %s'",node->name.value, sg->name.value);
  1984. sg_event_t sg_event;
  1985. switch (sg->avail_state) {
  1986. case SG_AC_Idle:
  1987. set_scope_for_failover_node(sg, node);
  1988. if (has_any_su_in_scope_active_workload (sg)) {
  1989. acsm_enter_deactivating_dependent_workload (sg);
  1990. } else {
  1991. amf_su_t **sus = sg->recovery_scope.sus;
  1992. /*
  1993. * Select next state depending on if some
  1994. * SU in the scope needs to be terminated.
  1995. */
  1996. while (*sus != NULL) {
  1997. amf_su_t *su = *sus;
  1998. ENTER("SU %s pr_state='%d'",su->name.value,
  1999. su->saAmfSUPresenceState);
  2000. if (su_presense_state_is_ored (su,
  2001. SA_AMF_PRESENCE_UNINSTANTIATED,
  2002. SA_AMF_PRESENCE_TERMINATION_FAILED,
  2003. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  2004. sus++;
  2005. continue;
  2006. }
  2007. break;
  2008. }
  2009. if (*sus != NULL) {
  2010. acsm_enter_terminating_suspected (sg);
  2011. } else {
  2012. delete_si_assignments_in_scope (sg);
  2013. acsm_enter_idle (sg);
  2014. }
  2015. }
  2016. break;
  2017. case SG_AC_DeactivatingDependantWorkload:
  2018. case SG_AC_TerminatingSuspected:
  2019. case SG_AC_ActivatingStandby:
  2020. case SG_AC_AssigningStandbyToSpare:
  2021. case SG_AC_ReparingComponent:
  2022. case SG_AC_ReparingSu:
  2023. case SG_AC_AssigningOnRequest:
  2024. case SG_AC_InstantiatingServiceUnits:
  2025. case SG_AC_RemovingAssignment:
  2026. case SG_AC_AssigningActiveworkload:
  2027. case SG_AC_AssigningAutoAdjust:
  2028. case SG_AC_AssigningWorkload:
  2029. case SG_AC_WaitingAfterOperationFailed:
  2030. case SG_AC_RemovingStandbyAssignments:
  2031. sg_set_event (SG_FAILOVER_NODE_EV, sg, 0, 0, node, &sg_event);
  2032. sg_defer_event (SG_FAILOVER_NODE_EV, &sg_event);
  2033. break;
  2034. default:
  2035. assert (0);
  2036. break;
  2037. }
  2038. }
  2039. /**
  2040. * This function is called because an error has been detected and the analysis
  2041. * (done elsewhere) indicated that this error shall be recovered by an SU
  2042. * failover. This function initiates the recovery action 'SU failover'.
  2043. * @param sg
  2044. * @param su - SU to failover
  2045. * @param node -
  2046. */
  2047. void amf_sg_failover_su_req (struct amf_sg *sg, struct amf_su *su,
  2048. struct amf_node *node)
  2049. {
  2050. ENTER ("%s", su->name.value);
  2051. sg_event_t sg_event;
  2052. switch (sg->avail_state) {
  2053. case SG_AC_Idle:
  2054. su->su_failover_cnt += 1;
  2055. set_scope_for_failover_su (sg, su);
  2056. if (has_any_su_in_scope_active_workload (sg)) {
  2057. acsm_enter_deactivating_dependent_workload (sg);
  2058. } else {
  2059. acsm_enter_terminating_suspected (sg);
  2060. }
  2061. break;
  2062. case SG_AC_DeactivatingDependantWorkload:
  2063. case SG_AC_TerminatingSuspected:
  2064. case SG_AC_ActivatingStandby:
  2065. case SG_AC_AssigningStandbyToSpare:
  2066. case SG_AC_ReparingComponent:
  2067. case SG_AC_ReparingSu:
  2068. case SG_AC_AssigningOnRequest:
  2069. case SG_AC_InstantiatingServiceUnits:
  2070. case SG_AC_RemovingAssignment:
  2071. case SG_AC_AssigningActiveworkload:
  2072. case SG_AC_AssigningAutoAdjust:
  2073. case SG_AC_AssigningWorkload:
  2074. case SG_AC_WaitingAfterOperationFailed:
  2075. case SG_AC_RemovingStandbyAssignments:
  2076. sg_set_event (SG_FAILOVER_SU_EV, sg, su, 0, 0, &sg_event);
  2077. sg_defer_event (SG_FAILOVER_SU_EV, &sg_event);
  2078. break;
  2079. default:
  2080. assert (0);
  2081. break;
  2082. }
  2083. }
  2084. /******************************************************************************
  2085. * Event response methods
  2086. *****************************************************************************/
  2087. #ifdef COMPILE_OUT
  2088. void amf_sg_su_state_changed_2 (struct amf_sg *sg,
  2089. struct amf_su *su, SaAmfStateT type, int state)
  2090. {
  2091. ENTER ("'%s' SU '%s' state %s",
  2092. sg->name.value, su->name.value, amf_presence_state(state));
  2093. if (type == SA_AMF_PRESENCE_STATE) {
  2094. if (state == SA_AMF_PRESENCE_INSTANTIATED) {
  2095. if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
  2096. if (no_su_has_presence_state(sg, sg->node_to_start,
  2097. SA_AMF_PRESENCE_INSTANTIATING)) {
  2098. acsm_enter_idle (sg);
  2099. }
  2100. } else if (sg->avail_state == SG_AC_ReparingSu) {
  2101. if (all_su_in_scope_has_either_of_three_presence_state(
  2102. su->sg,
  2103. SA_AMF_PRESENCE_INSTANTIATED,
  2104. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  2105. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  2106. su->sg->avail_state = SG_AC_AssigningWorkload;
  2107. if (assign_si (sg, 0) == 0) {
  2108. acsm_enter_idle (sg);
  2109. }
  2110. } else {
  2111. dprintf ("avail-state: %u", sg->avail_state);
  2112. assert (0);
  2113. }
  2114. } else {
  2115. dprintf ("avail-state: %u", sg->avail_state);
  2116. assert (0);
  2117. }
  2118. } else if (state == SA_AMF_PRESENCE_UNINSTANTIATED) {
  2119. if (sg->avail_state == SG_AC_TerminatingSuspected) {
  2120. if (all_su_in_scope_has_either_two_presence_state (sg,
  2121. SA_AMF_PRESENCE_UNINSTANTIATED,
  2122. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  2123. delete_si_assignments_in_scope (sg);
  2124. if (is_any_si_in_scope_assigned_standby (sg)) {
  2125. remove_all_suspected_sus (sg);
  2126. acsm_enter_removing_standby_assignments (sg);
  2127. } else { /*is_no_si_in_scope_assigned_standby*/
  2128. remove_all_suspected_sus (sg);
  2129. acsm_enter_assigning_standby_to_spare (sg);
  2130. }
  2131. }
  2132. } else if (sg->avail_state == SG_AC_ReparingSu) {
  2133. if (all_su_in_scope_has_either_of_three_presence_state(
  2134. su->sg,
  2135. SA_AMF_PRESENCE_INSTANTIATED,
  2136. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  2137. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  2138. su->sg->avail_state = SG_AC_AssigningWorkload;
  2139. if (assign_si (sg, 0) == 0) {
  2140. acsm_enter_idle (sg);
  2141. }
  2142. } else {
  2143. dprintf("%d",sg->avail_state);
  2144. assert (0);
  2145. }
  2146. }
  2147. } else if (state == SA_AMF_PRESENCE_TERMINATION_FAILED) {
  2148. if (all_su_in_scope_has_either_two_presence_state (sg,
  2149. SA_AMF_PRESENCE_UNINSTANTIATED,
  2150. SA_AMF_PRESENCE_TERMINATION_FAILED) &&
  2151. is_any_si_in_scope_assigned_standby (sg)) {
  2152. remove_all_suspected_sus (sg);
  2153. acsm_enter_removing_standby_assignments (sg);
  2154. } else if (all_su_in_scope_has_either_two_presence_state (sg,
  2155. SA_AMF_PRESENCE_UNINSTANTIATED,
  2156. SA_AMF_PRESENCE_TERMINATION_FAILED) &&
  2157. !is_any_si_in_scope_assigned_standby (sg)) {
  2158. remove_all_suspected_sus (sg);
  2159. acsm_enter_assigning_standby_to_spare (sg);
  2160. } else {
  2161. remove_sis_for_term_failed_su_from_scope (sg, su);
  2162. }
  2163. } else if (state == SA_AMF_PRESENCE_INSTANTIATING) {
  2164. ; /* nop */
  2165. } else if (state == SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
  2166. if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
  2167. if (no_su_has_presence_state(sg, sg->node_to_start,
  2168. SA_AMF_PRESENCE_INSTANTIATING)) {
  2169. acsm_enter_idle (sg);
  2170. }
  2171. } else if (sg->avail_state == SG_AC_ReparingSu) {
  2172. if (all_su_in_scope_has_either_of_three_presence_state(
  2173. su->sg,
  2174. SA_AMF_PRESENCE_INSTANTIATED,
  2175. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  2176. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  2177. su->sg->avail_state = SG_AC_AssigningWorkload;
  2178. if (assign_si (sg, 0) == 0) {
  2179. acsm_enter_idle (sg);
  2180. }
  2181. }
  2182. } else {
  2183. /* TODO: Insert the assert (0) until solving defers in SU */
  2184. dprintf("sg->avail_state = %d, su instantiation state = %d",
  2185. sg->avail_state, state);
  2186. }
  2187. } else {
  2188. dprintf("sg->avail_state = %d, su instantiation state = %d",
  2189. sg->avail_state, state);
  2190. assert (0);
  2191. }
  2192. }
  2193. }
  2194. #endif
  2195. /**
  2196. * SU indicates to SG that one of its state machines has changed state.
  2197. * @param sg - SG which contains the SU that has changed state
  2198. * @param su - SU which has changed state
  2199. * @param type - Indicates which state machine that has changed state
  2200. * @param state - The new state that has been assumed.
  2201. *
  2202. */
  2203. void amf_sg_su_state_changed (struct amf_sg *sg, struct amf_su *su,
  2204. SaAmfStateT type, int state)
  2205. {
  2206. ENTER ("'%s' SU '%s' state %s",
  2207. sg->name.value, su->name.value, amf_presence_state(state));
  2208. if (sg->avail_state != SG_AC_Idle) {
  2209. if (type == SA_AMF_PRESENCE_STATE) {
  2210. switch (state) {
  2211. case SA_AMF_PRESENCE_INSTANTIATED:
  2212. sg_su_state_changed_to_instantiated(sg, su);
  2213. break;
  2214. case SA_AMF_PRESENCE_UNINSTANTIATED:
  2215. amf_sg_su_state_changed_to_uninstantiated(sg, su);
  2216. break;
  2217. case SA_AMF_PRESENCE_TERMINATION_FAILED:
  2218. amf_sg_su_state_changed_to_termination_failed(sg, su);
  2219. break;
  2220. case SA_AMF_PRESENCE_INSTANTIATING:
  2221. ; /* nop */
  2222. break;
  2223. case SA_AMF_PRESENCE_INSTANTIATION_FAILED:
  2224. amf_sg_su_state_changed_to_instantiation_failed(sg, su);
  2225. break;
  2226. case SA_AMF_PRESENCE_TERMINATING:
  2227. ; /* nop */
  2228. break;
  2229. default :
  2230. dprintf("sg->avail_state = %d, su instantiation state = %d",
  2231. sg->avail_state, state);
  2232. assert (0);
  2233. break;
  2234. }
  2235. }
  2236. }
  2237. }
  2238. /**
  2239. * Callback function used by SI when there is no dependent SI to
  2240. * deactivate.
  2241. * @param sg
  2242. */
  2243. static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg)
  2244. {
  2245. struct amf_su **sus = sg->recovery_scope.sus;
  2246. ENTER("'%s'", sg->name.value);
  2247. /*
  2248. * Select next state depending on if some
  2249. * SU in the scope needs to be terminated.
  2250. */
  2251. while (*sus != NULL) {
  2252. amf_su_t *su = *sus;
  2253. ENTER("SU %s pr_state='%d'",su->name.value,
  2254. su->saAmfSUPresenceState);
  2255. if (su_presense_state_is_ored (su,
  2256. SA_AMF_PRESENCE_UNINSTANTIATED,
  2257. SA_AMF_PRESENCE_TERMINATION_FAILED,
  2258. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  2259. sus++;
  2260. continue;
  2261. }
  2262. break;
  2263. }
  2264. if (*sus != NULL) {
  2265. acsm_enter_terminating_suspected (sg);
  2266. } else {
  2267. delete_si_assignments_in_scope(sg);
  2268. acsm_enter_removing_standby_assignments (sg);
  2269. }
  2270. }
  2271. /**
  2272. * Callback function used by SI when an SI has been deactivated, i.e.
  2273. * transitioned from active HA-state to any other state.
  2274. * @param si_assignment
  2275. * @param result - Indicates the result of the operation.
  2276. */
  2277. static void dependent_si_deactivated_cbfn (
  2278. struct amf_si_assignment *si_assignment, int result)
  2279. {
  2280. struct amf_sg *sg = si_assignment->su->sg;
  2281. struct amf_su **sus = sg->recovery_scope.sus;
  2282. struct amf_su *su;
  2283. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  2284. /*
  2285. * If all SI assignments for all SUs in the SG are not pending,
  2286. * goto next state (TerminatingSuspected).
  2287. */
  2288. for (su = sg->su_head ; su != NULL; su = su->next) {
  2289. struct amf_si_assignment *si_assignment;
  2290. si_assignment = amf_su_get_next_si_assignment(su, NULL);
  2291. while (si_assignment != NULL) {
  2292. if (si_assignment->saAmfSISUHAState !=
  2293. si_assignment->requested_ha_state) {
  2294. goto still_wating;
  2295. }
  2296. si_assignment = amf_su_get_next_si_assignment(su,
  2297. si_assignment);
  2298. }
  2299. }
  2300. still_wating:
  2301. if (su == NULL) {
  2302. sus = si_assignment->su->sg->recovery_scope.sus;
  2303. /*
  2304. * Select next state depending on if some
  2305. * SU in the scope is needs to be terminated.
  2306. */
  2307. while (*sus != NULL) {
  2308. if (su_presense_state_is_not (*sus,
  2309. SA_AMF_PRESENCE_UNINSTANTIATED,
  2310. SA_AMF_PRESENCE_TERMINATION_FAILED,
  2311. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  2312. break;
  2313. }
  2314. sus++;
  2315. }
  2316. if (*sus != NULL) {
  2317. acsm_enter_terminating_suspected (sg);
  2318. } else {
  2319. acsm_enter_removing_standby_assignments (sg);
  2320. }
  2321. }
  2322. LEAVE("");
  2323. }
  2324. /**
  2325. * Callback function used by SI to indicate an SI has assumed a new HA-state or
  2326. * that the attempt to do so failed.
  2327. * @param si_assignment
  2328. * @param result - Indicates the result of the operation.
  2329. */
  2330. static void assign_si_assumed_cbfn (
  2331. struct amf_si_assignment *si_assignment, int result)
  2332. {
  2333. struct amf_sg *sg = si_assignment->su->sg;
  2334. int si_assignment_cnt = 0;
  2335. int confirmed_assignments = 0;
  2336. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  2337. switch (sg->avail_state) {
  2338. case SG_AC_AssigningOnRequest:
  2339. if (is_all_si_assigned (sg)) {
  2340. acsm_enter_idle (sg);
  2341. amf_application_sg_assigned (sg->application, sg);
  2342. } else {
  2343. dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments);
  2344. }
  2345. break;
  2346. case SG_AC_AssigningWorkload:
  2347. {
  2348. if (is_all_si_assigned(sg)) {
  2349. acsm_enter_idle (sg);
  2350. }
  2351. break;
  2352. }
  2353. case SG_AC_AssigningStandbyToSpare:
  2354. {
  2355. if(is_all_si_assigned (sg)) {
  2356. /*
  2357. * All si_assignments has asumed
  2358. * Prescense state SA_AMF_HA_STANDBY
  2359. */
  2360. switch (sg->recovery_scope.event_type) {
  2361. case SG_FAILOVER_NODE_EV:
  2362. acsm_enter_idle (sg);
  2363. break;
  2364. case SG_FAILOVER_SU_EV:
  2365. if (sg->saAmfSGAutoRepair == SA_TRUE) {
  2366. acsm_enter_repairing_su (sg);
  2367. }
  2368. break;
  2369. default:
  2370. assert (0);
  2371. break;
  2372. }
  2373. } else {
  2374. si_assignment->saAmfSISUHAState = SA_AMF_HA_STANDBY;
  2375. }
  2376. }
  2377. break;
  2378. default:
  2379. dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt,
  2380. confirmed_assignments);
  2381. amf_runtime_attributes_print (amf_cluster);
  2382. assert (0);
  2383. break;
  2384. }
  2385. }
  2386. /**
  2387. * Callback function used by SI when an SI has been activated, i.e. transitioned
  2388. * from any HA-state to an active HA-state.
  2389. * @param si_assignment
  2390. * @param result - Indicates the result of the operation.
  2391. */
  2392. static void standby_su_activated_cbfn (
  2393. struct amf_si_assignment *si_assignment, int result)
  2394. {
  2395. struct amf_su **sus = si_assignment->su->sg->recovery_scope.sus;
  2396. struct amf_si **sis = si_assignment->su->sg->recovery_scope.sis;
  2397. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  2398. /*
  2399. * If all SI assignments for all SIs in the scope are activated, goto next
  2400. * state.
  2401. */
  2402. while (*sis != NULL) {
  2403. if ((*sis)->assigned_sis != NULL &&
  2404. (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) {
  2405. break;
  2406. }
  2407. sis++;
  2408. }
  2409. if (*sis == NULL) {
  2410. acsm_enter_assigning_standby_to_spare ((*sus)->sg);
  2411. }
  2412. }
  2413. /******************************************************************************
  2414. * General methods
  2415. *****************************************************************************/
  2416. /**
  2417. * Constructor for SG objects. Adds SG to the list owned by
  2418. * the specified application. Always returns a valid SG
  2419. * object, out-of-memory problems are handled here. Default
  2420. * values are initialized.
  2421. * @param sg
  2422. * @param name
  2423. *
  2424. * @return struct amf_sg*
  2425. */
  2426. struct amf_sg *amf_sg_new (struct amf_application *app, char *name)
  2427. {
  2428. struct amf_sg *sg = amf_calloc (1, sizeof (struct amf_sg));
  2429. setSaNameT (&sg->name, name);
  2430. sg->saAmfSGAdminState = SA_AMF_ADMIN_UNLOCKED;
  2431. sg->saAmfSGNumPrefActiveSUs = 1;
  2432. sg->saAmfSGNumPrefStandbySUs = 1;
  2433. sg->saAmfSGNumPrefInserviceSUs = ~0;
  2434. sg->saAmfSGNumPrefAssignedSUs = ~0;
  2435. sg->saAmfSGCompRestartProb = -1;
  2436. sg->saAmfSGCompRestartMax = ~0;
  2437. sg->saAmfSGSuRestartProb = -1;
  2438. sg->saAmfSGSuRestartMax = ~0;
  2439. sg->saAmfSGAutoAdjustProb = -1;
  2440. sg->saAmfSGAutoRepair = SA_TRUE;
  2441. sg->application = app;
  2442. sg->next = app->sg_head;
  2443. app->sg_head = sg;
  2444. sg->deferred_events = NULL;
  2445. return sg;
  2446. }
  2447. void amf_sg_delete (struct amf_sg *sg)
  2448. {
  2449. struct amf_su *su;
  2450. for (su = sg->su_head; su != NULL;) {
  2451. struct amf_su *tmp = su;
  2452. su = su->next;
  2453. amf_su_delete (tmp);
  2454. }
  2455. free (sg);
  2456. }
  2457. void *amf_sg_serialize (struct amf_sg *sg, int *len)
  2458. {
  2459. char *buf = NULL;
  2460. int offset = 0, size = 0;
  2461. TRACE8 ("%s", sg->name.value);
  2462. buf = amf_serialize_SaNameT (buf, &size, &offset, &sg->name);
  2463. buf = amf_serialize_SaUint32T (buf, &size, &offset, sg->saAmfSGRedundancyModel);
  2464. buf = amf_serialize_SaUint32T (
  2465. buf, &size, &offset, sg->saAmfSGAutoAdjust);
  2466. buf = amf_serialize_SaUint32T (
  2467. buf, &size, &offset, sg->saAmfSGNumPrefActiveSUs);
  2468. buf = amf_serialize_SaUint32T (
  2469. buf, &size, &offset, sg->saAmfSGNumPrefStandbySUs);
  2470. buf = amf_serialize_SaUint32T (
  2471. buf, &size, &offset, sg->saAmfSGNumPrefInserviceSUs);
  2472. buf = amf_serialize_SaUint32T (
  2473. buf, &size, &offset, sg->saAmfSGNumPrefAssignedSUs);
  2474. buf = amf_serialize_SaUint32T (
  2475. buf, &size, &offset, sg->saAmfSGMaxActiveSIsperSUs);
  2476. buf = amf_serialize_SaUint32T (
  2477. buf, &size, &offset, sg->saAmfSGMaxStandbySIsperSUs);
  2478. buf = amf_serialize_SaUint32T (
  2479. buf, &size, &offset, sg->saAmfSGCompRestartProb);
  2480. buf = amf_serialize_SaUint32T (
  2481. buf, &size, &offset, sg->saAmfSGCompRestartMax);
  2482. buf = amf_serialize_SaUint32T (
  2483. buf, &size, &offset, sg->saAmfSGSuRestartProb);
  2484. buf = amf_serialize_SaUint32T (
  2485. buf, &size, &offset, sg->saAmfSGSuRestartMax);
  2486. buf = amf_serialize_SaUint32T (
  2487. buf, &size, &offset, sg->saAmfSGAutoAdjustProb);
  2488. buf = amf_serialize_SaUint32T (
  2489. buf, &size, &offset, sg->saAmfSGAutoRepair);
  2490. buf = amf_serialize_SaUint32T (
  2491. buf, &size, &offset, sg->saAmfSGAdminState);
  2492. buf = amf_serialize_SaUint32T (
  2493. buf, &size, &offset, sg->saAmfSGNumCurrAssignedSUs);
  2494. buf = amf_serialize_SaUint32T (
  2495. buf, &size, &offset, sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
  2496. buf = amf_serialize_SaUint32T (
  2497. buf, &size, &offset, sg->saAmfSGNumCurrInstantiatedSpareSUs);
  2498. buf = amf_serialize_SaStringT (
  2499. buf, &size, &offset, sg->clccli_path);
  2500. buf = amf_serialize_SaUint32T (
  2501. buf, &size, &offset, sg->avail_state);
  2502. buf = amf_serialize_SaUint32T (
  2503. buf, &size, &offset, sg->recovery_scope.event_type);
  2504. *len = offset;
  2505. return buf;
  2506. }
  2507. struct amf_sg *amf_sg_deserialize (struct amf_application *app, char *buf)
  2508. {
  2509. char *tmp = buf;
  2510. struct amf_sg *sg = amf_sg_new (app, "");
  2511. tmp = amf_deserialize_SaNameT (tmp, &sg->name);
  2512. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGRedundancyModel);
  2513. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoAdjust);
  2514. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefActiveSUs);
  2515. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefStandbySUs);
  2516. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefInserviceSUs);
  2517. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefAssignedSUs);
  2518. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGMaxActiveSIsperSUs);
  2519. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGMaxStandbySIsperSUs);
  2520. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGCompRestartProb);
  2521. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGCompRestartMax);
  2522. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGSuRestartProb);
  2523. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGSuRestartMax);
  2524. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoAdjustProb);
  2525. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoRepair);
  2526. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAdminState);
  2527. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrAssignedSUs);
  2528. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
  2529. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrInstantiatedSpareSUs);
  2530. tmp = amf_deserialize_SaStringT (tmp, &sg->clccli_path);
  2531. tmp = amf_deserialize_SaUint32T (tmp, &sg->avail_state);
  2532. tmp = amf_deserialize_SaUint32T (tmp, &sg->recovery_scope.event_type);
  2533. return sg;
  2534. }
  2535. struct amf_sg *amf_sg_find (struct amf_application *app, char *name)
  2536. {
  2537. struct amf_sg *sg;
  2538. for (sg = app->sg_head; sg != NULL; sg = sg->next) {
  2539. if (sg->name.length == strlen(name) &&
  2540. strncmp (name, (char*)sg->name.value, sg->name.length) == 0) {
  2541. break;
  2542. }
  2543. }
  2544. return sg;
  2545. }