amfsg.c 82 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856
  1. /** @file amfsg.c
  2. *
  3. * Copyright (c) 2002-2006 MontaVista Software, Inc.
  4. * Author: Steven Dake (sdake@mvista.com)
  5. *
  6. * Copyright (c) 2006 Ericsson AB.
  7. * Author: Hans Feldt, Anders Eriksson, Lars Holm
  8. * - Introduced AMF B.02 information model
  9. * - Use DN in API and multicast messages
  10. * - (Re-)Introduction of event based multicast messages
  11. * - Refactoring of code into several AMF files
  12. * - Component/SU restart, SU failover
  13. * - Constructors/destructors
  14. * - Serializers/deserializers
  15. *
  16. * All rights reserved.
  17. *
  18. *
  19. * This software licensed under BSD license, the text of which follows:
  20. *
  21. * Redistribution and use in source and binary forms, with or without
  22. * modification, are permitted provided that the following conditions are met:
  23. *
  24. * - Redistributions of source code must retain the above copyright notice,
  25. * this list of conditions and the following disclaimer.
  26. * - Redistributions in binary form must reproduce the above copyright notice,
  27. * this list of conditions and the following disclaimer in the documentation
  28. * and/or other materials provided with the distribution.
  29. * - Neither the name of the MontaVista Software, Inc. nor the names of its
  30. * contributors may be used to endorse or promote products derived from this
  31. * software without specific prior written permission.
  32. *
  33. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  34. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  35. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  36. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  37. * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  38. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  39. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  40. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  41. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  42. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  43. * THE POSSIBILITY OF SUCH DAMAGE.
  44. *
  45. * AMF Service Group Class Implementation
  46. *
  47. * This file contains functions for handling AMF-service groups(SGs). It can be
  48. * viewed as the implementation of the AMF Service Group class (called SG)
  49. * as described in SAI-Overview-B.02.01. The SA Forum specification
  50. * SAI-AIS-AMF-B.02.01 has been used as specification of the behaviour
  51. * and is referred to as 'the spec' below.
  52. *
  53. * The functions in this file are responsible for:
  54. * -on request start the service group by instantiating the contained SUs
  55. * -on request assign the service instances it protects to the in-service
  56. * service units it contains respecting as many as possible of the configured
  57. * requirements for the group
  58. * -create and delete an SI-assignment object for each relation between
  59. * an SI and an SU
  60. * -order each contained SU to create and delete CSI-assignments
  61. * -request the Service Instance class (SI) to execute the transfer of the
  62. * HA-state set/remove requests to each component involved
  63. * -fully control the execution of component failover and SU failover
  64. * -on request control the execution of the initial steps of node switchover
  65. * and node failover
  66. * -fully handle the auto adjust procedure
  67. *
  68. * Currently only the 'n+m' redundancy model is implemented. It is the
  69. * ambition to identify n+m specific variables and functions and add the suffix
  70. * '_nplusm' to them so that they can be easily recognized.
  71. *
  72. * When SG is requested to assign workload to all SUs or all SUs hosted on
  73. * a specific node, a procedure containing several steps is executed:
  74. * <1> An algorithm is executed which assigns SIs to SUs respecting the rules
  75. * that has been configured for SG. The algorithm also has to consider
  76. * if assignments between som SIs and SUs already exist. The scope of this
  77. * algorithm is to create SI-assignments and set up requested HA-state for
  78. * each assignment but not to transfer those HA-states to the components.
  79. * <2> All SI-assignments with a requested HA state == ACTIVE are transferred
  80. * to the components concerned before any STANDBY assignments are
  81. * transferred. All components have to acknowledge the setting of the
  82. * ACTIVE HA state before the transfer of any STANDBY assignment is
  83. * initiated.
  84. * <3> All active assignments can not be transferred at the same time to the
  85. * different components because the rules for dependencies between SI and
  86. * SI cluster wide and CSI and CSI within one SI, has to be respected.
  87. *
  88. * SG is fully responsible for step <1> but not fully responsible for handling
  89. * step <2> and <3>. However, SG uses an attribute called 'dependency level'
  90. * when requsted to assign workload. This parameter refers to an integer that
  91. * has been calculated initially for each SI. The 'dependency level' indicates
  92. * to which extent an SI depends on other SIs such that an SI that depends on
  93. * no other SI is on dependecy_level == 1, an SI that depends only on an SI on
  94. * dependency_level == 1 is on dependency-level == 2.
  95. * An SI that depends on several SIs gets a
  96. * dependency_level that is one unit higher than the SI with the highest
  97. * dependency_level it depends on. When SG is requested to assign the workload
  98. * on a certain dependency level, it requests all SI objects on that level to
  99. * activate (all) SI-assignments that during step <1> has been requested to
  100. * assume the active HA state.
  101. *
  102. * SG contains the following state machines:
  103. * - administrative state machine (ADSM) (NOT IN THIS RELEASE)
  104. * - availability control state machine (ACSM)
  105. *
  106. * The availability control state machine contains three states and one of them
  107. * is composite. Being a composite state means that it contains substates.
  108. * The states are:
  109. * - IDLE (non composite state)
  110. * - INSTANTIATING_SERVICE_UNITS
  111. * - MANAGING_SG (composite state)
  112. * MANAGING_SG is entered at several different events which has in common
  113. * the need to set up or change the assignment of SIs to SUs. Only one such
  114. * event can be handled at the time. If new events occur while one event is
  115. * being handled then the new event is saved and will be handled after the
  116. * handling of the first event is ready (return to IDLE state has been done).
  117. * MANAGING_SG handles the following events:
  118. * - start (requests SG to order SU to instantiate all SUs in SG and waits
  119. * for SU to indicate presence state change reports from the SUs and
  120. * finally responds 'started' to the requester)
  121. * - assign_si (requests SG to assign SIs to SUs according to pre-configured
  122. * rules (if not already done) and transfer the HA state of
  123. * the SIs on the requested SI dependency level. Then SG waits for
  124. * confirmation that the HA state has been succesfully set and
  125. * finally responds 'assigned' to the reqeuster)
  126. * - auto_adjust (this event indicates that the auto-adjust probation timer has
  127. * expired and that SG should evaluate current assignments of
  128. * SIs to SUs and if needed remove current assignments and
  129. * create new according to what is specified in paragraph
  130. * 3.7.1.2)
  131. * - failover_comp (requests SG to failover a specific component according to
  132. * the procedure described in paragraph 3.12.1.3)
  133. * - failover_su (requests SG to failover a specific SU according to the
  134. * procedure described in paragraph 3.12.1.3 and 3.12.1.4)
  135. * - switchover_node (requests SG to execute the recovery actions described
  136. * in 3.12.1.3 and respond to the requester when recovery
  137. * is completed)
  138. * - failover_node (requests SG to execute the recovery actions described
  139. * in 3.12.1.3 and respond to the requester when recovery is
  140. * completed)
  141. *
  142. * 1. SG Availability Control State Machine
  143. * ==========================================
  144. *
  145. * 1.1 State Transition Table
  146. *
  147. * State: Event: Action: New state:
  148. * ============================================================================
  149. * IDLE start A48,A28 INSTANTIATING_SUs
  150. * IDLE assign_si A48,A31 ASSIGNING_ON_REQ
  151. * IDLE failover_su A48,[C22]A10,A11 DEACTIVATING_DEP
  152. * IDLE failover_su A48,[!C22]A12 TERMINATING_SUSP
  153. * IDLE failover_node A48,[!C22]A12 TERMINATING_SUSP
  154. * IDLE failover_node A48,[C22]A10,A11 DEACTIVATING_DEP
  155. * IDLE failover_node A48,[C100]A34 IDLE
  156. * INSTANTIATING_SUs start A48,A28 INSTANTIATING_SUs
  157. * INSTANTIATING_SUs su_state_chg [C101]A26,A53 IDLE
  158. * INSTANTIATING_SUs su_state_chg [C102]A26,A53 IDLE
  159. * INSTANTIATING_SUs assign_si A31 ASSIGNING_ON_REQ
  160. * INSTANTIATING_SUs failover_su A52 INSTANTIATING_SUs
  161. * INSTANTIATING_SUs failover_node A52 INSTANTIATING_SUs
  162. * ASSIGNING_ON_REQ ha_state_assumed [C15]A54 IDLE
  163. * ASSIGNING_ON_REQ failover_su A52 ASSIGNING_ON_REQ
  164. * ASSIGNING_ON_REQ failover_node A52 ASSIGNING_ON_REQ
  165. * DEACTIVATING_DEP si_deactivated [C20] REMOVING_S-BY_ASS
  166. * DEACTIVATING_DEP si_deactivated [!C20]A12 TERMINATING_SUSP
  167. * TERMINATING_SUSP su_state_chg [C103]A24,A20 ASS_S-BY_TO_SPARE
  168. * TERMINATING_SUSP su_state_chg [C104]A24,A50 REMOVING_S-BY_ASS
  169. * TERMINATING_SUSP su_state_chg [C105]A16,A17 ACTIVATING_S-BY
  170. * TERMINATING_SUSP su_state_chg [C106]A20 ASS_S-BY_TO_SPARE
  171. * TERMINATING_SUSP su_state_chg [C108]A23 REPAIRING_SU
  172. * TERMINATING_SUSP su_state_chg [C109] IDLE
  173. * TERMINATING_SUSP failover_su A52 TERMINATING_SUSP
  174. * TERMINATING_SUSP failover_node A52 TERMINATING_SUSP
  175. * REMOVING_S-BY_ASS assignment_removed A51 REMOVING_S-BY_ASS
  176. * REMOVING_S-BY_ASS assignment_removed [C27]&[C24] ACTIVATING_S-BY
  177. * REMOVING_S-BY_ASS assignment_removed [C110]A20 ASS_S-BY_TO_SPARE
  178. * REMOVING_S-BY_ASS assignment_removed [C111]A23 REPAIRING_SU
  179. * REMOVING_S-BY_ASS assignment_removed [C112] IDLE
  180. * REMOVING_S-BY_ASS failover_su A52 REMOVING_S-BY_ASS
  181. * REMOVING_S-BY_ASS failover_node A52 REMOVING_S-BY_ASS
  182. * ACTIVATING_S-BY su_activated [C2]&[C11]A20 ASS_S-BY_TO_SPARE
  183. * ACTIVATING_S-BY su_activated [C113]A23 REPAIRING_SU
  184. * ACTIVATING_S-BY su_activated [C114] IDLE
  185. * ACTIVATING_S-BY failover_su A52 ACTIVATING_S-BY
  186. * ACTIVATING_S-BY failover_node A52 ACTIVATING_S-BY
  187. * ASS_S-BY_TO_SPARE ha_state_assumed [C115]A23 REPAIRING_SU
  188. * ASS_S-BY_TO_SPARE ha_state_assumed [C116] IDLE
  189. * ASS_S-BY_TO_SPARE failover_su A52 ASS_S-BY_TO_SPARE
  190. * ASS_S-BY_TO_SPARE failover_node A52 ASS_S-BY_TO_SPARE
  191. * REPAIRING_SU su_state_chg [C28]A36,A37,A31 ASSIGNING_WL
  192. * REPAIRING_SU su_state_chg [C28][C31] IDLE
  193. * REPAIRING_SU failover_su A52 REPAIRING_SU
  194. * REPAIRING_SU failover_node A52 REPAIRING_SU
  195. * ASSIGNING_WL ha_state_assumed [C15] IDLE
  196. * ASSIGNING_WL failover_su A52 ASSIGNING_WL
  197. * ASSIGNING_WL failover_node A52 ASSIGNING_WL
  198. *
  199. * 1.2 State Description
  200. * =====================
  201. * IDLE - SG is synchronized and idle. When IDLE state is set,
  202. * the oldest deferred event (if any) is recalled.
  203. * INSTANTIATING_SUs - INSTANTIATING_SERVICE_UNITS
  204. * SG has ordered all contained SUs to instantiate and
  205. * waits for the SUs to report a change of their
  206. * presence state. SG is also prepared to accept an
  207. * order to assign workload in this state.
  208. * ASSIGNING_ON_REQ - ASSIGNING_ON_REQUEST
  209. * SG has on request assigned workload to all service units
  210. * on the requested dependency level and waits for SIs to
  211. * indicate that the requested HA-state have been set to the
  212. * appropriate components.
  213. * TERMINATING_SUSP - TERMINATING_SUSPECTED
  214. * SG has cleaned up all components suspected to be
  215. * erroneous and waits for the concerned SUs to report a
  216. * change of their presence states.
  217. * REMOVING_S-BY_ASS - REMOVING_STANDBY_ASSIGNMENTS
  218. * This state is only applicable to the n-plus-m redundancy
  219. * model. In this state, SG has initiated the removal of
  220. * those assignments from standby SUs that do not match the
  221. * assignments of error suspected SUs. The reason for this
  222. * removal is a preparation for not violating the rule which
  223. * says an SU can not have both active and standby assign-
  224. * ments simultaneously in the n-plus-m redundancy model.
  225. * ACTIVATING_S-BY - ACTIVATING_STANDBY
  226. * SG has located all standby SI-assignments in the recovery
  227. * scope and ordered the corresponding SI to set the active
  228. * HA-state and waits for SI to indicate that the requested
  229. * HA-state has been set to the appropriate components.
  230. * ASS_S-BY_TO_SPARE - ASSIGNING_STANDBY_TO_SPARE
  231. * Current SG is configured with a spare SU. In this state,
  232. * SG has requested all SIs to assign the standby HA-state
  233. * to the spare SU and waits for the SIs to indicate that
  234. * the standby HA-state have been set.
  235. * REPAIRING_SU - REPAIRING_SU
  236. * In this state SG has initiated instantiation of all SUs
  237. * in current recovery scope until the configured preference
  238. * of number of instantiated SUs is fulfiled. SG then waits
  239. * for the concerned SUs to report a change of presence
  240. * state.
  241. * ASSIGNING_WL - ASSIGNING_WORKLOAD
  242. * In this state SG has initiated the assignment of workload
  243. * to all or a subset of its contained SUs and waits for the
  244. * concerned SIs to indicated that the requested HA-state
  245. * has been assumed.
  246. *
  247. * 1.3 Actions
  248. * ===========
  249. * A1 -
  250. * A2 -
  251. * A3 -
  252. * A4 -
  253. * A5 -
  254. * A6 -
  255. * A7 -
  256. * A8 -
  257. * A9 -
  258. * A10 - [foreach SI in the scope]&[foreach SI-assignment with
  259. * confirmed-HA-state == ACTIVE]/set requested-ha-state = QUIESCED
  260. * A11 - [foreach SI in the scope]/deactivate SI
  261. * A12 - [foreach suspected SU]/terminate all components
  262. * A13 -
  263. * A14 -
  264. * A15 -
  265. * A16 - [foreach SI in the scope]&[foreach SI-assignment with
  266. * confirmed-HA-state == STANDBY]/set requested-ha-state = ACTIVE
  267. * A17 - [foreach SI in the scope]/activate SI
  268. * A18 -
  269. * A19 -
  270. * A20 -
  271. * A21 -
  272. * A22 -
  273. * A23 -
  274. * A24 -
  275. * A25 -
  276. * A26 -
  277. * A27 -
  278. * A28 -
  279. * A29 -
  280. * A30 -
  281. * A31 -
  282. * A32 -
  283. * A33 -
  284. * A34 -
  285. * A35 -
  286. * A36 -
  287. * A37 -
  288. * A38 -
  289. * A39 -
  290. * A40 -
  291. * A41 -
  292. * A42 -
  293. * A43 -
  294. * A44 -
  295. * A45 -
  296. * A46 -
  297. * A47 -
  298. * A48 -
  299. * A49 -
  300. *
  301. * 1.4 Composite Guards
  302. * ====================
  303. * The meaning with these guards is just to save space in the state transition
  304. * table above.
  305. * C100 - [C7]&[!C22]&[C20]
  306. * C101 - [C12]&[C28]
  307. * C102 - [C13]&[C28]
  308. * C103 - [C6]&[C21]&[C11]
  309. * C104 - [C6]&[C25]&[C26]
  310. * C105 - [C6]&(!([C25]|[C26]))&[C24]
  311. * C106 - [C6]&(!([C25]|[C26]))&[!C24]&[C11]
  312. * C107 -
  313. * C108 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[!C30]
  314. * C109 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[C30]
  315. * C110 - [C27]&[!C24]&[C11]
  316. * C111 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[!C30]
  317. * C112 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[C30]
  318. * C113 - [C2]&[!C11]&[C9]&[C10]&[!C30]
  319. * C114 - [C2]&[!C11]&[C9]&[C10]&[C30]
  320. * C115 - [C9]&[C10]&[!C30]
  321. * C116 - [C9]&[C10]&[C30]
  322. *
  323. * 1.4 Guards
  324. * ==========
  325. * C1 -
  326. * C2 - all SI in the recovery scope
  327. * C3 -
  328. * C4 -
  329. * C5 -
  330. * C6 - all supected SUs or components have presence state == UNINSTANTIATED
  331. * or presence state == INSTANTIATION_FAILED
  332. * C7 - original event == failover node
  333. * C8 -
  334. * C9 - original event == failover su
  335. * C10 - SaAmfSGAutoRepair == true
  336. * C11 - spare SUs exist
  337. * C12 - original event == start(all SUs)
  338. * C13 - original event == start(node)
  339. * C14 -
  340. * C15 - all SI-assignments on current dependency-level have requested-ha-state
  341. == confirmed-ha-state or operation failed
  342. * C16 -
  343. * C17 -
  344. * C18 -
  345. * C19 -
  346. * C20 - all suspected SUs have presence state == UNINSTANTIATED or
  347. presence state == xx_FAILED
  348. * C21 - no SI in the scope has an SI-assignment with HA-state == STANDBY
  349. * C22 - the concerned entity has ACTIVE HA-state
  350. * C23 -
  351. * C24 - at least one SI-assignment related to an SI in the scope has HA-state
  352. == STANDBY
  353. * C25 - redundancy model == N plus M
  354. * C26 - at least one SU has STANDBY assignments for more SIs than those SIs
  355. that are within the recovery scope
  356. * C27 - no SI-assignment related to SI protected by current SG has requested
  357. HA-state == REMOVED
  358. * C28 - no SU has presence state == INSTANTIATING
  359. * C29 -
  360. * C30 - more SUs not needed or the node hosting the SU to repair is disabled.
  361. * C31 - no new additional assignments needed or possible
  362. */
  363. #include <stdlib.h>
  364. #include <errno.h>
  365. #include "amf.h"
  366. #include "print.h"
  367. #include "main.h"
  368. #include "util.h"
  369. static int assign_si (struct amf_sg *sg, int dependency_level);
  370. static void acsm_enter_activating_standby (struct amf_sg *sg);
  371. static void delete_si_assignments_in_scope (struct amf_sg *sg);
  372. static void acsm_enter_repairing_su (struct amf_sg *sg);
  373. static void standby_su_activated_cbfn (
  374. struct amf_si_assignment *si_assignment, int result);
  375. static void dependent_si_deactivated_cbfn (
  376. struct amf_si_assignment *si_assignment, int result);
  377. static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg);
  378. static void assign_si_assumed_cbfn (
  379. struct amf_si_assignment *si_assignment, int result);
  380. static void acsm_enter_removing_standby_assignments (amf_sg_t *sg);
  381. static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg);
  382. static const char *sg_event_type_text[] = {
  383. "Unknown",
  384. "Failover SU",
  385. "Failover node",
  386. "Failover comp",
  387. "Switchover node",
  388. "Start",
  389. "Autoadjust",
  390. "Assign SI"
  391. };
  392. typedef struct sg_event {
  393. amf_sg_event_type_t event_type;
  394. amf_sg_t *sg;
  395. amf_su_t *su;
  396. amf_comp_t *comp;
  397. amf_node_t *node;
  398. } sg_event_t;
  399. /******************************************************************************
  400. * Internal (static) utility functions
  401. *****************************************************************************/
  402. static int is_cluster_start(amf_node_t *node_to_start)
  403. {
  404. return node_to_start == NULL;
  405. }
  406. static void sg_set_event (amf_sg_event_type_t sg_event_type,
  407. amf_sg_t *sg, amf_su_t *su, amf_comp_t *comp, amf_node_t * node,
  408. sg_event_t *sg_event)
  409. {
  410. sg_event->event_type = sg_event_type;
  411. sg_event->node = node;
  412. sg_event->su = su;
  413. sg_event->comp = comp;
  414. sg_event->sg = sg;
  415. }
  416. static void sg_defer_event (amf_sg_event_type_t event_type,
  417. sg_event_t *sg_event)
  418. {
  419. ENTER("Defered event = %d", event_type);
  420. amf_fifo_put (event_type, &sg_event->sg->deferred_events,
  421. sizeof (sg_event_t),
  422. sg_event);
  423. }
  424. static void sg_recall_deferred_events (amf_sg_t *sg)
  425. {
  426. sg_event_t sg_event;
  427. ENTER ("%s", sg->name.value);
  428. if (amf_fifo_get (&sg->deferred_events, &sg_event)) {
  429. switch (sg_event.event_type) {
  430. case SG_FAILOVER_SU_EV:
  431. amf_sg_failover_su_req (sg_event.sg,
  432. sg_event.su, sg_event.node);
  433. break;
  434. case SG_FAILOVER_NODE_EV:
  435. amf_sg_failover_node_req (sg_event.sg,
  436. sg_event.node);
  437. break;
  438. case SG_FAILOVER_COMP_EV:
  439. case SG_SWITCH_OVER_NODE_EV:
  440. case SG_START_EV:
  441. case SG_AUTO_ADJUST_EV:
  442. default:
  443. dprintf("event_type = %d", sg_event.event_type);
  444. break;
  445. }
  446. }
  447. }
  448. static void timer_function_sg_recall_deferred_events (void *data)
  449. {
  450. amf_sg_t *sg = (amf_sg_t*)data;
  451. ENTER ("");
  452. sg_recall_deferred_events (sg);
  453. }
  454. static void acsm_enter_idle (amf_sg_t *sg)
  455. {
  456. SaNameT dn;
  457. ENTER ("sg: %s state: %d", sg->name.value, sg->avail_state);
  458. sg->avail_state = SG_AC_Idle;
  459. if (sg->recovery_scope.event_type != 0) {
  460. switch (sg->recovery_scope.event_type) {
  461. case SG_FAILOVER_SU_EV:
  462. assert (sg->recovery_scope.sus[0] != NULL);
  463. amf_su_dn_make (sg->recovery_scope.sus[0], &dn);
  464. log_printf (
  465. LOG_NOTICE,
  466. "'%s' %s recovery action finished",
  467. dn.value,
  468. sg_event_type_text[sg->recovery_scope.event_type]);
  469. break;
  470. case SG_FAILOVER_NODE_EV:
  471. amf_node_sg_failed_over (
  472. sg->recovery_scope.node, sg);
  473. log_printf (
  474. LOG_NOTICE,
  475. "'%s for %s' recovery action finished",
  476. sg_event_type_text[sg->recovery_scope.event_type],
  477. sg->name.value);
  478. break;
  479. case SG_START_EV:
  480. amf_application_sg_started (sg->application,
  481. sg, this_amf_node);
  482. break;
  483. case SG_ASSIGN_SI_EV:
  484. log_printf (LOG_NOTICE, "All SI assigned");
  485. break;
  486. default:
  487. log_printf (
  488. LOG_NOTICE,
  489. "'%s' recovery action finished",
  490. sg_event_type_text[sg->recovery_scope.event_type]);
  491. break;
  492. }
  493. }
  494. if (sg->recovery_scope.sus != NULL) {
  495. free ((void *)sg->recovery_scope.sus);
  496. }
  497. if (sg->recovery_scope.sis != NULL) {
  498. free ((void *)sg->recovery_scope.sis);
  499. }
  500. memset (&sg->recovery_scope, 0, sizeof (struct sg_recovery_scope));
  501. sg->node_to_start = NULL;
  502. amf_call_function_asynchronous (
  503. timer_function_sg_recall_deferred_events, sg);
  504. }
  505. static int su_instantiated_count (struct amf_sg *sg)
  506. {
  507. int cnt = 0;
  508. struct amf_su *su;
  509. for (su = sg->su_head; su != NULL; su = su->next) {
  510. if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED)
  511. cnt++;
  512. }
  513. return cnt;
  514. }
  515. static int has_any_su_in_scope_active_workload (struct amf_sg *sg)
  516. {
  517. struct amf_su **sus= sg->recovery_scope.sus;
  518. struct amf_si_assignment *si_assignment;
  519. while (*sus != NULL) {
  520. si_assignment = amf_su_get_next_si_assignment (*sus, NULL);
  521. while (si_assignment != NULL) {
  522. if (si_assignment->saAmfSISUHAState !=
  523. SA_AMF_HA_ACTIVE) {
  524. break;
  525. }
  526. si_assignment = amf_su_get_next_si_assignment (
  527. *sus, si_assignment);
  528. }
  529. if (si_assignment != NULL) {
  530. break;
  531. }
  532. sus++;
  533. }
  534. return(*sus == NULL);
  535. }
  536. static int is_any_si_in_scope_assigned_standby (struct amf_sg *sg)
  537. {
  538. struct amf_si **sis= sg->recovery_scope.sis;
  539. struct amf_si_assignment *si_assignment;
  540. /*
  541. * Check if there is any si in the scope which has no
  542. * active assignment and at least one standby assignment.
  543. */
  544. while (*sis != NULL) {
  545. si_assignment = (*sis)->assigned_sis;
  546. while (si_assignment != NULL) {
  547. if (si_assignment->saAmfSISUHAState ==
  548. SA_AMF_HA_ACTIVE) {
  549. break;
  550. }
  551. si_assignment = si_assignment->next;
  552. }
  553. if (si_assignment == NULL) {
  554. /* There is no ACTIVE assignment ..*/
  555. si_assignment = (*sis)->assigned_sis;
  556. while (si_assignment != NULL) {
  557. if (si_assignment->saAmfSISUHAState ==
  558. SA_AMF_HA_STANDBY) {
  559. break;
  560. }
  561. si_assignment = si_assignment->next;
  562. }
  563. if (si_assignment != NULL) {
  564. /* .. and one STANDBY assignment*/
  565. break;
  566. }
  567. }
  568. sis++;
  569. }
  570. return(*sis != NULL);
  571. }
  572. static void acsm_enter_terminating_suspected (struct amf_sg *sg)
  573. {
  574. struct amf_su **sus= sg->recovery_scope.sus;
  575. ENTER("%s",sg->name.value);
  576. sg->avail_state = SG_AC_TerminatingSuspected;
  577. /*
  578. * Terminate suspected SU(s)
  579. */
  580. while (*sus != 0) {
  581. amf_su_terminate (*sus);
  582. sus++;
  583. }
  584. }
  585. static inline int su_presense_state_is_ored (amf_su_t *su,
  586. SaAmfPresenceStateT state1,SaAmfPresenceStateT state2,
  587. SaAmfPresenceStateT state3)
  588. {
  589. return(su->saAmfSUPresenceState == state1 || su->saAmfSUPresenceState ==
  590. state2 || su->saAmfSUPresenceState == state3) ? 1 : 0;
  591. }
  592. static inline int su_presense_state_is_not (amf_su_t *su,
  593. SaAmfPresenceStateT state1,SaAmfPresenceStateT state2,
  594. SaAmfPresenceStateT state3)
  595. {
  596. return(su->saAmfSUPresenceState != state1 && su->saAmfSUPresenceState !=
  597. state2 && su->saAmfSUPresenceState != state3) ? 1 : 0;
  598. }
  599. static void timer_function_dependent_si_deactivated2 (void *data)
  600. {
  601. amf_sg_t *sg = (amf_sg_t *)data;
  602. ENTER ("");
  603. dependent_si_deactivated_cbfn2 (sg);
  604. }
  605. static struct amf_si *si_get_dependent (struct amf_si *si)
  606. {
  607. struct amf_si *tmp_si = NULL;
  608. if (si->depends_on != NULL) {
  609. SaNameT res_arr[2];
  610. int is_match;
  611. if (si->depends_on->name.length < SA_MAX_NAME_LENGTH) {
  612. si->depends_on->name.value[si->depends_on->name.length] = '\0';
  613. }
  614. is_match = sa_amf_grep ((char*)si->depends_on->name.value,
  615. "safDepend=.*,safSi=(.*),safApp=.*",
  616. 2, res_arr);
  617. if (is_match) {
  618. tmp_si = amf_si_find (si->application, (char*)res_arr[1].value);
  619. } else {
  620. log_printf (LOG_LEVEL_ERROR, "distinguished name for "
  621. "amf_si_depedency failed\n");
  622. openais_exit_error (AIS_DONE_FATAL_ERR);
  623. }
  624. }
  625. return tmp_si;
  626. }
  627. static struct amf_si *amf_dependent_get_next (struct amf_si *si,
  628. struct amf_si *si_iter)
  629. {
  630. struct amf_si *tmp_si;
  631. struct amf_application *application;
  632. if (si_iter == NULL) {
  633. assert(amf_cluster != NULL);
  634. application = amf_cluster->application_head;
  635. assert(application != NULL);
  636. tmp_si = application->si_head;
  637. } else {
  638. tmp_si = si_iter->next;
  639. if (tmp_si == NULL) {
  640. application = si->application->next;
  641. if (application == NULL) {
  642. goto out;
  643. }
  644. }
  645. }
  646. for (; tmp_si != NULL; tmp_si = tmp_si->next) {
  647. struct amf_si *depends_on_si = si_get_dependent (tmp_si);
  648. while (depends_on_si != NULL) {
  649. if (depends_on_si == si) {
  650. goto out;
  651. }
  652. depends_on_si = depends_on_si->next;
  653. }
  654. }
  655. out:
  656. return tmp_si;
  657. }
  658. static void acsm_enter_deactivating_dependent_workload (amf_sg_t *sg)
  659. {
  660. struct amf_si **sis= sg->recovery_scope.sis;
  661. struct amf_si_assignment *si_assignment;
  662. int callback_pending = 0;
  663. sg->avail_state = SG_AC_DeactivatingDependantWorkload;
  664. ENTER("'%s'",sg->name.value);
  665. /*
  666. * For each SI in the recovery scope, find all active
  667. * assignments and request them to be deactivated.
  668. */
  669. while (*sis != NULL) {
  670. struct amf_si *dependent_si;
  671. struct amf_si *si = *sis;
  672. si_assignment = si->assigned_sis;
  673. dependent_si = amf_dependent_get_next (si, NULL);
  674. while (dependent_si != NULL) {
  675. si_assignment = dependent_si->assigned_sis;
  676. while (si_assignment != NULL) {
  677. if (si_assignment->saAmfSISUHAState ==
  678. SA_AMF_HA_ACTIVE) {
  679. si_assignment->requested_ha_state =
  680. SA_AMF_HA_QUIESCED;
  681. callback_pending = 1;
  682. amf_si_ha_state_assume (
  683. si_assignment,
  684. dependent_si_deactivated_cbfn);
  685. }
  686. si_assignment = si_assignment->next;
  687. }
  688. dependent_si = amf_dependent_get_next (si, dependent_si);
  689. }
  690. sis++;
  691. }
  692. if (callback_pending == 0) {
  693. static poll_timer_handle dependent_si_deactivated_handle;
  694. ENTER("");
  695. poll_timer_add (aisexec_poll_handle, 0, sg,
  696. timer_function_dependent_si_deactivated2,
  697. &dependent_si_deactivated_handle);
  698. }
  699. }
  700. /**
  701. * Enter function for state SG_AC_ActivatingStandby. It activates
  702. * one STANDBY assignment for each SI in the recovery scope.
  703. * @param sg
  704. */
  705. static void acsm_enter_activating_standby (struct amf_sg *sg)
  706. {
  707. struct amf_si **sis= sg->recovery_scope.sis;
  708. struct amf_si_assignment *si_assignment;
  709. int is_no_standby_activated = 1;
  710. ENTER("'%s'",sg->name.value);
  711. sg->avail_state = SG_AC_ActivatingStandby;
  712. /*
  713. * For each SI in the recovery scope, find one standby
  714. * SI assignment and activate it.
  715. */
  716. while (*sis != NULL) {
  717. si_assignment = (*sis)->assigned_sis;
  718. while (si_assignment != NULL) {
  719. if (si_assignment->saAmfSISUHAState ==
  720. SA_AMF_HA_STANDBY) {
  721. si_assignment->requested_ha_state =
  722. SA_AMF_HA_ACTIVE;
  723. amf_si_ha_state_assume (
  724. si_assignment, standby_su_activated_cbfn);
  725. is_no_standby_activated = 0;
  726. break;
  727. }
  728. si_assignment = si_assignment->next;
  729. }
  730. sis++;
  731. }
  732. if (is_no_standby_activated) {
  733. acsm_enter_assigning_standby_to_spare (sg);
  734. }
  735. }
  736. static void acsm_enter_repairing_su (struct amf_sg *sg)
  737. {
  738. struct amf_su **sus= sg->recovery_scope.sus;
  739. int is_any_su_instantiated = 0;
  740. const int PERFORMS_INSTANTIATING = 1;
  741. ENTER("'%s'",sg->name.value);
  742. sg->avail_state = SG_AC_ReparingSu;
  743. /*
  744. * Instantiate SUs in current recovery scope until the configured
  745. * preference is fulfiled.
  746. */
  747. while (*sus != NULL) {
  748. if (su_instantiated_count ((*sus)->sg) <
  749. (*sus)->sg->saAmfSGNumPrefInserviceSUs) {
  750. struct amf_node *node =
  751. amf_node_find(&((*sus)->saAmfSUHostedByNode));
  752. if (node == NULL) {
  753. log_printf (LOG_LEVEL_ERROR,
  754. "Su to recover not hosted on any node\n");
  755. openais_exit_error (AIS_DONE_FATAL_ERR);
  756. }
  757. if (node->saAmfNodeOperState == SA_AMF_OPERATIONAL_ENABLED) {
  758. /* node is synchronized */
  759. if (amf_su_instantiate ((*sus)) == PERFORMS_INSTANTIATING) {
  760. is_any_su_instantiated = 1;
  761. }
  762. }
  763. }
  764. sus++;
  765. }
  766. if (is_any_su_instantiated == 0) {
  767. acsm_enter_idle (sg);
  768. }
  769. }
  770. static inline void remove_all_suspected_sus (amf_sg_t *sg)
  771. {
  772. amf_su_t *su;
  773. ENTER("");
  774. for (su = sg->su_head; su != NULL; su =su->next) {
  775. amf_comp_t *component;
  776. for (component = su->comp_head; component != NULL;
  777. component = component->next) {
  778. amf_comp_error_suspected_clear (component);
  779. }
  780. }
  781. }
  782. static int is_all_si_assigned (amf_sg_t *sg)
  783. {
  784. struct amf_si_assignment *si_assignment;
  785. int si_assignment_cnt = 0;
  786. int confirmed_assignments = 0;
  787. amf_si_t *si;
  788. for (si = sg->application->si_head; si != NULL; si = si->next) {
  789. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  790. for (si_assignment = si->assigned_sis;
  791. si_assignment != NULL;
  792. si_assignment = si_assignment->next) {
  793. si_assignment_cnt++;
  794. if (si_assignment->requested_ha_state ==
  795. si_assignment->saAmfSISUHAState) {
  796. confirmed_assignments++;
  797. }
  798. }
  799. }
  800. }
  801. return (confirmed_assignments == si_assignment_cnt);
  802. }
  803. /**
  804. * Inquire if SI is assigned to SU
  805. * @param si
  806. * @param su
  807. *
  808. * @return int
  809. */
  810. static int is_si_assigned_to_su (amf_si_t *si, amf_su_t *su)
  811. {
  812. amf_si_assignment_t *si_assignment = 0;
  813. int si_assignment_assigned_to_su = 0;
  814. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  815. si_assignment = si_assignment->next) {
  816. if (si_assignment->su == su) {
  817. si_assignment_assigned_to_su = 1;
  818. break;
  819. }
  820. }
  821. return si_assignment_assigned_to_su;
  822. }
  823. /**
  824. * Inquire if SU is a spare.
  825. * @param sg
  826. * @param su
  827. *
  828. * @return int
  829. */
  830. static int is_spare_su (amf_sg_t *sg, amf_su_t *su)
  831. {
  832. amf_si_t *si;
  833. int spare_su = 1;
  834. for (si = sg->application->si_head; si != NULL; si = si->next) {
  835. if(name_match(&sg->name, &si->saAmfSIProtectedbySG)) {
  836. if (is_si_assigned_to_su (si, su)) {
  837. spare_su = 0;
  838. break;
  839. }
  840. }
  841. }
  842. return (spare_su && su->saAmfSUPresenceState ==
  843. SA_AMF_PRESENCE_INSTANTIATED);
  844. }
  845. /**
  846. * Inqure if it is any spare SUs covered by SG
  847. * @param sg
  848. *
  849. * @return int
  850. */
  851. static int is_spare_sus (amf_sg_t *sg)
  852. {
  853. amf_su_t *su = NULL;
  854. int spare_sus = 0;
  855. for (su = sg->su_head; su != NULL; su = su->next) {
  856. if (is_spare_su(sg, su)) {
  857. spare_sus = 1;
  858. break;
  859. }
  860. }
  861. return spare_sus;
  862. }
  863. /**
  864. * Provide standby assignments for the spare SUs in SG
  865. * @param sg
  866. */
  867. static void assume_standby_si_assignment_for_spare_sus (amf_sg_t *sg)
  868. {
  869. ENTER("");
  870. assign_si (sg, 0);
  871. }
  872. /**
  873. * Enter the AssigningStandbyToSpare state.
  874. * @param sg
  875. */
  876. static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg)
  877. {
  878. ENTER("%s", sg->name.value);
  879. sg->avail_state = SG_AC_AssigningStandbyToSpare;
  880. if (is_spare_sus (sg)) {
  881. assume_standby_si_assignment_for_spare_sus (sg);
  882. } else {
  883. switch (sg->recovery_scope.event_type) {
  884. case SG_FAILOVER_NODE_EV:
  885. acsm_enter_idle (sg);
  886. break;
  887. case SG_FAILOVER_SU_EV:
  888. acsm_enter_repairing_su (sg);
  889. break;
  890. default:
  891. dprintf("event_type %d",sg->recovery_scope.event_type);
  892. assert (0);
  893. break;
  894. }
  895. }
  896. }
  897. /**
  898. * Checks if the si pointed out is already in the scope.
  899. * @param sg
  900. * @param si
  901. */
  902. static int is_si_in_scope(struct amf_sg *sg, struct amf_si *si)
  903. {
  904. struct amf_si **tmp_sis= sg->recovery_scope.sis;
  905. while (*tmp_sis != NULL) {
  906. if (*tmp_sis == si) {
  907. break;
  908. }
  909. tmp_sis++;
  910. }
  911. return(*tmp_sis == si);
  912. }
  913. /**
  914. * Adds the si pointed out to the scope.
  915. * @param sg
  916. * @param si
  917. */
  918. static void add_si_to_scope ( struct amf_sg *sg, struct amf_si *si)
  919. {
  920. int number_of_si = 2; /* It shall be at least two */
  921. struct amf_si **tmp_sis= sg->recovery_scope.sis;
  922. ENTER ("'%s'", si->name.value);
  923. while (*tmp_sis != NULL) {
  924. number_of_si++;
  925. tmp_sis++;
  926. }
  927. sg->recovery_scope.sis = (struct amf_si **)
  928. realloc((void *)sg->recovery_scope.sis,
  929. sizeof (struct amf_si *)*number_of_si);
  930. assert (sg->recovery_scope.sis != NULL);
  931. tmp_sis= sg->recovery_scope.sis;
  932. while (*tmp_sis != NULL) {
  933. tmp_sis++;
  934. }
  935. *tmp_sis = si;
  936. *(++tmp_sis) = NULL;
  937. }
  938. /**
  939. * Adds the ssu pointed out to the scope.
  940. * @param sg
  941. * @param su
  942. */
  943. static void add_su_to_scope (struct amf_sg *sg, struct amf_su *su)
  944. {
  945. int number_of_su = 2; /* It shall be at least two */
  946. struct amf_su **tmp_sus= sg->recovery_scope.sus;
  947. ENTER ("'%s'", su->name.value);
  948. while (*tmp_sus != NULL) {
  949. number_of_su++;
  950. tmp_sus++;
  951. }
  952. sg->recovery_scope.sus = (struct amf_su **)
  953. realloc((void *)sg->recovery_scope.sus,
  954. sizeof (struct amf_su *)*number_of_su);
  955. assert (sg->recovery_scope.sus != NULL);
  956. tmp_sus= sg->recovery_scope.sus;
  957. while (*tmp_sus != NULL) {
  958. tmp_sus++;
  959. }
  960. *tmp_sus = su;
  961. *(++tmp_sus) = NULL;
  962. }
  963. /**
  964. * Set recovery scope for failover SU.
  965. * @param sg
  966. * @param su
  967. */
  968. static void set_scope_for_failover_su (struct amf_sg *sg, struct amf_su *su)
  969. {
  970. struct amf_si_assignment *si_assignment;
  971. struct amf_si **sis;
  972. struct amf_su **sus;
  973. SaNameT dn;
  974. sg->recovery_scope.event_type = SG_FAILOVER_SU_EV;
  975. sg->recovery_scope.node = NULL;
  976. sg->recovery_scope.comp = NULL;
  977. sg->recovery_scope.sus = (struct amf_su **)
  978. calloc (2, sizeof (struct amf_su *));
  979. sg->recovery_scope.sis = (struct amf_si **)
  980. calloc (1, sizeof (struct amf_si *));
  981. assert ((sg->recovery_scope.sus != NULL) &&
  982. (sg->recovery_scope.sis != NULL));
  983. sg->recovery_scope.sus[0] = su;
  984. amf_su_dn_make (sg->recovery_scope.sus[0], &dn);
  985. log_printf (
  986. LOG_NOTICE, "'%s' for %s recovery action started",
  987. sg_event_type_text[sg->recovery_scope.event_type],
  988. dn.value);
  989. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  990. while (si_assignment != NULL) {
  991. if (is_si_in_scope(sg, si_assignment->si) == 0) {
  992. add_si_to_scope(sg,si_assignment->si );
  993. }
  994. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  995. }
  996. sus = sg->recovery_scope.sus;
  997. dprintf("The following sus are within the scope:\n");
  998. while (*sus != NULL) {
  999. dprintf("%s\n", (*sus)->name.value);
  1000. sus++;
  1001. }
  1002. sis= sg->recovery_scope.sis;
  1003. dprintf("The following sis are within the scope:\n");
  1004. while (*sis != NULL) {
  1005. dprintf("%s\n", (*sis)->name.value);
  1006. sis++;
  1007. }
  1008. }
  1009. static void set_scope_for_failover_node (struct amf_sg *sg, struct amf_node *node)
  1010. {
  1011. struct amf_si_assignment *si_assignment;
  1012. struct amf_si **sis;
  1013. struct amf_su **sus;
  1014. struct amf_su *su;
  1015. ENTER ("'%s'", node->name.value);
  1016. sg->recovery_scope.event_type = SG_FAILOVER_NODE_EV;
  1017. sg->recovery_scope.node = node;
  1018. sg->recovery_scope.comp = NULL;
  1019. sg->recovery_scope.sus = (struct amf_su **)
  1020. calloc (1, sizeof (struct amf_su *));
  1021. sg->recovery_scope.sis = (struct amf_si **)
  1022. calloc (1, sizeof (struct amf_si *));
  1023. log_printf (
  1024. LOG_NOTICE, "'%s' for node %s recovery action started",
  1025. sg_event_type_text[sg->recovery_scope.event_type],
  1026. node->name.value);
  1027. assert ((sg->recovery_scope.sus != NULL) &&
  1028. (sg->recovery_scope.sis != NULL));
  1029. for (su = sg->su_head; su != NULL; su = su->next) {
  1030. if (name_match (&node->name, &su->saAmfSUHostedByNode)) {
  1031. add_su_to_scope (sg, su);
  1032. }
  1033. }
  1034. sus = sg->recovery_scope.sus;
  1035. while (*sus != 0) {
  1036. su = *sus;
  1037. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  1038. while (si_assignment != NULL) {
  1039. if (is_si_in_scope(sg, si_assignment->si) == 0) {
  1040. add_si_to_scope(sg, si_assignment->si );
  1041. }
  1042. si_assignment = amf_su_get_next_si_assignment (
  1043. su, si_assignment);
  1044. }
  1045. sus++;
  1046. }
  1047. sus = sg->recovery_scope.sus;
  1048. dprintf("The following sus are within the scope:\n");
  1049. while (*sus != NULL) {
  1050. dprintf("%s\n", (*sus)->name.value);
  1051. sus++;
  1052. }
  1053. sis = sg->recovery_scope.sis;
  1054. dprintf("The following sis are within the scope:\n");
  1055. while (*sis != NULL) {
  1056. dprintf("%s\n", (*sis)->name.value);
  1057. sis++;
  1058. }
  1059. }
  1060. static void delete_si_assignment (amf_si_assignment_t *si_assignment)
  1061. {
  1062. amf_csi_t *csi;
  1063. amf_si_assignment_t *si_assignment_tmp;
  1064. amf_si_assignment_t **prev = &si_assignment->si->assigned_sis;
  1065. for (csi = si_assignment->si->csi_head; csi != NULL; csi = csi->next) {
  1066. amf_csi_delete_assignments (csi, si_assignment->su);
  1067. }
  1068. for (si_assignment_tmp = si_assignment->si->assigned_sis;
  1069. si_assignment_tmp != NULL;
  1070. si_assignment_tmp = si_assignment_tmp->next) {
  1071. if (si_assignment_tmp == si_assignment) {
  1072. amf_si_assignment_t *to_be_removed = si_assignment_tmp;
  1073. *prev = si_assignment_tmp->next;
  1074. dprintf ("SI assignment %s unlinked",
  1075. to_be_removed->name.value);
  1076. free (to_be_removed);
  1077. } else {
  1078. prev = &si_assignment_tmp->next;
  1079. }
  1080. }
  1081. }
  1082. /**
  1083. * Delete all SI assignments and all CSI assignments
  1084. * by requesting all contained components.
  1085. * @param su
  1086. */
  1087. static void delete_si_assignments (struct amf_su *su)
  1088. {
  1089. struct amf_csi *csi;
  1090. struct amf_si *si;
  1091. struct amf_si_assignment *si_assignment;
  1092. struct amf_si_assignment **prev;
  1093. ENTER ("'%s'", su->name.value);
  1094. for (si = su->sg->application->si_head; si != NULL; si = si->next) {
  1095. prev = &si->assigned_sis;
  1096. if (!name_match (&si->saAmfSIProtectedbySG, &su->sg->name)) {
  1097. continue;
  1098. }
  1099. for (csi = si->csi_head; csi != NULL; csi = csi->next) {
  1100. amf_csi_delete_assignments (csi, su);
  1101. }
  1102. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  1103. si_assignment = si_assignment->next) {
  1104. if (si_assignment->su == su) {
  1105. struct amf_si_assignment *tmp = si_assignment;
  1106. *prev = si_assignment->next;
  1107. dprintf ("SI assignment %s unlinked", tmp->name.value);
  1108. free (tmp);
  1109. } else {
  1110. prev = &si_assignment->next;
  1111. }
  1112. }
  1113. }
  1114. }
  1115. /**
  1116. * Delete all SI assignments and all CSI assignments in current
  1117. * recovery scope.
  1118. * @param sg
  1119. */
  1120. static void delete_si_assignments_in_scope (struct amf_sg *sg)
  1121. {
  1122. struct amf_su **sus= sg->recovery_scope.sus;
  1123. while (*sus != NULL) {
  1124. delete_si_assignments (*sus);
  1125. sus++;
  1126. }
  1127. }
  1128. /**
  1129. * Given an SI, find and return the SU assigned as standby
  1130. * @param si
  1131. *
  1132. * @return amf_su_t*
  1133. */
  1134. static amf_su_t *find_standby_su (amf_si_t *si)
  1135. {
  1136. amf_si_assignment_t *si_assignment;
  1137. amf_su_t *standby_su = NULL;
  1138. si_assignment = si->assigned_sis;
  1139. while (si_assignment != NULL) {
  1140. if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
  1141. standby_su = si_assignment->su;
  1142. break;
  1143. }
  1144. si_assignment = si_assignment->next;
  1145. }
  1146. return standby_su;
  1147. }
  1148. static int no_si_assignment_is_requested_to_be_removed (amf_sg_t *sg)
  1149. {
  1150. amf_si_t *si;
  1151. int no_to_be_removed = 1;
  1152. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1153. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1154. amf_si_assignment_t *si_assignment = 0;
  1155. for (si_assignment = si->assigned_sis; si_assignment != NULL;
  1156. si_assignment = si_assignment->next) {
  1157. if (si_assignment->requested_ha_state ==
  1158. USR_AMF_HA_STATE_REMOVED) {
  1159. no_to_be_removed = 0;
  1160. goto out;
  1161. }
  1162. }
  1163. }
  1164. }
  1165. out:
  1166. return no_to_be_removed;
  1167. }
  1168. static void removed_si_assignment_callback_fn (void *si_assignment_in)
  1169. {
  1170. amf_si_assignment_t *si_assignment = si_assignment_in;
  1171. ENTER("");
  1172. delete_si_assignment (si_assignment);
  1173. /*
  1174. * if all si assignments are remove then change state
  1175. */
  1176. if (no_si_assignment_is_requested_to_be_removed (si_assignment->su->sg)) {
  1177. acsm_enter_activating_standby (si_assignment->su->sg);
  1178. }
  1179. }
  1180. /**
  1181. *
  1182. * @param sg
  1183. *
  1184. * @return int, number of removed SI assignments
  1185. */
  1186. static int remove_standby_si_assignments (amf_sg_t *sg)
  1187. {
  1188. struct amf_si **sis = sg->recovery_scope.sis;
  1189. struct amf_si_assignment *si_assignment;
  1190. amf_su_t *standby_su;
  1191. int removed = 0;
  1192. ENTER("'%s'", sg->name.value);
  1193. /*
  1194. * For each SI in the recovery scope, find a standby
  1195. * SU, then remove all 'standby SI assignment' not in
  1196. * the recovery scope.
  1197. */
  1198. while (*sis != NULL) {
  1199. standby_su = find_standby_su (*sis);
  1200. if (standby_su != NULL) {
  1201. si_assignment = amf_su_get_next_si_assignment (standby_su, NULL);
  1202. while (si_assignment != NULL) {
  1203. amf_si_t **sia;
  1204. int in_recovery_scope;
  1205. for (sia = sg->recovery_scope.sis, in_recovery_scope = 0;
  1206. *sia != NULL; sia++) {
  1207. if (name_match (&si_assignment->si->name, &(*sia)->name)) {
  1208. in_recovery_scope = 1;
  1209. }
  1210. }
  1211. /*
  1212. * The si_assignment found with standby hastate is not in the
  1213. * recovery scope. The found si_assignment will then be
  1214. * requested to be removed once.
  1215. */
  1216. if (!in_recovery_scope &&
  1217. si_assignment->requested_ha_state !=
  1218. USR_AMF_HA_STATE_REMOVED) {
  1219. amf_si_assignment_remove (si_assignment,
  1220. removed_si_assignment_callback_fn);
  1221. removed++;
  1222. }
  1223. si_assignment = amf_su_get_next_si_assignment (standby_su,
  1224. si_assignment);
  1225. }
  1226. }
  1227. sis++;
  1228. }
  1229. return removed;
  1230. }
  1231. /**
  1232. * Entry function for state 'removing standby assignments'
  1233. * @param sg
  1234. */
  1235. static void acsm_enter_removing_standby_assignments (amf_sg_t *sg)
  1236. {
  1237. ENTER("SG: %s", sg->name.value);
  1238. sg->avail_state = SG_AC_RemovingStandbyAssignments;
  1239. if (sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) {
  1240. if (!remove_standby_si_assignments (sg)) {
  1241. acsm_enter_activating_standby (sg);
  1242. }
  1243. }
  1244. }
  1245. static inline int div_round (int a, int b)
  1246. {
  1247. int res;
  1248. assert (b != 0);
  1249. res = a / b;
  1250. if ((a % b) != 0)
  1251. res++;
  1252. return res;
  1253. }
  1254. static int no_su_has_presence_state (
  1255. struct amf_sg *sg, struct amf_node *node_to_start,
  1256. SaAmfPresenceStateT state)
  1257. {
  1258. struct amf_su *su;
  1259. int no_su_has_presence_state = 1;
  1260. for (su = sg->su_head; su != NULL; su = su->next) {
  1261. if (su->saAmfSUPresenceState == state) {
  1262. if (node_to_start == NULL) {
  1263. no_su_has_presence_state = 0;
  1264. break;
  1265. } else {
  1266. if (name_match(&node_to_start->name,
  1267. &su->saAmfSUHostedByNode)) {
  1268. no_su_has_presence_state = 0;
  1269. break;
  1270. }
  1271. }
  1272. }
  1273. }
  1274. return no_su_has_presence_state;
  1275. }
  1276. #if COMPILE_OUT
  1277. static int all_su_in_scope_has_presence_state (
  1278. struct amf_sg *sg, SaAmfPresenceStateT state)
  1279. {
  1280. struct amf_su **sus= sg->recovery_scope.sus;
  1281. while (*sus != NULL) {
  1282. if ((*sus)->saAmfSUPresenceState != state) {
  1283. break;
  1284. }
  1285. sus++;
  1286. }
  1287. return(*sus == NULL);
  1288. }
  1289. #endif
  1290. static int all_su_in_scope_has_either_two_presence_state (
  1291. amf_sg_t *sg,
  1292. SaAmfPresenceStateT state1,
  1293. SaAmfPresenceStateT state2)
  1294. {
  1295. struct amf_su **sus = sg->recovery_scope.sus;
  1296. while (*sus != NULL) {
  1297. if (!((*sus)->saAmfSUPresenceState == state1 ||
  1298. (*sus)->saAmfSUPresenceState == state2)) {
  1299. break;
  1300. }
  1301. sus++;
  1302. }
  1303. return (*sus == NULL);
  1304. }
  1305. static int all_su_in_scope_has_either_of_three_presence_state (amf_sg_t *sg,
  1306. SaAmfPresenceStateT state1, SaAmfPresenceStateT state2,
  1307. SaAmfPresenceStateT state3)
  1308. {
  1309. struct amf_su **sus = sg->recovery_scope.sus;
  1310. while (*sus != NULL) {
  1311. if (!((*sus)->saAmfSUPresenceState == state1 ||
  1312. (*sus)->saAmfSUPresenceState == state2 ||
  1313. (*sus)->saAmfSUPresenceState == state3)) {
  1314. break;
  1315. }
  1316. sus++;
  1317. }
  1318. return (*sus == NULL);
  1319. }
  1320. /**
  1321. * Get number of SIs protected by the specified SG.
  1322. * @param sg
  1323. *
  1324. * @return int
  1325. */
  1326. static int sg_si_count_get (struct amf_sg *sg)
  1327. {
  1328. struct amf_si *si;
  1329. int cnt = 0;
  1330. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1331. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1332. cnt += 1;
  1333. }
  1334. }
  1335. return(cnt);
  1336. }
  1337. static int amf_si_get_saAmfSINumReqActiveAssignments(struct amf_si *si)
  1338. {
  1339. struct amf_si_assignment *si_assignment = si->assigned_sis;
  1340. int number_of_req_active_assignments = 0;
  1341. for (; si_assignment != NULL; si_assignment = si_assignment->next) {
  1342. if (si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) {
  1343. number_of_req_active_assignments++;
  1344. }
  1345. }
  1346. return number_of_req_active_assignments;
  1347. }
  1348. static int amf_si_get_saAmfSINumReqStandbyAssignments(struct amf_si *si)
  1349. {
  1350. struct amf_si_assignment *si_assignment = si->assigned_sis;
  1351. int number_of_req_active_assignments = 0;
  1352. for (; si_assignment != NULL; si_assignment = si_assignment->next) {
  1353. if (si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) {
  1354. number_of_req_active_assignments++;
  1355. }
  1356. }
  1357. return number_of_req_active_assignments;
  1358. }
  1359. static int sg_assign_active_nplusm (struct amf_sg *sg, int su_active_assign)
  1360. {
  1361. struct amf_su *su;
  1362. struct amf_si *si;
  1363. int assigned = 0;
  1364. int assign_to_su = 0;
  1365. int total_assigned = 0;
  1366. int si_left;
  1367. int si_total;
  1368. int su_left_to_assign = su_active_assign;
  1369. ENTER("SG: %s", sg->name.value);
  1370. si_total = sg_si_count_get (sg);
  1371. si_left = si_total;
  1372. assign_to_su = div_round (si_left, su_active_assign);
  1373. if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) {
  1374. assign_to_su = sg->saAmfSGMaxActiveSIsperSUs;
  1375. }
  1376. su = sg->su_head;
  1377. while (su != NULL && su_left_to_assign > 0) {
  1378. if (amf_su_get_saAmfSUReadinessState (su) !=
  1379. SA_AMF_READINESS_IN_SERVICE ||
  1380. amf_su_get_saAmfSUNumCurrActiveSIs (su) == assign_to_su ||
  1381. amf_su_get_saAmfSUNumCurrStandbySIs (su) > 0) {
  1382. su = su->next;
  1383. continue; /* Not in service */
  1384. }
  1385. si = sg->application->si_head;
  1386. assigned = 0;
  1387. assign_to_su = div_round (si_left, su_left_to_assign);
  1388. if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) {
  1389. assign_to_su = sg->saAmfSGMaxActiveSIsperSUs;
  1390. }
  1391. while (si != NULL) {
  1392. if (name_match (&si->saAmfSIProtectedbySG, &sg->name) &&
  1393. assigned < assign_to_su &&
  1394. amf_si_get_saAmfSINumReqActiveAssignments(si) == 0) {
  1395. assigned += 1;
  1396. total_assigned += 1;
  1397. amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE);
  1398. }
  1399. si = si->next;
  1400. }
  1401. su = su->next;
  1402. su_left_to_assign -= 1;
  1403. si_left -= assigned;
  1404. dprintf (" su_left_to_assign =%d, si_left=%d\n",
  1405. su_left_to_assign, si_left);
  1406. }
  1407. assert (total_assigned <= si_total);
  1408. if (total_assigned == 0) {
  1409. dprintf ("Info: No SIs assigned");
  1410. }
  1411. return total_assigned;
  1412. }
  1413. static int sg_assign_standby_nplusm (struct amf_sg *sg, int su_standby_assign)
  1414. {
  1415. struct amf_su *su;
  1416. struct amf_si *si;
  1417. int assigned = 0;
  1418. int assign_to_su = 0;
  1419. int total_assigned = 0;
  1420. int si_left;
  1421. int si_total;
  1422. int su_left_to_assign = su_standby_assign;
  1423. ENTER ("'%s'", sg->name.value);
  1424. if (su_standby_assign == 0) {
  1425. return 0;
  1426. }
  1427. si_total = sg_si_count_get (sg);
  1428. si_left = si_total;
  1429. assign_to_su = div_round (si_left, su_standby_assign);
  1430. if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) {
  1431. assign_to_su = sg->saAmfSGMaxStandbySIsperSUs;
  1432. }
  1433. su = sg->su_head;
  1434. while (su != NULL && su_left_to_assign > 0) {
  1435. if (amf_su_get_saAmfSUReadinessState (su) !=
  1436. SA_AMF_READINESS_IN_SERVICE ||
  1437. amf_su_get_saAmfSUNumCurrActiveSIs (su) > 0 ||
  1438. amf_su_get_saAmfSUNumCurrStandbySIs (su) ==
  1439. assign_to_su) {
  1440. su = su->next;
  1441. continue; /* Not available for assignment */
  1442. }
  1443. si = sg->application->si_head;
  1444. assigned = 0;
  1445. assign_to_su = div_round (si_left, su_left_to_assign);
  1446. if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) {
  1447. assign_to_su = sg->saAmfSGMaxStandbySIsperSUs;
  1448. }
  1449. while (si != NULL) {
  1450. if (name_match (&si->saAmfSIProtectedbySG, &sg->name) &&
  1451. assigned < assign_to_su &&
  1452. amf_si_get_saAmfSINumReqStandbyAssignments (si) == 0) {
  1453. assigned += 1;
  1454. total_assigned += 1;
  1455. amf_su_assign_si (su, si, SA_AMF_HA_STANDBY);
  1456. }
  1457. si = si->next;
  1458. }
  1459. su_left_to_assign -= 1;
  1460. si_left -= assigned;
  1461. dprintf (" su_left_to_assign =%d, si_left=%d\n",
  1462. su_left_to_assign, si_left);
  1463. su = su->next;
  1464. }
  1465. assert (total_assigned <= si_total);
  1466. if (total_assigned == 0) {
  1467. dprintf ("Info: No SIs assigned!");
  1468. }
  1469. return total_assigned;
  1470. }
  1471. static int su_inservice_count_get (struct amf_sg *sg)
  1472. {
  1473. struct amf_su *su;
  1474. int answer = 0;
  1475. for (su = sg->su_head; su != NULL; su = su->next) {
  1476. if (amf_su_get_saAmfSUReadinessState (su) ==
  1477. SA_AMF_READINESS_IN_SERVICE) {
  1478. answer += 1;
  1479. }
  1480. }
  1481. return(answer);
  1482. }
  1483. static int su_active_out_of_service_count_get (amf_sg_t *sg)
  1484. {
  1485. int active_out_of_service_count = 0;
  1486. amf_su_t *su;
  1487. for (su = sg->su_head; su != NULL; su = su->next) {
  1488. amf_si_assignment_t *si_assignment;
  1489. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  1490. while (si_assignment != NULL) {
  1491. if ((si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) &&
  1492. (amf_su_get_saAmfSUReadinessState (su) ==
  1493. SA_AMF_READINESS_OUT_OF_SERVICE)) {
  1494. active_out_of_service_count += 1;
  1495. }
  1496. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  1497. }
  1498. }
  1499. return active_out_of_service_count;
  1500. }
  1501. static int su_standby_out_of_service_count_get (amf_sg_t *sg)
  1502. {
  1503. int active_out_of_service_count = 0;
  1504. amf_su_t *su;
  1505. for (su = sg->su_head; su != NULL; su = su->next) {
  1506. amf_si_assignment_t *si_assignment;
  1507. si_assignment = amf_su_get_next_si_assignment (su, NULL);
  1508. while (si_assignment != NULL) {
  1509. if ((si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) &&
  1510. (amf_su_get_saAmfSUReadinessState (su) ==
  1511. SA_AMF_READINESS_OUT_OF_SERVICE)) {
  1512. active_out_of_service_count += 1;
  1513. }
  1514. si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
  1515. }
  1516. }
  1517. return active_out_of_service_count;
  1518. }
  1519. /**
  1520. * This function calculates the number of active and standby assignments that
  1521. * shall be done according to what is configured and the number of in-service
  1522. * SUs available. This function leaves possible existing assignments as they are
  1523. * but possibly adds new assignments. This function also initiates the transfer
  1524. * of the calculated assignments to the SUs that shall execute them.
  1525. *
  1526. * TODO: dependency_level not used, hard coded
  1527. * @param sg
  1528. * @param dependency_level
  1529. * @return - the sum of assignments initiated
  1530. */
  1531. static int assign_si (struct amf_sg *sg, int dependency_level)
  1532. {
  1533. int active_sus_needed = 0;
  1534. int standby_sus_needed = 0;
  1535. int inservice_count;
  1536. int su_active_assign;
  1537. int su_standby_assign;
  1538. int su_spare_assign;
  1539. int assigned = 0;
  1540. int active_out_of_service = 0;
  1541. int standby_out_of_service = 0;
  1542. ENTER ("'%s'", sg->name.value);
  1543. /**
  1544. * Phase 1: Calculate assignments and create all runtime objects in
  1545. * information model. Do not do the actual assignment, done in
  1546. * phase 2.
  1547. */
  1548. /**
  1549. * Calculate number of SUs to assign to active or standby state
  1550. */
  1551. inservice_count = su_inservice_count_get (sg);
  1552. active_out_of_service = su_active_out_of_service_count_get(sg);
  1553. standby_out_of_service = su_standby_out_of_service_count_get(sg);
  1554. if (sg->saAmfSGNumPrefActiveSUs > 0) {
  1555. active_sus_needed = div_round (
  1556. sg_si_count_get (sg),
  1557. sg->saAmfSGMaxActiveSIsperSUs);
  1558. } else {
  1559. log_printf (LOG_LEVEL_ERROR, "ERROR: saAmfSGNumPrefActiveSUs == 0 !!");
  1560. openais_exit_error (AIS_DONE_FATAL_ERR);
  1561. }
  1562. if (sg->saAmfSGNumPrefStandbySUs > 0) {
  1563. standby_sus_needed = div_round (
  1564. sg_si_count_get (sg),
  1565. sg->saAmfSGMaxStandbySIsperSUs);
  1566. } else {
  1567. log_printf (LOG_LEVEL_ERROR, "ERROR: saAmfSGNumPrefStandbySUs == 0 !!");
  1568. openais_exit_error (AIS_DONE_FATAL_ERR);
  1569. }
  1570. dprintf ("(inservice=%d) (active_sus_needed=%d) (standby_sus_needed=%d)"
  1571. "\n",
  1572. inservice_count, active_sus_needed, standby_sus_needed);
  1573. /* Determine number of active and standby service units
  1574. * to assign based upon reduction procedure
  1575. */
  1576. if ((inservice_count < active_sus_needed - active_out_of_service)) {
  1577. dprintf ("assignment VI - partial assignment with SIs drop outs\n");
  1578. su_active_assign = inservice_count;
  1579. su_standby_assign = 0;
  1580. su_spare_assign = 0;
  1581. } else
  1582. if ((inservice_count < active_sus_needed - active_out_of_service +
  1583. standby_sus_needed)) {
  1584. dprintf ("assignment V - partial assignment with reduction of"
  1585. " standby units\n");
  1586. su_active_assign = active_sus_needed;
  1587. su_standby_assign = inservice_count - active_sus_needed - active_out_of_service;
  1588. su_spare_assign = 0;
  1589. } else
  1590. if ((inservice_count < sg->saAmfSGNumPrefActiveSUs + standby_sus_needed)) {
  1591. dprintf ("IV: full assignment with reduction of active service"
  1592. " units\n");
  1593. su_active_assign = inservice_count - standby_sus_needed;
  1594. su_standby_assign = standby_sus_needed;
  1595. su_spare_assign = 0;
  1596. } else
  1597. if ((inservice_count <
  1598. sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) {
  1599. dprintf ("III: full assignment with reduction of standby service"
  1600. " units\n");
  1601. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1602. su_standby_assign = inservice_count - sg->saAmfSGNumPrefActiveSUs;
  1603. su_spare_assign = 0;
  1604. } else
  1605. if ((inservice_count ==
  1606. sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) {
  1607. if (sg->saAmfSGNumPrefInserviceSUs > inservice_count) {
  1608. dprintf ("II: full assignment with spare reduction\n");
  1609. } else {
  1610. dprintf ("II: full assignment without spares\n");
  1611. }
  1612. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1613. su_standby_assign = sg->saAmfSGNumPrefStandbySUs;
  1614. su_spare_assign = 0;
  1615. } else {
  1616. dprintf ("I: full assignment with spares\n");
  1617. su_active_assign = sg->saAmfSGNumPrefActiveSUs;
  1618. su_standby_assign = sg->saAmfSGNumPrefStandbySUs;
  1619. su_spare_assign = inservice_count -
  1620. sg->saAmfSGNumPrefActiveSUs - sg->saAmfSGNumPrefStandbySUs;
  1621. }
  1622. dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)"
  1623. " (assigning spares=%d)\n",
  1624. inservice_count, su_active_assign, su_standby_assign, su_spare_assign);
  1625. if (inservice_count > 0) {
  1626. assigned = sg_assign_active_nplusm (sg, su_active_assign);
  1627. assigned += sg_assign_standby_nplusm (sg, su_standby_assign);
  1628. sg->saAmfSGNumCurrAssignedSUs = inservice_count;
  1629. /**
  1630. * Phase 2: do the actual assignment to the component
  1631. * TODO: first do active, then standby
  1632. */
  1633. {
  1634. struct amf_si *si;
  1635. struct amf_si_assignment *si_assignment;
  1636. for (si = sg->application->si_head; si != NULL; si = si->next) {
  1637. if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) {
  1638. for (si_assignment = si->assigned_sis;
  1639. si_assignment != NULL;
  1640. si_assignment = si_assignment->next) {
  1641. if (si_assignment->requested_ha_state !=
  1642. si_assignment->saAmfSISUHAState) {
  1643. amf_si_ha_state_assume (
  1644. si_assignment, assign_si_assumed_cbfn);
  1645. }
  1646. }
  1647. }
  1648. }
  1649. }
  1650. }
  1651. LEAVE ("'%s'", sg->name.value);
  1652. return assigned;
  1653. }
  1654. #ifdef COMPILE_OUT
  1655. static void remove_si_in_scope (amf_sg_t *sg, amf_si_t *si)
  1656. {
  1657. int i;
  1658. int j;
  1659. amf_si_t **sis = sg->recovery_scope.sis;
  1660. amf_si_t **new_sis = amf_calloc (1, sizeof (amf_si_t*));
  1661. for (i = 0,j = 0; sis[i] != NULL; i++) {
  1662. if (sis[i] == si) {
  1663. continue;
  1664. }
  1665. new_sis[j] = sis[i];
  1666. new_sis = amf_realloc (new_sis, j + sizeof (amf_si_t *));
  1667. j++;
  1668. }
  1669. sg->recovery_scope.sis = new_sis;
  1670. }
  1671. #endif
  1672. #ifdef COMPILE_OUT
  1673. static void remove_sis_for_term_failed_su_from_scope (amf_sg_t *sg,
  1674. amf_su_t *su)
  1675. {
  1676. amf_comp_t *component;
  1677. /*
  1678. * foreach component with presense state termiantion failed in su
  1679. */
  1680. for (component = su->comp_head; component != NULL;
  1681. component = component->next) {
  1682. amf_csi_assignment_t *csi_assignment;
  1683. if (component->saAmfCompPresenceState !=
  1684. SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
  1685. continue;
  1686. }
  1687. csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
  1688. while (csi_assignment != NULL) {
  1689. remove_si_in_scope (sg, csi_assignment->csi->si);
  1690. csi_assignment = amf_comp_get_next_csi_assignment (component, NULL);
  1691. }
  1692. }
  1693. }
  1694. #endif
  1695. /**
  1696. * This function returns 1 if the redundancy model is N plus M and at least one
  1697. * component of the specified SU has an active HA-state, else the function
  1698. * returns 0.
  1699. * @param sg
  1700. * @param su
  1701. * @return int
  1702. */
  1703. static int is_comp_in_active_ha_state_nplusm (
  1704. amf_sg_t *sg, amf_su_t *su)
  1705. {
  1706. amf_comp_t *component;
  1707. amf_csi_assignment_t *csi_assignment;
  1708. int comp_is_in_active_ha_state = 0;
  1709. if(sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) {
  1710. for (component = su->comp_head; component != NULL;
  1711. component = component->next) {
  1712. csi_assignment = amf_comp_get_next_csi_assignment(component, NULL);
  1713. while (csi_assignment != NULL) {
  1714. if (csi_assignment->saAmfCSICompHAState == SA_AMF_HA_ACTIVE) {
  1715. comp_is_in_active_ha_state = 1;
  1716. goto out;
  1717. }
  1718. csi_assignment = amf_comp_get_next_csi_assignment(component,
  1719. csi_assignment);
  1720. }
  1721. }
  1722. }
  1723. out:
  1724. return comp_is_in_active_ha_state;
  1725. }
  1726. /**
  1727. * This function handles a change of presence state reported by an SU contained
  1728. * in specified SG. The new presence state is INSTANTIATED.
  1729. * @param sg
  1730. * @param su
  1731. */
  1732. static void sg_su_state_changed_to_instantiated (struct amf_sg *sg, struct amf_su *su)
  1733. {
  1734. ENTER("%s %s",sg->name.value, su->name.value);
  1735. switch (sg->avail_state) {
  1736. case SG_AC_InstantiatingServiceUnits:
  1737. if (no_su_has_presence_state(sg, sg->node_to_start,
  1738. SA_AMF_PRESENCE_INSTANTIATING)) {
  1739. acsm_enter_idle (sg);
  1740. }
  1741. break;
  1742. case SG_AC_ReparingSu:
  1743. if (no_su_has_presence_state(sg, sg->node_to_start,
  1744. SA_AMF_PRESENCE_INSTANTIATING)) {
  1745. if (all_su_in_scope_has_either_of_three_presence_state(
  1746. su->sg,
  1747. SA_AMF_PRESENCE_INSTANTIATED,
  1748. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1749. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1750. su->sg->avail_state = SG_AC_AssigningWorkload;
  1751. if (assign_si (sg, 0) == 0) {
  1752. acsm_enter_idle (sg);
  1753. }
  1754. } else {
  1755. dprintf ("avail-state: %u", sg->avail_state);
  1756. assert (0);
  1757. }
  1758. }
  1759. break;
  1760. default:
  1761. dprintf ("avail-state: %u", sg->avail_state);
  1762. assert (0);
  1763. break;
  1764. }
  1765. }
  1766. /**
  1767. * This function handles a change of presence state reported by an SU contained
  1768. * in specified SG. The new presence state is UNINSTANTIATED.
  1769. * @param sg
  1770. * @param su
  1771. */
  1772. static void amf_sg_su_state_changed_to_uninstantiated (amf_sg_t *sg,
  1773. amf_su_t *su)
  1774. {
  1775. ENTER("%s %s",sg->name.value, su->name.value);
  1776. switch (sg->avail_state) {
  1777. case SG_AC_TerminatingSuspected:
  1778. if (no_su_has_presence_state(sg, sg->node_to_start,
  1779. SA_AMF_PRESENCE_TERMINATING)) {
  1780. if (all_su_in_scope_has_either_two_presence_state (sg,
  1781. SA_AMF_PRESENCE_UNINSTANTIATED,
  1782. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  1783. delete_si_assignments_in_scope (sg);
  1784. if (is_any_si_in_scope_assigned_standby (sg)) {
  1785. remove_all_suspected_sus (sg);
  1786. acsm_enter_removing_standby_assignments (sg);
  1787. } else { /*is_no_si_in_scope_assigned_standby*/
  1788. remove_all_suspected_sus (sg);
  1789. acsm_enter_assigning_standby_to_spare (sg);
  1790. }
  1791. }
  1792. }
  1793. break;
  1794. case SG_AC_ReparingSu:
  1795. if (no_su_has_presence_state(sg, sg->node_to_start,
  1796. SA_AMF_PRESENCE_TERMINATING)) {
  1797. if (all_su_in_scope_has_either_of_three_presence_state(
  1798. su->sg,
  1799. SA_AMF_PRESENCE_INSTANTIATED,
  1800. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1801. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1802. su->sg->avail_state = SG_AC_AssigningWorkload;
  1803. if (assign_si (sg, 0) == 0) {
  1804. acsm_enter_idle (sg);
  1805. }
  1806. }
  1807. }
  1808. break;
  1809. default:
  1810. log_printf (LOG_ERR, "sg avail_state = %d", sg->avail_state);
  1811. assert (0);
  1812. break;
  1813. }
  1814. }
  1815. /**
  1816. * This function handles a change of presence state reported by an SU contained
  1817. * in specified SG. The new presence state is TERMINATION_FAILED.
  1818. * @param sg
  1819. * @param su
  1820. */
  1821. static void amf_sg_su_state_changed_to_termination_failed (amf_sg_t *sg,
  1822. amf_su_t *su)
  1823. {
  1824. ENTER("%s %s",sg->name.value, su->name.value);
  1825. if (no_su_has_presence_state(sg, sg->node_to_start,
  1826. SA_AMF_PRESENCE_TERMINATING)) {
  1827. if (is_comp_in_active_ha_state_nplusm (sg, su)) {
  1828. acsm_enter_idle (sg);
  1829. goto out;
  1830. }
  1831. if (all_su_in_scope_has_either_two_presence_state (sg,
  1832. SA_AMF_PRESENCE_UNINSTANTIATED,
  1833. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  1834. delete_si_assignments_in_scope (sg);
  1835. if (is_any_si_in_scope_assigned_standby (sg)) {
  1836. remove_all_suspected_sus (sg);
  1837. acsm_enter_removing_standby_assignments (sg);
  1838. } else { /*is_no_si_in_scope_assigned_standby*/
  1839. remove_all_suspected_sus (sg);
  1840. acsm_enter_assigning_standby_to_spare (sg);
  1841. }
  1842. }
  1843. }
  1844. out:
  1845. return;
  1846. }
  1847. /**
  1848. * This function handles a change of presence state reported by an SU contained
  1849. * in specified SG. The new presence state is INSTANTIATION_FAILED.
  1850. * @param sg
  1851. * @param su
  1852. */
  1853. static void amf_sg_su_state_changed_to_instantiation_failed (amf_sg_t *sg,
  1854. amf_su_t *su)
  1855. {
  1856. ENTER("%s %s",sg->name.value, su->name.value);
  1857. switch (sg->avail_state) {
  1858. case SG_AC_InstantiatingServiceUnits:
  1859. if (no_su_has_presence_state(sg, sg->node_to_start,
  1860. SA_AMF_PRESENCE_INSTANTIATING)) {
  1861. acsm_enter_idle (sg);
  1862. }
  1863. break;
  1864. case SG_AC_ReparingSu:
  1865. if (no_su_has_presence_state(sg, sg->node_to_start,
  1866. SA_AMF_PRESENCE_INSTANTIATING)) {
  1867. if (all_su_in_scope_has_either_of_three_presence_state(
  1868. su->sg,
  1869. SA_AMF_PRESENCE_INSTANTIATED,
  1870. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  1871. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  1872. su->sg->avail_state = SG_AC_AssigningWorkload;
  1873. if (assign_si (sg, 0) == 0) {
  1874. acsm_enter_idle (sg);
  1875. }
  1876. }
  1877. }
  1878. break;
  1879. default:
  1880. /* TODO: Insert the assert (0) until solving defers in SU */
  1881. dprintf("sg->avail_state = %d", sg->avail_state);
  1882. break;
  1883. }
  1884. }
  1885. /******************************************************************************
  1886. * Event methods
  1887. *****************************************************************************/
  1888. /**
  1889. * This function starts all SUs in the SG or all SUs on a specified node.
  1890. * @param sg
  1891. * @param node - Node on which all SUs shall be started or
  1892. * NULL indicating that all SUs on all nodes shall be started.
  1893. * @return - No of SUs that has been attempted to start
  1894. */
  1895. int amf_sg_start (struct amf_sg *sg, struct amf_node *node)
  1896. {
  1897. sg->recovery_scope.event_type = SG_START_EV;
  1898. ENTER ("'%s'", sg->name.value);
  1899. int instantiated_sus = 0;
  1900. switch (sg->avail_state) {
  1901. case SG_AC_InstantiatingServiceUnits:
  1902. case SG_AC_Idle: {
  1903. amf_su_t *su;
  1904. sg_avail_control_state_t old_avail_state = sg->avail_state;
  1905. ENTER ("'%s'", sg->name.value);
  1906. sg->node_to_start = node;
  1907. sg->avail_state = SG_AC_InstantiatingServiceUnits;
  1908. for (su = sg->su_head;
  1909. (su != NULL) &&
  1910. (instantiated_sus < sg->saAmfSGNumPrefInserviceSUs);
  1911. su = su->next) {
  1912. if (is_cluster_start (node)) {
  1913. amf_su_instantiate (su);
  1914. instantiated_sus++;
  1915. } else { /*is_not_cluster_start*/
  1916. /*
  1917. * Node start, match if SU is hosted on the
  1918. * specified node
  1919. */
  1920. if (name_match (&node->name,
  1921. &su->saAmfSUHostedByNode)) {
  1922. amf_su_instantiate (su);
  1923. instantiated_sus++;
  1924. }
  1925. }
  1926. }
  1927. if (instantiated_sus == 0) {
  1928. sg->avail_state = old_avail_state;
  1929. }
  1930. break;
  1931. }
  1932. case SG_AC_DeactivatingDependantWorkload:
  1933. case SG_AC_TerminatingSuspected:
  1934. case SG_AC_ActivatingStandby:
  1935. case SG_AC_AssigningStandbyToSpare:
  1936. case SG_AC_ReparingComponent:
  1937. case SG_AC_ReparingSu:
  1938. case SG_AC_AssigningOnRequest:
  1939. case SG_AC_RemovingAssignment:
  1940. case SG_AC_AssigningActiveworkload:
  1941. case SG_AC_AssigningAutoAdjust:
  1942. case SG_AC_AssigningWorkload:
  1943. case SG_AC_WaitingAfterOperationFailed:
  1944. case SG_AC_RemovingStandbyAssignments:
  1945. default:
  1946. assert (0);
  1947. break;
  1948. }
  1949. return instantiated_sus;
  1950. }
  1951. /**
  1952. * This function initiates assignment of the subset of the workload which
  1953. * matches the specified workload dependency level, to all SUs contained in the
  1954. * SG according to the requirements specified in the configuration.
  1955. * @param sg -
  1956. * @param dependency_level - Dependency level to assign
  1957. * @return - No of SUs that has been attempted to start
  1958. */
  1959. int amf_sg_assign_si_req (struct amf_sg *sg, int dependency_level)
  1960. {
  1961. /* TODO: Introduce state control in this function
  1962. */
  1963. int posible_to_assign_si;
  1964. sg->recovery_scope.event_type = SG_ASSIGN_SI_EV;
  1965. sg->avail_state = SG_AC_AssigningOnRequest;
  1966. if ((posible_to_assign_si = assign_si (sg, dependency_level)) == 0) {
  1967. acsm_enter_idle (sg);
  1968. }
  1969. return posible_to_assign_si;
  1970. }
  1971. /**
  1972. * This function is called because an error has been detected and the analysis
  1973. * (done elsewhere) indicated that this error shall be recovered by a Node
  1974. * failover. This function initiates the recovery action 'Node failover'.
  1975. * @param sg
  1976. * @param su - SU to failover
  1977. * @param node -
  1978. */
  1979. void amf_sg_failover_node_req (struct amf_sg *sg, struct amf_node *node)
  1980. {
  1981. ENTER("'%s, %s'",node->name.value, sg->name.value);
  1982. sg_event_t sg_event;
  1983. switch (sg->avail_state) {
  1984. case SG_AC_Idle:
  1985. set_scope_for_failover_node(sg, node);
  1986. if (has_any_su_in_scope_active_workload (sg)) {
  1987. acsm_enter_deactivating_dependent_workload (sg);
  1988. } else {
  1989. amf_su_t **sus = sg->recovery_scope.sus;
  1990. /*
  1991. * Select next state depending on if some
  1992. * SU in the scope needs to be terminated.
  1993. */
  1994. while (*sus != NULL) {
  1995. amf_su_t *su = *sus;
  1996. ENTER("SU %s pr_state='%d'",su->name.value,
  1997. su->saAmfSUPresenceState);
  1998. if (su_presense_state_is_ored (su,
  1999. SA_AMF_PRESENCE_UNINSTANTIATED,
  2000. SA_AMF_PRESENCE_TERMINATION_FAILED,
  2001. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  2002. sus++;
  2003. continue;
  2004. }
  2005. break;
  2006. }
  2007. if (*sus != NULL) {
  2008. acsm_enter_terminating_suspected (sg);
  2009. } else {
  2010. delete_si_assignments_in_scope (sg);
  2011. acsm_enter_idle (sg);
  2012. }
  2013. }
  2014. break;
  2015. case SG_AC_DeactivatingDependantWorkload:
  2016. case SG_AC_TerminatingSuspected:
  2017. case SG_AC_ActivatingStandby:
  2018. case SG_AC_AssigningStandbyToSpare:
  2019. case SG_AC_ReparingComponent:
  2020. case SG_AC_ReparingSu:
  2021. case SG_AC_AssigningOnRequest:
  2022. case SG_AC_InstantiatingServiceUnits:
  2023. case SG_AC_RemovingAssignment:
  2024. case SG_AC_AssigningActiveworkload:
  2025. case SG_AC_AssigningAutoAdjust:
  2026. case SG_AC_AssigningWorkload:
  2027. case SG_AC_WaitingAfterOperationFailed:
  2028. case SG_AC_RemovingStandbyAssignments:
  2029. sg_set_event (SG_FAILOVER_NODE_EV, sg, 0, 0, node, &sg_event);
  2030. sg_defer_event (SG_FAILOVER_NODE_EV, &sg_event);
  2031. break;
  2032. default:
  2033. assert (0);
  2034. break;
  2035. }
  2036. }
  2037. /**
  2038. * This function is called because an error has been detected and the analysis
  2039. * (done elsewhere) indicated that this error shall be recovered by an SU
  2040. * failover. This function initiates the recovery action 'SU failover'.
  2041. * @param sg
  2042. * @param su - SU to failover
  2043. * @param node -
  2044. */
  2045. void amf_sg_failover_su_req (struct amf_sg *sg, struct amf_su *su,
  2046. struct amf_node *node)
  2047. {
  2048. ENTER ("%s", su->name.value);
  2049. sg_event_t sg_event;
  2050. switch (sg->avail_state) {
  2051. case SG_AC_Idle:
  2052. su->su_failover_cnt += 1;
  2053. set_scope_for_failover_su (sg, su);
  2054. if (has_any_su_in_scope_active_workload (sg)) {
  2055. acsm_enter_deactivating_dependent_workload (sg);
  2056. } else {
  2057. acsm_enter_terminating_suspected (sg);
  2058. }
  2059. break;
  2060. case SG_AC_DeactivatingDependantWorkload:
  2061. case SG_AC_TerminatingSuspected:
  2062. case SG_AC_ActivatingStandby:
  2063. case SG_AC_AssigningStandbyToSpare:
  2064. case SG_AC_ReparingComponent:
  2065. case SG_AC_ReparingSu:
  2066. case SG_AC_AssigningOnRequest:
  2067. case SG_AC_InstantiatingServiceUnits:
  2068. case SG_AC_RemovingAssignment:
  2069. case SG_AC_AssigningActiveworkload:
  2070. case SG_AC_AssigningAutoAdjust:
  2071. case SG_AC_AssigningWorkload:
  2072. case SG_AC_WaitingAfterOperationFailed:
  2073. case SG_AC_RemovingStandbyAssignments:
  2074. sg_set_event (SG_FAILOVER_SU_EV, sg, su, 0, 0, &sg_event);
  2075. sg_defer_event (SG_FAILOVER_SU_EV, &sg_event);
  2076. break;
  2077. default:
  2078. assert (0);
  2079. break;
  2080. }
  2081. }
  2082. /******************************************************************************
  2083. * Event response methods
  2084. *****************************************************************************/
  2085. #ifdef COMPILE_OUT
  2086. void amf_sg_su_state_changed_2 (struct amf_sg *sg,
  2087. struct amf_su *su, SaAmfStateT type, int state)
  2088. {
  2089. ENTER ("'%s' SU '%s' state %s",
  2090. sg->name.value, su->name.value, amf_presence_state(state));
  2091. if (type == SA_AMF_PRESENCE_STATE) {
  2092. if (state == SA_AMF_PRESENCE_INSTANTIATED) {
  2093. if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
  2094. if (no_su_has_presence_state(sg, sg->node_to_start,
  2095. SA_AMF_PRESENCE_INSTANTIATING)) {
  2096. acsm_enter_idle (sg);
  2097. }
  2098. } else if (sg->avail_state == SG_AC_ReparingSu) {
  2099. if (all_su_in_scope_has_either_of_three_presence_state(
  2100. su->sg,
  2101. SA_AMF_PRESENCE_INSTANTIATED,
  2102. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  2103. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  2104. su->sg->avail_state = SG_AC_AssigningWorkload;
  2105. if (assign_si (sg, 0) == 0) {
  2106. acsm_enter_idle (sg);
  2107. }
  2108. } else {
  2109. dprintf ("avail-state: %u", sg->avail_state);
  2110. assert (0);
  2111. }
  2112. } else {
  2113. dprintf ("avail-state: %u", sg->avail_state);
  2114. assert (0);
  2115. }
  2116. } else if (state == SA_AMF_PRESENCE_UNINSTANTIATED) {
  2117. if (sg->avail_state == SG_AC_TerminatingSuspected) {
  2118. if (all_su_in_scope_has_either_two_presence_state (sg,
  2119. SA_AMF_PRESENCE_UNINSTANTIATED,
  2120. SA_AMF_PRESENCE_TERMINATION_FAILED)) {
  2121. delete_si_assignments_in_scope (sg);
  2122. if (is_any_si_in_scope_assigned_standby (sg)) {
  2123. remove_all_suspected_sus (sg);
  2124. acsm_enter_removing_standby_assignments (sg);
  2125. } else { /*is_no_si_in_scope_assigned_standby*/
  2126. remove_all_suspected_sus (sg);
  2127. acsm_enter_assigning_standby_to_spare (sg);
  2128. }
  2129. }
  2130. } else if (sg->avail_state == SG_AC_ReparingSu) {
  2131. if (all_su_in_scope_has_either_of_three_presence_state(
  2132. su->sg,
  2133. SA_AMF_PRESENCE_INSTANTIATED,
  2134. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  2135. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  2136. su->sg->avail_state = SG_AC_AssigningWorkload;
  2137. if (assign_si (sg, 0) == 0) {
  2138. acsm_enter_idle (sg);
  2139. }
  2140. } else {
  2141. dprintf("%d",sg->avail_state);
  2142. assert (0);
  2143. }
  2144. }
  2145. } else if (state == SA_AMF_PRESENCE_TERMINATION_FAILED) {
  2146. if (all_su_in_scope_has_either_two_presence_state (sg,
  2147. SA_AMF_PRESENCE_UNINSTANTIATED,
  2148. SA_AMF_PRESENCE_TERMINATION_FAILED) &&
  2149. is_any_si_in_scope_assigned_standby (sg)) {
  2150. remove_all_suspected_sus (sg);
  2151. acsm_enter_removing_standby_assignments (sg);
  2152. } else if (all_su_in_scope_has_either_two_presence_state (sg,
  2153. SA_AMF_PRESENCE_UNINSTANTIATED,
  2154. SA_AMF_PRESENCE_TERMINATION_FAILED) &&
  2155. !is_any_si_in_scope_assigned_standby (sg)) {
  2156. remove_all_suspected_sus (sg);
  2157. acsm_enter_assigning_standby_to_spare (sg);
  2158. } else {
  2159. remove_sis_for_term_failed_su_from_scope (sg, su);
  2160. }
  2161. } else if (state == SA_AMF_PRESENCE_INSTANTIATING) {
  2162. ; /* nop */
  2163. } else if (state == SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
  2164. if (sg->avail_state == SG_AC_InstantiatingServiceUnits) {
  2165. if (no_su_has_presence_state(sg, sg->node_to_start,
  2166. SA_AMF_PRESENCE_INSTANTIATING)) {
  2167. acsm_enter_idle (sg);
  2168. }
  2169. } else if (sg->avail_state == SG_AC_ReparingSu) {
  2170. if (all_su_in_scope_has_either_of_three_presence_state(
  2171. su->sg,
  2172. SA_AMF_PRESENCE_INSTANTIATED,
  2173. SA_AMF_PRESENCE_INSTANTIATION_FAILED,
  2174. SA_AMF_PRESENCE_UNINSTANTIATED)) {
  2175. su->sg->avail_state = SG_AC_AssigningWorkload;
  2176. if (assign_si (sg, 0) == 0) {
  2177. acsm_enter_idle (sg);
  2178. }
  2179. }
  2180. } else {
  2181. /* TODO: Insert the assert (0) until solving defers in SU */
  2182. dprintf("sg->avail_state = %d, su instantiation state = %d",
  2183. sg->avail_state, state);
  2184. }
  2185. } else {
  2186. dprintf("sg->avail_state = %d, su instantiation state = %d",
  2187. sg->avail_state, state);
  2188. assert (0);
  2189. }
  2190. }
  2191. }
  2192. #endif
  2193. /**
  2194. * SU indicates to SG that one of its state machines has changed state.
  2195. * @param sg - SG which contains the SU that has changed state
  2196. * @param su - SU which has changed state
  2197. * @param type - Indicates which state machine that has changed state
  2198. * @param state - The new state that has been assumed.
  2199. *
  2200. */
  2201. void amf_sg_su_state_changed (struct amf_sg *sg, struct amf_su *su,
  2202. SaAmfStateT type, int state)
  2203. {
  2204. ENTER ("'%s' SU '%s' state %s",
  2205. sg->name.value, su->name.value, amf_presence_state(state));
  2206. if (sg->avail_state != SG_AC_Idle) {
  2207. if (type == SA_AMF_PRESENCE_STATE) {
  2208. switch (state) {
  2209. case SA_AMF_PRESENCE_INSTANTIATED:
  2210. sg_su_state_changed_to_instantiated(sg, su);
  2211. break;
  2212. case SA_AMF_PRESENCE_UNINSTANTIATED:
  2213. amf_sg_su_state_changed_to_uninstantiated(sg, su);
  2214. break;
  2215. case SA_AMF_PRESENCE_TERMINATION_FAILED:
  2216. amf_sg_su_state_changed_to_termination_failed(sg, su);
  2217. break;
  2218. case SA_AMF_PRESENCE_INSTANTIATING:
  2219. ; /* nop */
  2220. break;
  2221. case SA_AMF_PRESENCE_INSTANTIATION_FAILED:
  2222. amf_sg_su_state_changed_to_instantiation_failed(sg, su);
  2223. break;
  2224. case SA_AMF_PRESENCE_TERMINATING:
  2225. ; /* nop */
  2226. break;
  2227. default :
  2228. dprintf("sg->avail_state = %d, su instantiation state = %d",
  2229. sg->avail_state, state);
  2230. assert (0);
  2231. break;
  2232. }
  2233. }
  2234. }
  2235. }
  2236. /**
  2237. * Callback function used by SI when there is no dependent SI to
  2238. * deactivate.
  2239. * @param sg
  2240. */
  2241. static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg)
  2242. {
  2243. struct amf_su **sus = sg->recovery_scope.sus;
  2244. ENTER("'%s'", sg->name.value);
  2245. /*
  2246. * Select next state depending on if some
  2247. * SU in the scope needs to be terminated.
  2248. */
  2249. while (*sus != NULL) {
  2250. amf_su_t *su = *sus;
  2251. ENTER("SU %s pr_state='%d'",su->name.value,
  2252. su->saAmfSUPresenceState);
  2253. if (su_presense_state_is_ored (su,
  2254. SA_AMF_PRESENCE_UNINSTANTIATED,
  2255. SA_AMF_PRESENCE_TERMINATION_FAILED,
  2256. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  2257. sus++;
  2258. continue;
  2259. }
  2260. break;
  2261. }
  2262. if (*sus != NULL) {
  2263. acsm_enter_terminating_suspected (sg);
  2264. } else {
  2265. delete_si_assignments_in_scope(sg);
  2266. acsm_enter_removing_standby_assignments (sg);
  2267. }
  2268. }
  2269. /**
  2270. * Callback function used by SI when an SI has been deactivated, i.e.
  2271. * transitioned from active HA-state to any other state.
  2272. * @param si_assignment
  2273. * @param result - Indicates the result of the operation.
  2274. */
  2275. static void dependent_si_deactivated_cbfn (
  2276. struct amf_si_assignment *si_assignment, int result)
  2277. {
  2278. struct amf_sg *sg = si_assignment->su->sg;
  2279. struct amf_su **sus = sg->recovery_scope.sus;
  2280. struct amf_su *su;
  2281. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  2282. /*
  2283. * If all SI assignments for all SUs in the SG are not pending,
  2284. * goto next state (TerminatingSuspected).
  2285. */
  2286. for (su = sg->su_head ; su != NULL; su = su->next) {
  2287. struct amf_si_assignment *si_assignment;
  2288. si_assignment = amf_su_get_next_si_assignment(su, NULL);
  2289. while (si_assignment != NULL) {
  2290. if (si_assignment->saAmfSISUHAState !=
  2291. si_assignment->requested_ha_state) {
  2292. goto still_wating;
  2293. }
  2294. si_assignment = amf_su_get_next_si_assignment(su,
  2295. si_assignment);
  2296. }
  2297. }
  2298. still_wating:
  2299. if (su == NULL) {
  2300. sus = si_assignment->su->sg->recovery_scope.sus;
  2301. /*
  2302. * Select next state depending on if some
  2303. * SU in the scope is needs to be terminated.
  2304. */
  2305. while (*sus != NULL) {
  2306. if (su_presense_state_is_not (*sus,
  2307. SA_AMF_PRESENCE_UNINSTANTIATED,
  2308. SA_AMF_PRESENCE_TERMINATION_FAILED,
  2309. SA_AMF_PRESENCE_INSTANTIATION_FAILED)) {
  2310. break;
  2311. }
  2312. sus++;
  2313. }
  2314. if (*sus != NULL) {
  2315. acsm_enter_terminating_suspected (sg);
  2316. } else {
  2317. acsm_enter_removing_standby_assignments (sg);
  2318. }
  2319. }
  2320. LEAVE("");
  2321. }
  2322. /**
  2323. * Callback function used by SI to indicate an SI has assumed a new HA-state or
  2324. * that the attempt to do so failed.
  2325. * @param si_assignment
  2326. * @param result - Indicates the result of the operation.
  2327. */
  2328. static void assign_si_assumed_cbfn (
  2329. struct amf_si_assignment *si_assignment, int result)
  2330. {
  2331. struct amf_sg *sg = si_assignment->su->sg;
  2332. int si_assignment_cnt = 0;
  2333. int confirmed_assignments = 0;
  2334. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  2335. switch (sg->avail_state) {
  2336. case SG_AC_AssigningOnRequest:
  2337. if (is_all_si_assigned (sg)) {
  2338. acsm_enter_idle (sg);
  2339. amf_application_sg_assigned (sg->application, sg);
  2340. } else {
  2341. dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments);
  2342. }
  2343. break;
  2344. case SG_AC_AssigningWorkload:
  2345. {
  2346. if (is_all_si_assigned(sg)) {
  2347. acsm_enter_idle (sg);
  2348. }
  2349. break;
  2350. }
  2351. case SG_AC_AssigningStandbyToSpare:
  2352. {
  2353. if(is_all_si_assigned (sg)) {
  2354. /*
  2355. * All si_assignments has asumed
  2356. * Prescense state SA_AMF_HA_STANDBY
  2357. */
  2358. switch (sg->recovery_scope.event_type) {
  2359. case SG_FAILOVER_NODE_EV:
  2360. acsm_enter_idle (sg);
  2361. break;
  2362. case SG_FAILOVER_SU_EV:
  2363. if (sg->saAmfSGAutoRepair == SA_TRUE) {
  2364. acsm_enter_repairing_su (sg);
  2365. }
  2366. break;
  2367. default:
  2368. assert (0);
  2369. break;
  2370. }
  2371. } else {
  2372. si_assignment->saAmfSISUHAState = SA_AMF_HA_STANDBY;
  2373. }
  2374. }
  2375. break;
  2376. default:
  2377. dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt,
  2378. confirmed_assignments);
  2379. amf_runtime_attributes_print (amf_cluster);
  2380. assert (0);
  2381. break;
  2382. }
  2383. }
  2384. /**
  2385. * Callback function used by SI when an SI has been activated, i.e. transitioned
  2386. * from any HA-state to an active HA-state.
  2387. * @param si_assignment
  2388. * @param result - Indicates the result of the operation.
  2389. */
  2390. static void standby_su_activated_cbfn (
  2391. struct amf_si_assignment *si_assignment, int result)
  2392. {
  2393. struct amf_su **sus = si_assignment->su->sg->recovery_scope.sus;
  2394. struct amf_si **sis = si_assignment->su->sg->recovery_scope.sis;
  2395. ENTER ("'%s', %d", si_assignment->si->name.value, result);
  2396. /*
  2397. * If all SI assignments for all SIs in the scope are activated, goto next
  2398. * state.
  2399. */
  2400. while (*sis != NULL) {
  2401. if ((*sis)->assigned_sis != NULL &&
  2402. (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) {
  2403. break;
  2404. }
  2405. sis++;
  2406. }
  2407. if (*sis == NULL) {
  2408. acsm_enter_assigning_standby_to_spare ((*sus)->sg);
  2409. }
  2410. }
  2411. /******************************************************************************
  2412. * General methods
  2413. *****************************************************************************/
  2414. void amf_sg_init (void)
  2415. {
  2416. log_init ("AMF");
  2417. }
  2418. /**
  2419. * Constructor for SG objects. Adds SG to the list owned by
  2420. * the specified application. Always returns a valid SG
  2421. * object, out-of-memory problems are handled here. Default
  2422. * values are initialized.
  2423. * @param sg
  2424. * @param name
  2425. *
  2426. * @return struct amf_sg*
  2427. */
  2428. struct amf_sg *amf_sg_new (struct amf_application *app, char *name)
  2429. {
  2430. struct amf_sg *sg = amf_calloc (1, sizeof (struct amf_sg));
  2431. setSaNameT (&sg->name, name);
  2432. sg->saAmfSGAdminState = SA_AMF_ADMIN_UNLOCKED;
  2433. sg->saAmfSGNumPrefActiveSUs = 1;
  2434. sg->saAmfSGNumPrefStandbySUs = 1;
  2435. sg->saAmfSGNumPrefInserviceSUs = ~0;
  2436. sg->saAmfSGNumPrefAssignedSUs = ~0;
  2437. sg->saAmfSGCompRestartProb = -1;
  2438. sg->saAmfSGCompRestartMax = ~0;
  2439. sg->saAmfSGSuRestartProb = -1;
  2440. sg->saAmfSGSuRestartMax = ~0;
  2441. sg->saAmfSGAutoAdjustProb = -1;
  2442. sg->saAmfSGAutoRepair = SA_TRUE;
  2443. sg->application = app;
  2444. sg->next = app->sg_head;
  2445. app->sg_head = sg;
  2446. sg->deferred_events = NULL;
  2447. return sg;
  2448. }
  2449. void amf_sg_delete (struct amf_sg *sg)
  2450. {
  2451. struct amf_su *su;
  2452. for (su = sg->su_head; su != NULL;) {
  2453. struct amf_su *tmp = su;
  2454. su = su->next;
  2455. amf_su_delete (tmp);
  2456. }
  2457. free (sg);
  2458. }
  2459. void *amf_sg_serialize (struct amf_sg *sg, int *len)
  2460. {
  2461. char *buf = NULL;
  2462. int offset = 0, size = 0;
  2463. TRACE8 ("%s", sg->name.value);
  2464. buf = amf_serialize_SaNameT (buf, &size, &offset, &sg->name);
  2465. buf = amf_serialize_SaUint32T (buf, &size, &offset, sg->saAmfSGRedundancyModel);
  2466. buf = amf_serialize_SaUint32T (
  2467. buf, &size, &offset, sg->saAmfSGAutoAdjust);
  2468. buf = amf_serialize_SaUint32T (
  2469. buf, &size, &offset, sg->saAmfSGNumPrefActiveSUs);
  2470. buf = amf_serialize_SaUint32T (
  2471. buf, &size, &offset, sg->saAmfSGNumPrefStandbySUs);
  2472. buf = amf_serialize_SaUint32T (
  2473. buf, &size, &offset, sg->saAmfSGNumPrefInserviceSUs);
  2474. buf = amf_serialize_SaUint32T (
  2475. buf, &size, &offset, sg->saAmfSGNumPrefAssignedSUs);
  2476. buf = amf_serialize_SaUint32T (
  2477. buf, &size, &offset, sg->saAmfSGMaxActiveSIsperSUs);
  2478. buf = amf_serialize_SaUint32T (
  2479. buf, &size, &offset, sg->saAmfSGMaxStandbySIsperSUs);
  2480. buf = amf_serialize_SaUint32T (
  2481. buf, &size, &offset, sg->saAmfSGCompRestartProb);
  2482. buf = amf_serialize_SaUint32T (
  2483. buf, &size, &offset, sg->saAmfSGCompRestartMax);
  2484. buf = amf_serialize_SaUint32T (
  2485. buf, &size, &offset, sg->saAmfSGSuRestartProb);
  2486. buf = amf_serialize_SaUint32T (
  2487. buf, &size, &offset, sg->saAmfSGSuRestartMax);
  2488. buf = amf_serialize_SaUint32T (
  2489. buf, &size, &offset, sg->saAmfSGAutoAdjustProb);
  2490. buf = amf_serialize_SaUint32T (
  2491. buf, &size, &offset, sg->saAmfSGAutoRepair);
  2492. buf = amf_serialize_SaUint32T (
  2493. buf, &size, &offset, sg->saAmfSGAdminState);
  2494. buf = amf_serialize_SaUint32T (
  2495. buf, &size, &offset, sg->saAmfSGNumCurrAssignedSUs);
  2496. buf = amf_serialize_SaUint32T (
  2497. buf, &size, &offset, sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
  2498. buf = amf_serialize_SaUint32T (
  2499. buf, &size, &offset, sg->saAmfSGNumCurrInstantiatedSpareSUs);
  2500. buf = amf_serialize_SaStringT (
  2501. buf, &size, &offset, sg->clccli_path);
  2502. buf = amf_serialize_SaUint32T (
  2503. buf, &size, &offset, sg->avail_state);
  2504. buf = amf_serialize_SaUint32T (
  2505. buf, &size, &offset, sg->recovery_scope.event_type);
  2506. *len = offset;
  2507. return buf;
  2508. }
  2509. struct amf_sg *amf_sg_deserialize (struct amf_application *app, char *buf)
  2510. {
  2511. char *tmp = buf;
  2512. struct amf_sg *sg = amf_sg_new (app, "");
  2513. tmp = amf_deserialize_SaNameT (tmp, &sg->name);
  2514. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGRedundancyModel);
  2515. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoAdjust);
  2516. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefActiveSUs);
  2517. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefStandbySUs);
  2518. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefInserviceSUs);
  2519. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumPrefAssignedSUs);
  2520. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGMaxActiveSIsperSUs);
  2521. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGMaxStandbySIsperSUs);
  2522. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGCompRestartProb);
  2523. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGCompRestartMax);
  2524. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGSuRestartProb);
  2525. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGSuRestartMax);
  2526. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoAdjustProb);
  2527. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAutoRepair);
  2528. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGAdminState);
  2529. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrAssignedSUs);
  2530. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrNonInstantiatedSpareSUs);
  2531. tmp = amf_deserialize_SaUint32T (tmp, &sg->saAmfSGNumCurrInstantiatedSpareSUs);
  2532. tmp = amf_deserialize_SaStringT (tmp, &sg->clccli_path);
  2533. tmp = amf_deserialize_SaUint32T (tmp, &sg->avail_state);
  2534. tmp = amf_deserialize_SaUint32T (tmp, &sg->recovery_scope.event_type);
  2535. return sg;
  2536. }
  2537. struct amf_sg *amf_sg_find (struct amf_application *app, char *name)
  2538. {
  2539. struct amf_sg *sg;
  2540. for (sg = app->sg_head; sg != NULL; sg = sg->next) {
  2541. if (sg->name.length == strlen(name) &&
  2542. strncmp (name, (char*)sg->name.value, sg->name.length) == 0) {
  2543. break;
  2544. }
  2545. }
  2546. return sg;
  2547. }