group_concurrency_test.go 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. package executor
  2. import (
  3. "sync"
  4. "testing"
  5. "time"
  6. "github.com/OliveTin/OliveTin/internal/auth"
  7. config "github.com/OliveTin/OliveTin/internal/config"
  8. "github.com/stretchr/testify/assert"
  9. "github.com/stretchr/testify/require"
  10. )
  11. func testGroupExecutor(actions []*config.Action, groups map[string]*config.ActionGroup) (*Executor, *config.Config) {
  12. cfg := config.DefaultConfig()
  13. cfg.ActionGroups = groups
  14. cfg.Actions = actions
  15. cfg.Sanitize()
  16. e := DefaultExecutor(cfg)
  17. e.RebuildActionMap()
  18. return e, cfg
  19. }
  20. func TestGroupConcurrencyQueuesSecondAction(t *testing.T) {
  21. t.Parallel()
  22. slowAction := &config.Action{
  23. Title: "Unity Job 1",
  24. Shell: "sleep 2",
  25. Groups: []string{"unity"},
  26. }
  27. fastAction := &config.Action{
  28. Title: "Unity Job 2",
  29. Shell: "echo queued-run",
  30. Groups: []string{"unity"},
  31. }
  32. e, cfg := testGroupExecutor(
  33. []*config.Action{slowAction, fastAction},
  34. map[string]*config.ActionGroup{
  35. "unity": {MaxConcurrent: 1},
  36. },
  37. )
  38. binding1 := e.FindBindingWithNoEntity(slowAction)
  39. binding2 := e.FindBindingWithNoEntity(fastAction)
  40. require.NotNil(t, binding1)
  41. require.NotNil(t, binding2)
  42. wg1, tracking1 := e.ExecRequest(&ExecutionRequest{
  43. Binding: binding1,
  44. Cfg: cfg,
  45. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  46. })
  47. waitUntilExecutionStarted(t, e, tracking1)
  48. wg2, tracking2 := e.ExecRequest(&ExecutionRequest{
  49. Binding: binding2,
  50. Cfg: cfg,
  51. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  52. })
  53. require.Eventually(t, func() bool {
  54. logEntry, ok := e.GetLog(tracking2)
  55. return ok && logEntry.Queued
  56. }, time.Second, 10*time.Millisecond)
  57. wg1.Wait()
  58. wg2.Wait()
  59. logEntry, ok := e.GetLog(tracking2)
  60. require.True(t, ok)
  61. assert.False(t, logEntry.Queued)
  62. assert.False(t, logEntry.Blocked)
  63. assert.Equal(t, int32(0), logEntry.ExitCode)
  64. assert.Contains(t, logEntry.Output, "queued-run")
  65. }
  66. func TestDifferentGroupsRunConcurrently(t *testing.T) {
  67. t.Parallel()
  68. actionA := &config.Action{
  69. Title: "Group A Job",
  70. Shell: "sleep 1",
  71. Groups: []string{"groupA"},
  72. }
  73. actionB := &config.Action{
  74. Title: "Group B Job",
  75. Shell: "echo group-b",
  76. Groups: []string{"groupB"},
  77. }
  78. e, cfg := testGroupExecutor(
  79. []*config.Action{actionA, actionB},
  80. map[string]*config.ActionGroup{
  81. "groupA": {MaxConcurrent: 1},
  82. "groupB": {MaxConcurrent: 1},
  83. },
  84. )
  85. wg1, _ := e.ExecRequest(&ExecutionRequest{
  86. Binding: e.FindBindingWithNoEntity(actionA),
  87. Cfg: cfg,
  88. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  89. })
  90. time.Sleep(50 * time.Millisecond)
  91. wg2, tracking2 := e.ExecRequest(&ExecutionRequest{
  92. Binding: e.FindBindingWithNoEntity(actionB),
  93. Cfg: cfg,
  94. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  95. })
  96. require.Eventually(t, func() bool {
  97. logEntry, ok := e.GetLog(tracking2)
  98. return ok && logEntry.ExecutionFinished && !logEntry.Queued
  99. }, 2*time.Second, 20*time.Millisecond)
  100. wg1.Wait()
  101. wg2.Wait()
  102. logEntry, ok := e.GetLog(tracking2)
  103. require.True(t, ok)
  104. assert.Contains(t, logEntry.Output, "group-b")
  105. }
  106. func TestPerActionConcurrencyStillBlocksWithoutQueue(t *testing.T) {
  107. t.Parallel()
  108. action := &config.Action{
  109. Title: "Single binding",
  110. Shell: "sleep 1",
  111. MaxConcurrent: 1,
  112. }
  113. e, cfg := testGroupExecutor([]*config.Action{action}, nil)
  114. binding := e.FindBindingWithNoEntity(action)
  115. wg1, _ := e.ExecRequest(&ExecutionRequest{
  116. Binding: binding,
  117. Cfg: cfg,
  118. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  119. })
  120. time.Sleep(50 * time.Millisecond)
  121. wg2, tracking2 := e.ExecRequest(&ExecutionRequest{
  122. Binding: binding,
  123. Cfg: cfg,
  124. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  125. })
  126. wg1.Wait()
  127. wg2.Wait()
  128. logEntry, ok := e.GetLog(tracking2)
  129. require.True(t, ok)
  130. assert.True(t, logEntry.Blocked)
  131. assert.False(t, logEntry.Queued)
  132. }
  133. func waitUntilExecutionStarted(t *testing.T, e *Executor, trackingID string) {
  134. t.Helper()
  135. require.Eventually(t, func() bool {
  136. logEntry, ok := e.GetLog(trackingID)
  137. return ok && logEntry.ExecutionStarted
  138. }, 2*time.Second, 10*time.Millisecond)
  139. }
  140. func assertWaitGroupPending(t *testing.T, wg *sync.WaitGroup) {
  141. t.Helper()
  142. done := make(chan struct{})
  143. go func() {
  144. wg.Wait()
  145. close(done)
  146. }()
  147. select {
  148. case <-done:
  149. t.Fatal("wait group completed before queued execution finished")
  150. case <-time.After(100 * time.Millisecond):
  151. }
  152. }
  153. func assertWaitGroupCompletes(t *testing.T, wg *sync.WaitGroup) {
  154. t.Helper()
  155. done := make(chan struct{})
  156. go func() {
  157. wg.Wait()
  158. close(done)
  159. }()
  160. select {
  161. case <-done:
  162. case <-time.After(3 * time.Second):
  163. t.Fatal("wait group did not complete after queue drained")
  164. }
  165. }
  166. func TestStartActionAndWaitWaitsForQueuedExecution(t *testing.T) {
  167. t.Parallel()
  168. first := &config.Action{
  169. Title: "Hold group",
  170. Shell: "sleep 1",
  171. Groups: []string{"unity"},
  172. }
  173. second := &config.Action{
  174. Title: "Wait in queue",
  175. Shell: "echo waited",
  176. Groups: []string{"unity"},
  177. }
  178. e, cfg := testGroupExecutor(
  179. []*config.Action{first, second},
  180. map[string]*config.ActionGroup{
  181. "unity": {MaxConcurrent: 1},
  182. },
  183. )
  184. wg1, tracking1 := e.ExecRequest(&ExecutionRequest{
  185. Binding: e.FindBindingWithNoEntity(first),
  186. Cfg: cfg,
  187. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  188. })
  189. waitUntilExecutionStarted(t, e, tracking1)
  190. wg2, tracking2 := e.ExecRequest(&ExecutionRequest{
  191. Binding: e.FindBindingWithNoEntity(second),
  192. Cfg: cfg,
  193. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  194. })
  195. assertWaitGroupPending(t, wg2)
  196. wg1.Wait()
  197. assertWaitGroupCompletes(t, wg2)
  198. logEntry, ok := e.GetLog(tracking2)
  199. require.True(t, ok)
  200. assert.Contains(t, logEntry.Output, "waited")
  201. }
  202. func TestUnknownActionGroupReferenceWarnsAndSkipsLimit(t *testing.T) {
  203. t.Parallel()
  204. action := &config.Action{
  205. Title: "Unknown group action",
  206. Shell: "echo ok",
  207. Groups: []string{"missing"},
  208. }
  209. e, cfg := testGroupExecutor([]*config.Action{action}, map[string]*config.ActionGroup{})
  210. wg, tracking := e.ExecRequest(&ExecutionRequest{
  211. Binding: e.FindBindingWithNoEntity(action),
  212. Cfg: cfg,
  213. AuthenticatedUser: auth.UserFromSystem(cfg, "testuser"),
  214. })
  215. wg.Wait()
  216. logEntry, ok := e.GetLog(tracking)
  217. require.True(t, ok)
  218. assert.False(t, logEntry.Queued)
  219. assert.Equal(t, int32(0), logEntry.ExitCode)
  220. }