git.iko.soy/crush
on main help sign in
fix(test,race): probe server health during the run, not after
eb03b3bChristian Rocha
internal/cmd/clientserverrace/race_test.go+54−12
compare with main
unifiedsplit
@@ -18,6 +18,7 @@ import (
18 "runtime"
18 "runtime"
19 "strings"
19 "strings"
20 "sync"
20 "sync"
21+ "sync/atomic"
21 "testing"
22 "testing"
22 "time"
23 "time"
23)
24)
@@ -81,7 +82,8 @@ func TestClientServerSpawnRace(t *testing.T) {
81 }
82 }
82 }
83 }
83
84
84− env := append(os.Environ(),
85+ env := append(
86+ os.Environ(),
85 "CRUSH_CLIENT_SERVER=1",
87 "CRUSH_CLIENT_SERVER=1",
86 "XDG_CACHE_HOME="+cacheHome,
88 "XDG_CACHE_HOME="+cacheHome,
87 "XDG_DATA_HOME="+dataHome,
89 "XDG_DATA_HOME="+dataHome,
@@ -108,8 +110,43 @@ func TestClientServerSpawnRace(t *testing.T) {
108 }
110 }
109 results := make(chan result, numClients)
111 results := make(chan result, numClients)
110
112
113+ // Probe /v1/health concurrently while the clients are still
114+ // running. The server self-shuts-down when the last workspace is
115+ // released (internal/backend/backend.go:DeleteWorkspace), so once
116+ // all clients exit cleanly the socket may legitimately be gone —
117+ // asserting the socket post-hoc would race with that documented
118+ // self-shutdown. Instead we require that during the parallel run
119+ // at least one /v1/health probe got a 2xx, which proves the
120+ // spawn-and-readiness path actually produced a live server.
121+ var sawHealthy atomic.Bool
122+ probeDone := make(chan struct{})
123+ stopProbe := make(chan struct{})
124+
111 var wg sync.WaitGroup
125 var wg sync.WaitGroup
112 start := make(chan struct{})
126 start := make(chan struct{})
127+
128+ go func() {
129+ defer close(probeDone)
130+ <-start
131+ deadline := time.Now().Add(clientTimeout)
132+ for time.Now().Before(deadline) {
133+ select {
134+ case <-stopProbe:
135+ return
136+ default:
137+ }
138+ if err := pingHealth(socketPath); err == nil {
139+ sawHealthy.Store(true)
140+ return
141+ }
142+ select {
143+ case <-stopProbe:
144+ return
145+ case <-time.After(50 * time.Millisecond):
146+ }
147+ }
148+ }()
149+
113 for i := range numClients {
150 for i := range numClients {
114 wg.Add(1)
151 wg.Add(1)
115 go func(i int) {
152 go func(i int) {
@@ -133,7 +170,8 @@ func TestClientServerSpawnRace(t *testing.T) {
133 // (e.g. waiting on event subscriptions); the context
170 // (e.g. waiting on event subscriptions); the context
134 // timeout above bounds that. We assert race outcomes
171 // timeout above bounds that. We assert race outcomes
135 // purely from output, not exit code.
172 // purely from output, not exit code.
136− c := exec.CommandContext(ctx, bin,
173+ c := exec.CommandContext(
174+ ctx, bin,
137 "--host", host,
175 "--host", host,
138 "--cwd", cwd,
176 "--cwd", cwd,
139 "run", "hi",
177 "run", "hi",
@@ -156,6 +194,8 @@ func TestClientServerSpawnRace(t *testing.T) {
156 close(start) // release all clients as simultaneously as possible
194 close(start) // release all clients as simultaneously as possible
157 wg.Wait()
195 wg.Wait()
158 close(results)
196 close(results)
197+ close(stopProbe)
198+ <-probeDone
159
199
160 var raceFailures []string
200 var raceFailures []string
161 for r := range results {
201 for r := range results {
@@ -169,21 +209,23 @@ func TestClientServerSpawnRace(t *testing.T) {
169 }
209 }
170
210
171 if len(raceFailures) > 0 {
211 if len(raceFailures) > 0 {
212+ t.Fatalf(
172− t.Fatalf("client/server spawn race regressed: %d/%d clients failed\n\n%s",
213+ "client/server spawn race regressed: %d/%d clients failed\n\n%s",
173 len(raceFailures), numClients,
214 len(raceFailures), numClients,
174 strings.Join(raceFailures, "\n---\n"),
215 strings.Join(raceFailures, "\n---\n"),
175 )
216 )
176 }
217 }
177
218
178− // Positive sanity check: a single live server should be
219+ // Positive sanity check: at some point during the parallel run a
179− // reachable on the socket. If the race had fired, we'd either
220+ // /v1/health probe must have succeeded. We deliberately do *not*
180− // have no socket or a stale one with no listener.
221+ // stat the socket post-hoc: when every client returns cleanly
181− if _, err := os.Stat(socketPath); err != nil {
222+ // (e.g. exits early because no providers are configured), the
182− t.Fatalf("expected socket %s to exist after parallel clients ran, got: %v",
223+ // last DeleteWorkspace triggers the server's self-shutdown and
224+ // the socket disappears. That is correct behaviour, not a race
225+ // regression.
226+ if !sawHealthy.Load() {
227+ t.Fatalf("no /v1/health probe succeeded on %s while %d clients were running",
183− socketPath, err)
228+ socketPath, numClients)
184− }
185− if err := pingHealth(socketPath); err != nil {
186− t.Fatalf("server is not responding on %s: %v", socketPath, err)
187 }
229 }
188}
230}
189
231