mirror of
https://github.com/F1bonacc1/process-compose.git
synced 2024-10-03 22:27:13 +03:00
Issue #107 - Fix goroutine leak when readiness probe on
This commit is contained in:
Berger Eugene
parent
5836dc8ea0
commit
f058176b4a
22
issues/issue_107/after.4h.alloc_space.txt
Normal file
22
issues/issue_107/after.4h.alloc_space.txt
Normal file
@ -0,0 +1,22 @@
|
||||
> go tool pprof -alloc_space heap.out
|
||||
File: process-compose
|
||||
Type: alloc_space
|
||||
Time: Dec 2, 2023 at 2:32am (IST)
|
||||
Duration: 30.01s, Total samples = 206.33MB
|
||||
Entering interactive mode (type "help" for commands, "o" for options)
|
||||
(pprof) top
|
||||
Showing nodes accounting for 199.44MB, 96.66% of 206.33MB total
|
||||
Dropped 72 nodes (cum <= 1.03MB)
|
||||
Showing top 10 nodes out of 26
|
||||
flat flat% sum% cum cum%
|
||||
187.04MB 90.65% 90.65% 187.04MB 90.65% github.com/rivo/tview.(*TextView).parseAhead
|
||||
5.90MB 2.86% 93.51% 5.90MB 2.86% strings.(*Builder).WriteString (inline)
|
||||
3MB 1.45% 94.97% 3.50MB 1.70% github.com/rivo/tview.step
|
||||
3MB 1.45% 96.42% 196.44MB 95.21% github.com/rivo/tview.(*TextView).Draw
|
||||
0.50MB 0.24% 96.66% 3.50MB 1.70% github.com/rivo/tview.(*Table).Draw
|
||||
0 0% 96.66% 2MB 0.97% github.com/InVisionApp/go-health/v2.(*Health).startRunner.func1
|
||||
0 0% 96.66% 2MB 0.97% github.com/InVisionApp/go-health/v2.(*Health).startRunner.func2
|
||||
0 0% 96.66% 200.44MB 97.15% github.com/f1bonacc1/process-compose/src/cmd.startTui
|
||||
0 0% 96.66% 140.93MB 68.30% github.com/f1bonacc1/process-compose/src/tui.(*pcView).updateLogs.(*Application).QueueUpdateDraw.func4
|
||||
0 0% 96.66% 59.51MB 28.84% github.com/f1bonacc1/process-compose/src/tui.(*pcView).updateTable.(*Application).QueueUpdateDraw.func4
|
||||
(pprof)
|
||||
20
issues/issue_107/after.4h.txt
Normal file
20
issues/issue_107/after.4h.txt
Normal file
@ -0,0 +1,20 @@
|
||||
> go tool pprof heap.out
|
||||
File: process-compose
|
||||
Type: inuse_space
|
||||
Time: Dec 2, 2023 at 2:32am (IST)
|
||||
Duration: 30.01s, Total samples = 1.52MB
|
||||
Entering interactive mode (type "help" for commands, "o" for options)
|
||||
(pprof) top
|
||||
Showing nodes accounting for -528.06kB, 34.02% of 1552.40kB total
|
||||
Showing top 10 nodes out of 30
|
||||
flat flat% sum% cum cum%
|
||||
-528.17kB 34.02% 34.02% -528.17kB 34.02% regexp.(*bitState).reset
|
||||
512.17kB 32.99% 1.03% 512.17kB 32.99% internal/profile.(*Profile).postDecode
|
||||
-512.06kB 32.99% 34.02% -512.06kB 32.99% github.com/rivo/tview.(*TextView).parseAhead
|
||||
0 0% 34.02% -528.17kB 34.02% github.com/f1bonacc1/process-compose/src/app.(*Process).handleInfo
|
||||
0 0% 34.02% -528.17kB 34.02% github.com/f1bonacc1/process-compose/src/app.(*Process).handleOutput
|
||||
0 0% 34.02% -512.06kB 32.99% github.com/f1bonacc1/process-compose/src/cmd.startTui
|
||||
0 0% 34.02% -528.17kB 34.02% github.com/f1bonacc1/process-compose/src/pclog.(*ProcessLogBuffer).Write
|
||||
0 0% 34.02% -528.17kB 34.02% github.com/f1bonacc1/process-compose/src/tui.(*LogView).WriteString
|
||||
0 0% 34.02% -512.06kB 32.99% github.com/f1bonacc1/process-compose/src/tui.(*pcView).updateLogs.(*Application).QueueUpdateDraw.func4
|
||||
0 0% 34.02% -512.06kB 32.99% github.com/f1bonacc1/process-compose/src/tui.SetupTui
|
||||
104
issues/issue_107/process-compose.yaml
Normal file
104
issues/issue_107/process-compose.yaml
Normal file
@ -0,0 +1,104 @@
|
||||
version: "0.5"
|
||||
|
||||
log_level: debug
|
||||
log_length: 1000
|
||||
|
||||
processes:
|
||||
redis:
|
||||
command: "sleep 999999999"
|
||||
readiness_probe:
|
||||
exec:
|
||||
command: "pidof process-compose"
|
||||
initial_delay_seconds: 1
|
||||
period_seconds: 1
|
||||
timeout_seconds: 1
|
||||
success_threshold: 1
|
||||
failure_threshold: 20
|
||||
|
||||
webserver-a:
|
||||
command: "sleep 999999999"
|
||||
readiness_probe:
|
||||
http_get:
|
||||
host: localhost
|
||||
scheme: http
|
||||
path: "/live"
|
||||
port: 8080
|
||||
initial_delay_seconds: 1
|
||||
period_seconds: 1
|
||||
timeout_seconds: 1
|
||||
success_threshold: 1
|
||||
failure_threshold: 20
|
||||
depends_on:
|
||||
redis:
|
||||
condition: process_healthy
|
||||
|
||||
webserver-b:
|
||||
command: "sleep 999999999"
|
||||
readiness_probe:
|
||||
http_get:
|
||||
host: localhost
|
||||
scheme: http
|
||||
path: "/live"
|
||||
port: 8080
|
||||
initial_delay_seconds: 1
|
||||
period_seconds: 1
|
||||
timeout_seconds: 1
|
||||
success_threshold: 1
|
||||
failure_threshold: 20
|
||||
depends_on:
|
||||
redis:
|
||||
condition: process_started
|
||||
shutdown:
|
||||
parent_only: true
|
||||
|
||||
rpyc-client:
|
||||
command: "sleep 999999999"
|
||||
depends_on:
|
||||
webserver-b:
|
||||
condition: process_healthy
|
||||
shutdown:
|
||||
signal: 1 # SIGHUP
|
||||
timeout_seconds: 5
|
||||
availability:
|
||||
restart: on_failure
|
||||
backoff_seconds: 10
|
||||
max_restarts: 3
|
||||
|
||||
webserver-c:
|
||||
command: "sleep 999999999"
|
||||
readiness_probe:
|
||||
http_get:
|
||||
host: localhost
|
||||
scheme: http
|
||||
path: "/live"
|
||||
port: 8080
|
||||
initial_delay_seconds: 20
|
||||
period_seconds: 30
|
||||
timeout_seconds: 10
|
||||
success_threshold: 1
|
||||
failure_threshold: 4
|
||||
depends_on:
|
||||
webserver-b:
|
||||
condition: process_healthy
|
||||
availability:
|
||||
restart: on_failure
|
||||
backoff_seconds: 10
|
||||
max_restarts: 3
|
||||
|
||||
httpclient:
|
||||
command: "sleep 999999999"
|
||||
depends_on:
|
||||
webserver-b:
|
||||
condition: process_healthy
|
||||
|
||||
pc_log:
|
||||
command: "tail -f -n100 process-compose-${USER}.log"
|
||||
working_dir: "/tmp"
|
||||
namespace: debug
|
||||
|
||||
memory:
|
||||
command: "./bin/process-compose project state --with-memory"
|
||||
availability:
|
||||
restart: always
|
||||
backoff_seconds: 60
|
||||
namespace: debug
|
||||
@ -38,9 +38,10 @@ type Process struct {
|
||||
stateMtx sync.Mutex
|
||||
procCond sync.Cond
|
||||
procStateChan chan string
|
||||
procReadyChan chan string
|
||||
procReadyCtx context.Context
|
||||
readyCancelFn context.CancelFunc
|
||||
procRunCtx context.Context
|
||||
runCancelFn context.CancelFunc
|
||||
procColor func(a ...interface{}) string
|
||||
noColor func(a ...interface{}) string
|
||||
redColor func(a ...interface{}) string
|
||||
@ -83,13 +84,13 @@ func NewProcess(
|
||||
logBuffer: procLog,
|
||||
shellConfig: shellConfig,
|
||||
procStateChan: make(chan string, 1),
|
||||
procReadyChan: make(chan string, 1),
|
||||
printLogs: printLogs,
|
||||
isMain: isMain,
|
||||
extraArgs: extraArgs,
|
||||
}
|
||||
|
||||
proc.procReadyCtx, proc.readyCancelFn = context.WithCancel(context.Background())
|
||||
proc.procRunCtx, proc.runCancelFn = context.WithCancel(context.Background())
|
||||
proc.setUpProbes()
|
||||
proc.procCond = *sync.NewCond(proc)
|
||||
return proc
|
||||
@ -153,7 +154,14 @@ func (p *Process) run() int {
|
||||
log.Info().Msgf("Restarting %s in %v second(s)... Restarts: %d",
|
||||
p.getName(), p.getBackoff().Seconds(), p.procState.Restarts)
|
||||
|
||||
time.Sleep(p.getBackoff())
|
||||
select {
|
||||
case <-p.procRunCtx.Done():
|
||||
log.Debug().Str("process", p.getName()).Msg("process stopped while waiting to restart")
|
||||
break
|
||||
case <-time.After(p.getBackoff()):
|
||||
p.handleInfo("\n")
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.onProcessEnd(types.ProcessStateCompleted)
|
||||
return p.procState.ExitCode
|
||||
@ -262,19 +270,17 @@ func (p *Process) waitUntilReady() bool {
|
||||
for {
|
||||
select {
|
||||
case <-p.procReadyCtx.Done():
|
||||
log.Error().Msgf("Process %s was aborted and won't become ready", p.getName())
|
||||
return false
|
||||
case ready := <-p.procReadyChan:
|
||||
if ready == types.ProcessHealthReady {
|
||||
if p.procState.Health == types.ProcessHealthReady {
|
||||
return true
|
||||
}
|
||||
log.Error().Msgf("Process %s was aborted and won't become ready", p.getName())
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Process) wontRun() {
|
||||
p.onProcessEnd(types.ProcessStateCompleted)
|
||||
|
||||
}
|
||||
|
||||
// perform graceful process shutdown if defined in configuration
|
||||
@ -285,6 +291,7 @@ func (p *Process) shutDownNoRestart() error {
|
||||
|
||||
// perform graceful process shutdown if defined in configuration
|
||||
func (p *Process) shutDown() error {
|
||||
p.runCancelFn()
|
||||
if !p.isRunning() {
|
||||
state := p.getState()
|
||||
log.Debug().Msgf("process %s is in state %s not shutting down", p.getName(), state.Status)
|
||||
@ -579,7 +586,7 @@ func (p *Process) onReadinessCheckEnd(isOk, isFatal bool, err string) {
|
||||
_ = p.shutDown()
|
||||
} else if isOk {
|
||||
p.procState.Health = types.ProcessHealthReady
|
||||
p.procReadyChan <- types.ProcessHealthReady
|
||||
p.readyCancelFn()
|
||||
} else {
|
||||
p.procState.Health = types.ProcessHealthNotReady
|
||||
}
|
||||
|
||||
@ -81,6 +81,9 @@ func (p *Prober) healthCheckCompleted(state *health.State) {
|
||||
if state.Status == OK {
|
||||
ok = true
|
||||
}
|
||||
if p.stopped {
|
||||
return
|
||||
}
|
||||
p.onCheckEndFunc(ok, fatal, state.Err)
|
||||
}
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#trap "echo ERROR: The program is terminated ; exit" SIGTERM
|
||||
trap 'echo CODE: $?; exit $EXIT_CODE' 1 2 3 15
|
||||
|
||||
LOOPS=30000
|
||||
LOOPS=3000000
|
||||
for (( i=1; i<=LOOPS; i++ ))
|
||||
do
|
||||
sleep 0.1
|
||||
|
||||
Loading…
Reference in New Issue
Block a user