feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
bircni
2026-07-22 15:10:45 +00:00
parent 7bec310002
commit c43cbe87ca
16 changed files with 753 additions and 32 deletions

View File

@@ -0,0 +1,148 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package run
import (
"context"
"errors"
"testing"
"time"
"gitea.com/gitea/runner/internal/pkg/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConfiguredHealthCheckCachesAndRecovers(t *testing.T) {
cfg, err := config.LoadDefault("")
require.NoError(t, err)
cfg.HealthCheck.Enabled = true
cfg.HealthCheck.Script = "/health-check"
cfg.HealthCheck.Interval = time.Minute
cfg.HealthCheck.Timeout = time.Second
now := time.Now()
calls := 0
fail := true
r := &Runner{
cfg: cfg,
name: "runner-1",
now: func() time.Time { return now },
runHealthCheck: func(_ context.Context, script string, timeout time.Duration, env map[string]string) error {
calls++
assert.Equal(t, "/health-check", script)
assert.Equal(t, time.Second, timeout)
assert.Equal(t, "true", env["GITEA_RUNNER_HEALTH_CHECK"])
assert.Equal(t, "runner-1", env["GITEA_RUNNER_NAME"])
if fail {
return errors.New("unhealthy")
}
return nil
},
}
ready, reason := r.CanAcceptTask(t.Context())
assert.False(t, ready)
assert.Contains(t, reason, "unhealthy")
assert.Equal(t, 1, calls)
fail = false
ready, _ = r.CanAcceptTask(t.Context())
assert.False(t, ready, "the failed result should remain cached")
assert.Equal(t, 1, calls)
now = now.Add(time.Minute)
ready, reason = r.CanAcceptTask(t.Context())
assert.True(t, ready)
assert.Equal(t, "ok", reason)
assert.Equal(t, 2, calls)
}
func TestConfiguredHealthCheckDisabled(t *testing.T) {
cfg, err := config.LoadDefault("")
require.NoError(t, err)
cfg.HealthCheck.MinFreeDiskSpaceMB = 1 << 40
cfg.HealthCheck.Script = "/must-not-run"
r := &Runner{
cfg: cfg,
runHealthCheck: func(_ context.Context, _ string, _ time.Duration, _ map[string]string) error {
t.Fatal("disabled health check executed")
return nil
},
}
ready, reason := r.CanAcceptTask(t.Context())
assert.True(t, ready)
assert.Equal(t, "ok", reason)
}
func TestConfiguredHealthCheckDeferredWhileJobRuns(t *testing.T) {
cfg, err := config.LoadDefault("")
require.NoError(t, err)
cfg.HealthCheck.Enabled = true
cfg.HealthCheck.Script = "/health-check"
cfg.HealthCheck.Interval = time.Minute
now := time.Now()
calls := 0
fail := false
r := &Runner{
cfg: cfg,
now: func() time.Time { return now },
runHealthCheck: func(_ context.Context, _ string, _ time.Duration, _ map[string]string) error {
calls++
if fail {
return errors.New("unhealthy")
}
return nil
},
}
ready, reason := r.CanAcceptTask(t.Context())
assert.True(t, ready)
assert.Equal(t, "ok", reason)
assert.Equal(t, 1, calls)
now = now.Add(time.Minute)
fail = true
r.runningCount.Store(1)
ready, reason = r.CanAcceptTask(t.Context())
assert.True(t, ready, "the last result should be reused while a job runs")
assert.Equal(t, "ok", reason)
assert.Equal(t, 1, calls)
r.runningCount.Store(0)
ready, reason = r.CanAcceptTask(t.Context())
assert.False(t, ready)
assert.Contains(t, reason, "unhealthy")
assert.Equal(t, 2, calls)
}
func TestConfiguredHealthCheckInitialRunDeferredWhileJobRuns(t *testing.T) {
cfg, err := config.LoadDefault("")
require.NoError(t, err)
cfg.HealthCheck.Enabled = true
cfg.HealthCheck.Script = "/health-check"
calls := 0
r := &Runner{
cfg: cfg,
runHealthCheck: func(_ context.Context, _ string, _ time.Duration, _ map[string]string) error {
calls++
return nil
},
}
r.runningCount.Store(1)
ready, reason := r.CanAcceptTask(t.Context())
assert.True(t, ready)
assert.Contains(t, reason, "deferred")
assert.Zero(t, calls)
r.runningCount.Store(0)
ready, reason = r.CanAcceptTask(t.Context())
assert.True(t, ready)
assert.Equal(t, "ok", reason)
assert.Equal(t, 1, calls)
}