feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
bircni
2026-07-22 15:10:45 +00:00
parent 7bec310002
commit c43cbe87ca
16 changed files with 753 additions and 32 deletions

View File

@@ -159,6 +159,81 @@ type idleAwareRunner struct {
idleCalls atomic.Int64
}
type availabilityRunner struct {
mockRunner
ready atomic.Bool
reason string
}
func (r *availabilityRunner) CanAcceptTask(_ context.Context) (bool, string) {
if r.ready.Load() {
return true, "ok"
}
return false, r.reason
}
func TestPollerReady(t *testing.T) {
cfg, err := config.LoadDefault("")
require.NoError(t, err)
poller := New(cfg, nil, &availabilityRunner{})
// /readyz reuses the availability the poll loop last recorded.
ready, reason := poller.Ready(time.Second)
assert.True(t, ready)
assert.Equal(t, "ok", reason)
poller.reportAvailability(false, "low disk space")
ready, reason = poller.Ready(time.Second)
assert.False(t, ready)
assert.Equal(t, "low disk space", reason)
poller.reportAvailability(true, "ok")
poller.lastPollFailed.Store(true)
poller.lastHealthyPoll.Store(time.Now().UnixNano())
ready, _ = poller.Ready(time.Second)
assert.True(t, ready, "transient polling errors should remain ready during grace")
poller.lastHealthyPoll.Store(time.Now().Add(-2 * time.Second).UnixNano())
ready, reason = poller.Ready(time.Second)
assert.False(t, ready)
assert.Equal(t, "unable to poll Gitea", reason)
poller.unregistered.Store(true)
ready, reason = poller.Ready(time.Second)
assert.False(t, ready)
assert.Equal(t, "runner is no longer registered", reason)
}
func TestPollerPausesAndResumesForLocalAvailability(t *testing.T) {
var fetches atomic.Int64
cli := mocks.NewClient(t)
cli.On("FetchTask", mock.Anything, mock.Anything).Maybe().Run(func(mock.Arguments) {
fetches.Add(1)
}).Return(connect_go.NewResponse(&runnerv1.FetchTaskResponse{}), nil)
cfg, err := config.LoadDefault("")
require.NoError(t, err)
cfg.Runner.FetchInterval = 10 * time.Millisecond
cfg.Runner.FetchIntervalMax = 10 * time.Millisecond
runner := &availabilityRunner{reason: "low disk space"}
poller := New(cfg, cli, runner)
var wg sync.WaitGroup
wg.Go(poller.Poll)
time.Sleep(40 * time.Millisecond)
assert.Zero(t, fetches.Load(), "an unavailable runner must not fetch a task")
runner.ready.Store(true)
require.Eventually(t, func() bool {
return fetches.Load() > 0
}, time.Second, 10*time.Millisecond, "polling should resume after local recovery")
shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
require.NoError(t, poller.Shutdown(shutdownCtx))
wg.Wait()
}
func (m *mockRunner) Run(ctx context.Context, _ *runnerv1.Task) error {
atomicMax(&m.maxConcurrent, m.running.Add(1))
select {