mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-07-22 20:47:45 +00:00
feat: add runner health admission checks (#1090)
Opt-in local task-admission checks under a `health_check` config section (disabled by default): - pause new task fetching when free disk space on the workspace volume is below the configured minimum - optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable - checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery - `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint --------- Co-authored-by: silverwind <me@silverwind.io> Reviewed-on: https://gitea.com/gitea/runner/pulls/1090 Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
@@ -159,6 +159,81 @@ type idleAwareRunner struct {
|
||||
idleCalls atomic.Int64
|
||||
}
|
||||
|
||||
type availabilityRunner struct {
|
||||
mockRunner
|
||||
ready atomic.Bool
|
||||
reason string
|
||||
}
|
||||
|
||||
func (r *availabilityRunner) CanAcceptTask(_ context.Context) (bool, string) {
|
||||
if r.ready.Load() {
|
||||
return true, "ok"
|
||||
}
|
||||
return false, r.reason
|
||||
}
|
||||
|
||||
func TestPollerReady(t *testing.T) {
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
poller := New(cfg, nil, &availabilityRunner{})
|
||||
|
||||
// /readyz reuses the availability the poll loop last recorded.
|
||||
ready, reason := poller.Ready(time.Second)
|
||||
assert.True(t, ready)
|
||||
assert.Equal(t, "ok", reason)
|
||||
|
||||
poller.reportAvailability(false, "low disk space")
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "low disk space", reason)
|
||||
|
||||
poller.reportAvailability(true, "ok")
|
||||
poller.lastPollFailed.Store(true)
|
||||
poller.lastHealthyPoll.Store(time.Now().UnixNano())
|
||||
ready, _ = poller.Ready(time.Second)
|
||||
assert.True(t, ready, "transient polling errors should remain ready during grace")
|
||||
|
||||
poller.lastHealthyPoll.Store(time.Now().Add(-2 * time.Second).UnixNano())
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "unable to poll Gitea", reason)
|
||||
|
||||
poller.unregistered.Store(true)
|
||||
ready, reason = poller.Ready(time.Second)
|
||||
assert.False(t, ready)
|
||||
assert.Equal(t, "runner is no longer registered", reason)
|
||||
}
|
||||
|
||||
func TestPollerPausesAndResumesForLocalAvailability(t *testing.T) {
|
||||
var fetches atomic.Int64
|
||||
cli := mocks.NewClient(t)
|
||||
cli.On("FetchTask", mock.Anything, mock.Anything).Maybe().Run(func(mock.Arguments) {
|
||||
fetches.Add(1)
|
||||
}).Return(connect_go.NewResponse(&runnerv1.FetchTaskResponse{}), nil)
|
||||
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
cfg.Runner.FetchInterval = 10 * time.Millisecond
|
||||
cfg.Runner.FetchIntervalMax = 10 * time.Millisecond
|
||||
runner := &availabilityRunner{reason: "low disk space"}
|
||||
poller := New(cfg, cli, runner)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(poller.Poll)
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
assert.Zero(t, fetches.Load(), "an unavailable runner must not fetch a task")
|
||||
|
||||
runner.ready.Store(true)
|
||||
require.Eventually(t, func() bool {
|
||||
return fetches.Load() > 0
|
||||
}, time.Second, 10*time.Millisecond, "polling should resume after local recovery")
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, poller.Shutdown(shutdownCtx))
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (m *mockRunner) Run(ctx context.Context, _ *runnerv1.Task) error {
|
||||
atomicMax(&m.maxConcurrent, m.running.Add(1))
|
||||
select {
|
||||
|
||||
Reference in New Issue
Block a user