feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

internal/app/poll/poller.go

@@ -32,6 +32,12 @@ type IdleRunner interface {
 	OnIdle(ctx context.Context)
 }
 
+// AvailabilityRunner can temporarily pause task fetching for local resource
+// conditions such as low disk space without changing server-side scheduling.
+type AvailabilityRunner interface {
+	CanAcceptTask(ctx context.Context) (bool, string)
+}
+
 type Poller struct {
 	client       client.Client
 	runner       TaskRunner
@@ -48,7 +54,12 @@ type Poller struct {
 
 	// unregistered is set when the server rejects the runner with an
 	// Unauthenticated response, meaning the runner is no longer registered.
-	unregistered atomic.Bool
+	unregistered       atomic.Bool
+	lastHealthyPoll    atomic.Int64
+	lastPollFailed     atomic.Bool
+	availabilityMu     sync.Mutex
+	availabilityReady  bool
+	availabilityReason string
 }
 
 // workerState holds the single poller's backoff state. Consecutive empty or
@@ -70,7 +81,7 @@ func New(cfg *config.Config, client client.Client, runner TaskRunner) *Poller {
 
 	done := make(chan struct{})
 
-	return &Poller{
+	p := &Poller{
 		client: client,
 		runner: runner,
 		cfg:    cfg,
@@ -83,6 +94,10 @@ func New(cfg *config.Config, client client.Client, runner TaskRunner) *Poller {
 
 		done: done,
 	}
+	p.lastHealthyPoll.Store(time.Now().UnixNano())
+	p.availabilityReady = true
+	p.availabilityReason = "ok"
+	return p
 }
 
 func (p *Poller) Poll() {
@@ -102,6 +117,17 @@ func (p *Poller) Poll() {
 			return
 		}
 
+		ready, reason := p.localAvailability(p.pollingCtx)
+		p.reportAvailability(ready, reason)
+		if !ready {
+			p.runIdleMaintenance()
+			<-sem
+			if !p.waitBackoff(s) {
+				return
+			}
+			continue
+		}
+
 		task, ok := p.fetchTask(p.pollingCtx, s)
 		if !ok {
 			p.runIdleMaintenance()
@@ -127,6 +153,15 @@ func (p *Poller) PollOnce() {
 	defer close(p.done)
 	s := &workerState{}
 	for {
+		ready, reason := p.localAvailability(p.pollingCtx)
+		p.reportAvailability(ready, reason)
+		if !ready {
+			p.runIdleMaintenance()
+			if !p.waitBackoff(s) {
+				return
+			}
+			continue
+		}
 		task, ok := p.fetchTask(p.pollingCtx, s)
 		if !ok {
 			p.runIdleMaintenance()
@@ -154,6 +189,48 @@ func (p *Poller) Unregistered() bool {
 	return p.unregistered.Load()
 }
 
+// Ready reports whether the daemon can currently communicate with Gitea and
+// accept work, reusing the availability the poll loop last observed rather than
+// re-running the check. Transient transport failures are tolerated for grace.
+func (p *Poller) Ready(grace time.Duration) (bool, string) {
+	if p.unregistered.Load() {
+		return false, "runner is no longer registered"
+	}
+	p.availabilityMu.Lock()
+	ready, reason := p.availabilityReady, p.availabilityReason
+	p.availabilityMu.Unlock()
+	if !ready {
+		return false, reason
+	}
+	if !p.lastPollFailed.Load() {
+		return true, "ok"
+	}
+	if time.Since(time.Unix(0, p.lastHealthyPoll.Load())) <= grace {
+		return true, "polling errors within grace period"
+	}
+	return false, "unable to poll Gitea"
+}
+
+func (p *Poller) localAvailability(ctx context.Context) (bool, string) {
+	if available, ok := p.runner.(AvailabilityRunner); ok {
+		return available.CanAcceptTask(ctx)
+	}
+	return true, "ok"
+}
+
+func (p *Poller) reportAvailability(ready bool, reason string) {
+	p.availabilityMu.Lock()
+	defer p.availabilityMu.Unlock()
+	switch {
+	case !ready && p.availabilityReady:
+		log.Warnf("runner temporarily unavailable: %s", reason)
+	case ready && !p.availabilityReady:
+		log.Info("runner local health recovered, resuming task polling")
+	}
+	p.availabilityReady = ready
+	p.availabilityReason = reason
+}
+
 func (p *Poller) runIdleMaintenance() {
 	if idleRunner, ok := p.runner.(IdleRunner); ok {
 		idleRunner.OnIdle(p.jobsCtx)
@@ -273,6 +350,7 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 	// found no work within FetchTimeout. Treat it as an empty response and do
 	// not record the duration — the timeout value would swamp the histogram.
 	if errors.Is(err, context.DeadlineExceeded) {
+		p.markHealthyPoll()
 		s.consecutiveEmpty++
 		s.consecutiveErrors = 0 // timeout is a healthy idle response
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultEmpty).Inc()
@@ -291,11 +369,13 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 			return nil, false
 		}
 		log.WithError(err).Error("failed to fetch task")
+		p.lastPollFailed.Store(true)
 		s.consecutiveErrors++
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()
 		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask).Inc()
 		return nil, false
 	}
+	p.markHealthyPoll()
 
 	// Successful response — reset error counter.
 	s.consecutiveErrors = 0
@@ -322,3 +402,8 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 	metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultTask).Inc()
 	return resp.Msg.Task, true
 }
+
+func (p *Poller) markHealthyPoll() {
+	p.lastHealthyPoll.Store(time.Now().UnixNano())
+	p.lastPollFailed.Store(false)
+}

internal/app/poll/poller_test.go

@@ -159,6 +159,81 @@ type idleAwareRunner struct {
 	idleCalls atomic.Int64
 }
 
+type availabilityRunner struct {
+	mockRunner
+	ready  atomic.Bool
+	reason string
+}
+
+func (r *availabilityRunner) CanAcceptTask(_ context.Context) (bool, string) {
+	if r.ready.Load() {
+		return true, "ok"
+	}
+	return false, r.reason
+}
+
+func TestPollerReady(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	poller := New(cfg, nil, &availabilityRunner{})
+
+	// /readyz reuses the availability the poll loop last recorded.
+	ready, reason := poller.Ready(time.Second)
+	assert.True(t, ready)
+	assert.Equal(t, "ok", reason)
+
+	poller.reportAvailability(false, "low disk space")
+	ready, reason = poller.Ready(time.Second)
+	assert.False(t, ready)
+	assert.Equal(t, "low disk space", reason)
+
+	poller.reportAvailability(true, "ok")
+	poller.lastPollFailed.Store(true)
+	poller.lastHealthyPoll.Store(time.Now().UnixNano())
+	ready, _ = poller.Ready(time.Second)
+	assert.True(t, ready, "transient polling errors should remain ready during grace")
+
+	poller.lastHealthyPoll.Store(time.Now().Add(-2 * time.Second).UnixNano())
+	ready, reason = poller.Ready(time.Second)
+	assert.False(t, ready)
+	assert.Equal(t, "unable to poll Gitea", reason)
+
+	poller.unregistered.Store(true)
+	ready, reason = poller.Ready(time.Second)
+	assert.False(t, ready)
+	assert.Equal(t, "runner is no longer registered", reason)
+}
+
+func TestPollerPausesAndResumesForLocalAvailability(t *testing.T) {
+	var fetches atomic.Int64
+	cli := mocks.NewClient(t)
+	cli.On("FetchTask", mock.Anything, mock.Anything).Maybe().Run(func(mock.Arguments) {
+		fetches.Add(1)
+	}).Return(connect_go.NewResponse(&runnerv1.FetchTaskResponse{}), nil)
+
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.Runner.FetchInterval = 10 * time.Millisecond
+	cfg.Runner.FetchIntervalMax = 10 * time.Millisecond
+	runner := &availabilityRunner{reason: "low disk space"}
+	poller := New(cfg, cli, runner)
+
+	var wg sync.WaitGroup
+	wg.Go(poller.Poll)
+	time.Sleep(40 * time.Millisecond)
+	assert.Zero(t, fetches.Load(), "an unavailable runner must not fetch a task")
+
+	runner.ready.Store(true)
+	require.Eventually(t, func() bool {
+		return fetches.Load() > 0
+	}, time.Second, 10*time.Millisecond, "polling should resume after local recovery")
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	require.NoError(t, poller.Shutdown(shutdownCtx))
+	wg.Wait()
+}
+
 func (m *mockRunner) Run(ctx context.Context, _ *runnerv1.Task) error {
 	atomicMax(&m.maxConcurrent, m.running.Add(1))
 	select {