feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

internal/app/poll/poller.go

@@ -32,6 +32,12 @@ type IdleRunner interface {
 	OnIdle(ctx context.Context)
 }
 
+// AvailabilityRunner can temporarily pause task fetching for local resource
+// conditions such as low disk space without changing server-side scheduling.
+type AvailabilityRunner interface {
+	CanAcceptTask(ctx context.Context) (bool, string)
+}
+
 type Poller struct {
 	client       client.Client
 	runner       TaskRunner
@@ -48,7 +54,12 @@ type Poller struct {
 
 	// unregistered is set when the server rejects the runner with an
 	// Unauthenticated response, meaning the runner is no longer registered.
-	unregistered atomic.Bool
+	unregistered       atomic.Bool
+	lastHealthyPoll    atomic.Int64
+	lastPollFailed     atomic.Bool
+	availabilityMu     sync.Mutex
+	availabilityReady  bool
+	availabilityReason string
 }
 
 // workerState holds the single poller's backoff state. Consecutive empty or
@@ -70,7 +81,7 @@ func New(cfg *config.Config, client client.Client, runner TaskRunner) *Poller {
 
 	done := make(chan struct{})
 
-	return &Poller{
+	p := &Poller{
 		client: client,
 		runner: runner,
 		cfg:    cfg,
@@ -83,6 +94,10 @@ func New(cfg *config.Config, client client.Client, runner TaskRunner) *Poller {
 
 		done: done,
 	}
+	p.lastHealthyPoll.Store(time.Now().UnixNano())
+	p.availabilityReady = true
+	p.availabilityReason = "ok"
+	return p
 }
 
 func (p *Poller) Poll() {
@@ -102,6 +117,17 @@ func (p *Poller) Poll() {
 			return
 		}
 
+		ready, reason := p.localAvailability(p.pollingCtx)
+		p.reportAvailability(ready, reason)
+		if !ready {
+			p.runIdleMaintenance()
+			<-sem
+			if !p.waitBackoff(s) {
+				return
+			}
+			continue
+		}
+
 		task, ok := p.fetchTask(p.pollingCtx, s)
 		if !ok {
 			p.runIdleMaintenance()
@@ -127,6 +153,15 @@ func (p *Poller) PollOnce() {
 	defer close(p.done)
 	s := &workerState{}
 	for {
+		ready, reason := p.localAvailability(p.pollingCtx)
+		p.reportAvailability(ready, reason)
+		if !ready {
+			p.runIdleMaintenance()
+			if !p.waitBackoff(s) {
+				return
+			}
+			continue
+		}
 		task, ok := p.fetchTask(p.pollingCtx, s)
 		if !ok {
 			p.runIdleMaintenance()
@@ -154,6 +189,48 @@ func (p *Poller) Unregistered() bool {
 	return p.unregistered.Load()
 }
 
+// Ready reports whether the daemon can currently communicate with Gitea and
+// accept work, reusing the availability the poll loop last observed rather than
+// re-running the check. Transient transport failures are tolerated for grace.
+func (p *Poller) Ready(grace time.Duration) (bool, string) {
+	if p.unregistered.Load() {
+		return false, "runner is no longer registered"
+	}
+	p.availabilityMu.Lock()
+	ready, reason := p.availabilityReady, p.availabilityReason
+	p.availabilityMu.Unlock()
+	if !ready {
+		return false, reason
+	}
+	if !p.lastPollFailed.Load() {
+		return true, "ok"
+	}
+	if time.Since(time.Unix(0, p.lastHealthyPoll.Load())) <= grace {
+		return true, "polling errors within grace period"
+	}
+	return false, "unable to poll Gitea"
+}
+
+func (p *Poller) localAvailability(ctx context.Context) (bool, string) {
+	if available, ok := p.runner.(AvailabilityRunner); ok {
+		return available.CanAcceptTask(ctx)
+	}
+	return true, "ok"
+}
+
+func (p *Poller) reportAvailability(ready bool, reason string) {
+	p.availabilityMu.Lock()
+	defer p.availabilityMu.Unlock()
+	switch {
+	case !ready && p.availabilityReady:
+		log.Warnf("runner temporarily unavailable: %s", reason)
+	case ready && !p.availabilityReady:
+		log.Info("runner local health recovered, resuming task polling")
+	}
+	p.availabilityReady = ready
+	p.availabilityReason = reason
+}
+
 func (p *Poller) runIdleMaintenance() {
 	if idleRunner, ok := p.runner.(IdleRunner); ok {
 		idleRunner.OnIdle(p.jobsCtx)
@@ -273,6 +350,7 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 	// found no work within FetchTimeout. Treat it as an empty response and do
 	// not record the duration — the timeout value would swamp the histogram.
 	if errors.Is(err, context.DeadlineExceeded) {
+		p.markHealthyPoll()
 		s.consecutiveEmpty++
 		s.consecutiveErrors = 0 // timeout is a healthy idle response
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultEmpty).Inc()
@@ -291,11 +369,13 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 			return nil, false
 		}
 		log.WithError(err).Error("failed to fetch task")
+		p.lastPollFailed.Store(true)
 		s.consecutiveErrors++
 		metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()
 		metrics.ClientErrors.WithLabelValues(metrics.LabelMethodFetchTask).Inc()
 		return nil, false
 	}
+	p.markHealthyPoll()
 
 	// Successful response — reset error counter.
 	s.consecutiveErrors = 0
@@ -322,3 +402,8 @@ func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task,
 	metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultTask).Inc()
 	return resp.Msg.Task, true
 }
+
+func (p *Poller) markHealthyPoll() {
+	p.lastHealthyPoll.Store(time.Now().UnixNano())
+	p.lastPollFailed.Store(false)
+}