feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

internal/pkg/config/config.example.yaml

@@ -199,11 +199,29 @@ host:
   # If it's empty, $HOME/.cache/act/ will be used.
   workdir_parent:
 
+# Optional local task-admission checks. Disabled by default. When enabled, low
+# disk space or a failing script pauses new task fetching; existing jobs continue.
+# No health checks run while any job is active; the last result is reused until idle.
+health_check:
+  enabled: false
+  # Minimum free space required on the filesystem holding runner workspaces.
+  # Defaults to 1024 MiB when omitted or set to zero.
+  min_free_disk_space_mb: 1024
+  # Optional additional executable. A non-zero exit, timeout, or startup failure
+  # marks the runner unavailable.
+  script: ''
+  # How long a script result is cached and its maximum execution time.
+  interval: 30s
+  timeout: 10s
+
 metrics:
   # Enable the Prometheus metrics endpoint.
-  # When enabled, metrics are served at http://<addr>/metrics and a liveness check at /healthz.
+  # When enabled, metrics are served at /metrics, liveness at /healthz, and
+  # task-admission readiness at /readyz.
   enabled: false
   # The address for the metrics HTTP server to listen on.
   # Defaults to localhost only. Set to ":9101" to allow external access,
   # but ensure the port is firewall-protected as there is no authentication.
   addr: "127.0.0.1:9101"
+  # Consecutive polling failures may last this long before /readyz returns 503.
+  readiness_grace: 30s

internal/pkg/config/config.go

@@ -95,18 +95,30 @@ type Host struct {
 
 // Metrics represents the configuration for the Prometheus metrics endpoint.
 type Metrics struct {
-	Enabled bool   `yaml:"enabled"` // Enabled indicates whether the metrics endpoint is exposed.
-	Addr    string `yaml:"addr"`    // Addr specifies the listen address for the metrics HTTP server (e.g., ":9101").
+	Enabled        bool          `yaml:"enabled"`         // Enabled indicates whether the metrics endpoint is exposed.
+	Addr           string        `yaml:"addr"`            // Addr specifies the listen address for the metrics HTTP server (e.g., ":9101").
+	ReadinessGrace time.Duration `yaml:"readiness_grace"` // ReadinessGrace permits transient polling errors before /readyz becomes unhealthy.
+}
+
+// HealthCheck represents local checks that control whether the runner accepts
+// new tasks. The entire feature is opt-in through Enabled.
+type HealthCheck struct {
+	Enabled            bool          `yaml:"enabled"`                // Enabled activates local task-admission health checks.
+	MinFreeDiskSpaceMB int64         `yaml:"min_free_disk_space_mb"` // MinFreeDiskSpaceMB is the minimum free space required on the work volume.
+	Script             string        `yaml:"script"`                 // Script is an optional executable used as an additional health check.
+	Interval           time.Duration `yaml:"interval"`               // Interval controls how long a script result is cached.
+	Timeout            time.Duration `yaml:"timeout"`                // Timeout caps one health-check script invocation.
 }
 
 // Config represents the overall configuration.
 type Config struct {
-	Log       Log       `yaml:"log"`       // Log represents the configuration for logging.
-	Runner    Runner    `yaml:"runner"`    // Runner represents the configuration for the runner.
-	Cache     Cache     `yaml:"cache"`     // Cache represents the configuration for caching.
-	Container Container `yaml:"container"` // Container represents the configuration for the container.
-	Host      Host      `yaml:"host"`      // Host represents the configuration for the host.
-	Metrics   Metrics   `yaml:"metrics"`   // Metrics represents the configuration for the Prometheus metrics endpoint.
+	Log         Log         `yaml:"log"`          // Log represents the configuration for logging.
+	Runner      Runner      `yaml:"runner"`       // Runner represents the configuration for the runner.
+	Cache       Cache       `yaml:"cache"`        // Cache represents the configuration for caching.
+	Container   Container   `yaml:"container"`    // Container represents the configuration for the container.
+	Host        Host        `yaml:"host"`         // Host represents the configuration for the host.
+	Metrics     Metrics     `yaml:"metrics"`      // Metrics represents the configuration for the Prometheus metrics endpoint.
+	HealthCheck HealthCheck `yaml:"health_check"` // HealthCheck controls opt-in local task-admission checks.
 }
 
 // LoadDefault returns the default configuration.
@@ -220,9 +232,21 @@ func LoadDefault(file string) (*Config, error) {
 	if cfg.Runner.PostTaskScript != "" && cfg.Runner.PostTaskScriptTimeout <= 0 {
 		cfg.Runner.PostTaskScriptTimeout = DefaultPostTaskScriptTimeout
 	}
+	if cfg.HealthCheck.MinFreeDiskSpaceMB <= 0 {
+		cfg.HealthCheck.MinFreeDiskSpaceMB = 1024
+	}
+	if cfg.HealthCheck.Interval <= 0 {
+		cfg.HealthCheck.Interval = 30 * time.Second
+	}
+	if cfg.HealthCheck.Timeout <= 0 {
+		cfg.HealthCheck.Timeout = 10 * time.Second
+	}
 	if cfg.Metrics.Addr == "" {
 		cfg.Metrics.Addr = "127.0.0.1:9101"
 	}
+	if cfg.Metrics.ReadinessGrace <= 0 {
+		cfg.Metrics.ReadinessGrace = 30 * time.Second
+	}
 
 	// Validate and fix invalid config combinations to prevent confusing behavior.
 	if cfg.Runner.FetchIntervalMax < cfg.Runner.FetchInterval {

internal/pkg/config/config_test.go

@@ -48,6 +48,41 @@ func TestLoadDefault_DefaultsWorkdirCleanupAge(t *testing.T) {
 	assert.Equal(t, 10*time.Minute, cfg.Runner.IdleCleanupInterval)
 }
 
+func TestLoadDefault_HealthChecksAreOptIn(t *testing.T) {
+	cfg, err := LoadDefault("")
+	require.NoError(t, err)
+	assert.False(t, cfg.HealthCheck.Enabled)
+	assert.Equal(t, int64(1024), cfg.HealthCheck.MinFreeDiskSpaceMB)
+	assert.Empty(t, cfg.HealthCheck.Script)
+	assert.Equal(t, 30*time.Second, cfg.HealthCheck.Interval)
+	assert.Equal(t, 10*time.Second, cfg.HealthCheck.Timeout)
+	assert.False(t, cfg.Metrics.Enabled)
+}
+
+func TestLoadDefault_DiskAndReadinessSettings(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`
+health_check:
+  enabled: true
+  min_free_disk_space_mb: 4096
+  script: /usr/local/bin/runner-health
+  interval: 15s
+  timeout: 3s
+metrics:
+  readiness_grace: 45s
+`), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.True(t, cfg.HealthCheck.Enabled)
+	assert.Equal(t, int64(4096), cfg.HealthCheck.MinFreeDiskSpaceMB)
+	assert.Equal(t, "/usr/local/bin/runner-health", cfg.HealthCheck.Script)
+	assert.Equal(t, 15*time.Second, cfg.HealthCheck.Interval)
+	assert.Equal(t, 3*time.Second, cfg.HealthCheck.Timeout)
+	assert.Equal(t, 45*time.Second, cfg.Metrics.ReadinessGrace)
+}
+
 func TestLoadDefault_UsesConfiguredWorkdirCleanupAge(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "config.yaml")

internal/pkg/metrics/metrics_test.go

@@ -81,7 +81,7 @@ func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {
 
 func TestStartServerCanBeCancelled(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
-	StartServer(ctx, "127.0.0.1:0")
+	StartServer(ctx, "127.0.0.1:0", nil)
 	cancel()
 }
 

internal/pkg/metrics/server.go

@@ -13,19 +13,12 @@ import (
 )
 
 // StartServer starts an HTTP server that serves Prometheus metrics on /metrics
-// and a liveness check on /healthz. The server shuts down when ctx is cancelled.
+// and health checks. The server shuts down when ctx is cancelled.
 // Call Init() before StartServer to register metrics with the Registry.
-func StartServer(ctx context.Context, addr string) {
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
-	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		_, _ = w.Write([]byte("ok"))
-	})
-
+func StartServer(ctx context.Context, addr string, readiness func() (bool, string)) {
 	srv := &http.Server{
 		Addr:              addr,
-		Handler:           mux,
+		Handler:           NewHTTPHandler(readiness),
 		ReadHeaderTimeout: 5 * time.Second,
 		ReadTimeout:       10 * time.Second,
 		WriteTimeout:      10 * time.Second,
@@ -48,3 +41,25 @@ func StartServer(ctx context.Context, addr string) {
 		}
 	}()
 }
+
+// NewHTTPHandler returns the metrics and health endpoints used by StartServer.
+func NewHTTPHandler(readiness func() (bool, string)) http.Handler {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	})
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
+		ready, reason := true, "ok"
+		if readiness != nil {
+			ready, reason = readiness()
+		}
+		if !ready {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}
+		_, _ = w.Write([]byte(reason))
+	})
+
+	return mux
+}

internal/pkg/metrics/server_test.go

@@ -0,0 +1,43 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package metrics
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadyEndpoint(t *testing.T) {
+	tests := []struct {
+		name   string
+		ready  bool
+		reason string
+		status int
+	}{
+		{"ready", true, "ok", http.StatusOK},
+		{"not ready", false, "low disk space", http.StatusServiceUnavailable},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			handler := NewHTTPHandler(func() (bool, string) { return test.ready, test.reason })
+			request := httptest.NewRequest(http.MethodGet, "/readyz", nil)
+			response := httptest.NewRecorder()
+			handler.ServeHTTP(response, request)
+			assert.Equal(t, test.status, response.Code)
+			assert.Equal(t, test.reason, response.Body.String())
+		})
+	}
+}
+
+func TestHealthEndpointStaysLiveWhenNotReady(t *testing.T) {
+	handler := NewHTTPHandler(func() (bool, string) { return false, "low disk space" })
+	request := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, request)
+	assert.Equal(t, http.StatusOK, response.Code)
+	assert.Equal(t, "ok", response.Body.String())
+}