feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

internal/app/run/disk_other.go

@@ -0,0 +1,12 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows
+
+package run
+
+import "fmt"
+
+func freeDiskBytes(path string) (uint64, error) {
+	return 0, fmt.Errorf("free disk space checks are not supported for %s", path)
+}

internal/app/run/disk_test.go

@@ -0,0 +1,53 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package run
+
+import (
+	"testing"
+
+	"gitea.com/gitea/runner/internal/pkg/config"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCanAcceptTaskDiskGuard(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.Host.WorkdirParent = t.TempDir()
+	cfg.HealthCheck.Enabled = true
+	r := &Runner{cfg: cfg}
+
+	ready, _ := r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+
+	cfg.HealthCheck.MinFreeDiskSpaceMB = 1 << 40
+	ready, reason := r.CanAcceptTask(t.Context())
+	assert.False(t, ready)
+	assert.Contains(t, reason, "low disk space")
+}
+
+func TestDiskCheckDeferredWhileJobRuns(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.HealthCheck.Enabled = true
+	cfg.HealthCheck.MinFreeDiskSpaceMB = 1
+	cfg.Host.WorkdirParent = t.TempDir()
+	r := &Runner{cfg: cfg}
+
+	ready, reason := r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+	assert.Equal(t, "ok", reason)
+
+	cfg.HealthCheck.MinFreeDiskSpaceMB = 1 << 40
+	r.runningCount.Store(1)
+	ready, reason = r.CanAcceptTask(t.Context())
+	assert.True(t, ready, "the disk check must not run while a job is active")
+	assert.Equal(t, "ok", reason)
+
+	r.runningCount.Store(0)
+	ready, reason = r.CanAcceptTask(t.Context())
+	assert.False(t, ready)
+	assert.Contains(t, reason, "low disk space")
+}

internal/app/run/disk_unix.go

@@ -0,0 +1,16 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+
+package run
+
+import "golang.org/x/sys/unix"
+
+func freeDiskBytes(path string) (uint64, error) {
+	var stat unix.Statfs_t
+	if err := unix.Statfs(path, &stat); err != nil {
+		return 0, err
+	}
+	return stat.Bavail * uint64(stat.Bsize), nil
+}

internal/app/run/disk_windows.go

@@ -0,0 +1,20 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build windows
+
+package run
+
+import "golang.org/x/sys/windows"
+
+func freeDiskBytes(path string) (uint64, error) {
+	pathPtr, err := windows.UTF16PtrFromString(path)
+	if err != nil {
+		return 0, err
+	}
+	var available uint64
+	if err := windows.GetDiskFreeSpaceEx(pathPtr, &available, nil, nil); err != nil {
+		return 0, err
+	}
+	return available, nil
+}

internal/app/run/health_check.go

@@ -0,0 +1,106 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package run
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.com/gitea/runner/act/common"
+	"gitea.com/gitea/runner/internal/pkg/process"
+)
+
+func (r *Runner) checkConfiguredHealth(ctx context.Context) (bool, string) {
+	script := r.cfg.HealthCheck.Script
+	if script == "" {
+		return true, "ok"
+	}
+
+	now := time.Now()
+	if r.now != nil {
+		now = r.now()
+	}
+	if !r.healthCheckLast.IsZero() && now.Sub(r.healthCheckLast) < r.cfg.HealthCheck.Interval {
+		return r.healthCheckReady, r.healthCheckReason
+	}
+
+	env := processEnvironment()
+	maps.Copy(env, r.cloneEnvs())
+	env["GITEA_RUNNER_HEALTH_CHECK"] = "true"
+	env["GITEA_RUNNER_NAME"] = r.name
+	if r.client != nil {
+		env["GITEA_INSTANCE_URL"] = r.client.Address()
+	}
+	env["GITEA_RUNNER_RUNNING_JOBS"] = strconv.FormatInt(r.RunningCount(), 10)
+
+	runner := r.runHealthCheck
+	if runner == nil {
+		runner = executeHealthCheck
+	}
+	err := runner(ctx, script, r.cfg.HealthCheck.Timeout, env)
+	r.healthCheckLast = now
+	r.healthCheckReady = err == nil
+	if err != nil {
+		r.healthCheckReason = "runner health check failed: " + err.Error()
+	} else {
+		r.healthCheckReason = "ok"
+	}
+	return r.healthCheckReady, r.healthCheckReason
+}
+
+func executeHealthCheck(ctx context.Context, script string, timeout time.Duration, env map[string]string) error {
+	checkCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	cmd := exec.CommandContext(checkCtx, script)
+	cmd.Env = envListFromMap(env)
+	cmd.SysProcAttr = process.SysProcAttr(script, false)
+	writer := common.NewLineWriter(func(line string) bool {
+		line = strings.TrimRight(line, "\r\n")
+		if line != "" {
+			common.Logger(ctx).Infof("health check: %s", line)
+		}
+		return true
+	})
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	treeKill := process.NewTreeKill(cmd)
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("start: %w", err)
+	}
+	if killer, err := treeKill.Capture(cmd.Process); err == nil {
+		defer killer.Close()
+	}
+	err := cmd.Wait()
+	common.FlushWriter(writer)
+	if errors.Is(checkCtx.Err(), context.DeadlineExceeded) {
+		return fmt.Errorf("timed out after %s", timeout)
+	}
+	if err == nil {
+		return nil
+	}
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		return fmt.Errorf("exited with code %d", exitErr.ExitCode())
+	}
+	return err
+}
+
+func processEnvironment() map[string]string {
+	environ := os.Environ()
+	env := make(map[string]string, len(environ))
+	for _, value := range environ {
+		if key, item, ok := strings.Cut(value, "="); ok {
+			env[key] = item
+		}
+	}
+	return env
+}

internal/app/run/health_check_test.go

@@ -0,0 +1,148 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package run
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"gitea.com/gitea/runner/internal/pkg/config"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfiguredHealthCheckCachesAndRecovers(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.HealthCheck.Enabled = true
+	cfg.HealthCheck.Script = "/health-check"
+	cfg.HealthCheck.Interval = time.Minute
+	cfg.HealthCheck.Timeout = time.Second
+
+	now := time.Now()
+	calls := 0
+	fail := true
+	r := &Runner{
+		cfg:  cfg,
+		name: "runner-1",
+		now:  func() time.Time { return now },
+		runHealthCheck: func(_ context.Context, script string, timeout time.Duration, env map[string]string) error {
+			calls++
+			assert.Equal(t, "/health-check", script)
+			assert.Equal(t, time.Second, timeout)
+			assert.Equal(t, "true", env["GITEA_RUNNER_HEALTH_CHECK"])
+			assert.Equal(t, "runner-1", env["GITEA_RUNNER_NAME"])
+			if fail {
+				return errors.New("unhealthy")
+			}
+			return nil
+		},
+	}
+
+	ready, reason := r.CanAcceptTask(t.Context())
+	assert.False(t, ready)
+	assert.Contains(t, reason, "unhealthy")
+	assert.Equal(t, 1, calls)
+
+	fail = false
+	ready, _ = r.CanAcceptTask(t.Context())
+	assert.False(t, ready, "the failed result should remain cached")
+	assert.Equal(t, 1, calls)
+
+	now = now.Add(time.Minute)
+	ready, reason = r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+	assert.Equal(t, "ok", reason)
+	assert.Equal(t, 2, calls)
+}
+
+func TestConfiguredHealthCheckDisabled(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.HealthCheck.MinFreeDiskSpaceMB = 1 << 40
+	cfg.HealthCheck.Script = "/must-not-run"
+	r := &Runner{
+		cfg: cfg,
+		runHealthCheck: func(_ context.Context, _ string, _ time.Duration, _ map[string]string) error {
+			t.Fatal("disabled health check executed")
+			return nil
+		},
+	}
+	ready, reason := r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+	assert.Equal(t, "ok", reason)
+}
+
+func TestConfiguredHealthCheckDeferredWhileJobRuns(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.HealthCheck.Enabled = true
+	cfg.HealthCheck.Script = "/health-check"
+	cfg.HealthCheck.Interval = time.Minute
+
+	now := time.Now()
+	calls := 0
+	fail := false
+	r := &Runner{
+		cfg: cfg,
+		now: func() time.Time { return now },
+		runHealthCheck: func(_ context.Context, _ string, _ time.Duration, _ map[string]string) error {
+			calls++
+			if fail {
+				return errors.New("unhealthy")
+			}
+			return nil
+		},
+	}
+
+	ready, reason := r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+	assert.Equal(t, "ok", reason)
+	assert.Equal(t, 1, calls)
+
+	now = now.Add(time.Minute)
+	fail = true
+	r.runningCount.Store(1)
+	ready, reason = r.CanAcceptTask(t.Context())
+	assert.True(t, ready, "the last result should be reused while a job runs")
+	assert.Equal(t, "ok", reason)
+	assert.Equal(t, 1, calls)
+
+	r.runningCount.Store(0)
+	ready, reason = r.CanAcceptTask(t.Context())
+	assert.False(t, ready)
+	assert.Contains(t, reason, "unhealthy")
+	assert.Equal(t, 2, calls)
+}
+
+func TestConfiguredHealthCheckInitialRunDeferredWhileJobRuns(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.HealthCheck.Enabled = true
+	cfg.HealthCheck.Script = "/health-check"
+
+	calls := 0
+	r := &Runner{
+		cfg: cfg,
+		runHealthCheck: func(_ context.Context, _ string, _ time.Duration, _ map[string]string) error {
+			calls++
+			return nil
+		},
+	}
+	r.runningCount.Store(1)
+
+	ready, reason := r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+	assert.Contains(t, reason, "deferred")
+	assert.Zero(t, calls)
+
+	r.runningCount.Store(0)
+	ready, reason = r.CanAcceptTask(t.Context())
+	assert.True(t, ready)
+	assert.Equal(t, "ok", reason)
+	assert.Equal(t, 1, calls)
+}

internal/app/run/runner.go

@@ -66,6 +66,13 @@ type Runner struct {
 	runningCount            atomic.Int64
 	lastIdleCleanupUnixNano atomic.Int64
 	now                     func() time.Time
+	healthCheckLast         time.Time
+	healthCheckReady        bool
+	healthCheckReason       string
+	healthStatusSet         bool
+	healthStatusReady       bool
+	healthStatusReason      string
+	runHealthCheck          func(context.Context, string, time.Duration, map[string]string) error
 }
 
 func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client) *Runner {
@@ -109,13 +116,14 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
 	envs["GITEA_ACTIONS_RUNNER_VERSION"] = ver.Version()
 
 	runner := &Runner{
-		name:         reg.Name,
-		cfg:          cfg,
-		client:       cli,
-		labels:       ls,
-		envs:         envs,
-		cacheHandler: cacheHandler,
-		now:          time.Now,
+		name:           reg.Name,
+		cfg:            cfg,
+		client:         cli,
+		labels:         ls,
+		envs:           envs,
+		cacheHandler:   cacheHandler,
+		now:            time.Now,
+		runHealthCheck: executeHealthCheck,
 	}
 	return runner
 }
@@ -579,6 +587,67 @@ func (r *Runner) RunningCount() int64 {
 	return r.runningCount.Load()
 }
 
+// CanAcceptTask checks local admission conditions without consuming a task. It is
+// called only from the poll loop, so the cached health fields need no lock.
+func (r *Runner) CanAcceptTask(ctx context.Context) (bool, string) {
+	if !r.cfg.HealthCheck.Enabled {
+		return true, "ok"
+	}
+
+	if r.RunningCount() > 0 {
+		if !r.healthStatusSet {
+			return true, "health checks deferred while jobs are running"
+		}
+		return r.healthStatusReady, r.healthStatusReason
+	}
+
+	if ready, reason := checkFreeDisk(r.cfg); !ready {
+		r.setHealthStatus(ready, reason)
+		return false, reason
+	}
+	ready, reason := r.checkConfiguredHealth(ctx)
+	r.setHealthStatus(ready, reason)
+	return ready, reason
+}
+
+func (r *Runner) setHealthStatus(ready bool, reason string) {
+	r.healthStatusSet = true
+	r.healthStatusReady = ready
+	r.healthStatusReason = reason
+}
+
+// checkFreeDisk evaluates the configured task-admission disk threshold.
+func checkFreeDisk(cfg *config.Config) (bool, string) {
+	root := cfg.Host.WorkdirParent
+	if cfg.Container.BindWorkdir {
+		root = filepath.FromSlash("/" + strings.TrimLeft(cfg.Container.WorkdirParent, "/"))
+	}
+	root = nearestExistingPath(root)
+	available, err := freeDiskBytes(root)
+	if err != nil {
+		return false, fmt.Sprintf("cannot determine free disk space for %s: %v", root, err)
+	}
+	availableMB := available / (1024 * 1024)
+	if availableMB < uint64(cfg.HealthCheck.MinFreeDiskSpaceMB) {
+		return false, fmt.Sprintf("low disk space on %s: %d MiB available, %d MiB required", root, availableMB, cfg.HealthCheck.MinFreeDiskSpaceMB)
+	}
+	return true, "ok"
+}
+
+func nearestExistingPath(path string) string {
+	for path != "" {
+		if _, err := os.Stat(path); err == nil {
+			return path
+		}
+		parent := filepath.Dir(path)
+		if parent == path {
+			return parent
+		}
+		path = parent
+	}
+	return "."
+}
+
 func (r *Runner) Declare(ctx context.Context, labels []string) (*connect.Response[runnerv1.DeclareResponse], error) {
 	return r.client.Declare(ctx, connect.NewRequest(&runnerv1.DeclareRequest{
 		Version:      ver.Version(),