feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
bircni
2026-07-22 15:10:45 +00:00
parent 7bec310002
commit c43cbe87ca
16 changed files with 753 additions and 32 deletions

View File

@@ -66,6 +66,13 @@ type Runner struct {
runningCount atomic.Int64
lastIdleCleanupUnixNano atomic.Int64
now func() time.Time
healthCheckLast time.Time
healthCheckReady bool
healthCheckReason string
healthStatusSet bool
healthStatusReady bool
healthStatusReason string
runHealthCheck func(context.Context, string, time.Duration, map[string]string) error
}
func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client) *Runner {
@@ -109,13 +116,14 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
envs["GITEA_ACTIONS_RUNNER_VERSION"] = ver.Version()
runner := &Runner{
name: reg.Name,
cfg: cfg,
client: cli,
labels: ls,
envs: envs,
cacheHandler: cacheHandler,
now: time.Now,
name: reg.Name,
cfg: cfg,
client: cli,
labels: ls,
envs: envs,
cacheHandler: cacheHandler,
now: time.Now,
runHealthCheck: executeHealthCheck,
}
return runner
}
@@ -579,6 +587,67 @@ func (r *Runner) RunningCount() int64 {
return r.runningCount.Load()
}
// CanAcceptTask checks local admission conditions without consuming a task. It is
// called only from the poll loop, so the cached health fields need no lock.
func (r *Runner) CanAcceptTask(ctx context.Context) (bool, string) {
if !r.cfg.HealthCheck.Enabled {
return true, "ok"
}
if r.RunningCount() > 0 {
if !r.healthStatusSet {
return true, "health checks deferred while jobs are running"
}
return r.healthStatusReady, r.healthStatusReason
}
if ready, reason := checkFreeDisk(r.cfg); !ready {
r.setHealthStatus(ready, reason)
return false, reason
}
ready, reason := r.checkConfiguredHealth(ctx)
r.setHealthStatus(ready, reason)
return ready, reason
}
func (r *Runner) setHealthStatus(ready bool, reason string) {
r.healthStatusSet = true
r.healthStatusReady = ready
r.healthStatusReason = reason
}
// checkFreeDisk evaluates the configured task-admission disk threshold.
func checkFreeDisk(cfg *config.Config) (bool, string) {
root := cfg.Host.WorkdirParent
if cfg.Container.BindWorkdir {
root = filepath.FromSlash("/" + strings.TrimLeft(cfg.Container.WorkdirParent, "/"))
}
root = nearestExistingPath(root)
available, err := freeDiskBytes(root)
if err != nil {
return false, fmt.Sprintf("cannot determine free disk space for %s: %v", root, err)
}
availableMB := available / (1024 * 1024)
if availableMB < uint64(cfg.HealthCheck.MinFreeDiskSpaceMB) {
return false, fmt.Sprintf("low disk space on %s: %d MiB available, %d MiB required", root, availableMB, cfg.HealthCheck.MinFreeDiskSpaceMB)
}
return true, "ok"
}
func nearestExistingPath(path string) string {
for path != "" {
if _, err := os.Stat(path); err == nil {
return path
}
parent := filepath.Dir(path)
if parent == path {
return parent
}
path = parent
}
return "."
}
func (r *Runner) Declare(ctx context.Context, labels []string) (*connect.Response[runnerv1.DeclareResponse], error) {
return r.client.Declare(ctx, connect.NewRequest(&runnerv1.DeclareRequest{
Version: ver.Version(),