mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-07-22 20:47:45 +00:00
feat: add runner health admission checks (#1090)
Opt-in local task-admission checks under a `health_check` config section (disabled by default): - pause new task fetching when free disk space on the workspace volume is below the configured minimum - optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable - checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery - `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint --------- Co-authored-by: silverwind <me@silverwind.io> Reviewed-on: https://gitea.com/gitea/runner/pulls/1090 Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
@@ -66,6 +66,13 @@ type Runner struct {
|
||||
runningCount atomic.Int64
|
||||
lastIdleCleanupUnixNano atomic.Int64
|
||||
now func() time.Time
|
||||
healthCheckLast time.Time
|
||||
healthCheckReady bool
|
||||
healthCheckReason string
|
||||
healthStatusSet bool
|
||||
healthStatusReady bool
|
||||
healthStatusReason string
|
||||
runHealthCheck func(context.Context, string, time.Duration, map[string]string) error
|
||||
}
|
||||
|
||||
func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client) *Runner {
|
||||
@@ -109,13 +116,14 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
envs["GITEA_ACTIONS_RUNNER_VERSION"] = ver.Version()
|
||||
|
||||
runner := &Runner{
|
||||
name: reg.Name,
|
||||
cfg: cfg,
|
||||
client: cli,
|
||||
labels: ls,
|
||||
envs: envs,
|
||||
cacheHandler: cacheHandler,
|
||||
now: time.Now,
|
||||
name: reg.Name,
|
||||
cfg: cfg,
|
||||
client: cli,
|
||||
labels: ls,
|
||||
envs: envs,
|
||||
cacheHandler: cacheHandler,
|
||||
now: time.Now,
|
||||
runHealthCheck: executeHealthCheck,
|
||||
}
|
||||
return runner
|
||||
}
|
||||
@@ -579,6 +587,67 @@ func (r *Runner) RunningCount() int64 {
|
||||
return r.runningCount.Load()
|
||||
}
|
||||
|
||||
// CanAcceptTask checks local admission conditions without consuming a task. It is
|
||||
// called only from the poll loop, so the cached health fields need no lock.
|
||||
func (r *Runner) CanAcceptTask(ctx context.Context) (bool, string) {
|
||||
if !r.cfg.HealthCheck.Enabled {
|
||||
return true, "ok"
|
||||
}
|
||||
|
||||
if r.RunningCount() > 0 {
|
||||
if !r.healthStatusSet {
|
||||
return true, "health checks deferred while jobs are running"
|
||||
}
|
||||
return r.healthStatusReady, r.healthStatusReason
|
||||
}
|
||||
|
||||
if ready, reason := checkFreeDisk(r.cfg); !ready {
|
||||
r.setHealthStatus(ready, reason)
|
||||
return false, reason
|
||||
}
|
||||
ready, reason := r.checkConfiguredHealth(ctx)
|
||||
r.setHealthStatus(ready, reason)
|
||||
return ready, reason
|
||||
}
|
||||
|
||||
func (r *Runner) setHealthStatus(ready bool, reason string) {
|
||||
r.healthStatusSet = true
|
||||
r.healthStatusReady = ready
|
||||
r.healthStatusReason = reason
|
||||
}
|
||||
|
||||
// checkFreeDisk evaluates the configured task-admission disk threshold.
|
||||
func checkFreeDisk(cfg *config.Config) (bool, string) {
|
||||
root := cfg.Host.WorkdirParent
|
||||
if cfg.Container.BindWorkdir {
|
||||
root = filepath.FromSlash("/" + strings.TrimLeft(cfg.Container.WorkdirParent, "/"))
|
||||
}
|
||||
root = nearestExistingPath(root)
|
||||
available, err := freeDiskBytes(root)
|
||||
if err != nil {
|
||||
return false, fmt.Sprintf("cannot determine free disk space for %s: %v", root, err)
|
||||
}
|
||||
availableMB := available / (1024 * 1024)
|
||||
if availableMB < uint64(cfg.HealthCheck.MinFreeDiskSpaceMB) {
|
||||
return false, fmt.Sprintf("low disk space on %s: %d MiB available, %d MiB required", root, availableMB, cfg.HealthCheck.MinFreeDiskSpaceMB)
|
||||
}
|
||||
return true, "ok"
|
||||
}
|
||||
|
||||
func nearestExistingPath(path string) string {
|
||||
for path != "" {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path
|
||||
}
|
||||
parent := filepath.Dir(path)
|
||||
if parent == path {
|
||||
return parent
|
||||
}
|
||||
path = parent
|
||||
}
|
||||
return "."
|
||||
}
|
||||
|
||||
func (r *Runner) Declare(ctx context.Context, labels []string) (*connect.Response[runnerv1.DeclareResponse], error) {
|
||||
return r.client.Declare(ctx, connect.NewRequest(&runnerv1.DeclareRequest{
|
||||
Version: ver.Version(),
|
||||
|
||||
Reference in New Issue
Block a user