feat: add runner health admission checks (#1090)

Opt-in local task-admission checks under a `health_check` config section (disabled by default):

- pause new task fetching when free disk space on the workspace volume is below the configured minimum
- optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable
- checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery
- `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1090
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
bircni
2026-07-22 15:10:45 +00:00
parent 7bec310002
commit c43cbe87ca
16 changed files with 753 additions and 32 deletions

View File

@@ -81,7 +81,7 @@ func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {
func TestStartServerCanBeCancelled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
StartServer(ctx, "127.0.0.1:0")
StartServer(ctx, "127.0.0.1:0", nil)
cancel()
}

View File

@@ -13,19 +13,12 @@ import (
)
// StartServer starts an HTTP server that serves Prometheus metrics on /metrics
// and a liveness check on /healthz. The server shuts down when ctx is cancelled.
// and health checks. The server shuts down when ctx is cancelled.
// Call Init() before StartServer to register metrics with the Registry.
func StartServer(ctx context.Context, addr string) {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
})
func StartServer(ctx context.Context, addr string, readiness func() (bool, string)) {
srv := &http.Server{
Addr: addr,
Handler: mux,
Handler: NewHTTPHandler(readiness),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
@@ -48,3 +41,25 @@ func StartServer(ctx context.Context, addr string) {
}
}()
}
// NewHTTPHandler returns the metrics and health endpoints used by StartServer.
func NewHTTPHandler(readiness func() (bool, string)) http.Handler {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
})
mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
ready, reason := true, "ok"
if readiness != nil {
ready, reason = readiness()
}
if !ready {
w.WriteHeader(http.StatusServiceUnavailable)
}
_, _ = w.Write([]byte(reason))
})
return mux
}

View File

@@ -0,0 +1,43 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package metrics
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
)
func TestReadyEndpoint(t *testing.T) {
tests := []struct {
name string
ready bool
reason string
status int
}{
{"ready", true, "ok", http.StatusOK},
{"not ready", false, "low disk space", http.StatusServiceUnavailable},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
handler := NewHTTPHandler(func() (bool, string) { return test.ready, test.reason })
request := httptest.NewRequest(http.MethodGet, "/readyz", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
assert.Equal(t, test.status, response.Code)
assert.Equal(t, test.reason, response.Body.String())
})
}
}
func TestHealthEndpointStaysLiveWhenNotReady(t *testing.T) {
handler := NewHTTPHandler(func() (bool, string) { return false, "low disk space" })
request := httptest.NewRequest(http.MethodGet, "/healthz", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
assert.Equal(t, http.StatusOK, response.Code)
assert.Equal(t, "ok", response.Body.String())
}