mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-07-22 20:47:45 +00:00
feat: add runner health admission checks (#1090)
Opt-in local task-admission checks under a `health_check` config section (disabled by default): - pause new task fetching when free disk space on the workspace volume is below the configured minimum - optional executable health-check script — a non-zero exit, timeout, or start failure marks the runner unavailable - checks run only while the runner is idle; the last result is reused while a job is active, and polling resumes automatically on recovery - `/readyz` reports task-admission readiness (reusing the poll loop's last check); `/healthz` stays a process-liveness endpoint --------- Co-authored-by: silverwind <me@silverwind.io> Reviewed-on: https://gitea.com/gitea/runner/pulls/1090 Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
This commit is contained in:
@@ -81,7 +81,7 @@ func TestRegisterRunningJobsFuncZeroCapacity(t *testing.T) {
|
||||
|
||||
func TestStartServerCanBeCancelled(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
StartServer(ctx, "127.0.0.1:0")
|
||||
StartServer(ctx, "127.0.0.1:0", nil)
|
||||
cancel()
|
||||
}
|
||||
|
||||
|
||||
@@ -13,19 +13,12 @@ import (
|
||||
)
|
||||
|
||||
// StartServer starts an HTTP server that serves Prometheus metrics on /metrics
|
||||
// and a liveness check on /healthz. The server shuts down when ctx is cancelled.
|
||||
// and health checks. The server shuts down when ctx is cancelled.
|
||||
// Call Init() before StartServer to register metrics with the Registry.
|
||||
func StartServer(ctx context.Context, addr string) {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
|
||||
func StartServer(ctx context.Context, addr string, readiness func() (bool, string)) {
|
||||
srv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: mux,
|
||||
Handler: NewHTTPHandler(readiness),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
@@ -48,3 +41,25 @@ func StartServer(ctx context.Context, addr string) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// NewHTTPHandler returns the metrics and health endpoints used by StartServer.
|
||||
func NewHTTPHandler(readiness func() (bool, string)) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
ready, reason := true, "ok"
|
||||
if readiness != nil {
|
||||
ready, reason = readiness()
|
||||
}
|
||||
if !ready {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
_, _ = w.Write([]byte(reason))
|
||||
})
|
||||
|
||||
return mux
|
||||
}
|
||||
|
||||
43
internal/pkg/metrics/server_test.go
Normal file
43
internal/pkg/metrics/server_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestReadyEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ready bool
|
||||
reason string
|
||||
status int
|
||||
}{
|
||||
{"ready", true, "ok", http.StatusOK},
|
||||
{"not ready", false, "low disk space", http.StatusServiceUnavailable},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
handler := NewHTTPHandler(func() (bool, string) { return test.ready, test.reason })
|
||||
request := httptest.NewRequest(http.MethodGet, "/readyz", nil)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
assert.Equal(t, test.status, response.Code)
|
||||
assert.Equal(t, test.reason, response.Body.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthEndpointStaysLiveWhenNotReady(t *testing.T) {
|
||||
handler := NewHTTPHandler(func() (bool, string) { return false, "low disk space" })
|
||||
request := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
assert.Equal(t, "ok", response.Body.String())
|
||||
}
|
||||
Reference in New Issue
Block a user