feat!: add cache service v2, add toolkit patches (#1110)

Serves `github.actions.results.api.v1.CacheService` next to the v1 cache API, sharing its store, plus the subset of the Azure blob protocol the toolkit uploads with. On by default via `cache.v2`, and works with `external_server`.

Clients reach it through two edits in the action's own bundle: the GHES check is opened, and the cache service URL is taken from `ACTIONS_CACHE_URL`.

The same GHES check is what makes the stock `actions/upload-artifact` and `download-artifact` abort on Gitea. Opening it makes them work without the `gitea-upload-artifact` fork, from `upload-artifact@v4.4.0` on.

Verified against 118 real bundles, every major version of 16 actions: 92 patched, the rest deliberately left alone, and every patched bundle checked with `node --check`. Also end to end against pinned `actions/cache@v6.1.0` with an unreachable results URL, so only the patch can make the cache work.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: bircni <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1110
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

internal/app/cmd/daemon.go

@@ -141,6 +141,11 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
 		)
 
 		runner := run.NewRunner(cfg, reg, cli)
+		defer func() {
+			if err := runner.Close(); err != nil {
+				log.Warnf("runner %s: cache server shutdown: %v", reg.Name, err)
+			}
+		}()
 
 		// declare the labels of the runner before fetching tasks
 		resp, err := runner.Declare(ctx, ls.Names())

internal/app/cmd/exec.go

@@ -132,6 +132,8 @@ func (i *executeArgs) LoadEnvs() map[string]string {
 	_ = readEnvs(i.Envfile(), envs)
 
 	envs["ACTIONS_CACHE_URL"] = i.cacheHandler.ExternalURL() + "/"
+	// The same server answers the cache service v2 API, so let the actions reach it.
+	envs[runner.CacheServiceV2Env] = "true"
 
 	return envs
 }

internal/app/run/runner.go

@@ -109,6 +109,12 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
 		}
 	}
 
+	if envs["ACTIONS_CACHE_URL"] != "" && (cfg.Cache.V2 == nil || *cfg.Cache.V2) {
+		// act patches the GHES check out of an action's bundle when it sees this, so the client
+		// uses the cache service v2 API this server also answers; see act/runner/toolkit_patch.go.
+		envs[runner.CacheServiceV2Env] = "true"
+	}
+
 	// set artifact gitea api
 	artifactGiteaAPI := strings.TrimSuffix(cli.Address(), "/") + "/api/actions_pipeline/"
 	envs["ACTIONS_RUNTIME_URL"] = artifactGiteaAPI
@@ -132,6 +138,11 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
 	return runner
 }
 
+// Close shuts down the cache server this runner exposes to job containers.
+func (r *Runner) Close() error {
+	return r.cacheHandler.Close()
+}
+
 // removeOrphanNetworks is a variable so tests can substitute one that needs no Docker daemon.
 var removeOrphanNetworks = container.RemoveOrphanNetworks
 

internal/app/run/runner_test.go

@@ -7,12 +7,14 @@ import (
 	"context"
 	"testing"
 
+	"gitea.com/gitea/runner/act/runner"
 	clientmocks "gitea.com/gitea/runner/internal/pkg/client/mocks"
 	"gitea.com/gitea/runner/internal/pkg/config"
 	"gitea.com/gitea/runner/internal/pkg/ver"
 
 	"connectrpc.com/connect"
 	runnerv1 "gitea.dev/actions-proto-go/runner/v1"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -91,6 +93,7 @@ func TestNewRunnerInitializesLabelsAndEnvironment(t *testing.T) {
 	require.Equal(t, "true", r.envs["GITEA_ACTIONS"])
 	require.NotEmpty(t, r.envs["GITEA_ACTIONS_RUNNER_VERSION"])
 	require.Nil(t, r.cacheHandler)
+	require.Empty(t, r.envs[runner.CacheServiceV2Env], "no cache server, nothing to serve v2 from")
 }
 
 // Proxy variables are assembled per task, because a job's service containers have to be
@@ -120,3 +123,22 @@ func taskWithDefaultActionsURL(url string) *runnerv1.Task {
 		},
 	}
 }
+
+// The cache service v2 API is announced to jobs unless it is turned off. Announcing it is what
+// makes act patch the GHES check out of an action's bundle, so the client can reach it.
+func TestNewRunnerCacheServiceV2(t *testing.T) {
+	announced := func(v2 *bool) string {
+		cfg := &config.Config{}
+		cfg.Cache.V2, cfg.Cache.Dir, cfg.Cache.Host = v2, t.TempDir(), "127.0.0.1"
+		cli := clientmocks.NewClient(t)
+		cli.On("Address").Return("https://gitea.example/").Maybe()
+
+		r := NewRunner(cfg, &config.Registration{Name: "runner"}, cli)
+		t.Cleanup(func() { _ = r.Close() })
+		return r.envs[runner.CacheServiceV2Env]
+	}
+
+	off := false
+	assert.Equal(t, "true", announced(nil))
+	assert.Empty(t, announced(&off))
+}

internal/pkg/config/config.example.yaml

@@ -155,6 +155,11 @@ cache:
   # A moved tag (e.g. a re-tagged "v6") or an updated branch stays at the cached commit
   # until its cache entry expires or is manually removed.
   offline_mode: false
+  # Serve the actions cache service v2 API, used by actions/cache@v4.2 and later. Those actions
+  # refuse any host they do not take for GitHub, so reaching it means editing that check out of
+  # the action's own bundle, keeping the untouched copy beside it. The same edit lets the stock
+  # upload-artifact and download-artifact work here. A bundle that does not match is left alone.
+  v2: true
 
 container:
   # Specifies the network to which the container will connect.

internal/pkg/config/config.go

@@ -74,6 +74,7 @@ type Cache struct {
 	ExternalSecret     string `yaml:"external_secret"`      // ExternalSecret is a shared secret between this runner and an external gitea-runner cache-server, enabling per-job ACTIONS_RUNTIME_TOKEN authentication and repo scoping over the network. Required whenever ExternalServer is set; ExternalSecretFile is the alternative way to provide it.
 	ExternalSecretFile string `yaml:"external_secret_file"` // ExternalSecretFile is the path to a file holding the ExternalSecret value, so the secret can be mounted instead of stored in the config file. LoadDefault reads it into ExternalSecret; setting both is an error.
 	OfflineMode        bool   `yaml:"offline_mode"`         // OfflineMode reuses a cached action without fetching from the remote; a moved tag or branch stays at the cached commit until the cache entry is removed.
+	V2                 *bool  `yaml:"v2"`                   // V2 serves the actions cache service v2 API to jobs, used by actions/cache@v4.2 and later, and edits the action bundles that would otherwise refuse it. Unset means enabled.
 }
 
 // Container represents the configuration for the container.

internal/pkg/config/config_test.go

@@ -347,3 +347,13 @@ cache:
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "contains no secret")
 }
+
+// The shipped example must parse, and every key in it must be one the config knows.
+func TestLoadDefault_ExampleConfigParses(t *testing.T) {
+	hook := test.NewGlobal()
+	defer hook.Reset()
+
+	_, err := LoadDefault("config.example.yaml")
+	require.NoError(t, err)
+	assert.Empty(t, hook.AllEntries())
+}