fix: stop leaking per-job docker networks (#1124)

Leaked per-job networks each hold a subnet of the daemon's address pool and nothing reclaimed them, so a host eventually fails every job with `all predefined address pools have been fully subnetted`. This is what CI hit in https://gitea.com/gitea/runner/actions/runs/742177.

- teardown no longer loses a container, and its network with it: a failed `ContainerRemove` was reported as success, a container whose id was never learned was skipped, and the daemon's own `AutoRemove` teardown was raced
- the idle cleanup reclaims what teardown cannot: networks carry `com.gitea.runner.uuid`, so a runner only touches its own, and a cutoff keeps a job starting during the pass out of scope
- pull failures reported mid-stream were discarded, surfacing later as a confusing `No such image`; they now propagate, and fall back to a local copy instead of failing the job
- `NetworkCreate` retries pool exhaustion briefly, then says which knobs to turn
- digest-pinned images are not re-pulled, and removal kills first so it never waits out Podman's stop timeout (measured 10.1s → 0.09s per container)

---------

Co-authored-by: bircni <bircni@icloud.com>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1124
Reviewed-by: bircni <bircni@icloud.com>

internal/app/run/runner.go

@@ -54,6 +54,7 @@ func RunnerCapabilities() []string {
 // Runner runs the pipeline.
 type Runner struct {
 	name string
+	uuid string
 
 	cfg *config.Config
 
@@ -119,6 +120,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
 
 	runner := &Runner{
 		name:           reg.Name,
+		uuid:           reg.UUID,
 		cfg:            cfg,
 		client:         cli,
 		labels:         ls,
@@ -130,6 +132,9 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
 	return runner
 }
 
+// removeOrphanNetworks is a variable so tests can substitute one that needs no Docker daemon.
+var removeOrphanNetworks = container.RemoveOrphanNetworks
+
 // OnIdle performs lightweight maintenance during polling idle windows.
 // It runs synchronously on the poller goroutine; shouldRunIdleCleanup
 // throttles invocations to runner.idle_cleanup_interval so the impact on
@@ -151,6 +156,21 @@ func (r *Runner) OnIdle(ctx context.Context) {
 	if hostRoot := filepath.FromSlash(r.cfg.Host.WorkdirParent); hostRoot != "" {
 		r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
 	}
+	r.cleanupOrphanNetworks(ctx)
+}
+
+// cleanupOrphanNetworks reclaims the per-job networks of jobs this runner did not live to
+// tear down. A labelled network with no containers on it is finished with, and as for the
+// directories above, a task beginning during the pass is safe because the cutoff keeps a
+// network it has created but not yet attached a container to out of scope.
+func (r *Runner) cleanupOrphanNetworks(ctx context.Context) {
+	if r.uuid == "" || !r.labels.RequireDocker() && !r.cfg.Container.RequireDocker {
+		return
+	}
+	cutoff := r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)
+	if err := removeOrphanNetworks(ctx, r.uuid, cutoff); err != nil {
+		log.Warnf("failed to clean up networks left behind by earlier jobs: %v", err)
+	}
 }
 
 func (r *Runner) shouldRunIdleCleanup() bool {
@@ -472,6 +492,9 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		ContainerNetworkCreateOptions: container.NewDockerNetworkCreateExecutorInput{
 			EnableIPv4: r.cfg.Container.NetworkCreateOptions.EnableIPv4,
 			EnableIPv6: r.cfg.Container.NetworkCreateOptions.EnableIPv6,
+			// so a network this job leaks, if the runner dies before its teardown, can be
+			// told apart from one belonging to another runner on the same daemon
+			RunnerUUID: r.uuid,
 		},
 		ContainerOptions:                  r.cfg.Container.Options,
 		ContainerDaemonSocket:             r.cfg.Container.DockerHost,

internal/app/run/runner_idle_cleanup_test.go

@@ -297,3 +297,37 @@ func TestRunnerOnIdleSkipsWhenAlreadyCancelled(t *testing.T) {
 
 	assert.DirExists(t, stale)
 }
+
+// The idle sweep reclaims the docker networks of jobs this runner did not live to tear down,
+// and stays out of the way of runners that share the daemon but not the registration.
+func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
+	now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
+	cfg := &config.Config{
+		Container: config.Container{RequireDocker: true},
+		Runner: config.Runner{
+			WorkdirCleanupAge:   24 * time.Hour,
+			IdleCleanupInterval: time.Minute,
+		},
+	}
+
+	var swept []string
+	var sweptCutoff time.Time
+	origRemoveOrphanNetworks := removeOrphanNetworks
+	removeOrphanNetworks = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
+		swept = append(swept, runnerUUID)
+		sweptCutoff = createdBefore
+		return nil
+	}
+	t.Cleanup(func() { removeOrphanNetworks = origRemoveOrphanNetworks })
+
+	r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
+	r.OnIdle(context.Background())
+	assert.Equal(t, []string{"runner-1"}, swept)
+	// a network of a job starting during the pass is younger than this and so out of scope
+	assert.Equal(t, now.Add(-24*time.Hour), sweptCutoff)
+
+	// a host-only runner has no daemon to sweep
+	hostOnly := &Runner{uuid: "runner-2", cfg: &config.Config{Runner: cfg.Runner}, now: func() time.Time { return now }}
+	hostOnly.OnIdle(context.Background())
+	assert.Equal(t, []string{"runner-1"}, swept)
+}