fix: stop leaking per-job docker networks (#1124)

Leaked per-job networks each hold a subnet of the daemon's address pool and nothing reclaimed them, so a host eventually fails every job with `all predefined address pools have been fully subnetted`. This is what CI hit in https://gitea.com/gitea/runner/actions/runs/742177.

- teardown no longer loses a container, and its network with it: a failed `ContainerRemove` was reported as success, a container whose id was never learned was skipped, and the daemon's own `AutoRemove` teardown was raced
- the idle cleanup reclaims what teardown cannot: networks carry `com.gitea.runner.uuid`, so a runner only touches its own, and a cutoff keeps a job starting during the pass out of scope
- pull failures reported mid-stream were discarded, surfacing later as a confusing `No such image`; they now propagate, and fall back to a local copy instead of failing the job
- `NetworkCreate` retries pool exhaustion briefly, then says which knobs to turn
- digest-pinned images are not re-pulled, and removal kills first so it never waits out Podman's stop timeout (measured 10.1s → 0.09s per container)

---------

Co-authored-by: bircni <bircni@icloud.com>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1124
Reviewed-by: bircni <bircni@icloud.com>

internal/pkg/config/config.example.yaml

@@ -15,6 +15,9 @@ runner:
   # Where to store the registration result.
   file: .runner
   # Execute how many tasks concurrently at the same time.
+  # With `container.network` empty, each concurrent docker job takes a subnet from the
+  # daemon's address pool, so a high capacity can exhaust it. See `default-address-pools`
+  # in the docker daemon config.
   capacity: 1
   # Extra environment variables to run jobs.
   envs:
@@ -43,9 +46,12 @@ runner:
   # While idle, remove stale bind-workdir task directories and orphaned host-mode
   # scratch directories (left behind when a host cleanup delete stalls) older than
   # this duration. Setting either workdir_cleanup_age or idle_cleanup_interval to 0
-  # (or any non-positive value) disables stale-directory cleanup entirely.
+  # (or any non-positive value) disables stale-directory cleanup entirely, along with
+  # the docker network cleanup below.
   workdir_cleanup_age: 24h
-  # Cadence for the idle stale-directory cleanup pass.
+  # Cadence for the idle cleanup pass. Besides the directories above, on runners that use
+  # docker it removes the per-job networks of jobs this runner did not live to tear down,
+  # which would otherwise hold a subnet of the daemon address pool until the host is rebuilt.
   idle_cleanup_interval: 10m
   # The base interval for periodic log flush to the Gitea instance.
   # Logs may be sent earlier if the buffer reaches log_report_batch_size
@@ -161,7 +167,9 @@ container:
   # network_create_options only apply when `network` is left empty and the runner
   # auto-creates a per-job network that does not already exist. They have no effect
   # when a custom `network` name is set, because that network is used as-is and never
-  # created by the runner. Omit the entire block to use Docker's defaults.
+  # created by the runner. Omit the entire block to use Docker's defaults. An auto-created
+  # network is labelled com.gitea.runner.uuid=<this runner's uuid>, which is how the idle
+  # cleanup tells its own leftovers apart from those of other runners on the same daemon.
   network_create_options:
     enable_ipv4: true  # Omit to use Docker's default (IPv4 enabled). Set false to disable IPv4.
     enable_ipv6: false # Omit to use Docker's default (IPv6 disabled). Enabling it requires dockerd started with --ipv6.
@@ -197,6 +205,9 @@ container:
   docker_host: ""
   # Pull docker image(s) even if already present.
   # Defaults to false when the key is omitted.
+  # Two exceptions: an image pinned by digest (image@sha256:...) cannot change, so it is never
+  # re-pulled, and a pull that fails while a copy is already on the host does not fail the job,
+  # which runs on that copy with a warning in its log.
   force_pull: false
   # Rebuild docker image(s) even if already present
   force_rebuild: false

internal/pkg/config/config.go

@@ -42,7 +42,7 @@ type Runner struct {
 	FetchInterval         time.Duration     `yaml:"fetch_interval"`           // FetchInterval specifies the interval duration for fetching resources.
 	FetchIntervalMax      time.Duration     `yaml:"fetch_interval_max"`       // FetchIntervalMax specifies the maximum backoff interval when idle.
 	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories and orphaned host-mode scratch dirs older than this duration during idle cleanup.
-	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs stale-directory cleanup periodically while the runner is idle. Set to 0 to disable cleanup cadence.
+	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs the idle cleanup (stale directories and orphaned docker networks) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
 	LogReportInterval     time.Duration     `yaml:"log_report_interval"`      // LogReportInterval specifies the base interval for periodic log flush.
 	LogReportMaxLatency   time.Duration     `yaml:"log_report_max_latency"`   // LogReportMaxLatency specifies the max time a log row can wait before being sent.
 	LogReportBatchSize    int               `yaml:"log_report_batch_size"`    // LogReportBatchSize triggers immediate log flush when buffer reaches this size.
@@ -86,7 +86,7 @@ type Container struct {
 	WorkdirParent        string                        `yaml:"workdir_parent"`         // WorkdirParent specifies the parent directory for the container's working directory.
 	ValidVolumes         []string                      `yaml:"valid_volumes"`          // ValidVolumes specifies the volumes (including bind mounts) can be mounted to containers.
 	DockerHost           string                        `yaml:"docker_host"`            // DockerHost specifies the Docker host. It overrides the value specified in environment variable DOCKER_HOST.
-	ForcePull            bool                          `yaml:"force_pull"`             // Pull docker image(s) even if already present
+	ForcePull            bool                          `yaml:"force_pull"`             // Pull docker image(s) even if already present, except digest-pinned ones. A pull that fails while a local copy exists is a warning, not a job failure.
 	ForceRebuild         bool                          `yaml:"force_rebuild"`          // Rebuild docker image(s) even if already present
 	RequireDocker        bool                          `yaml:"require_docker"`         // Always require a reachable docker daemon, even if not required by runner
 	DockerTimeout        time.Duration                 `yaml:"docker_timeout"`         // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or runner