mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-08-02 21:03:09 +00:00
b7a3bf98bc
Leaked per-job networks each hold a subnet of the daemon's address pool and nothing reclaimed them, so a host eventually fails every job with `all predefined address pools have been fully subnetted`. This is what CI hit in https://gitea.com/gitea/runner/actions/runs/742177. - teardown no longer loses a container, and its network with it: a failed `ContainerRemove` was reported as success, a container whose id was never learned was skipped, and the daemon's own `AutoRemove` teardown was raced - the idle cleanup reclaims what teardown cannot: networks carry `com.gitea.runner.uuid`, so a runner only touches its own, and a cutoff keeps a job starting during the pass out of scope - pull failures reported mid-stream were discarded, surfacing later as a confusing `No such image`; they now propagate, and fall back to a local copy instead of failing the job - `NetworkCreate` retries pool exhaustion briefly, then says which knobs to turn - digest-pinned images are not re-pulled, and removal kills first so it never waits out Podman's stop timeout (measured 10.1s → 0.09s per container) --------- Co-authored-by: bircni <bircni@icloud.com> Reviewed-on: https://gitea.com/gitea/runner/pulls/1124 Reviewed-by: bircni <bircni@icloud.com>
51 lines
2.4 KiB
Go
51 lines
2.4 KiB
Go
// Copyright 2026 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package container
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
cerrdefs "github.com/containerd/errdefs"
|
|
"github.com/moby/moby/api/types/network"
|
|
mobyclient "github.com/moby/moby/client"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestIsAddressPoolExhausted(t *testing.T) {
|
|
assert.True(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("Error response from daemon: all predefined address pools have been fully subnetted")))
|
|
assert.True(t, isAddressPoolExhausted(errors.New("could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network")))
|
|
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
|
|
}
|
|
|
|
// Of this runner's networks, only the ones nothing is attached to and old enough to predate
|
|
// any job now starting are the runner's to reclaim. An unexpected NetworkRemove fails the
|
|
// test on its own, since testify has no expectation to match it against.
|
|
func TestRemoveOrphanNetworks(t *testing.T) {
|
|
ctx := context.Background()
|
|
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
client := &mockDockerClient{}
|
|
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
|
|
Filters: make(mobyclient.Filters).Add("label", runnerUUIDLabel+"=runner-1"),
|
|
}).Return(mobyclient.NetworkListResult{Items: []network.Summary{
|
|
{Network: network.Network{ID: "orphan"}},
|
|
{Network: network.Network{ID: "busy"}},
|
|
{Network: network.Network{ID: "starting"}},
|
|
}}, nil)
|
|
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
|
|
Return(mobyclient.NetworkInspectResult{}, nil)
|
|
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
|
|
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
|
|
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
|
|
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: network.Network{Created: cutoff.Add(time.Second)}}}, nil)
|
|
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
|
|
Return(mobyclient.NetworkRemoveResult{}, nil)
|
|
|
|
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", cutoff))
|
|
client.AssertExpectations(t)
|
|
}
|