From dc5806cd949178f776dec8dc83e51dd1feea65a3 Mon Sep 17 00:00:00 2001 From: Hasan Siddiqui Date: Mon, 23 Mar 2026 20:52:21 +0000 Subject: [PATCH 1/3] Propagate OpenTelemetry traces in outgoing RPCs from plugin clients While incoming gRPC requests are already being traced via the server-side handler, outgoing RPCs to proxy plugins are missing the client-side equivalent. Adding otelgrpc.NewClientHandler() ensures trace context is successfully propagated to the plugins. Signed-off-by: Hasan Siddiqui --- cmd/containerd/server/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/containerd/server/server.go b/cmd/containerd/server/server.go index 746e6804d6756..08ea9ac38b386 100644 --- a/cmd/containerd/server/server.go +++ b/cmd/containerd/server/server.go @@ -555,6 +555,7 @@ func (pc *proxyClients) getClient(address string) (*grpc.ClientConn, error) { Backoff: backoffConfig, } gopts := []grpc.DialOption{ + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithConnectParams(connParams), grpc.WithContextDialer(dialer.ContextDialer), From e6c7f37235b30a3e697a3411045a9d00ab876c63 Mon Sep 17 00:00:00 2001 From: Chris Henzie Date: Tue, 24 Mar 2026 14:47:46 -0700 Subject: [PATCH 2/3] Add unit tests for CRI resource updates Provides test coverage for existing UpdatePodSandboxResources behavior. Assisted-by: Antigravity Signed-off-by: Chris Henzie --- .../server/sandbox_update_resources_test.go | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 internal/cri/server/sandbox_update_resources_test.go diff --git a/internal/cri/server/sandbox_update_resources_test.go b/internal/cri/server/sandbox_update_resources_test.go new file mode 100644 index 0000000000000..2860914e31d9a --- /dev/null +++ b/internal/cri/server/sandbox_update_resources_test.go @@ -0,0 +1,153 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "context" + "testing" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/core/sandbox" + "github.com/containerd/containerd/v2/internal/cri/server/podsandbox" + sstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox" + "github.com/containerd/errdefs" + "github.com/containerd/typeurl/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +type fakeSandboxStore struct { + sandbox.Store + updatedSandbox *sandbox.Sandbox + getErr error + updateErr error +} + +func (f *fakeSandboxStore) Get(_ context.Context, id string) (sandbox.Sandbox, error) { + if f.getErr != nil { + return sandbox.Sandbox{}, f.getErr + } + return sandbox.Sandbox{ID: id, Extensions: map[string]typeurl.Any{}}, nil +} + +func (f *fakeSandboxStore) Update(_ context.Context, sb sandbox.Sandbox, _ ...string) (sandbox.Sandbox, error) { + if f.updateErr != nil { + return sandbox.Sandbox{}, f.updateErr + } + f.updatedSandbox = &sb + return sb, nil +} + +func TestUpdatePodSandboxResources(t *testing.T) { + mySandbox := sstore.Metadata{ + ID: "test-sandbox-id", + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{Name: "test-name"}, + }, + } + + t.Run("fails when sandbox not found in local store", func(t *testing.T) { + c := newTestCRIService(func(*criService) {}) + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "missing-sandbox-id", + } + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to find sandbox") + }) + + t.Run("fails when core store get fails", func(t *testing.T) { + fakeStore := &fakeSandboxStore{getErr: errdefs.ErrNotFound} + c := newTestCRIService(func(service *criService) { + client, _ := containerd.New("", containerd.WithServices(containerd.WithSandboxStore(fakeStore))) + service.client = client + }) + s := sstore.NewSandbox(mySandbox, sstore.Status{}) + c.sandboxStore.Add(s) + + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "test-sandbox-id", + } + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get sandbox test-sandbox-id from sandbox store") + }) + + t.Run("fails when core store update fails", func(t *testing.T) { + fakeStore := &fakeSandboxStore{updateErr: errdefs.ErrUnavailable} + c := newTestCRIService(func(service *criService) { + client, _ := containerd.New("", containerd.WithServices(containerd.WithSandboxStore(fakeStore))) + service.client = client + }) + s := sstore.NewSandbox(mySandbox, sstore.Status{}) + c.sandboxStore.Add(s) + + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "test-sandbox-id", + } + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to update sandbox test-sandbox-id in core store") + }) + + t.Run("successfully extracts and delegates resources to stores", func(t *testing.T) { + fakeStore := &fakeSandboxStore{} + c := newTestCRIService(func(service *criService) { + client, _ := containerd.New("", containerd.WithServices(containerd.WithSandboxStore(fakeStore))) + service.client = client + }) + s := sstore.NewSandbox(mySandbox, sstore.Status{}) + c.sandboxStore.Add(s) + + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "test-sandbox-id", + Overhead: &runtime.LinuxContainerResources{ + MemoryLimitInBytes: 100, + }, + Resources: &runtime.LinuxContainerResources{ + CpuQuota: 200, + }, + } + + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.NoError(t, err) + + // Assert local mem store mutated correctly. + localSb, err := c.sandboxStore.Get("test-sandbox-id") + require.NoError(t, err) + status := localSb.Status.Get() + require.NotNil(t, status.Overhead) + require.NotNil(t, status.Resources) + assert.Equal(t, int64(100), status.Overhead.Linux.MemoryLimitInBytes) + assert.Equal(t, int64(200), status.Resources.Linux.CpuQuota) + + // Assert core store was updated with correct extensions. + require.NotNil(t, fakeStore.updatedSandbox) + ext, ok := fakeStore.updatedSandbox.Extensions[podsandbox.UpdatedResourcesKey] + require.True(t, ok, "expected UpdatedResourcesKey in extensions") + + var updatedRes podsandbox.UpdatedResources + err = typeurl.UnmarshalTo(ext, &updatedRes) + require.NoError(t, err) + + require.NotNil(t, updatedRes.Overhead) + require.NotNil(t, updatedRes.Resources) + assert.Equal(t, int64(100), updatedRes.Overhead.MemoryLimitInBytes) + assert.Equal(t, int64(200), updatedRes.Resources.CpuQuota) + }) +} From 33db836a8b06000e8afb5bba947c299ae721878a Mon Sep 17 00:00:00 2001 From: Chris Henzie Date: Tue, 24 Mar 2026 14:48:28 -0700 Subject: [PATCH 3/3] Wire UpdatePodSandboxResources to Sandbox API Integrates CRI container resource updates with core Sandbox API plugin. Delegates payload to out-of-tree controllers via UpdateSandbox API. Gracefully tolerates ErrNotImplemented to preserve backwards compatibility for legacy sandboxers. Assisted-by: Antigravity Signed-off-by: Chris Henzie --- internal/cri/server/sandbox_service.go | 8 +++ .../cri/server/sandbox_update_resources.go | 9 +++ .../server/sandbox_update_resources_test.go | 70 +++++++++++++++++++ internal/cri/server/service.go | 1 + internal/cri/server/service_test.go | 4 ++ 5 files changed, 92 insertions(+) diff --git a/internal/cri/server/sandbox_service.go b/internal/cri/server/sandbox_service.go index 8cec187caa337..1510aaabedb52 100644 --- a/internal/cri/server/sandbox_service.go +++ b/internal/cri/server/sandbox_service.go @@ -110,6 +110,14 @@ func (c *criSandboxService) ShutdownSandbox(ctx context.Context, sandboxer strin return ctrl.Shutdown(ctx, sandboxID) } +func (c *criSandboxService) UpdateSandbox(ctx context.Context, sandboxer string, sandboxID string, sandbox sandbox.Sandbox, fields ...string) error { + ctrl, err := c.SandboxController(sandboxer) + if err != nil { + return err + } + return ctrl.Update(ctx, sandboxID, sandbox, fields...) +} + func (c *criSandboxService) StopSandbox(ctx context.Context, sandboxer, sandboxID string, opts ...sandbox.StopOpt) error { ctrl, err := c.SandboxController(sandboxer) if err != nil { diff --git a/internal/cri/server/sandbox_update_resources.go b/internal/cri/server/sandbox_update_resources.go index a38a3030a1849..67bf432089166 100644 --- a/internal/cri/server/sandbox_update_resources.go +++ b/internal/cri/server/sandbox_update_resources.go @@ -25,6 +25,7 @@ import ( "github.com/containerd/containerd/v2/internal/cri/server/podsandbox" sstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox" "github.com/containerd/containerd/v2/pkg/tracing" + "github.com/containerd/errdefs" "github.com/containerd/log" ) @@ -75,6 +76,14 @@ func (c *criService) UpdatePodSandboxResources(ctx context.Context, r *runtime.U return nil, fmt.Errorf("failed to update sandbox %s in core store: %w", sandbox.ID, err) } + if err := c.sandboxService.UpdateSandbox(ctx, sandboxInfo.Sandboxer, sandboxInfo.ID, sandboxInfo, "extensions"); err != nil { + // Tolerate these errors for older sandbox controllers that may not support this yet. + if !errdefs.IsNotImplemented(err) { + return nil, fmt.Errorf("failed to update sandbox controller: %w", err) + } + log.G(ctx).Tracef("sandbox controller %q does not implement Update endpoint", sandboxInfo.Sandboxer) + } + err = c.nri.PostUpdatePodSandboxResources(ctx, &sandbox) if err != nil { log.G(ctx).WithError(err).Errorf("NRI post-update notification failed") diff --git a/internal/cri/server/sandbox_update_resources_test.go b/internal/cri/server/sandbox_update_resources_test.go index 2860914e31d9a..be6e1d99421c3 100644 --- a/internal/cri/server/sandbox_update_resources_test.go +++ b/internal/cri/server/sandbox_update_resources_test.go @@ -53,6 +53,17 @@ func (f *fakeSandboxStore) Update(_ context.Context, sb sandbox.Sandbox, _ ...st return sb, nil } +type recordSandboxService struct { + fakeSandboxService + calledID string + returnErr error +} + +func (s *recordSandboxService) UpdateSandbox(_ context.Context, _ string, sandboxID string, _ sandbox.Sandbox, _ ...string) error { + s.calledID = sandboxID + return s.returnErr +} + func TestUpdatePodSandboxResources(t *testing.T) { mySandbox := sstore.Metadata{ ID: "test-sandbox-id", @@ -150,4 +161,63 @@ func TestUpdatePodSandboxResources(t *testing.T) { assert.Equal(t, int64(100), updatedRes.Overhead.MemoryLimitInBytes) assert.Equal(t, int64(200), updatedRes.Resources.CpuQuota) }) + + t.Run("success fallback when sandbox controller does not implement update", func(t *testing.T) { + recordService := &recordSandboxService{returnErr: errdefs.ErrNotImplemented} + c := newTestCRIService(func(service *criService) { + service.sandboxService = recordService + client, _ := containerd.New("", containerd.WithServices(containerd.WithSandboxStore(&fakeSandboxStore{}))) + service.client = client + }) + s := sstore.NewSandbox(mySandbox, sstore.Status{}) + c.sandboxStore.Add(s) + + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "test-sandbox-id", + } + + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.NoError(t, err) + + assert.Equal(t, "test-sandbox-id", recordService.calledID) + }) + + t.Run("success when sandbox controller implements update", func(t *testing.T) { + recordService := &recordSandboxService{returnErr: nil} + c := newTestCRIService(func(service *criService) { + service.sandboxService = recordService + client, _ := containerd.New("", containerd.WithServices(containerd.WithSandboxStore(&fakeSandboxStore{}))) + service.client = client + }) + s := sstore.NewSandbox(mySandbox, sstore.Status{}) + c.sandboxStore.Add(s) + + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "test-sandbox-id", + } + + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.NoError(t, err) + + assert.Equal(t, "test-sandbox-id", recordService.calledID) + }) + + t.Run("fails when sandbox controller update fails", func(t *testing.T) { + recordService := &recordSandboxService{returnErr: errdefs.ErrInvalidArgument} + c := newTestCRIService(func(service *criService) { + service.sandboxService = recordService + client, _ := containerd.New("", containerd.WithServices(containerd.WithSandboxStore(&fakeSandboxStore{}))) + service.client = client + }) + s := sstore.NewSandbox(mySandbox, sstore.Status{}) + c.sandboxStore.Add(s) + + req := &runtime.UpdatePodSandboxResourcesRequest{ + PodSandboxId: "test-sandbox-id", + } + + _, err := c.UpdatePodSandboxResources(context.Background(), req) + require.Error(t, err) + assert.Contains(t, err.Error(), errdefs.ErrInvalidArgument.Error()) + }) } diff --git a/internal/cri/server/service.go b/internal/cri/server/service.go index c462d46dc09da..953e665d28ede 100644 --- a/internal/cri/server/service.go +++ b/internal/cri/server/service.go @@ -78,6 +78,7 @@ type sandboxService interface { CreateSandbox(ctx context.Context, info sandbox.Sandbox, opts ...sandbox.CreateOpt) error StartSandbox(ctx context.Context, sandboxer string, sandboxID string) (sandbox.ControllerInstance, error) WaitSandbox(ctx context.Context, sandboxer string, sandboxID string) (<-chan containerd.ExitStatus, error) + UpdateSandbox(ctx context.Context, sandboxer string, sandboxID string, sandbox sandbox.Sandbox, fields ...string) error StopSandbox(ctx context.Context, sandboxer, sandboxID string, opts ...sandbox.StopOpt) error ShutdownSandbox(ctx context.Context, sandboxer string, sandboxID string) error SandboxStatus(ctx context.Context, sandboxer string, sandboxID string, verbose bool) (sandbox.ControllerStatus, error) diff --git a/internal/cri/server/service_test.go b/internal/cri/server/service_test.go index 5eaa7abff5d3d..21efe3c7aba5c 100644 --- a/internal/cri/server/service_test.go +++ b/internal/cri/server/service_test.go @@ -47,6 +47,10 @@ func (f *fakeSandboxService) StartSandbox(ctx context.Context, sandboxer string, return sandbox.ControllerInstance{}, errdefs.ErrNotImplemented } +func (f *fakeSandboxService) UpdateSandbox(ctx context.Context, sandboxer string, sandboxID string, sandbox sandbox.Sandbox, fields ...string) error { + return errdefs.ErrNotImplemented +} + func (f *fakeSandboxService) StopSandbox(ctx context.Context, sandboxer, sandboxID string, opts ...sandbox.StopOpt) error { return errdefs.ErrNotImplemented }