diff --git a/SPECS/containerd2/add-tardev-support.patch b/SPECS/containerd2/add-tardev-support.patch new file mode 100644 index 00000000000..7ddc24bb813 --- /dev/null +++ b/SPECS/containerd2/add-tardev-support.patch @@ -0,0 +1,288 @@ +From 33ac7f6f513934a0ef59cc6739eee0efdac3631c Mon Sep 17 00:00:00 2001 +From: Mitch Zhu +Date: Fri, 22 Nov 2024 20:41:27 +0000 +Subject: [PATCH] Enhance snapshot handling and CRI runtime compatibility for + tardev-snapshotter + +--- + client/image.go | 4 +- + internal/cri/server/container_status_test.go | 2 +- + internal/cri/server/images/image_pull.go | 37 +++++++++++-------- + internal/cri/server/podsandbox/controller.go | 2 +- + internal/cri/server/podsandbox/sandbox_run.go | 30 ++++++++------- + internal/cri/server/service.go | 2 +- + internal/cri/store/image/image.go | 29 ++++++++++++--- + 7 files changed, 68 insertions(+), 38 deletions(-) + +diff --git a/client/image.go b/client/image.go +index 355bcba..791db88 100644 +--- a/client/image.go ++++ b/client/image.go +@@ -31,6 +31,7 @@ import ( + "github.com/containerd/containerd/v2/internal/kmutex" + "github.com/containerd/containerd/v2/pkg/labels" + "github.com/containerd/containerd/v2/pkg/rootfs" ++ "github.com/containerd/containerd/v2/pkg/snapshotters" + "github.com/containerd/errdefs" + "github.com/containerd/platforms" + "github.com/opencontainers/go-digest" +@@ -333,7 +334,8 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...Unpa + } + + for _, layer := range layers { +- unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, config.SnapshotOpts, config.ApplyOpts) ++ snOpts := append(config.SnapshotOpts, snapshots.WithLabels(map[string]string{snapshotters.TargetLayerDigestLabel: layer.Blob.Digest.String()})) ++ unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, snOpts, config.ApplyOpts) + if err != nil { + return fmt.Errorf("apply layer error for %q: %w", i.Name(), err) + } +diff --git a/internal/cri/server/container_status_test.go b/internal/cri/server/container_status_test.go +index 05b1650..71dcc10 100644 +--- a/internal/cri/server/container_status_test.go ++++ b/internal/cri/server/container_status_test.go +@@ -302,7 +302,7 @@ func (s *fakeImageService) LocalResolve(refOrID string) (imagestore.Image, error + + func (s *fakeImageService) ImageFSPaths() map[string]string { return make(map[string]string) } + +-func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string) (string, error) { ++func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string, string) (string, error) { + return "", errors.New("not implemented") + } + +diff --git a/internal/cri/server/images/image_pull.go b/internal/cri/server/images/image_pull.go +index e59b88b..f9c90b7 100644 +--- a/internal/cri/server/images/image_pull.go ++++ b/internal/cri/server/images/image_pull.go +@@ -96,6 +96,15 @@ import ( + + // PullImage pulls an image with authentication config. + func (c *GRPCCRIImageService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (_ *runtime.PullImageResponse, err error) { ++ imageRef := r.GetImage().GetImage() ++ snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, imageRef, r.SandboxConfig, r.GetImage().GetRuntimeHandler()) ++ if err != nil { ++ return nil, err ++ } ++ return c.pullImage(ctx, r, snapshotter) ++} ++ ++func (c *GRPCCRIImageService) pullImage(ctx context.Context, r *runtime.PullImageRequest, snapshotter string) (_ *runtime.PullImageResponse, err error) { + + imageRef := r.GetImage().GetImage() + +@@ -110,14 +119,14 @@ func (c *GRPCCRIImageService) PullImage(ctx context.Context, r *runtime.PullImag + return ParseAuth(hostauth, host) + } + +- ref, err := c.CRIImageService.PullImage(ctx, imageRef, credentials, r.SandboxConfig, r.GetImage().GetRuntimeHandler()) ++ ref, err := c.CRIImageService.PullImage(ctx, imageRef, credentials, r.SandboxConfig, r.GetImage().GetRuntimeHandler(), snapshotter) + if err != nil { + return nil, err + } + return &runtime.PullImageResponse{ImageRef: ref}, nil + } + +-func (c *CRIImageService) PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (_ string, err error) { ++func (c *CRIImageService) PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (_ string, err error) { + span := tracing.SpanFromContext(ctx) + defer func() { + // TODO: add domain label for imagePulls metrics, and we may need to provide a mechanism +@@ -167,10 +176,6 @@ func (c *CRIImageService) PullImage(ctx context.Context, name string, credential + ) + + defer pcancel() +- snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, ref, sandboxConfig) +- if err != nil { +- return "", err +- } + log.G(ctx).Debugf("PullImage %q with snapshotter %s", ref, snapshotter) + span.SetAttributes( + tracing.Attribute("image.ref", ref), +@@ -761,17 +766,19 @@ func (rt *pullRequestReporterRoundTripper) RoundTrip(req *http.Request) (*http.R + // Once we know the runtime, try to override default snapshotter if it is set for this runtime. + // See https://github.com/containerd/containerd/issues/6657 + func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, imageRef string, +- s *runtime.PodSandboxConfig) (string, error) { ++ s *runtime.PodSandboxConfig, runtimeHandler string) (string, error) { + snapshotter := c.config.Snapshotter +- if s == nil || s.Annotations == nil { +- return snapshotter, nil +- } + +- // TODO(kiashok): honor the new CRI runtime handler field added to v0.29.0 +- // for image pull per runtime class support. +- runtimeHandler, ok := s.Annotations[annotations.RuntimeHandler] +- if !ok { +- return snapshotter, nil ++ if runtimeHandler == "" { ++ if s == nil || s.Annotations == nil { ++ return snapshotter, nil ++ } else { ++ ok := false ++ runtimeHandler, ok = s.Annotations[annotations.RuntimeHandler] ++ if !ok { ++ return snapshotter, nil ++ } ++ } + } + + // TODO: Ensure error is returned if runtime not found? +diff --git a/internal/cri/server/podsandbox/controller.go b/internal/cri/server/podsandbox/controller.go +index a185a4c..8fd032b 100644 +--- a/internal/cri/server/podsandbox/controller.go ++++ b/internal/cri/server/podsandbox/controller.go +@@ -110,7 +110,7 @@ type RuntimeService interface { + type ImageService interface { + LocalResolve(refOrID string) (imagestore.Image, error) + GetImage(id string) (imagestore.Image, error) +- PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string) (string, error) ++ PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (string, error) + RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string + PinnedImage(string) string + } +diff --git a/internal/cri/server/podsandbox/sandbox_run.go b/internal/cri/server/podsandbox/sandbox_run.go +index 53d949f..b296061 100644 +--- a/internal/cri/server/podsandbox/sandbox_run.go ++++ b/internal/cri/server/podsandbox/sandbox_run.go +@@ -77,23 +77,25 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll + + sandboxImage := c.getSandboxImageName() + // Ensure sandbox container image snapshot. +- image, err := c.ensureImageExists(ctx, sandboxImage, config, metadata.RuntimeHandler) ++ ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler) + if err != nil { +- return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err) ++ return cin, fmt.Errorf("failed to get sandbox runtime: %w", err) + } ++ log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime) + +- containerdImage, err := c.toContainerdImage(ctx, *image) ++ labels["oci_runtime_type"] = ociRuntime.Type ++ ++ snapshotter := c.imageService.RuntimeSnapshotter(ctx, ociRuntime) ++ ++ image, err := c.ensureImageExists(ctx, sandboxImage, config, metadata.RuntimeHandler, snapshotter) + if err != nil { +- return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err) ++ return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err) + } + +- ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler) ++ containerdImage, err := c.toContainerdImage(ctx, *image) + if err != nil { +- return cin, fmt.Errorf("failed to get sandbox runtime: %w", err) ++ return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err) + } +- log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime) +- +- labels["oci_runtime_type"] = ociRuntime.Type + + // Create sandbox container root directories. + sandboxRootDir := c.getSandboxRootDir(id) +@@ -173,7 +175,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll + snapshotterOpt = append(snapshotterOpt, extraSOpts...) + + opts := []containerd.NewContainerOpts{ +- containerd.WithSnapshotter(c.imageService.RuntimeSnapshotter(ctx, ociRuntime)), ++ containerd.WithSnapshotter(snapshotter), + customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt...), + containerd.WithSpec(spec, specOpts...), + containerd.WithContainerLabels(sandboxLabels), +@@ -299,17 +301,19 @@ func (c *Controller) Create(_ctx context.Context, info sandbox.Sandbox, opts ... + return c.store.Save(podSandbox) + } + +-func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig, runtimeHandler string) (*imagestore.Image, error) { ++func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (*imagestore.Image, error) { + image, err := c.imageService.LocalResolve(ref) + if err != nil && !errdefs.IsNotFound(err) { + return nil, fmt.Errorf("failed to get image %q: %w", ref, err) + } + if err == nil { +- return &image, nil ++ if _, ok := image.Snapshotters[snapshotter]; ok { ++ return &image, nil ++ } + } + // Pull image to ensure the image exists + // TODO: Cleaner interface +- imageID, err := c.imageService.PullImage(ctx, ref, nil, config, runtimeHandler) ++ imageID, err := c.imageService.PullImage(ctx, ref, nil, config, runtimeHandler, snapshotter) + if err != nil { + return nil, fmt.Errorf("failed to pull image %q: %w", ref, err) + } +diff --git a/internal/cri/server/service.go b/internal/cri/server/service.go +index 37d66f0..5d1546e 100644 +--- a/internal/cri/server/service.go ++++ b/internal/cri/server/service.go +@@ -97,7 +97,7 @@ type RuntimeService interface { + type ImageService interface { + RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string + +- PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (string, error) ++ PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (string, error) + UpdateImage(ctx context.Context, r string) error + + CheckImages(ctx context.Context) error +diff --git a/internal/cri/store/image/image.go b/internal/cri/store/image/image.go +index 5887e75..43ecf0d 100644 +--- a/internal/cri/store/image/image.go ++++ b/internal/cri/store/image/image.go +@@ -20,6 +20,7 @@ import ( + "context" + "encoding/json" + "fmt" ++ "strings" + "sync" + + "github.com/containerd/containerd/v2/core/content" +@@ -53,6 +54,8 @@ type Image struct { + ImageSpec imagespec.Image + // Pinned image to prevent it from garbage collection + Pinned bool ++ // Snapshotters is a map whose keys are snapshotters for which this image has a snapshot. ++ Snapshotters map[string]struct{} + } + + // Getter is used to get images but does not make changes +@@ -170,6 +173,19 @@ func (s *Store) getImage(ctx context.Context, i images.Image) (*Image, error) { + return nil, fmt.Errorf("read image config from content store: %w", err) + } + ++ info, err := s.provider.Info(ctx, desc.Digest) ++ if err != nil { ++ return nil, fmt.Errorf("get content store config info: %w", err) ++ } ++ ++ snapshotters := make(map[string]struct{}) ++ for label := range info.Labels { ++ const Prefix = "containerd.io/gc.ref.snapshot." ++ if strings.HasPrefix(label, Prefix) { ++ snapshotters[label[len(Prefix):]] = struct{}{} ++ } ++ } ++ + var spec imagespec.Image + if err := json.Unmarshal(blob, &spec); err != nil { + return nil, fmt.Errorf("unmarshal image config %s: %w", blob, err) +@@ -178,12 +194,13 @@ func (s *Store) getImage(ctx context.Context, i images.Image) (*Image, error) { + pinned := i.Labels[labels.PinnedImageLabelKey] == labels.PinnedImageLabelValue + + return &Image{ +- ID: id, +- References: []string{i.Name}, +- ChainID: chainID.String(), +- Size: size, +- ImageSpec: spec, +- Pinned: pinned, ++ ID: id, ++ References: []string{i.Name}, ++ ChainID: chainID.String(), ++ Size: size, ++ ImageSpec: spec, ++ Pinned: pinned, ++ Snapshotters: snapshotters, + }, nil + + } +-- +2.34.1 \ No newline at end of file diff --git a/SPECS/containerd2/containerd2.spec b/SPECS/containerd2/containerd2.spec index 79013cc94f4..643f01da834 100644 --- a/SPECS/containerd2/containerd2.spec +++ b/SPECS/containerd2/containerd2.spec @@ -5,7 +5,7 @@ Summary: Industry-standard container runtime Name: %{upstream_name}2 Version: 2.0.0 -Release: 1%{?dist} +Release: 2%{?dist} License: ASL 2.0 Group: Tools/Container URL: https://www.containerd.io @@ -15,6 +15,8 @@ Distribution: Azure Linux Source0: https://github.com/containerd/containerd/archive/v%{version}.tar.gz#/%{upstream_name}-%{version}.tar.gz Source1: containerd.service Source2: containerd.toml +# Added patch to support tardev-snapshotter for Kata CC +Patch0: add-tardev-support.patch %{?systemd_requires} @@ -79,6 +81,9 @@ fi %dir /opt/containerd/lib %changelog +* Thu Jan 09 2024 Mitch Zhu - 2.0.0-2 +- Added patch to support tardev-snapshotter for Kata CC. + * Wed Dec 11 2024 Nan Liu - 2.0.0-1 - Created a standalone package for containerd 2.0.0 - Initial CBL-Mariner import from Azure