Skip to content

Commit

Permalink
Add tardev support patch
Browse files Browse the repository at this point in the history
  • Loading branch information
@miz060
miz060 committed Jan 9, 2025
1 parent db2fd14 commit fb2813c
Show file tree
Hide file tree
Showing 2 changed files with 294 additions and 1 deletion.
288 changes: 288 additions & 0 deletions SPECS/containerd2/add-tardev-support.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,288 @@
From 33ac7f6f513934a0ef59cc6739eee0efdac3631c Mon Sep 17 00:00:00 2001
From: Mitch Zhu <[email protected]>
Date: Fri, 22 Nov 2024 20:41:27 +0000
Subject: [PATCH] Enhance snapshot handling and CRI runtime compatibility for
tardev-snapshotter

---
client/image.go | 4 +-
internal/cri/server/container_status_test.go | 2 +-
internal/cri/server/images/image_pull.go | 37 +++++++++++--------
internal/cri/server/podsandbox/controller.go | 2 +-
internal/cri/server/podsandbox/sandbox_run.go | 30 ++++++++-------
internal/cri/server/service.go | 2 +-
internal/cri/store/image/image.go | 29 ++++++++++++---
7 files changed, 68 insertions(+), 38 deletions(-)

diff --git a/client/image.go b/client/image.go
index 355bcba..791db88 100644
--- a/client/image.go
+++ b/client/image.go
@@ -31,6 +31,7 @@ import (
"github.com/containerd/containerd/v2/internal/kmutex"
"github.com/containerd/containerd/v2/pkg/labels"
"github.com/containerd/containerd/v2/pkg/rootfs"
+ "github.com/containerd/containerd/v2/pkg/snapshotters"
"github.com/containerd/errdefs"
"github.com/containerd/platforms"
"github.com/opencontainers/go-digest"
@@ -333,7 +334,8 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...Unpa
}

for _, layer := range layers {
- unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, config.SnapshotOpts, config.ApplyOpts)
+ snOpts := append(config.SnapshotOpts, snapshots.WithLabels(map[string]string{snapshotters.TargetLayerDigestLabel: layer.Blob.Digest.String()}))
+ unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, snOpts, config.ApplyOpts)
if err != nil {
return fmt.Errorf("apply layer error for %q: %w", i.Name(), err)
}
diff --git a/internal/cri/server/container_status_test.go b/internal/cri/server/container_status_test.go
index 05b1650..71dcc10 100644
--- a/internal/cri/server/container_status_test.go
+++ b/internal/cri/server/container_status_test.go
@@ -302,7 +302,7 @@ func (s *fakeImageService) LocalResolve(refOrID string) (imagestore.Image, error

func (s *fakeImageService) ImageFSPaths() map[string]string { return make(map[string]string) }

-func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string) (string, error) {
+func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string, string) (string, error) {
return "", errors.New("not implemented")
}

diff --git a/internal/cri/server/images/image_pull.go b/internal/cri/server/images/image_pull.go
index e59b88b..f9c90b7 100644
--- a/internal/cri/server/images/image_pull.go
+++ b/internal/cri/server/images/image_pull.go
@@ -96,6 +96,15 @@ import (

// PullImage pulls an image with authentication config.
func (c *GRPCCRIImageService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (_ *runtime.PullImageResponse, err error) {
+ imageRef := r.GetImage().GetImage()
+ snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, imageRef, r.SandboxConfig, r.GetImage().GetRuntimeHandler())
+ if err != nil {
+ return nil, err
+ }
+ return c.pullImage(ctx, r, snapshotter)
+}
+
+func (c *GRPCCRIImageService) pullImage(ctx context.Context, r *runtime.PullImageRequest, snapshotter string) (_ *runtime.PullImageResponse, err error) {

imageRef := r.GetImage().GetImage()

@@ -110,14 +119,14 @@ func (c *GRPCCRIImageService) PullImage(ctx context.Context, r *runtime.PullImag
return ParseAuth(hostauth, host)
}

- ref, err := c.CRIImageService.PullImage(ctx, imageRef, credentials, r.SandboxConfig, r.GetImage().GetRuntimeHandler())
+ ref, err := c.CRIImageService.PullImage(ctx, imageRef, credentials, r.SandboxConfig, r.GetImage().GetRuntimeHandler(), snapshotter)
if err != nil {
return nil, err
}
return &runtime.PullImageResponse{ImageRef: ref}, nil
}

-func (c *CRIImageService) PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (_ string, err error) {
+func (c *CRIImageService) PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (_ string, err error) {
span := tracing.SpanFromContext(ctx)
defer func() {
// TODO: add domain label for imagePulls metrics, and we may need to provide a mechanism
@@ -167,10 +176,6 @@ func (c *CRIImageService) PullImage(ctx context.Context, name string, credential
)

defer pcancel()
- snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, ref, sandboxConfig)
- if err != nil {
- return "", err
- }
log.G(ctx).Debugf("PullImage %q with snapshotter %s", ref, snapshotter)
span.SetAttributes(
tracing.Attribute("image.ref", ref),
@@ -761,17 +766,19 @@ func (rt *pullRequestReporterRoundTripper) RoundTrip(req *http.Request) (*http.R
// Once we know the runtime, try to override default snapshotter if it is set for this runtime.
// See https://github.com/containerd/containerd/issues/6657
func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, imageRef string,
- s *runtime.PodSandboxConfig) (string, error) {
+ s *runtime.PodSandboxConfig, runtimeHandler string) (string, error) {
snapshotter := c.config.Snapshotter
- if s == nil || s.Annotations == nil {
- return snapshotter, nil
- }

- // TODO(kiashok): honor the new CRI runtime handler field added to v0.29.0
- // for image pull per runtime class support.
- runtimeHandler, ok := s.Annotations[annotations.RuntimeHandler]
- if !ok {
- return snapshotter, nil
+ if runtimeHandler == "" {
+ if s == nil || s.Annotations == nil {
+ return snapshotter, nil
+ } else {
+ ok := false
+ runtimeHandler, ok = s.Annotations[annotations.RuntimeHandler]
+ if !ok {
+ return snapshotter, nil
+ }
+ }
}

// TODO: Ensure error is returned if runtime not found?
diff --git a/internal/cri/server/podsandbox/controller.go b/internal/cri/server/podsandbox/controller.go
index a185a4c..8fd032b 100644
--- a/internal/cri/server/podsandbox/controller.go
+++ b/internal/cri/server/podsandbox/controller.go
@@ -110,7 +110,7 @@ type RuntimeService interface {
type ImageService interface {
LocalResolve(refOrID string) (imagestore.Image, error)
GetImage(id string) (imagestore.Image, error)
- PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string) (string, error)
+ PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (string, error)
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
PinnedImage(string) string
}
diff --git a/internal/cri/server/podsandbox/sandbox_run.go b/internal/cri/server/podsandbox/sandbox_run.go
index 53d949f..b296061 100644
--- a/internal/cri/server/podsandbox/sandbox_run.go
+++ b/internal/cri/server/podsandbox/sandbox_run.go
@@ -77,23 +77,25 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll

sandboxImage := c.getSandboxImageName()
// Ensure sandbox container image snapshot.
- image, err := c.ensureImageExists(ctx, sandboxImage, config, metadata.RuntimeHandler)
+ ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler)
if err != nil {
- return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err)
+ return cin, fmt.Errorf("failed to get sandbox runtime: %w", err)
}
+ log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime)

- containerdImage, err := c.toContainerdImage(ctx, *image)
+ labels["oci_runtime_type"] = ociRuntime.Type
+
+ snapshotter := c.imageService.RuntimeSnapshotter(ctx, ociRuntime)
+
+ image, err := c.ensureImageExists(ctx, sandboxImage, config, metadata.RuntimeHandler, snapshotter)
if err != nil {
- return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
+ return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err)
}

- ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler)
+ containerdImage, err := c.toContainerdImage(ctx, *image)
if err != nil {
- return cin, fmt.Errorf("failed to get sandbox runtime: %w", err)
+ return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
}
- log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime)
-
- labels["oci_runtime_type"] = ociRuntime.Type

// Create sandbox container root directories.
sandboxRootDir := c.getSandboxRootDir(id)
@@ -173,7 +175,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
snapshotterOpt = append(snapshotterOpt, extraSOpts...)

opts := []containerd.NewContainerOpts{
- containerd.WithSnapshotter(c.imageService.RuntimeSnapshotter(ctx, ociRuntime)),
+ containerd.WithSnapshotter(snapshotter),
customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt...),
containerd.WithSpec(spec, specOpts...),
containerd.WithContainerLabels(sandboxLabels),
@@ -299,17 +301,19 @@ func (c *Controller) Create(_ctx context.Context, info sandbox.Sandbox, opts ...
return c.store.Save(podSandbox)
}

-func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig, runtimeHandler string) (*imagestore.Image, error) {
+func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (*imagestore.Image, error) {
image, err := c.imageService.LocalResolve(ref)
if err != nil && !errdefs.IsNotFound(err) {
return nil, fmt.Errorf("failed to get image %q: %w", ref, err)
}
if err == nil {
- return &image, nil
+ if _, ok := image.Snapshotters[snapshotter]; ok {
+ return &image, nil
+ }
}
// Pull image to ensure the image exists
// TODO: Cleaner interface
- imageID, err := c.imageService.PullImage(ctx, ref, nil, config, runtimeHandler)
+ imageID, err := c.imageService.PullImage(ctx, ref, nil, config, runtimeHandler, snapshotter)
if err != nil {
return nil, fmt.Errorf("failed to pull image %q: %w", ref, err)
}
diff --git a/internal/cri/server/service.go b/internal/cri/server/service.go
index 37d66f0..5d1546e 100644
--- a/internal/cri/server/service.go
+++ b/internal/cri/server/service.go
@@ -97,7 +97,7 @@ type RuntimeService interface {
type ImageService interface {
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string

- PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (string, error)
+ PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (string, error)
UpdateImage(ctx context.Context, r string) error

CheckImages(ctx context.Context) error
diff --git a/internal/cri/store/image/image.go b/internal/cri/store/image/image.go
index 5887e75..43ecf0d 100644
--- a/internal/cri/store/image/image.go
+++ b/internal/cri/store/image/image.go
@@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "strings"
"sync"

"github.com/containerd/containerd/v2/core/content"
@@ -53,6 +54,8 @@ type Image struct {
ImageSpec imagespec.Image
// Pinned image to prevent it from garbage collection
Pinned bool
+ // Snapshotters is a map whose keys are snapshotters for which this image has a snapshot.
+ Snapshotters map[string]struct{}
}

// Getter is used to get images but does not make changes
@@ -170,6 +173,19 @@ func (s *Store) getImage(ctx context.Context, i images.Image) (*Image, error) {
return nil, fmt.Errorf("read image config from content store: %w", err)
}

+ info, err := s.provider.Info(ctx, desc.Digest)
+ if err != nil {
+ return nil, fmt.Errorf("get content store config info: %w", err)
+ }
+
+ snapshotters := make(map[string]struct{})
+ for label := range info.Labels {
+ const Prefix = "containerd.io/gc.ref.snapshot."
+ if strings.HasPrefix(label, Prefix) {
+ snapshotters[label[len(Prefix):]] = struct{}{}
+ }
+ }
+
var spec imagespec.Image
if err := json.Unmarshal(blob, &spec); err != nil {
return nil, fmt.Errorf("unmarshal image config %s: %w", blob, err)
@@ -178,12 +194,13 @@ func (s *Store) getImage(ctx context.Context, i images.Image) (*Image, error) {
pinned := i.Labels[labels.PinnedImageLabelKey] == labels.PinnedImageLabelValue

return &Image{
- ID: id,
- References: []string{i.Name},
- ChainID: chainID.String(),
- Size: size,
- ImageSpec: spec,
- Pinned: pinned,
+ ID: id,
+ References: []string{i.Name},
+ ChainID: chainID.String(),
+ Size: size,
+ ImageSpec: spec,
+ Pinned: pinned,
+ Snapshotters: snapshotters,
}, nil

}
--
2.34.1
7 changes: 6 additions & 1 deletion SPECS/containerd2/containerd2.spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
Summary: Industry-standard container runtime
Name: %{upstream_name}2
Version: 2.0.0
Release: 1%{?dist}
Release: 2%{?dist}
License: ASL 2.0
Group: Tools/Container
URL: https://www.containerd.io
Expand All @@ -15,6 +15,8 @@ Distribution: Azure Linux
Source0: https://github.com/containerd/containerd/archive/v%{version}.tar.gz#/%{upstream_name}-%{version}.tar.gz
Source1: containerd.service
Source2: containerd.toml
# Added patch to support tardev-snapshotter for Kata CC
Patch0: add-tardev-support.patch

%{?systemd_requires}

Expand Down Expand Up @@ -79,6 +81,9 @@ fi
%dir /opt/containerd/lib

%changelog
* Thu Jan 09 2024 Mitch Zhu <[email protected]> - 2.0.0-2
- Added patch to support tardev-snapshotter for Kata CC.

* Wed Dec 11 2024 Nan Liu <[email protected]> - 2.0.0-1
- Created a standalone package for containerd 2.0.0
- Initial CBL-Mariner import from Azure
Expand Down

0 comments on commit fb2813c

Please sign in to comment.