From 475c10604ec4010e3df66f7cdfd5eba61c7a9dfc Mon Sep 17 00:00:00 2001 From: Jiwoo Ahn Date: Fri, 17 Apr 2026 17:23:30 +0900 Subject: [PATCH] feat: enable reading OCI manifest annotations in shim via containerd API refactor(shim): reuse sessionm atomic patching and read labels from container metadata chore: add todo for decode in config.go chore: add error log Signed-off-by: Jiwoo Ahn --- .github/linters/urunc-dict.txt | 5 +- go.mod | 4 +- pkg/containerd-shim/containerd/annotations.go | 235 ++++++++++++++++++ pkg/containerd-shim/containerd/session.go | 2 - pkg/containerd-shim/task_service.go | 20 +- pkg/unikontainers/config.go | 16 +- 6 files changed, 263 insertions(+), 19 deletions(-) create mode 100644 pkg/containerd-shim/containerd/annotations.go diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index bf838cdf..7c72147c 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -183,6 +183,7 @@ iface ifaces ifname imagesapi +imagespec initpipe initrds inlinehilite @@ -294,6 +295,7 @@ rumprun runbindable runp runtimeclasses +runtimespec sandboxing scontroller seabios @@ -328,6 +330,7 @@ traefik triger ttrpc twemoji +typesapi uidmap ukernel ukvmif @@ -415,4 +418,4 @@ Logr onsi ESRCH Prafful -praffq \ No newline at end of file +praffq diff --git a/go.mod b/go.mod index 82252800..1e9de679 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/cavaliergopher/cpio v1.0.1 github.com/containerd/containerd v1.7.30 github.com/containerd/containerd/api v1.10.0 + github.com/containerd/log v0.1.0 github.com/containerd/ttrpc v1.2.7 github.com/creack/pty v1.1.24 github.com/elastic/go-seccomp-bpf v1.6.0 @@ -18,6 +19,7 @@ require ( github.com/nubificus/hedge_cli v0.0.3 github.com/onsi/ginkgo/v2 v2.28.1 github.com/onsi/gomega v1.39.1 + github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runc v1.3.4 github.com/opencontainers/runtime-spec v1.2.1 github.com/prometheus-community/pro-bing v0.8.0 @@ -44,7 +46,6 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/go-runc v1.0.0 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect @@ -67,7 +68,6 @@ require ( github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/stretchr/objx v0.5.3 // indirect diff --git a/pkg/containerd-shim/containerd/annotations.go b/pkg/containerd-shim/containerd/annotations.go new file mode 100644 index 00000000..5d980c96 --- /dev/null +++ b/pkg/containerd-shim/containerd/annotations.go @@ -0,0 +1,235 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package containerd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + contentapi "github.com/containerd/containerd/api/services/content/v1" + imagesapi "github.com/containerd/containerd/api/services/images/v1" + typesapi "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/images" + + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + runtimespec "github.com/opencontainers/runtime-spec/specs-go" +) + +const uruncPrefix = "com.urunc.unikernel." + +type annotationFetcher struct { + containerLabels map[string]string + contentClient contentapi.ContentClient + namespace string + target *typesapi.Descriptor +} + +func newAnnotationFetcher(ctx context.Context, session *Session) (*annotationFetcher, error) { + container := session.GetContainer() + if container == nil { + return nil, fmt.Errorf("container metadata is not loaded") + } + + fetcher := &annotationFetcher{ + containerLabels: container.Labels, + contentClient: session.contentClient(), + namespace: session.GetNamespace(), + } + + if container.Image == "" { + // TODO: Add Docker fallback. When Docker does not use containerd's image/content + // store, the containerd container metadata may not include an image reference. + // In that case, resolve the image reference through the Docker Engine API and + // fetch manifest annotations from a Docker-compatible path. + return fetcher, nil + } + + imageResp, err := session.imagesClient().Get( + withNamespace(ctx, session.GetNamespace()), + &imagesapi.GetImageRequest{Name: container.Image}, + ) + if err != nil { + return fetcher, nil + } + + fetcher.target = imageResp.GetImage().GetTarget() + return fetcher, nil +} + +func InjectUruncAnnotations(ctx context.Context, session *Session, bundlePath string) error { + fetcher, err := newAnnotationFetcher(ctx, session) + if err != nil { + return fmt.Errorf("create annotation fetcher: %w", err) + } + annotations, err := fetcher.fetchUruncAnnotations(ctx) + if err != nil { + return fmt.Errorf("fetch urunc annotations: %w", err) + } + if len(annotations) == 0 { + return nil + } + + return patchConfigJSON(bundlePath, annotations) +} + +func (f *annotationFetcher) fetchUruncAnnotations(ctx context.Context) (map[string]string, error) { + filtered := make(map[string]string) + + // Collect urunc annotations from container labels. + for k, v := range f.containerLabels { + if strings.HasPrefix(k, uruncPrefix) { + filtered[k] = v + } + } + + // Collect urunc annotations from manifest + if f.target == nil || !images.IsManifestType(f.target.MediaType) { + // If the image target is missing or does not point to a manifest, + // keep the labels collected so far. + return filtered, nil + } + + manifestRaw, err := readBlob(ctx, f.namespace, f.contentClient, f.target.Digest, f.target.Size) + if err != nil { + return nil, fmt.Errorf("read manifest blob: %w", err) + } + + var manifest imagespec.Manifest + if err := json.Unmarshal(manifestRaw, &manifest); err != nil { + return nil, fmt.Errorf("unmarshal manifest: %w", err) + } + + // Manifest annotations override config labels on duplicate keys. + for k, v := range manifest.Annotations { + if strings.HasPrefix(k, uruncPrefix) { + filtered[k] = v + } + } + + return filtered, nil +} + +// readBlob reads a blob with the given digest from containerd's content store +// and returns it as a byte slice. +func readBlob(ctx context.Context, namespace string, contentClient contentapi.ContentClient, digest string, size int64) ([]byte, error) { + stream, err := contentClient.Read(withNamespace(ctx, namespace), &contentapi.ReadContentRequest{ + Digest: digest, + Size: size, + }) + if err != nil { + return nil, containerdErr(err) + } + + var raw []byte + for { + resp, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return nil, containerdErr(err) + } + raw = append(raw, resp.Data...) + } + + return raw, nil +} + +// patchConfigJSON injects missing annotations into the OCI runtime spec +// stored in the bundle's config.json. +// +// Existing annotations in config.json are preserved. Only annotation keys that +// are not already present in the runtime spec are added. +func patchConfigJSON(bundlePath string, annotations map[string]string) error { + configPath := filepath.Join(bundlePath, "config.json") + + fi, err := os.Stat(configPath) + if err != nil { + return fmt.Errorf("stat config.json: %w", err) + } + + data, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("read config.json: %w", err) + } + + var spec runtimespec.Spec + if err := json.Unmarshal(data, &spec); err != nil { + return fmt.Errorf("unmarshal spec: %w", err) + } + + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + + for k, v := range annotations { + if _, exists := spec.Annotations[k]; exists { + continue + } + spec.Annotations[k] = v + } + + patched, err := json.MarshalIndent(spec, "", " ") + if err != nil { + return fmt.Errorf("marshal spec: %w", err) + } + + if err := atomicWriteFile(configPath, patched, fi.Mode()); err != nil { + return fmt.Errorf("write config.json atomically: %w", err) + } + return nil +} + +func atomicWriteFile(path string, data []byte, mode os.FileMode) error { + tmpDir := filepath.Dir(path) + + f, err := os.CreateTemp(tmpDir, "."+filepath.Base(path)+".tmp-*") + if err != nil { + return err + } + + tmpName := f.Name() + defer os.Remove(tmpName) + + if err := f.Chmod(mode); err != nil { + _ = f.Close() + return err + } + + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + + if err := f.Close(); err != nil { + return err + } + + if err := os.Rename(tmpName, path); err != nil { + return err + } + + return nil +} diff --git a/pkg/containerd-shim/containerd/session.go b/pkg/containerd-shim/containerd/session.go index 996c0be8..e7168ffa 100644 --- a/pkg/containerd-shim/containerd/session.go +++ b/pkg/containerd-shim/containerd/session.go @@ -150,12 +150,10 @@ func (s *Session) containersClient() containersapi.ContainersClient { return containersapi.NewContainersClient(s.conn) } -//nolint:unused // Used by follow-up feature-specific access constructors. func (s *Session) imagesClient() imagesapi.ImagesClient { return imagesapi.NewImagesClient(s.conn) } -//nolint:unused // Used by follow-up feature-specific access constructors. func (s *Session) contentClient() contentapi.ContentClient { return contentapi.NewContentClient(s.conn) } diff --git a/pkg/containerd-shim/task_service.go b/pkg/containerd-shim/task_service.go index 70fb44a7..2c3e53b5 100644 --- a/pkg/containerd-shim/task_service.go +++ b/pkg/containerd-shim/task_service.go @@ -18,12 +18,14 @@ import ( "context" taskAPI "github.com/containerd/containerd/api/runtime/task/v2" + "github.com/containerd/log" "github.com/containerd/ttrpc" + containerdShim "github.com/urunc-dev/urunc/pkg/containerd-shim/containerd" ) // taskService is urunc's shim-side wrapper around containerd's runc task -// service. It currently forwards calls to the wrapped service while keeping a -// urunc-owned place for task-level feature wiring. +// service. It wires urunc task setup before forwarding calls to the wrapped +// service. type taskService struct { taskAPI.TaskService @@ -31,6 +33,20 @@ type taskService struct { } func (s *taskService) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (*taskAPI.CreateTaskResponse, error) { + session, err := containerdShim.OpenSession(ctx, s.containerdAddress, r.ID) + if err != nil { + log.G(ctx).WithError(err).Warn("urunc(shim): failed to open containerd session") + } else { + defer func() { + if err := session.Close(); err != nil { + log.G(ctx).WithError(err).Warn("urunc(shim): failed to close containerd session") + } + }() + if err := containerdShim.InjectUruncAnnotations(ctx, session, r.Bundle); err != nil { + log.G(ctx).WithError(err).Warn("urunc(shim): failed to inject annotations to spec") + } + } + return s.TaskService.Create(ctx, r) } diff --git a/pkg/unikontainers/config.go b/pkg/unikontainers/config.go index 7751aa27..f8066eea 100644 --- a/pkg/unikontainers/config.go +++ b/pkg/unikontainers/config.go @@ -76,23 +76,15 @@ func (c *UnikernelConfig) validate() error { // GetUnikernelConfig tries to get the Unikernel config from the bundle annotations. // If that fails, it gets the Unikernel config from the urunc.json file inside the rootfs. -// FIXME: custom annotations are unreachable, we need to investigate why to skip adding the urunc.json file -// For more details, see: https://github.com/urunc-dev/urunc/issues/12 -// GetUnikernelConfig tries to get a valid Unikernel config from the bundle annotations. -// If the annotations do not provide a valid config, it falls back to the urunc.json file. func GetUnikernelConfig(bundleDir string, spec *specs.Spec) (*UnikernelConfig, error) { - conf := getConfigFromSpec(spec) - - err := conf.validate() - if err == nil { - - if err := conf.decode(); err != nil { - return nil, err - } + if err := conf.validate(); err == nil { + // TODO: in case of urunc executed without shim, the annotations would remain endcoded return conf, nil } + // Failed to fetch urunc annotations from spec, fallback to urunc.json + uniklog.Info("failed to fetch urunc annotations from spec, fallback to urunc.json") rootFSDir := spec.Root.Path var jsonFilePath string if filepath.IsAbs(rootFSDir) {