diff --git a/nodeinstaller/internal/kataconfig/snp-id-blocks.json b/cli/cmd/assets/snp-id-blocks.json similarity index 100% rename from nodeinstaller/internal/kataconfig/snp-id-blocks.json rename to cli/cmd/assets/snp-id-blocks.json diff --git a/cli/cmd/common.go b/cli/cmd/common.go index 79fdbc7ce0..b9c37ec9cc 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -17,6 +17,7 @@ import ( "github.com/edgelesssys/contrast/internal/attestation/certcache" "github.com/edgelesssys/contrast/internal/constants" "github.com/edgelesssys/contrast/internal/fsstore" + "github.com/google/go-sev-guest/abi" "github.com/spf13/afero" "github.com/spf13/cobra" "golang.org/x/term" @@ -41,6 +42,19 @@ const ( //go:embed assets/image-replacements.txt var ReleaseImageReplacements []byte +// SNPIDBlocks contains the SNP ID blocks for different vCPU counts and CPU generations +// as a JSON map of platform -> vCPU count -> CPU generation -> SnpIDBlock. +// +//go:embed assets/snp-id-blocks.json +var SNPIDBlocks []byte + +// SnpIDBlock represents the SNP ID block and ID auth used for SEV-SNP guests. +type SnpIDBlock struct { + IDBlock string `json:"idBlock"` + IDAuth string `json:"idAuth"` + GuestPolicy abi.SnpPolicy `json:"guestPolicy"` +} + func commandOut() io.Writer { if term.IsTerminal(int(os.Stdout.Fd())) { return nil // use out writer of parent diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index 585aa6fa3e..49f48d435b 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "github.com/edgelesssys/contrast/cli/genpolicy" @@ -24,6 +25,8 @@ import ( "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" "github.com/edgelesssys/contrast/internal/platforms" + "github.com/google/go-sev-guest/abi" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1" applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" @@ -33,8 +36,13 @@ import ( ) const ( - contrastRoleAnnotationKey = "contrast.edgeless.systems/pod-role" - workloadSecretIDAnnotationKey = "contrast.edgeless.systems/workload-secret-id" + annotationPrefix = "contrast.edgeless.systems/" + kataAnnotationPrefix = "io.katacontainers.config.hypervisor." + contrastRoleAnnotationKey = annotationPrefix + "pod-role" + workloadSecretIDAnnotationKey = annotationPrefix + "workload-secret-id" + idBlockAnnotation = kataAnnotationPrefix + "snp_id_block_" + idAuthAnnotationKey = kataAnnotationPrefix + "snp_id_auth_" + guestPolicyAnnotationKey = kataAnnotationPrefix + "snp_guest_policy_" ) // NewGenerateCmd creates the contrast generate subcommand. @@ -511,6 +519,10 @@ func patchTargets(logger *slog.Logger, fileMap map[string][]*unstructured.Unstru replaceRuntimeClassName := patchRuntimeClassName(logger, runtimeHandler) kuberesource.MapPodSpec(res, replaceRuntimeClassName) + if err := patchIDBlockAnnotation(res, logger); err != nil { + return nil, fmt.Errorf("injecting ID block annotations: %w", err) + } + return res, nil }) } @@ -706,6 +718,84 @@ func patchRuntimeClassName(logger *slog.Logger, defaultRuntimeHandler string) fu } } +func patchIDBlockAnnotation(res any, logger *slog.Logger) error { + // runtime -> cpu_count -> product_line -> ID block + var snpIDBlocks map[string]map[string]map[string]SnpIDBlock + if err := json.Unmarshal(SNPIDBlocks, &snpIDBlocks); err != nil { + return fmt.Errorf("unmarshal SNP ID blocks: %w", err) + } + + var mapErr error + mapFunc := func(meta *applymetav1.ObjectMetaApplyConfiguration, spec *applycorev1.PodSpecApplyConfiguration) (*applymetav1.ObjectMetaApplyConfiguration, *applycorev1.PodSpecApplyConfiguration) { + if spec == nil || spec.RuntimeClassName == nil { + return meta, spec + } + + targetPlatform, err := platforms.FromRuntimeClassString(*spec.RuntimeClassName) + if err != nil { + logger.Error("could not determine platform for runtime class", "runtime-class-name", *spec.RuntimeClassName, "err", err) + return meta, spec + } + if !platforms.IsSNP(targetPlatform) { + return meta, spec + } + + var regularContainersCPU int64 + for _, container := range spec.Containers { + regularContainersCPU += getCPUCount(container.Resources) + } + var initContainersCPU int64 + for _, container := range spec.InitContainers { + cpuCount := getCPUCount(container.Resources) + initContainersCPU += cpuCount + // Sidecar containers remain running alongside the actual application, consuming CPU resources + if container.RestartPolicy != nil && *container.RestartPolicy == corev1.ContainerRestartPolicyAlways { + regularContainersCPU += cpuCount + } + } + podLevelCPU := getCPUCount(spec.Resources) + + // Convert milliCPUs to number of CPUs (rounding up), and add 1 for hypervisor overhead + totalMilliCPUs := max(regularContainersCPU, initContainersCPU, podLevelCPU) + cpuCount := strconv.FormatInt((totalMilliCPUs+999)/1000+1, 10) + + platform := strings.ToLower(targetPlatform.String()) + + genoa, milan := string(manifest.Genoa), string(manifest.Milan) + // Ensure we pre-calculated the required blocks + if snpIDBlocks[platform] == nil || snpIDBlocks[platform][cpuCount] == nil || + snpIDBlocks[platform][cpuCount][genoa].IDBlock == "" || snpIDBlocks[platform][cpuCount][milan].IDBlock == "" { + mapErr = fmt.Errorf("missing ID block configuration for runtime %s with %s CPUs", platform, cpuCount) + return meta, spec + } + + if meta == nil { + meta = &applymetav1.ObjectMetaApplyConfiguration{} + } + if meta.Annotations == nil { + meta.Annotations = make(map[string]string, 6) + } + meta.Annotations[idBlockAnnotation+genoa] = snpIDBlocks[platform][cpuCount][genoa].IDBlock + meta.Annotations[idBlockAnnotation+milan] = snpIDBlocks[platform][cpuCount][milan].IDBlock + meta.Annotations[idAuthAnnotationKey+genoa] = snpIDBlocks[platform][cpuCount][genoa].IDAuth + meta.Annotations[idAuthAnnotationKey+milan] = snpIDBlocks[platform][cpuCount][milan].IDAuth + meta.Annotations[guestPolicyAnnotationKey+genoa] = strconv.FormatUint(abi.SnpPolicyToBytes(snpIDBlocks[platform][cpuCount][genoa].GuestPolicy), 10) + meta.Annotations[guestPolicyAnnotationKey+milan] = strconv.FormatUint(abi.SnpPolicyToBytes(snpIDBlocks[platform][cpuCount][milan].GuestPolicy), 10) + + return meta, spec + } + + kuberesource.MapPodSpecWithMeta(res, mapFunc) + return mapErr +} + +func getCPUCount(resources *applycorev1.ResourceRequirementsApplyConfiguration) int64 { + if resources != nil && resources.Limits != nil { + return resources.Limits.Cpu().MilliValue() + } + return 0 +} + type generateFlags struct { policyPath string settingsPath string diff --git a/cli/main.go b/cli/main.go index e17f6094d2..08a7634906 100644 --- a/cli/main.go +++ b/cli/main.go @@ -78,6 +78,7 @@ func buildVersionString() (string, error) { } for _, snp := range values.SNP { fmt.Fprintf(versionsWriter, "\t- product name:\t%s\n", snp.ProductName) + fmt.Fprintf(versionsWriter, "\t vCPUs:\t%d\n", snp.CPUs) fmt.Fprintf(versionsWriter, "\t launch digest:\t%s\n", snp.TrustedMeasurement.String()) fmt.Fprint(versionsWriter, "\t default SNP TCB:\t\n") printOptionalSVN("bootloader", snp.MinimumTCB.BootloaderVersion) diff --git a/internal/kuberesource/sets.go b/internal/kuberesource/sets.go index 8ac2e4a63a..c2a6d9ba89 100644 --- a/internal/kuberesource/sets.go +++ b/internal/kuberesource/sets.go @@ -156,11 +156,15 @@ func MultiCPU() []any { Container(). WithName("multi-cpu"). WithImage("ghcr.io/edgelesssys/contrast/ubuntu:24.04@sha256:0f9e2b7901aa01cf394f9e1af69387e2fd4ee256fd8a95fb9ce3ae87375a31e6"). - WithCommand("/usr/bin/bash", "-c", "sleep infinity"). + WithCommand("/bin/sh", "-c", "sleep inf"). WithResources(ResourceRequirements(). - // Explicitly set a CPU limit to test assignement of CPUs to VMs. + WithRequests(corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("200Mi"), + }). + // Explicitly set a CPU limit to test assignment of CPUs to VMs. WithLimits(corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceCPU: resource.MustParse("1"), }), ), ), diff --git a/internal/manifest/referencevalues.go b/internal/manifest/referencevalues.go index f9fa75ceff..715cbae7d5 100644 --- a/internal/manifest/referencevalues.go +++ b/internal/manifest/referencevalues.go @@ -200,6 +200,10 @@ type SNPReferenceValues struct { PlatformInfo abi.SnpPlatformInfo MinimumMitigationVector uint64 AllowedChipIDs []HexString + // CPUs is the number of vCPUs assigned to the VM. + // This field is purely informative as [SNPReferenceValues.TrustedMeasurement] + // already implicitly contains the number of vCPUs + CPUs uint64 } // Validate checks the validity of all fields in the AKS reference values. diff --git a/nodeinstaller/internal/kataconfig/config.go b/nodeinstaller/internal/kataconfig/config.go index 829bb0b9ee..471ff884a6 100644 --- a/nodeinstaller/internal/kataconfig/config.go +++ b/nodeinstaller/internal/kataconfig/config.go @@ -4,17 +4,11 @@ package kataconfig import ( - "bytes" _ "embed" - "encoding/json" "fmt" "path/filepath" - "strings" "github.com/edgelesssys/contrast/internal/platforms" - "github.com/google/go-sev-guest/abi" - "github.com/google/go-sev-guest/kds" - "github.com/google/go-sev-guest/proto/sevsnp" "github.com/pelletier/go-toml/v2" ) @@ -31,9 +25,6 @@ var ( //go:embed configuration-qemu-snp.toml kataBareMetalQEMUSNPBaseConfig string - //go:embed snp-id-blocks.json - snpIDBlocks string - // RuntimeNamePlaceholder is the placeholder for the per-runtime path (i.e. /opt/edgeless/contrast-cc...) in the target file paths. RuntimeNamePlaceholder = "@@runtimeName@@" ) @@ -43,10 +34,10 @@ func KataRuntimeConfig( baseDir string, platform platforms.Platform, qemuExtraKernelParams string, - snpIDBlock SnpIDBlock, imagepullerConfigPath string, debug bool, ) (*Config, error) { + var customContrastAnnotations []string var config Config switch { case platforms.IsTDX(platform): @@ -59,11 +50,13 @@ func KataRuntimeConfig( return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) } + for _, productLine := range []string{"_Milan", "_Genoa"} { + for _, annotationType := range []string{"snp_id_block", "snp_id_auth", "snp_guest_policy"} { + customContrastAnnotations = append(customContrastAnnotations, annotationType+productLine) + } + } + config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "snp", "share", "OVMF.fd") - // Add SNP ID block to protect against migration attacks. - config.Hypervisor["qemu"]["snp_id_block"] = snpIDBlock.IDBlock - config.Hypervisor["qemu"]["snp_id_auth"] = snpIDBlock.IDAuth - config.Hypervisor["qemu"]["snp_guest_policy"] = abi.SnpPolicyToBytes(snpIDBlock.GuestPolicy) default: return nil, fmt.Errorf("unsupported platform: %s", platform) } @@ -105,7 +98,7 @@ func KataRuntimeConfig( config.Hypervisor["qemu"]["enable_debug"] = debug // Disable all annotations, as we don't support these. Some will mess up measurements, // others bypass things you can archive via correct resource declaration anyway. - config.Hypervisor["qemu"]["enable_annotations"] = []string{"cc_init_data"} + config.Hypervisor["qemu"]["enable_annotations"] = append(customContrastAnnotations, "cc_init_data") // Fix and align guest memory calculation. config.Hypervisor["qemu"]["default_memory"] = platforms.DefaultMemoryInMebiBytes(platform) config.Runtime["sandbox_cgroup_only"] = true @@ -115,36 +108,6 @@ func KataRuntimeConfig( return &config, nil } -// SnpIDBlock represents the SNP ID block and ID auth used for SEV-SNP guests. -type SnpIDBlock struct { - IDBlock string `json:"idBlock"` - IDAuth string `json:"idAuth"` - GuestPolicy abi.SnpPolicy `json:"guestPolicy"` -} - -// platform -> product -> snpIDBlock. -type snpIDBlockMap map[string]map[string]SnpIDBlock - -// SnpIDBlockForPlatform returns the embedded SNP ID block and ID auth for the given platform and product. -func SnpIDBlockForPlatform(platform platforms.Platform, productName sevsnp.SevProduct_SevProductName) (SnpIDBlock, error) { - var blocks snpIDBlockMap - decoder := json.NewDecoder(bytes.NewReader([]byte(snpIDBlocks))) - decoder.DisallowUnknownFields() - if err := decoder.Decode(&blocks); err != nil { - return SnpIDBlock{}, fmt.Errorf("unmarshaling embedded SNP ID blocks: %w", err) - } - blockForPlatform, ok := blocks[strings.ToLower(platform.String())] - if !ok { - return SnpIDBlock{}, fmt.Errorf("no SNP ID block found for platform %s", platform) - } - productLine := kds.ProductLine(&sevsnp.SevProduct{Name: productName}) - block, ok := blockForPlatform[productLine] - if !ok { - return SnpIDBlock{}, fmt.Errorf("no SNP ID block found for product %s", productLine) - } - return block, nil -} - // Config is the configuration for the Kata runtime. // Source: https://github.com/kata-containers/kata-containers/blob/4029d154ba0c26fcf4a8f9371275f802e3ef522c/src/runtime/pkg/katautils/Config.go // This is a simplified version of the actual configuration. diff --git a/nodeinstaller/internal/kataconfig/config_test.go b/nodeinstaller/internal/kataconfig/config_test.go index dc46388d38..5a2a19d10d 100644 --- a/nodeinstaller/internal/kataconfig/config_test.go +++ b/nodeinstaller/internal/kataconfig/config_test.go @@ -51,11 +51,7 @@ func TestKataRuntimeConfig(t *testing.T) { require := require.New(t) assert := assert.New(t) - snpIDBlock := kataconfig.SnpIDBlock{ - IDAuth: "PLACEHOLDER_ID_AUTH", - IDBlock: "PLACEHOLDER_ID_BLOCK", - } - cfg, err := kataconfig.KataRuntimeConfig("/", platform, "", snpIDBlock, "", false) + cfg, err := kataconfig.KataRuntimeConfig("/", platform, "", "", false) require.NoError(err) configBytes, err := cfg.Marshal() diff --git a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu.toml b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu.toml index ab478913e8..6b3e11747b 100644 --- a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu.toml +++ b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu.toml @@ -20,7 +20,7 @@ disable_image_nvdimm = true disable_nesting_checks = true disable_selinux = false disable_vhost_net = false -enable_annotations = ['cc_init_data'] +enable_annotations = ['snp_id_block_Milan', 'snp_id_auth_Milan', 'snp_guest_policy_Milan', 'snp_id_block_Genoa', 'snp_id_auth_Genoa', 'snp_guest_policy_Genoa', 'cc_init_data'] enable_debug = false enable_guest_swap = false enable_hugepages = false @@ -57,9 +57,9 @@ rx_rate_limiter_max_rate = 0 seccompsandbox = '' sev_snp_guest = true shared_fs = 'none' -snp_guest_policy = 131072 -snp_id_auth = 'PLACEHOLDER_ID_AUTH' -snp_id_block = 'PLACEHOLDER_ID_BLOCK' +snp_guest_policy = 196608 +snp_id_auth = '' +snp_id_block = '' tx_rate_limiter_max_rate = 0 use_legacy_serial = false valid_entropy_sources = ['/dev/urandom', '/dev/random', ''] diff --git a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp.toml b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp.toml index 28e1ee352b..0ae6c17825 100644 --- a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp.toml +++ b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp.toml @@ -19,7 +19,7 @@ disable_image_nvdimm = true disable_nesting_checks = true disable_selinux = false disable_vhost_net = false -enable_annotations = ['cc_init_data'] +enable_annotations = ['snp_id_block_Milan', 'snp_id_auth_Milan', 'snp_guest_policy_Milan', 'snp_id_block_Genoa', 'snp_id_auth_Genoa', 'snp_guest_policy_Genoa', 'cc_init_data'] enable_debug = false enable_guest_swap = false enable_hugepages = false @@ -56,9 +56,9 @@ rx_rate_limiter_max_rate = 0 seccompsandbox = '' sev_snp_guest = true shared_fs = 'none' -snp_guest_policy = 131072 -snp_id_auth = 'PLACEHOLDER_ID_AUTH' -snp_id_block = 'PLACEHOLDER_ID_BLOCK' +snp_guest_policy = 196608 +snp_id_auth = '' +snp_id_block = '' tx_rate_limiter_max_rate = 0 use_legacy_serial = false valid_entropy_sources = ['/dev/urandom', '/dev/random', ''] diff --git a/nodeinstaller/internal/kataconfig/update-testdata/main.go b/nodeinstaller/internal/kataconfig/update-testdata/main.go index 31858d72a4..5745844b73 100644 --- a/nodeinstaller/internal/kataconfig/update-testdata/main.go +++ b/nodeinstaller/internal/kataconfig/update-testdata/main.go @@ -48,11 +48,6 @@ func main() { }, } - snpIDBlock := kataconfig.SnpIDBlock{ - IDAuth: "PLACEHOLDER_ID_AUTH", - IDBlock: "PLACEHOLDER_ID_BLOCK", - } - for platform, platformConfig := range platforms { upstreamFile := filepath.Join(tarball, "opt", "kata", "share", "defaults", "kata-containers", fmt.Sprintf("configuration-%s.toml", platformConfig.upstream)) configFile := filepath.Join(gitroot, "nodeinstaller", "internal", "kataconfig", fmt.Sprintf("configuration-%s.toml", platformConfig.config)) @@ -80,7 +75,7 @@ func main() { log.Fatalf("failed to write new config: %s", err) } - cfg, err := kataconfig.KataRuntimeConfig("/", platform, "", snpIDBlock, "", false) + cfg, err := kataconfig.KataRuntimeConfig("/", platform, "", "", false) if err != nil { log.Fatalf("failed to create config: %s", err) } diff --git a/nodeinstaller/node-installer.go b/nodeinstaller/node-installer.go index 44cf89c0b4..e7b6fe4ab4 100644 --- a/nodeinstaller/node-installer.go +++ b/nodeinstaller/node-installer.go @@ -24,7 +24,6 @@ import ( "github.com/edgelesssys/contrast/nodeinstaller/internal/containerdconfig" "github.com/edgelesssys/contrast/nodeinstaller/internal/kataconfig" "github.com/edgelesssys/contrast/nodeinstaller/internal/targetconfig" - "github.com/google/go-sev-guest/abi" ) func main() { @@ -170,15 +169,7 @@ func envWithDefault(key, dflt string) string { } func containerdRuntimeConfig(basePath, configPath string, platform platforms.Platform, qemuExtraKernelParams, imagepullerConfigPath string, debugRuntime bool) error { - var snpIDBlock kataconfig.SnpIDBlock - if platforms.IsSNP(platform) && platforms.IsQEMU(platform) { - var err error - snpIDBlock, err = kataconfig.SnpIDBlockForPlatform(platform, abi.SevProduct().Name) - if err != nil { - return fmt.Errorf("getting SNP ID block for platform %q: %w", platform, err) - } - } - kataRuntimeConfig, err := kataconfig.KataRuntimeConfig(basePath, platform, qemuExtraKernelParams, snpIDBlock, imagepullerConfigPath, debugRuntime) + kataRuntimeConfig, err := kataconfig.KataRuntimeConfig(basePath, platform, qemuExtraKernelParams, imagepullerConfigPath, debugRuntime) if err != nil { return fmt.Errorf("generating kata runtime config: %w", err) } diff --git a/packages/by-name/contrast/cli/package.nix b/packages/by-name/contrast/cli/package.nix index 8bedb037ac..689b7bc54e 100644 --- a/packages/by-name/contrast/cli/package.nix +++ b/packages/by-name/contrast/cli/package.nix @@ -9,6 +9,7 @@ installShellFiles, contrastPkgsStatic, reference-values, + snp-id-blocks, }: buildGoModule (finalAttrs: { @@ -56,6 +57,7 @@ buildGoModule (finalAttrs: { install -D ${lib.getExe contrastPkgsStatic.kata.genpolicy} cli/genpolicy/assets/genpolicy-kata install -D ${kata.genpolicy.rules}/genpolicy-rules.rego cli/genpolicy/assets/genpolicy-rules-kata.rego install -D ${reference-values} internal/manifest/assets/reference-values.json + install -D ${snp-id-blocks} cli/cmd/assets/snp-id-blocks.json ''; # postPatch will be overwritten by the .#base.contrast.cli-release derivation, prePatch won't. diff --git a/packages/by-name/contrast/nodeinstaller/package.nix b/packages/by-name/contrast/nodeinstaller/package.nix index 1693fca962..f8cbc563bd 100644 --- a/packages/by-name/contrast/nodeinstaller/package.nix +++ b/packages/by-name/contrast/nodeinstaller/package.nix @@ -6,7 +6,6 @@ buildGoModule, contrast, reference-values, - snp-id-blocks, }: buildGoModule (finalAttrs: { @@ -44,7 +43,6 @@ buildGoModule (finalAttrs: { prePatch = '' install -D ${reference-values} internal/manifest/assets/reference-values.json - install -D ${snp-id-blocks} nodeinstaller/internal/kataconfig/snp-id-blocks.json ''; env.CGO_ENABLED = 0; diff --git a/packages/by-name/contrast/reference-values/package.nix b/packages/by-name/contrast/reference-values/package.nix index f21e59cbb9..c071d2119e 100644 --- a/packages/by-name/contrast/reference-values/package.nix +++ b/packages/by-name/contrast/reference-values/package.nix @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BUSL-1.1 { + lib, kata, OVMF-TDX, node-installer-image, @@ -24,23 +25,29 @@ let platformInfo = { SMTEnabled = true; }; - launch-digest = kata.calculateSnpLaunchDigest { - inherit os-image; - inherit (node-installer-image) withDebug; - }; + vcpuCounts = lib.range 1 8; + products = [ + "Milan" + "Genoa" + ]; + + generateRefVal = + vcpus: product: + let + launch-digest = kata.calculateSnpLaunchDigest { + inherit os-image vcpus; + inherit (node-installer-image) withDebug; + }; + filename = "${lib.toLower product}.hex"; + in + { + inherit guestPolicy platformInfo; + trustedMeasurement = builtins.readFile "${launch-digest}/${filename}"; + productName = product; + cpus = vcpus; + }; in - [ - { - inherit guestPolicy platformInfo; - trustedMeasurement = builtins.readFile "${launch-digest}/milan.hex"; - productName = "Milan"; - } - { - inherit guestPolicy platformInfo; - trustedMeasurement = builtins.readFile "${launch-digest}/genoa.hex"; - productName = "Genoa"; - } - ]; + builtins.concatLists (map (vcpus: map (product: generateRefVal vcpus product) products) vcpuCounts); }; snpRefVals = snpRefValsWith node-installer-image.os-image; diff --git a/packages/by-name/contrast/snp-id-blocks/package.nix b/packages/by-name/contrast/snp-id-blocks/package.nix index f6368b06ae..1a5f18cb7d 100644 --- a/packages/by-name/contrast/snp-id-blocks/package.nix +++ b/packages/by-name/contrast/snp-id-blocks/package.nix @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BUSL-1.1 { + lib, kata, calculateSnpIDBlock, node-installer-image, @@ -12,27 +13,40 @@ let os-image: let guestPolicy = builtins.fromJSON (builtins.readFile ../reference-values/snpGuestPolicyQEMU.json); - launch-digest = kata.calculateSnpLaunchDigest { - inherit os-image; - inherit (node-installer-image) withDebug; - }; - idBlocks = calculateSnpIDBlock { - snp-launch-digest = launch-digest; - snp-guest-policy = ../reference-values/snpGuestPolicyQEMU.json; - }; + idBlocksForVcpus = + vcpus: + let + launch-digest = kata.calculateSnpLaunchDigest { + inherit os-image vcpus; + inherit (node-installer-image) withDebug; + }; + idBlocks = calculateSnpIDBlock { + snp-launch-digest = launch-digest; + snp-guest-policy = ../reference-values/snpGuestPolicyQEMU.json; + }; + in + { + Milan = { + idBlock = builtins.readFile "${idBlocks}/id-block-milan.base64"; + idAuth = builtins.readFile "${idBlocks}/id-auth-milan.base64"; + inherit guestPolicy; + }; + Genoa = { + idBlock = builtins.readFile "${idBlocks}/id-block-genoa.base64"; + idAuth = builtins.readFile "${idBlocks}/id-auth-genoa.base64"; + inherit guestPolicy; + }; + }; + + vcpuCounts = lib.range 1 8; + allVcpuBlocks = builtins.listToAttrs ( + map (vcpus: { + name = toString vcpus; + value = idBlocksForVcpus vcpus; + }) vcpuCounts + ); in - { - Milan = { - idBlock = builtins.readFile "${idBlocks}/id-block-milan.base64"; - idAuth = builtins.readFile "${idBlocks}/id-auth-milan.base64"; - inherit guestPolicy; - }; - Genoa = { - idBlock = builtins.readFile "${idBlocks}/id-block-genoa.base64"; - idAuth = builtins.readFile "${idBlocks}/id-auth-genoa.base64"; - inherit guestPolicy; - }; - }; + allVcpuBlocks; in builtins.toFile "snp-id-blocks.json" ( diff --git a/packages/by-name/kata/calculateSnpLaunchDigest/package.nix b/packages/by-name/kata/calculateSnpLaunchDigest/package.nix index a972c40c6f..bc1a0483f6 100644 --- a/packages/by-name/kata/calculateSnpLaunchDigest/package.nix +++ b/packages/by-name/kata/calculateSnpLaunchDigest/package.nix @@ -12,6 +12,7 @@ { os-image, withDebug ? false, + vcpus, }: let @@ -21,7 +22,8 @@ let # Kata uses a base command line and then appends the command line from the kata config (i.e. also our node-installer config). # Thus, we need to perform the same steps when calculating the digest. - baseCmdline = if withDebug then kata.runtime.cmdline.debug else kata.runtime.cmdline.default; + runtime = kata.runtime.override { inherit vcpus; }; + baseCmdline = if withDebug then runtime.cmdline.debug else runtime.cmdline.default; cmdline = lib.strings.concatStringsSep " " [ baseCmdline os-image.cmdline @@ -40,7 +42,7 @@ stdenvNoCC.mkDerivation { ${lib.getExe python3Packages.sev-snp-measure} \ --mode snp \ --ovmf ${ovmf-snp} \ - --vcpus 1 \ + --vcpus ${toString vcpus} \ --vcpu-type EPYC-Milan \ --kernel ${kernel} \ --initrd ${initrd} \ @@ -49,7 +51,7 @@ stdenvNoCC.mkDerivation { ${lib.getExe python3Packages.sev-snp-measure} \ --mode snp \ --ovmf ${ovmf-snp} \ - --vcpus 1 \ + --vcpus ${toString vcpus} \ --vcpu-type EPYC-Genoa \ --kernel ${kernel} \ --initrd ${initrd} \ diff --git a/packages/by-name/kata/runtime/0029-runtime-add-SNP-ID-block-from-Pod-annotations.patch b/packages/by-name/kata/runtime/0029-runtime-add-SNP-ID-block-from-Pod-annotations.patch new file mode 100644 index 0000000000..f8a2c251b1 --- /dev/null +++ b/packages/by-name/kata/runtime/0029-runtime-add-SNP-ID-block-from-Pod-annotations.patch @@ -0,0 +1,192 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= +Date: Wed, 4 Mar 2026 10:42:36 +0100 +Subject: [PATCH] runtime: add SNP ID block from Pod annotations +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +A Pod VM requires different ID blocks depending +on CPU model and number of CPUs assigned to the VM. +Since CPU model is not known at the time the K8s +yaml files are written, we opted to add ID Blocks +for all supported CPU models as annotations. +At runtime we can then choose the correct value +and start the VM with the requested amount of CPUs. + +Signed-off-by: Daniel Weiße +--- + src/runtime/pkg/oci/utils.go | 27 ++++++++++++++++ + src/runtime/virtcontainers/hypervisor.go | 22 +++++++++++++ + .../pkg/annotations/annotations.go | 19 ++++++++++++ + src/runtime/virtcontainers/qemu_amd64.go | 31 ++++++++++++++----- + 4 files changed, 92 insertions(+), 7 deletions(-) + +diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go +index 66f8c60c7dfb78f61d7e22fae50746a7b9ca54db..7b42e5f26d25c440cc4722b88554b4d3e102c0c4 100644 +--- a/src/runtime/pkg/oci/utils.go ++++ b/src/runtime/pkg/oci/utils.go +@@ -590,6 +590,33 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, + config.HypervisorConfig.SGXEPCSize = size + } + ++ if value := ocispec.Annotations[vcAnnotations.SNPIDBlock+vcAnnotations.MilanSuffix]; value != "" { ++ config.HypervisorConfig.SnpIdBlockMilan = value ++ } ++ if value := ocispec.Annotations[vcAnnotations.SNPIDBlock+vcAnnotations.GenoaSuffix]; value != "" { ++ config.HypervisorConfig.SnpIdBlockGenoa = value ++ } ++ if value := ocispec.Annotations[vcAnnotations.SNPIDAuth+vcAnnotations.MilanSuffix]; value != "" { ++ config.HypervisorConfig.SnpIdAuthMilan = value ++ } ++ if value := ocispec.Annotations[vcAnnotations.SNPIDAuth+vcAnnotations.GenoaSuffix]; value != "" { ++ config.HypervisorConfig.SnpIdAuthGenoa = value ++ } ++ if value := ocispec.Annotations[vcAnnotations.SNPGuestPolicy+vcAnnotations.MilanSuffix]; value != "" { ++ guestPolicy, err := strconv.ParseUint(value, 10, 64) ++ if err != nil { ++ return fmt.Errorf("Couldn't parse Milan SNP guest policy '%s': %w", value, err) ++ } ++ config.HypervisorConfig.SnpGuestPolicyMilan = &guestPolicy ++ } ++ if value := ocispec.Annotations[vcAnnotations.SNPGuestPolicy+vcAnnotations.GenoaSuffix]; value != "" { ++ guestPolicy, err := strconv.ParseUint(value, 10, 64) ++ if err != nil { ++ return fmt.Errorf("Couldn't parse Genoa SNP guest policy '%s': %w", value, err) ++ } ++ config.HypervisorConfig.SnpGuestPolicyGenoa = &guestPolicy ++ } ++ + if err := addHypervisorGPUOverrides(ocispec, config); err != nil { + return err + } +diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go +index 911f671cae117d33d263aea566d90f5741dfdb54..141697960cc644fb51327e064c45fa3025f2f7ed 100644 +--- a/src/runtime/virtcontainers/hypervisor.go ++++ b/src/runtime/virtcontainers/hypervisor.go +@@ -473,13 +473,35 @@ type HypervisorConfig struct { + // for the SNP_LAUNCH_FINISH command defined in the SEV-SNP firmware ABI (default: all-zero) + SnpIdBlock string + ++ // SnpIdBlockMilan is the 96-byte, base64-encoded blob to provide the `ID Block` structure ++ // specifically for Milan CPUs ++ SnpIdBlockMilan string ++ ++ // SnpIdBlockGenoa is the 96-byte, base64-encoded blob to provide the `ID Block` structure ++ // specifically for Genoa CPUs ++ SnpIdBlockGenoa string ++ + // SnpIdAuth is the 4096-byte, base64-encoded blob to provide the ‘ID Authentication Information Structure’ + // for the SNP_LAUNCH_FINISH command defined in the SEV-SNP firmware ABI (default: all-zero) + SnpIdAuth string + ++ // SnpIdAuthMilan is the 4096-byte, base64-encoded blob to provide the ‘ID Authentication Information Structure’ ++ // specifically for Milan CPUs. ++ SnpIdAuthMilan string ++ ++ // SnpIdAuthGenoa is the 4096-byte, base64-encoded blob to provide the ‘ID Authentication Information Structure’ ++ // specifically for Genoa CPUs. ++ SnpIdAuthGenoa string ++ + // SnpGuestPolicy is the integer representation of the SEV-SNP guest policy. + SnpGuestPolicy *uint64 + ++ // SnpGuestPolicyMilan is the integer representation of the SEV-SNP guest policy specifically for Milan CPUs. ++ SnpGuestPolicyMilan *uint64 ++ ++ // SnpGuestPolicyGenoa is the integer representation of the SEV-SNP guest policy specifically for Genoa CPUs. ++ SnpGuestPolicyGenoa *uint64 ++ + // KernelParams are additional guest kernel parameters. + KernelParams []Param + +diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go +index a63646413455958f6a87e5c24da2e05e83eefa69..b115bcf3fececaa29f88f51c1d2b7f7d2bfd56b0 100644 +--- a/src/runtime/virtcontainers/pkg/annotations/annotations.go ++++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go +@@ -250,6 +250,25 @@ const ( + + // Initdata is the initdata passed in when CreateVM + Initdata = kataConfAnnotationsPrefix + "hypervisor.cc_init_data" ++ ++ // ++ // Contrast specific annotations ++ // ++ ++ // MilanSuffix is a suffix added to SNP related annotations for the Milan platform. ++ MilanSuffix = "_Milan" ++ ++ // GenoaSuffix is a suffix added to SNP related annotations for the Genoa platform. ++ GenoaSuffix = "_Genoa" ++ ++ // SNPIDBlock is a sandbox annotation to provide a custom SNP ID block for the VM. ++ SNPIDBlock = kataAnnotHypervisorPrefix + "snp_id_block" ++ ++ // SNPIDAuth is a sandbox annotation to provide a custom SNP ID auth for the VM. ++ SNPIDAuth = kataAnnotHypervisorPrefix + "snp_id_auth" ++ ++ // SNPGuestPolicy is a sandbox annotation to specify the SNP guest policy for the VM. ++ SNPGuestPolicy = kataAnnotHypervisorPrefix + "snp_guest_policy" + ) + + // Runtime related annotations +diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go +index 8ccf503b6d7ccae59bb116ac0f5b1317ee8223a1..5923f5ed47abecc7dd92b7ea1147e2efdfc0cee5 100644 +--- a/src/runtime/virtcontainers/qemu_amd64.go ++++ b/src/runtime/virtcontainers/qemu_amd64.go +@@ -21,6 +21,12 @@ import ( + govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" + ) + ++const ( ++ cpuModelEpycGenoa = "EPYC-Genoa" ++ cpuModelEpycMilan = "EPYC-Milan" ++ cpuModelEpycGeneric = "EPYC-v4" ++) ++ + type qemuAmd64 struct { + // inherit from qemuArchBase, overwrite methods if needed + qemuArchBase +@@ -131,9 +137,21 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { + vmFactory: factory, + snpGuest: config.SevSnpGuest, + qgsPort: config.QgsPort, +- snpIdBlock: config.SnpIdBlock, +- snpIdAuth: config.SnpIdAuth, +- snpGuestPolicy: config.SnpGuestPolicy, ++ } ++ ++ switch q.cpuModel() { ++ case cpuModelEpycMilan: ++ q.snpIdBlock = config.SnpIdBlockMilan ++ q.snpIdAuth = config.SnpIdAuthMilan ++ q.snpGuestPolicy = config.SnpGuestPolicyMilan ++ case cpuModelEpycGenoa: ++ q.snpIdBlock = config.SnpIdBlockGenoa ++ q.snpIdAuth = config.SnpIdAuthGenoa ++ q.snpGuestPolicy = config.SnpGuestPolicyGenoa ++ default: ++ q.snpIdBlock = config.SnpIdBlock ++ q.snpIdAuth = config.SnpIdAuth ++ q.snpGuestPolicy = config.SnpGuestPolicy + } + + if config.ConfidentialGuest { +@@ -197,14 +215,13 @@ func (q *qemuAmd64) cpuModel() string { + // model value. + switch cpuid.DisplayModel { + case 0x01: +- cpuModel = "EPYC-Milan" ++ cpuModel = cpuModelEpycMilan + case 0x11: +- cpuModel = "EPYC-Genoa" ++ cpuModel = cpuModelEpycGenoa + default: + // Fall back to a generic CPU. +- cpuModel = "EPYC-v4" ++ cpuModel = cpuModelEpycGeneric + } +- + } + } + diff --git a/packages/by-name/kata/runtime/package.nix b/packages/by-name/kata/runtime/package.nix index 567eb882c2..ebaa83cf3b 100644 --- a/packages/by-name/kata/runtime/package.nix +++ b/packages/by-name/kata/runtime/package.nix @@ -8,6 +8,7 @@ yq-go, git, applyPatches, + vcpus ? 1, }: buildGoModule (finalAttrs: { @@ -195,6 +196,12 @@ buildGoModule (finalAttrs: { # mismatch. # Upstream PR: https://github.com/kata-containers/kata-containers/pull/12570. ./0028-runtime-don-t-stop-after-first-VFIO-mismatch.patch + + # Enables the Kata runtime to set the SNP ID blocks for the CPU model it is running on + # based on Pod annotations. This allows us to run Pods with multiple CPUs. + # This patch relies on changes made by 0001-emulate-CPU-model-that-most-closely-matches-the-host.patch + # together with being specific to our use case. There are no plans to upstream it. + ./0029-runtime-add-SNP-ID-block-from-Pod-annotations.patch ]; }; @@ -288,7 +295,7 @@ buildGoModule (finalAttrs: { ] ++ [ "panic=1" - "nr_cpus=1" + "nr_cpus=${toString vcpus}" "selinux=0" "systemd.unit=kata-containers.target" "systemd.mask=systemd-networkd.service"