From 95ad702a3310f95dbeded327cec134140f25a1ab Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 20 Apr 2026 21:19:51 +0900 Subject: [PATCH 01/10] feat!: add Kyverno-based cc_init_data injection for OSC 1.12 / Trustee 1.1 Replace broken io.katacontainers.config.agent.policy annotation with Kyverno MutatingPolicy that injects cc_init_data from ConfigMaps into pods with kata runtime classes. - Add coco-kyverno-policies chart with MutatingPolicy, ValidatingPolicy, and ClusterPolicy for ConfigMap namespace propagation - Update imperative job to generate both default and debug initdata ConfigMaps with Kyverno validation fields - Update workload pod templates to use coco.io/initdata-configmap annotation instead of inline policy - Update values-simple.yaml: OSC 1.12, Trustee 1.1, Kyverno Helm app - Add conditional memory annotation for non-Azure platforms - Delete insecure-policy.rego (policy now embedded in cc_init_data) BREAKING CHANGE: Requires Kyverno and OSC 1.12 / Trustee 1.1.0 Co-Authored-By: Claude Opus 4.6 --- ansible/init-data-gzipper.yaml | 115 +++++++++++++++--- .../initdata-debug.toml.tpl | 32 ++++- charts/all/coco-kyverno-policies/Chart.yaml | 6 + .../initdata-namespace-propagation.yaml | 31 +++++ .../templates/inject-coco-initdata.yaml | 52 ++++++++ .../validate-initdata-configmap.yaml | 46 +++++++ charts/all/coco-kyverno-policies/values.yaml | 5 + .../templates/insecure-policy-pod.yaml | 2 +- .../hello-openshift/templates/secure-pod.yaml | 1 + .../kbs-access/templates/secure-pod.yaml | 4 + charts/coco-supported/kbs-access/values.yaml | 2 + values-simple.yaml | 41 ++++++- 12 files changed, 318 insertions(+), 19 deletions(-) rename charts/coco-supported/hello-openshift/insecure-policy.rego => ansible/initdata-debug.toml.tpl (69%) create mode 100644 charts/all/coco-kyverno-policies/Chart.yaml create mode 100644 charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml create mode 100644 charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml create mode 100644 charts/all/coco-kyverno-policies/templates/validate-initdata-configmap.yaml create mode 100644 charts/all/coco-kyverno-policies/values.yaml diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index c5a161ca..f0e10cf6 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -9,6 +9,7 @@ hub_domain: "{{ global.hubClusterDomain | default('none') | lower}}" security_policy_flavour: "{{ global.coco.securityPolicyFlavour | default('insecure') }}" template_src: "initdata-default.toml.tpl" + debug_template_src: "initdata-debug.toml.tpl" tasks: - name: Create temporary working directory ansible.builtin.tempfile: @@ -37,32 +38,82 @@ - name: Define temp file paths ansible.builtin.set_fact: rendered_path: "{{ tmpdir.path }}/rendered.toml" + debug_rendered_path: "{{ tmpdir.path }}/debug-rendered.toml" - - name: Render template to temp file + - name: Render default template to temp file ansible.builtin.template: src: "{{ template_src }}" dest: "{{ rendered_path }}" mode: "0600" + - name: Render debug template to temp file + ansible.builtin.template: + src: "{{ debug_template_src }}" + dest: "{{ debug_rendered_path }}" + mode: "0600" + + - name: Read raw aa.toml from rendered template + ansible.builtin.shell: | + set -o pipefail + python3 -c " + import tomllib + with open('{{ rendered_path }}', 'rb') as f: + data = tomllib.load(f) + print(data['data']['aa.toml'], end='') + " + register: raw_aa_toml + changed_when: false + + - name: Read raw cdh.toml from rendered template + ansible.builtin.shell: | + set -o pipefail + python3 -c " + import tomllib + with open('{{ rendered_path }}', 'rb') as f: + data = tomllib.load(f) + print(data['data']['cdh.toml'], end='') + " + register: raw_cdh_toml + changed_when: false - - name: Gzip and base64 encode the rendered content + - name: Read raw policy.rego from default template + ansible.builtin.shell: | + set -o pipefail + python3 -c " + import tomllib + with open('{{ rendered_path }}', 'rb') as f: + data = tomllib.load(f) + print(data['data']['policy.rego'], end='') + " + register: raw_policy_rego + changed_when: false + + - name: Read raw policy.rego from debug template + ansible.builtin.shell: | + set -o pipefail + python3 -c " + import tomllib + with open('{{ debug_rendered_path }}', 'rb') as f: + data = tomllib.load(f) + print(data['data']['policy.rego'], end='') + " + register: debug_raw_policy_rego + changed_when: false + + - name: Gzip and base64 encode the default rendered content ansible.builtin.shell: | set -o pipefail cat "{{ rendered_path }}" | gzip | base64 -w0 register: initdata_encoded changed_when: false - # This block runs a shell script that calculates a hash value (PCR8_HASH) derived from the contents of 'initdata.toml'. - # The script performs the following steps: - # 1. hash=$(sha256sum initdata.toml | cut -d' ' -f1): Computes the sha256 hash of 'initdata.toml' and assigns it to $hash. - # 2. initial_pcr=0000000000000000000000000000000000000000000000000000000000000000: Initializes a string of zeros as the initial PCR value. - # 3. PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1): - # Concatenates initial_pcr and $hash, converts from hex to binary, - # computes its sha256 hash, and stores the result as PCR8_HASH. - # 4. echo $PCR8_HASH: Outputs the PCR hash value. - # The important part: The 'register: pcr8_hash' registers the **stdout of the command**, - # which is the value output by 'echo $PCR8_HASH', as 'pcr8_hash.stdout' in Ansible. - # It does NOT register an environment variable, but rather the value actually printed by 'echo'. + - name: Gzip and base64 encode the debug rendered content + ansible.builtin.shell: | + set -o pipefail + cat "{{ debug_rendered_path }}" | gzip | base64 -w0 + register: debug_initdata_encoded + changed_when: false + - name: Register init data pcr into a var ansible.builtin.shell: | set -o pipefail @@ -72,8 +123,16 @@ register: pcr8_hash changed_when: false + - name: Register debug init data pcr into a var + ansible.builtin.shell: | + set -o pipefail + hash=$(sha256sum "{{ debug_rendered_path }}" | cut -d' ' -f1) + initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 + PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + register: debug_pcr8_hash + changed_when: false - - name: Create/update ConfigMap with gzipped+base64 content + - name: Create/update default initdata ConfigMap kubernetes.core.k8s: kubeconfig: "{{ kubeconfig | default(omit) }}" state: present @@ -83,6 +142,34 @@ metadata: name: "initdata" namespace: "imperative" + labels: + coco.io/type: initdata data: INITDATA: "{{ initdata_encoded.stdout }}" PCR8_HASH: "{{ pcr8_hash.stdout }}" + version: "0.1.0" + algorithm: "sha256" + aa.toml: "{{ raw_aa_toml.stdout }}" + cdh.toml: "{{ raw_cdh_toml.stdout }}" + policy.rego: "{{ raw_policy_rego.stdout }}" + + - name: Create/update debug initdata ConfigMap + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig | default(omit) }}" + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "debug-initdata" + namespace: "imperative" + labels: + coco.io/type: initdata + data: + INITDATA: "{{ debug_initdata_encoded.stdout }}" + PCR8_HASH: "{{ debug_pcr8_hash.stdout }}" + version: "0.1.0" + algorithm: "sha256" + aa.toml: "{{ raw_aa_toml.stdout }}" + cdh.toml: "{{ raw_cdh_toml.stdout }}" + policy.rego: "{{ debug_raw_policy_rego.stdout }}" diff --git a/charts/coco-supported/hello-openshift/insecure-policy.rego b/ansible/initdata-debug.toml.tpl similarity index 69% rename from charts/coco-supported/hello-openshift/insecure-policy.rego rename to ansible/initdata-debug.toml.tpl index b82a0e93..b49fae8f 100644 --- a/charts/coco-supported/hello-openshift/insecure-policy.rego +++ b/ansible/initdata-debug.toml.tpl @@ -1,3 +1,32 @@ +algorithm = "sha256" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = "https://kbs.{{ hub_domain }}" + +[token_configs.kbs] +url = "https://kbs.{{ hub_domain }}" +cert = """{{ trustee_cert }}""" +''' + +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] + +[kbc] +name = "cc_kbc" +url = "https://kbs.{{ hub_domain }}" +kbs_cert = """{{ trustee_cert }}""" + + +[image] +image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_flavour }}' +''' + +"policy.rego" = ''' package agent_policy default AddARPNeighborsRequest := true @@ -35,4 +64,5 @@ default UpdateEphemeralMountsRequest := true default UpdateInterfaceRequest := true default UpdateRoutesRequest := true default WaitProcessRequest := true -default WriteStreamRequest := true \ No newline at end of file +default WriteStreamRequest := true +''' diff --git a/charts/all/coco-kyverno-policies/Chart.yaml b/charts/all/coco-kyverno-policies/Chart.yaml new file mode 100644 index 00000000..5ce30d00 --- /dev/null +++ b/charts/all/coco-kyverno-policies/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: coco-kyverno-policies +description: Kyverno policies for CoCo cc_init_data injection and validation +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml b/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml new file mode 100644 index 00000000..a3a42bcb --- /dev/null +++ b/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml @@ -0,0 +1,31 @@ +{{- range .Values.workloadNamespaces }} +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: propagate-initdata-to-{{ . }} + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + rules: + - name: copy-initdata + match: + any: + - resources: + kinds: + - ConfigMap + namespaces: + - {{ $.Values.initdataSourceNamespace }} + selector: + matchLabels: + coco.io/type: initdata + generate: + synchronize: true + apiVersion: v1 + kind: ConfigMap + name: "{{ "{{" }}request.object.metadata.name{{ "}}" }}" + namespace: {{ . }} + clone: + namespace: {{ $.Values.initdataSourceNamespace }} + name: "{{ "{{" }}request.object.metadata.name{{ "}}" }}" +{{- end }} diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml new file mode 100644 index 00000000..518e9a61 --- /dev/null +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -0,0 +1,52 @@ +apiVersion: policies.kyverno.io/v1 +kind: MutatingPolicy +metadata: + name: inject-coco-initdata + annotations: + policies.kyverno.io/title: Inject CoCo InitData + policies.kyverno.io/category: Confidential Computing + policies.kyverno.io/severity: medium + policies.kyverno.io/subject: Pod + policies.kyverno.io/description: >- + Injects cc_init_data annotation into pods with a kata runtime class + by reading from a ConfigMap specified via the coco.io/initdata-configmap + annotation. Adapted from upstream kyverno inject-coco-initdata policy. + argocd.argoproj.io/sync-wave: "1" +spec: + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["pods"] + matchConditions: + - name: has-kata-runtime + expression: >- + has(object.spec.runtimeClassName) && + object.spec.runtimeClassName.startsWith("kata") + - name: has-initdata-configmap-annotation + expression: >- + has(object.metadata.annotations) && + 'coco.io/initdata-configmap' in object.metadata.annotations && + object.metadata.annotations['coco.io/initdata-configmap'] != '' + - name: no-existing-cc-init-data + expression: >- + !has(object.metadata.annotations) || + !('io.katacontainers.config.hypervisor.cc_init_data' in object.metadata.annotations) + variables: + - name: configMapName + expression: "object.metadata.annotations['coco.io/initdata-configmap']" + - name: configMap + expression: >- + namespaceObject.get('configmaps', variables.configMapName) + mutations: + - patchType: JSONPatch + jsonPatch: + expression: >- + [ + JSONPatch{ + op: "add", + path: "/metadata/annotations/io.katacontainers.config.hypervisor.cc_init_data", + value: variables.configMap.data['INITDATA'] + } + ] diff --git a/charts/all/coco-kyverno-policies/templates/validate-initdata-configmap.yaml b/charts/all/coco-kyverno-policies/templates/validate-initdata-configmap.yaml new file mode 100644 index 00000000..4b588289 --- /dev/null +++ b/charts/all/coco-kyverno-policies/templates/validate-initdata-configmap.yaml @@ -0,0 +1,46 @@ +apiVersion: policies.kyverno.io/v1 +kind: ValidatingPolicy +metadata: + name: validate-initdata-configmap + annotations: + policies.kyverno.io/title: Validate InitData ConfigMap Required Fields + policies.kyverno.io/category: Confidential Computing + policies.kyverno.io/severity: medium + policies.kyverno.io/subject: ConfigMap + policies.kyverno.io/description: >- + Validates that ConfigMaps with label coco.io/type=initdata contain + all required fields with proper values. + argocd.argoproj.io/sync-wave: "1" +spec: + evaluation: + background: + enabled: true + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["configmaps"] + matchConditions: + - name: has-initdata-label + expression: >- + has(object.metadata.labels) && + 'coco.io/type' in object.metadata.labels && + object.metadata.labels['coco.io/type'] == 'initdata' + variables: + - name: configData + expression: "object.data.orValue({})" + - name: allowedAlgorithms + expression: "['sha256', 'sha384', 'sha512']" + validationActions: ["Audit"] + validations: + - expression: "'version' in variables.configData && variables.configData['version'] != ''" + messageExpression: "'ConfigMap must contain a non-empty version field'" + - expression: "'algorithm' in variables.configData && variables.configData['algorithm'] in variables.allowedAlgorithms" + messageExpression: "'ConfigMap must contain an algorithm field with value sha256, sha384, or sha512, found: ' + ('algorithm' in variables.configData ? variables.configData['algorithm'] : 'missing')" + - expression: "'policy.rego' in variables.configData && variables.configData['policy.rego'] != ''" + messageExpression: "'ConfigMap must contain a non-empty policy.rego field'" + - expression: "'aa.toml' in variables.configData && variables.configData['aa.toml'] != ''" + messageExpression: "'ConfigMap must contain a non-empty aa.toml field'" + - expression: "'cdh.toml' in variables.configData && variables.configData['cdh.toml'] != ''" + messageExpression: "'ConfigMap must contain a non-empty cdh.toml field'" diff --git a/charts/all/coco-kyverno-policies/values.yaml b/charts/all/coco-kyverno-policies/values.yaml new file mode 100644 index 00000000..ef78f81e --- /dev/null +++ b/charts/all/coco-kyverno-policies/values.yaml @@ -0,0 +1,5 @@ +workloadNamespaces: + - hello-openshift + - kbs-access + +initdataSourceNamespace: imperative diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml index 205a9350..8ffa890b 100644 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml @@ -5,7 +5,7 @@ metadata: labels: app: insecure-policy annotations: - io.katacontainers.config.agent.policy: '{{ tpl ( .Files.Get "insecure-policy.rego") . | b64enc }}' + coco.io/initdata-configmap: initdata spec: runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} containers: diff --git a/charts/coco-supported/hello-openshift/templates/secure-pod.yaml b/charts/coco-supported/hello-openshift/templates/secure-pod.yaml index 28e26bf5..7ff21bb1 100644 --- a/charts/coco-supported/hello-openshift/templates/secure-pod.yaml +++ b/charts/coco-supported/hello-openshift/templates/secure-pod.yaml @@ -6,6 +6,7 @@ metadata: app: secure annotations: peerpods: "true" + coco.io/initdata-configmap: initdata spec: runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} containers: diff --git a/charts/coco-supported/kbs-access/templates/secure-pod.yaml b/charts/coco-supported/kbs-access/templates/secure-pod.yaml index 663408bd..a99f2461 100644 --- a/charts/coco-supported/kbs-access/templates/secure-pod.yaml +++ b/charts/coco-supported/kbs-access/templates/secure-pod.yaml @@ -6,6 +6,10 @@ metadata: app: secure annotations: peerpods: "true" + coco.io/initdata-configmap: initdata + {{- if .Values.defaultMemory }} + io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} + {{- end }} spec: runtimeClassName: kata-remote containers: diff --git a/charts/coco-supported/kbs-access/values.yaml b/charts/coco-supported/kbs-access/values.yaml index fdaa4d74..22a134cd 100644 --- a/charts/coco-supported/kbs-access/values.yaml +++ b/charts/coco-supported/kbs-access/values.yaml @@ -2,6 +2,8 @@ # Common values are inherited from values-global.yaml # Global values used by this chart (overridden by values-global.yaml) +defaultMemory: "" + global: coco: runtimeClassName: "" # Runtime class for confidential containers diff --git a/values-simple.yaml b/values-simple.yaml index fed2b1d3..a998f59a 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -16,6 +16,7 @@ clusterGroup: - cert-manager - kbs-access - encrypted-storage + - kyverno subscriptions: # ACM is kept anticipating acm: @@ -27,14 +28,14 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.11.0 + csv: sandboxed-containers-operator.v1.12.0 trustee: name: trustee-operator namespace: trustee-operator-system source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v1.0.0 + csv: trustee-operator.v1.1.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -78,7 +79,10 @@ clusterGroup: namespace: trustee-operator-system #upstream config project: trustee chart: trustee - chartVersion: 0.2.* + chartVersion: 0.2.* + overrides: + - name: kbs.admin.format + value: "v1.1" sandbox: name: sandbox namespace: openshift-sandboxed-containers-operator #upstream config @@ -107,6 +111,37 @@ clusterGroup: project: workloads path: charts/coco-supported/kbs-access + kyverno: + name: kyverno + namespace: kyverno + project: hub + repoURL: https://kyverno.github.io/kyverno/ + chart: kyverno + chartVersion: 3.7.* + overrides: + - name: admissionController.container.securityContext + value: "null" + - name: admissionController.initContainer.securityContext + value: "null" + - name: backgroundController.securityContext + value: "null" + - name: cleanupController.securityContext + value: "null" + - name: reportsController.securityContext + value: "null" + - name: crds.migration.securityContext + value: "null" + - name: webhooksCleanup.securityContext + value: "null" + - name: test.securityContext + value: "null" + + coco-kyverno-policies: + name: coco-kyverno-policies + namespace: openshift-sandboxed-containers-operator + project: sandbox + path: charts/all/coco-kyverno-policies + imperative: # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm # The default schedule is every 10 minutes: imperative.schedule From 41dfd80df3735ba8f5492f8064d430c7087852c4 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 20 Apr 2026 21:20:38 +0900 Subject: [PATCH 02/10] chore: point trustee app to feature branch for testing Temporarily targets butler54/trustee-chart feature/trustee-1.1-compat branch instead of released chart version. Revert to chart: trustee + chartVersion: 0.2.* after trustee-chart PR is merged and released. Co-Authored-By: Claude Opus 4.6 --- values-simple.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index a998f59a..90fabe8e 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -78,8 +78,9 @@ clusterGroup: name: trustee namespace: trustee-operator-system #upstream config project: trustee - chart: trustee - chartVersion: 0.2.* + repoURL: https://github.com/butler54/trustee-chart.git + path: . + chartVersion: feature/trustee-1.1-compat overrides: - name: kbs.admin.format value: "v1.1" From 72510aad14bdb9672468a099693139500695676a Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 10:05:48 +0900 Subject: [PATCH 03/10] fix: differentiate kata vs kata-cc runtime classes by platform Cloud platforms (Azure/AWS) use "kata" runtime class for peer-pods while baremetal uses "kata-cc" for confidential containers. Updates Kyverno MutatingPolicy to explicitly match both classes instead of startsWith, and adds ServerSideApply syncPolicy for Kyverno CRDs. Co-Authored-By: Claude Opus 4.6 --- .../templates/inject-coco-initdata.yaml | 2 +- .../coco-supported/hello-openshift/templates/_helpers.tpl | 6 +++--- charts/coco-supported/kbs-access/templates/secure-pod.yaml | 2 +- charts/coco-supported/kbs-access/values.yaml | 4 +--- values-simple.yaml | 6 ++++++ 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index 518e9a61..26aeb106 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -23,7 +23,7 @@ spec: - name: has-kata-runtime expression: >- has(object.spec.runtimeClassName) && - object.spec.runtimeClassName.startsWith("kata") + (object.spec.runtimeClassName == "kata" || object.spec.runtimeClassName == "kata-cc") - name: has-initdata-configmap-annotation expression: >- has(object.metadata.annotations) && diff --git a/charts/coco-supported/hello-openshift/templates/_helpers.tpl b/charts/coco-supported/hello-openshift/templates/_helpers.tpl index 6082b80a..f5aa6d02 100644 --- a/charts/coco-supported/hello-openshift/templates/_helpers.tpl +++ b/charts/coco-supported/hello-openshift/templates/_helpers.tpl @@ -51,12 +51,12 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Determine runtime class name based on cluster platform -Returns "kata-remote" for Azure/AWS, "kata-cc" for other platforms +Determine runtime class name based on cluster platform. +Cloud (Azure/AWS) uses "kata" for peer-pods; baremetal uses "kata-cc" for confidential containers. */}} {{- define "hello-openshift.runtimeClassName" -}} {{- if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") -}} -kata-remote +kata {{- else -}} kata-cc {{- end -}} diff --git a/charts/coco-supported/kbs-access/templates/secure-pod.yaml b/charts/coco-supported/kbs-access/templates/secure-pod.yaml index a99f2461..ce3d033f 100644 --- a/charts/coco-supported/kbs-access/templates/secure-pod.yaml +++ b/charts/coco-supported/kbs-access/templates/secure-pod.yaml @@ -11,7 +11,7 @@ metadata: io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} {{- end }} spec: - runtimeClassName: kata-remote + runtimeClassName: {{ if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}kata{{ else }}kata-cc{{ end }} containers: - name: python-access image: ghcr.io/butler54/kbs-access-app:latest diff --git a/charts/coco-supported/kbs-access/values.yaml b/charts/coco-supported/kbs-access/values.yaml index 22a134cd..6d3496ac 100644 --- a/charts/coco-supported/kbs-access/values.yaml +++ b/charts/coco-supported/kbs-access/values.yaml @@ -1,9 +1,7 @@ # Chart-specific values # Common values are inherited from values-global.yaml -# Global values used by this chart (overridden by values-global.yaml) defaultMemory: "" global: - coco: - runtimeClassName: "" # Runtime class for confidential containers + clusterPlatform: "" # Cluster platform: "Azure", "AWS", etc. - determines runtime class diff --git a/values-simple.yaml b/values-simple.yaml index 90fabe8e..468927a1 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -119,6 +119,12 @@ clusterGroup: repoURL: https://kyverno.github.io/kyverno/ chart: kyverno chartVersion: 3.7.* + syncPolicy: + automated: {} + retry: + limit: 20 + syncOptions: + - ServerSideApply=true overrides: - name: admissionController.container.securityContext value: "null" From a58de90e94e414cca83146dc96dc09f44c94e9d2 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 10:12:00 +0900 Subject: [PATCH 04/10] fix: exclude policyreport CRDs from Kyverno to avoid ACM conflict ACM owns the policyreports.wgpolicyk8s.io CRD with v1beta1 as a stored version. Kyverno's version drops v1beta1, causing a storage migration error. Disable Kyverno's policyreport CRD installation since ACM already provides it. Co-Authored-By: Claude Opus 4.6 --- values-simple.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/values-simple.yaml b/values-simple.yaml index 468927a1..20f34e9e 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -142,6 +142,10 @@ clusterGroup: value: "null" - name: test.securityContext value: "null" + - name: crds.groups.wgpolicyk8s.policyreports + value: "false" + - name: crds.groups.wgpolicyk8s.clusterpolicyreports + value: "false" coco-kyverno-policies: name: coco-kyverno-policies From 51e1358613e7e303fbe1c9d90726798ec9697af1 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 10:16:19 +0900 Subject: [PATCH 05/10] fix: disable Kyverno reportsController to match disabled policyreport CRDs The Kyverno chart validates that reportsController and policyreport CRDs are consistent. Since ACM owns the policyreport CRDs and we disabled their installation, we must also disable the reportsController. Mutation and validation policies still work without it. Co-Authored-By: Claude Opus 4.6 --- values-simple.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/values-simple.yaml b/values-simple.yaml index 20f34e9e..f6d173b5 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -146,6 +146,8 @@ clusterGroup: value: "false" - name: crds.groups.wgpolicyk8s.clusterpolicyreports value: "false" + - name: reportsController.enabled + value: "false" coco-kyverno-policies: name: coco-kyverno-policies From e0b936e90ddc8f3cfe5827bcfe9443b5347d9a1a Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 10:46:52 +0900 Subject: [PATCH 06/10] feat: convert bare Pods to Deployments, retarget Kyverno to Deployments Replace bare Pod templates with Deployments (replicas: 1) in hello-openshift and kbs-access charts. Update the Kyverno MutatingPolicy to target Deployments instead of Pods since the policies.kyverno.io/v1 API does not support autogen and confidential pods can't be easily mutated. Add generateExisting to namespace propagation ClusterPolicies and enable ArgoCD pruning to clean up old Pod resources. Co-Authored-By: Claude Opus 4.6 --- .../initdata-namespace-propagation.yaml | 1 + .../templates/inject-coco-initdata.yaml | 33 +++++++------- .../templates/insecure-policy-deployment.yaml | 34 ++++++++++++++ .../templates/insecure-policy-pod.yaml | 27 ------------ .../templates/secure-deployment.yaml | 35 +++++++++++++++ .../hello-openshift/templates/secure-pod.yaml | 27 ------------ .../templates/standard-deployment.yaml | 31 +++++++++++++ .../templates/standard-pod.yaml | 22 ---------- .../templates/secure-deployment.yaml | 44 +++++++++++++++++++ .../kbs-access/templates/secure-pod.yaml | 36 --------------- values-simple.yaml | 6 +++ 11 files changed, 168 insertions(+), 128 deletions(-) create mode 100644 charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml delete mode 100644 charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml create mode 100644 charts/coco-supported/hello-openshift/templates/secure-deployment.yaml delete mode 100644 charts/coco-supported/hello-openshift/templates/secure-pod.yaml create mode 100644 charts/coco-supported/hello-openshift/templates/standard-deployment.yaml delete mode 100644 charts/coco-supported/hello-openshift/templates/standard-pod.yaml create mode 100644 charts/coco-supported/kbs-access/templates/secure-deployment.yaml delete mode 100644 charts/coco-supported/kbs-access/templates/secure-pod.yaml diff --git a/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml b/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml index a3a42bcb..90d6e445 100644 --- a/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml +++ b/charts/all/coco-kyverno-policies/templates/initdata-namespace-propagation.yaml @@ -20,6 +20,7 @@ spec: matchLabels: coco.io/type: initdata generate: + generateExisting: true synchronize: true apiVersion: v1 kind: ConfigMap diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index 26aeb106..41d0d649 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -6,36 +6,37 @@ metadata: policies.kyverno.io/title: Inject CoCo InitData policies.kyverno.io/category: Confidential Computing policies.kyverno.io/severity: medium - policies.kyverno.io/subject: Pod + policies.kyverno.io/subject: Deployment policies.kyverno.io/description: >- - Injects cc_init_data annotation into pods with a kata runtime class - by reading from a ConfigMap specified via the coco.io/initdata-configmap - annotation. Adapted from upstream kyverno inject-coco-initdata policy. + Injects cc_init_data annotation into Deployment pod templates with a kata + runtime class by reading from a ConfigMap specified via the + coco.io/initdata-configmap annotation on the pod template. + Adapted from upstream kyverno inject-coco-initdata policy. argocd.argoproj.io/sync-wave: "1" spec: matchConstraints: resourceRules: - - apiGroups: [""] + - apiGroups: ["apps"] apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["pods"] + operations: ["CREATE", "UPDATE"] + resources: ["deployments"] matchConditions: - name: has-kata-runtime expression: >- - has(object.spec.runtimeClassName) && - (object.spec.runtimeClassName == "kata" || object.spec.runtimeClassName == "kata-cc") + has(object.spec.template.spec.runtimeClassName) && + (object.spec.template.spec.runtimeClassName == "kata" || object.spec.template.spec.runtimeClassName == "kata-cc") - name: has-initdata-configmap-annotation expression: >- - has(object.metadata.annotations) && - 'coco.io/initdata-configmap' in object.metadata.annotations && - object.metadata.annotations['coco.io/initdata-configmap'] != '' + has(object.spec.template.metadata.annotations) && + 'coco.io/initdata-configmap' in object.spec.template.metadata.annotations && + object.spec.template.metadata.annotations['coco.io/initdata-configmap'] != '' - name: no-existing-cc-init-data expression: >- - !has(object.metadata.annotations) || - !('io.katacontainers.config.hypervisor.cc_init_data' in object.metadata.annotations) + !has(object.spec.template.metadata.annotations) || + !('io.katacontainers.config.hypervisor.cc_init_data' in object.spec.template.metadata.annotations) variables: - name: configMapName - expression: "object.metadata.annotations['coco.io/initdata-configmap']" + expression: "object.spec.template.metadata.annotations['coco.io/initdata-configmap']" - name: configMap expression: >- namespaceObject.get('configmaps', variables.configMapName) @@ -46,7 +47,7 @@ spec: [ JSONPatch{ op: "add", - path: "/metadata/annotations/io.katacontainers.config.hypervisor.cc_init_data", + path: "/spec/template/metadata/annotations/io.katacontainers.config.hypervisor.cc_init_data", value: variables.configMap.data['INITDATA'] } ] diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml new file mode 100644 index 00000000..ad4f93a6 --- /dev/null +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: insecure-policy + labels: + app: insecure-policy +spec: + replicas: 1 + selector: + matchLabels: + app: insecure-policy + template: + metadata: + labels: + app: insecure-policy + annotations: + coco.io/initdata-configmap: initdata + spec: + runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} + containers: + - name: hello-openshift + image: quay.io/openshift/origin-hello-openshift + ports: + - containerPort: 8888 + securityContext: + privileged: false + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml deleted file mode 100644 index 8ffa890b..00000000 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-pod.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: insecure-policy - labels: - app: insecure-policy - annotations: - coco.io/initdata-configmap: initdata -spec: - runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} - containers: - - name: hello-openshift - image: quay.io/openshift/origin-hello-openshift - ports: - - containerPort: 8888 - securityContext: - privileged: false - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1001 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - ---- diff --git a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml new file mode 100644 index 00000000..3fdf30c6 --- /dev/null +++ b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: secure + labels: + app: secure +spec: + replicas: 1 + selector: + matchLabels: + app: secure + template: + metadata: + labels: + app: secure + annotations: + peerpods: "true" + coco.io/initdata-configmap: initdata + spec: + runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} + containers: + - name: hello-openshift + image: quay.io/openshift/origin-hello-openshift + ports: + - containerPort: 8888 + securityContext: + privileged: false + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault diff --git a/charts/coco-supported/hello-openshift/templates/secure-pod.yaml b/charts/coco-supported/hello-openshift/templates/secure-pod.yaml deleted file mode 100644 index 7ff21bb1..00000000 --- a/charts/coco-supported/hello-openshift/templates/secure-pod.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: secure - labels: - app: secure - annotations: - peerpods: "true" - coco.io/initdata-configmap: initdata -spec: - runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} - containers: - - name: hello-openshift - image: quay.io/openshift/origin-hello-openshift - ports: - - containerPort: 8888 - securityContext: - privileged: false - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1001 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - diff --git a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml new file mode 100644 index 00000000..a2b0b823 --- /dev/null +++ b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: standard + labels: + app: standard +spec: + replicas: 1 + selector: + matchLabels: + app: standard + template: + metadata: + labels: + app: standard + spec: + containers: + - name: hello-openshift + image: quay.io/openshift/origin-hello-openshift + ports: + - containerPort: 8888 + securityContext: + privileged: false + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault diff --git a/charts/coco-supported/hello-openshift/templates/standard-pod.yaml b/charts/coco-supported/hello-openshift/templates/standard-pod.yaml deleted file mode 100644 index 4a92a8a0..00000000 --- a/charts/coco-supported/hello-openshift/templates/standard-pod.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: standard - labels: - app: standard -spec: - containers: - - name: hello-openshift - image: quay.io/openshift/origin-hello-openshift - ports: - - containerPort: 8888 - securityContext: - privileged: false - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 1001 - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault \ No newline at end of file diff --git a/charts/coco-supported/kbs-access/templates/secure-deployment.yaml b/charts/coco-supported/kbs-access/templates/secure-deployment.yaml new file mode 100644 index 00000000..e655b99d --- /dev/null +++ b/charts/coco-supported/kbs-access/templates/secure-deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: secure + labels: + app: secure +spec: + replicas: 1 + selector: + matchLabels: + app: secure + template: + metadata: + labels: + app: secure + annotations: + peerpods: "true" + coco.io/initdata-configmap: initdata + {{- if .Values.defaultMemory }} + io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} + {{- end }} + spec: + runtimeClassName: {{ if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}kata{{ else }}kata-cc{{ end }} + containers: + - name: python-access + image: ghcr.io/butler54/kbs-access-app:latest + ports: + - containerPort: 5000 + volumeMounts: + - name: output-volume + mountPath: /output + envFrom: + - configMapRef: + name: kbsref + initContainers: + - name: curl + image: registry.access.redhat.com/ubi9/ubi:latest + command: ['sh', '-c', 'curl -s http://127.0.0.1:8006/cdh/resource/default/kbsres1/key3 > /output/kbsres1.txt'] + volumeMounts: + - name: output-volume + mountPath: /output + volumes: + - name: output-volume + emptyDir: {} diff --git a/charts/coco-supported/kbs-access/templates/secure-pod.yaml b/charts/coco-supported/kbs-access/templates/secure-pod.yaml deleted file mode 100644 index ce3d033f..00000000 --- a/charts/coco-supported/kbs-access/templates/secure-pod.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: secure - labels: - app: secure - annotations: - peerpods: "true" - coco.io/initdata-configmap: initdata - {{- if .Values.defaultMemory }} - io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} - {{- end }} -spec: - runtimeClassName: {{ if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}kata{{ else }}kata-cc{{ end }} - containers: - - name: python-access - image: ghcr.io/butler54/kbs-access-app:latest - ports: - - containerPort: 5000 - volumeMounts: - - name: output-volume - mountPath: /output - envFrom: - - configMapRef: - name: kbsref - initContainers: - - name: curl - image: registry.access.redhat.com/ubi9/ubi:latest # Lightweight image with curl installed - command: ['sh', '-c', 'curl -s http://127.0.0.1:8006/cdh/resource/default/kbsres1/key3 > /output/kbsres1.txt'] - volumeMounts: - - name: output-volume - mountPath: /output - volumes: - - name: output-volume - emptyDir: {} - diff --git a/values-simple.yaml b/values-simple.yaml index f6d173b5..0af23a36 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -105,12 +105,18 @@ clusterGroup: namespace: hello-openshift project: workloads path: charts/coco-supported/hello-openshift + syncPolicy: + automated: + prune: true kbs-access: name: kbs-access namespace: kbs-access project: workloads path: charts/coco-supported/kbs-access + syncPolicy: + automated: + prune: true kyverno: name: kyverno From ea2d8fda626cee867efdfc4b4453896671879584 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 10:49:25 +0900 Subject: [PATCH 07/10] fix: remove fixed runAsUser from hello-openshift Deployments Bare Pods ran under kube:admin with anyuid SCC, but Deployment-created pods use the default ServiceAccount which only has restricted-v2 SCC. Remove the fixed runAsUser: 1001 and let OpenShift assign a UID from the namespace range. runAsNonRoot: true ensures non-root execution. Co-Authored-By: Claude Opus 4.6 --- .../hello-openshift/templates/insecure-policy-deployment.yaml | 1 - .../hello-openshift/templates/secure-deployment.yaml | 1 - .../hello-openshift/templates/standard-deployment.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml index ad4f93a6..9069fd77 100644 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml @@ -26,7 +26,6 @@ spec: privileged: false allowPrivilegeEscalation: false runAsNonRoot: true - runAsUser: 1001 capabilities: drop: - ALL diff --git a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml index 3fdf30c6..4fa15564 100644 --- a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml @@ -27,7 +27,6 @@ spec: privileged: false allowPrivilegeEscalation: false runAsNonRoot: true - runAsUser: 1001 capabilities: drop: - ALL diff --git a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml index a2b0b823..b8b5689b 100644 --- a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml @@ -23,7 +23,6 @@ spec: privileged: false allowPrivilegeEscalation: false runAsNonRoot: true - runAsUser: 1001 capabilities: drop: - ALL From cb3098420eb48d0da6de3da2b53b914b413cfa20 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 10:59:14 +0900 Subject: [PATCH 08/10] fix: switch initdata injection from MutatingPolicy to ClusterPolicy The policies.kyverno.io/v1 MutatingPolicy API doesn't support ConfigMap lookups (namespaceObject.get is not available). Switch to kyverno.io/v1 ClusterPolicy which supports configMap context lookups and has autogen for Deployments/StatefulSets/DaemonSets/Jobs. Co-Authored-By: Claude Opus 4.6 --- .../templates/inject-coco-initdata.yaml | 82 +++++++++---------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index 41d0d649..b0533485 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -1,53 +1,47 @@ -apiVersion: policies.kyverno.io/v1 -kind: MutatingPolicy +apiVersion: kyverno.io/v1 +kind: ClusterPolicy metadata: name: inject-coco-initdata annotations: policies.kyverno.io/title: Inject CoCo InitData policies.kyverno.io/category: Confidential Computing policies.kyverno.io/severity: medium - policies.kyverno.io/subject: Deployment + policies.kyverno.io/subject: Pod,Deployment policies.kyverno.io/description: >- - Injects cc_init_data annotation into Deployment pod templates with a kata - runtime class by reading from a ConfigMap specified via the - coco.io/initdata-configmap annotation on the pod template. - Adapted from upstream kyverno inject-coco-initdata policy. + Injects cc_init_data annotation into pods with a kata runtime class + by reading from a ConfigMap specified via the coco.io/initdata-configmap + annotation. Kyverno autogen extends this to Deployments, StatefulSets, + DaemonSets, and Jobs automatically. argocd.argoproj.io/sync-wave: "1" + pod-policies.kyverno.io/autogen-controllers: Deployment,StatefulSet,DaemonSet,Job spec: - matchConstraints: - resourceRules: - - apiGroups: ["apps"] - apiVersions: ["v1"] - operations: ["CREATE", "UPDATE"] - resources: ["deployments"] - matchConditions: - - name: has-kata-runtime - expression: >- - has(object.spec.template.spec.runtimeClassName) && - (object.spec.template.spec.runtimeClassName == "kata" || object.spec.template.spec.runtimeClassName == "kata-cc") - - name: has-initdata-configmap-annotation - expression: >- - has(object.spec.template.metadata.annotations) && - 'coco.io/initdata-configmap' in object.spec.template.metadata.annotations && - object.spec.template.metadata.annotations['coco.io/initdata-configmap'] != '' - - name: no-existing-cc-init-data - expression: >- - !has(object.spec.template.metadata.annotations) || - !('io.katacontainers.config.hypervisor.cc_init_data' in object.spec.template.metadata.annotations) - variables: - - name: configMapName - expression: "object.spec.template.metadata.annotations['coco.io/initdata-configmap']" - - name: configMap - expression: >- - namespaceObject.get('configmaps', variables.configMapName) - mutations: - - patchType: JSONPatch - jsonPatch: - expression: >- - [ - JSONPatch{ - op: "add", - path: "/spec/template/metadata/annotations/io.katacontainers.config.hypervisor.cc_init_data", - value: variables.configMap.data['INITDATA'] - } - ] + rules: + - name: inject-initdata + match: + any: + - resources: + kinds: + - Pod + operations: + - CREATE + preconditions: + all: + - key: "{{ "{{" }}request.object.spec.runtimeClassName || '' {{ "}}" }}" + operator: AnyIn + value: ["kata", "kata-cc"] + - key: "{{ "{{" }}request.object.metadata.annotations.\"coco.io/initdata-configmap\" || '' {{ "}}" }}" + operator: NotEquals + value: "" + - key: "{{ "{{" }}request.object.metadata.annotations.\"io.katacontainers.config.hypervisor.cc_init_data\" || '' {{ "}}" }}" + operator: Equals + value: "" + context: + - name: initdata + configMap: + name: "{{ "{{" }}request.object.metadata.annotations.\"coco.io/initdata-configmap\"{{ "}}" }}" + namespace: "{{ "{{" }}request.namespace{{ "}}" }}" + mutate: + patchStrategicMerge: + metadata: + annotations: + io.katacontainers.config.hypervisor.cc_init_data: "{{ "{{" }}initdata.data.INITDATA{{ "}}" }}" From f53261d2b069cae2d391cb5ec0addaa88d01b4d5 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 11:07:12 +0900 Subject: [PATCH 09/10] fix: use kata-remote runtime class for cloud peer-pods The kata runtime class runs a local QEMU VM without CDH. Cloud peer-pods require kata-remote to spawn a VM in Azure/AWS with CDH available at 127.0.0.1:8006. Also add kata-remote to the Kyverno ClusterPolicy match list. Co-Authored-By: Claude Opus 4.6 --- .../coco-kyverno-policies/templates/inject-coco-initdata.yaml | 2 +- charts/coco-supported/hello-openshift/templates/_helpers.tpl | 4 ++-- .../kbs-access/templates/secure-deployment.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index b0533485..397ba39b 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -28,7 +28,7 @@ spec: all: - key: "{{ "{{" }}request.object.spec.runtimeClassName || '' {{ "}}" }}" operator: AnyIn - value: ["kata", "kata-cc"] + value: ["kata", "kata-cc", "kata-remote"] - key: "{{ "{{" }}request.object.metadata.annotations.\"coco.io/initdata-configmap\" || '' {{ "}}" }}" operator: NotEquals value: "" diff --git a/charts/coco-supported/hello-openshift/templates/_helpers.tpl b/charts/coco-supported/hello-openshift/templates/_helpers.tpl index f5aa6d02..cc79680e 100644 --- a/charts/coco-supported/hello-openshift/templates/_helpers.tpl +++ b/charts/coco-supported/hello-openshift/templates/_helpers.tpl @@ -52,11 +52,11 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{/* Determine runtime class name based on cluster platform. -Cloud (Azure/AWS) uses "kata" for peer-pods; baremetal uses "kata-cc" for confidential containers. +Cloud (Azure/AWS) uses "kata-remote" for peer-pods; baremetal uses "kata-cc" for confidential containers. */}} {{- define "hello-openshift.runtimeClassName" -}} {{- if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") -}} -kata +kata-remote {{- else -}} kata-cc {{- end -}} diff --git a/charts/coco-supported/kbs-access/templates/secure-deployment.yaml b/charts/coco-supported/kbs-access/templates/secure-deployment.yaml index e655b99d..4a63170c 100644 --- a/charts/coco-supported/kbs-access/templates/secure-deployment.yaml +++ b/charts/coco-supported/kbs-access/templates/secure-deployment.yaml @@ -20,7 +20,7 @@ spec: io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} {{- end }} spec: - runtimeClassName: {{ if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}kata{{ else }}kata-cc{{ end }} + runtimeClassName: {{ if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}kata-remote{{ else }}kata-cc{{ end }} containers: - name: python-access image: ghcr.io/butler54/kbs-access-app:latest From 87cad95819fa6e7e873c93052ff04b29395a0c42 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 21 Apr 2026 13:37:05 +0900 Subject: [PATCH 10/10] fix: minor fixes Signed-off-by: Chris Butler --- README.md | 4 ++-- scripts/get-pcr.sh | 4 ++-- values-spoke.yaml | 2 +- values-trusted-hub.yaml | 9 ++++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 32ebe80b..08c98e82 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ Currently supports Azure via peer-pods. Peer-pods provision confidential VMs (`S Breaking change from v3. This is the first version using GA (Generally Available) releases of the CoCo stack: -- **OpenShift Sandboxed Containers 1.11+** (requires OCP 4.17+) -- **Red Hat Build of Trustee 1.0** (first GA release; all prior versions were Technology Preview) +- **OpenShift Sandboxed Containers 1.12+** (requires OCP 4.17+) +- **Red Hat Build of Trustee 1.1** (GA release; all versions prior to 1.0 were Technology Preview) - External chart repositories for [Trustee](https://github.com/validatedpatterns/trustee-chart), [sandboxed-containers](https://github.com/validatedpatterns/sandboxed-containers-chart), and [sandboxed-policies](https://github.com/validatedpatterns/sandboxed-policies-chart) - Self-signed certificates via cert-manager (Let's Encrypt no longer required) - Multi-cluster support via ACM diff --git a/scripts/get-pcr.sh b/scripts/get-pcr.sh index c432aa42..b509edbd 100755 --- a/scripts/get-pcr.sh +++ b/scripts/get-pcr.sh @@ -76,14 +76,14 @@ if [ -z "$SANDBOX_CSV" ] || [ "$SANDBOX_CSV" == "null" ]; then exit 0 fi -# Extract version from CSV (e.g., "sandboxed-containers-operator.v1.11.0" -> "1.11.0") +# Extract version from CSV (e.g., "sandboxed-containers-operator.v1.12.0" -> "1.12.0") # Remove everything up to and including ".v" SANDBOX_VERSION="${SANDBOX_CSV##*.v}" echo "Sandboxed container operator CSV: $SANDBOX_CSV" echo "Version: $SANDBOX_VERSION" # alternatively, use the operator-version tag. -# OSC_VERSION=1.11.1 +# OSC_VERSION=1.12.0 VERITY_IMAGE=registry.redhat.io/openshift-sandboxed-containers/osc-dm-verity-image TAG=$(skopeo inspect --authfile $PULL_SECRET_PATH docker://${VERITY_IMAGE}:${SANDBOX_VERSION} | jq -r .Digest) diff --git a/values-spoke.yaml b/values-spoke.yaml index c604a5a6..0dab1011 100644 --- a/values-spoke.yaml +++ b/values-spoke.yaml @@ -17,7 +17,7 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.11.0 + csv: sandboxed-containers-operator.v1.12.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator diff --git a/values-trusted-hub.yaml b/values-trusted-hub.yaml index c3531ba1..0806bd77 100644 --- a/values-trusted-hub.yaml +++ b/values-trusted-hub.yaml @@ -22,7 +22,7 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v1.0.0 + csv: trustee-operator.v1.1.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -66,11 +66,14 @@ clusterGroup: name: trustee namespace: trustee-operator-system #upstream config project: trustee - chart: trustee - chartVersion: 0.2.* + repoURL: https://github.com/butler54/trustee-chart.git + path: . + chartVersion: feature/trustee-1.1-compat overrides: - name: global.coco.secured value: "true" + - name: kbs.admin.format + value: "v1.1" sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator #upstream config