From d2c1da4219139c0f38e5e16b73d9260a75756c02 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 7 Jun 2026 16:46:55 +0000 Subject: [PATCH] fix: Helm/k8s design gaps, kind CI, and container-binary-service tests --- .github/workflows/CI.yaml | 8 + AGENTS.md | 8 +- README.md | 11 +- defaults/main.yml | 24 ++- handlers/main.yml | 2 + helm/README.md | 11 ++ helm/templates/_container.tpl | 125 ++++++++++++++++ helm/templates/deployment.yaml | 106 +++---------- helm/templates/hpa.yml | 44 ++++-- helm/templates/ingress.yaml | 2 +- helm/templates/poddisruptionbudget.yaml | 3 +- helm/templates/secret.yaml | 7 +- helm/templates/service.yaml | 2 +- helm/templates/servicemonitor.yaml | 95 ++++++------ helm/templates/statefulset.yaml | 141 +++++------------- helm/templates/tests/test-connect.yaml | 2 +- helm/values-test.yaml | 11 +- helm/values.yaml | 46 +++++- tasks/common/custom-facts.yml | 4 + tasks/common/facts-k8s.yml | 71 +++++++++ tasks/k8s/render-values.yml | 34 +++++ tasks/k8s/setup.yml | 15 +- templates/k8s-values.yaml.j2 | 125 ++++++++++++++++ tests/fixtures/k8s-rendered-values.yaml | 33 ++++ tests/helm-validate.sh | 21 +++ tests/molecule/cleanup-kind.yml | 18 +++ .../container-binary-service/converge.yml | 43 ++++++ .../container-binary-service/molecule.yml | 37 +++++ .../tests/test_container_binary_service.py | 54 +++++++ tests/molecule/k8s-basic/converge.yml | 21 +++ tests/molecule/k8s-basic/molecule.yml | 40 +++++ .../k8s-basic/tests/test_k8s_basic.py | 56 +++++++ tests/molecule/prepare-docker.yml | 4 +- tests/molecule/prepare-kind.yml | 70 +++++++++ tests/molecule/requirements.yml | 5 + 35 files changed, 1033 insertions(+), 266 deletions(-) create mode 100644 helm/templates/_container.tpl create mode 100644 tasks/common/facts-k8s.yml create mode 100644 tasks/k8s/render-values.yml create mode 100644 templates/k8s-values.yaml.j2 create mode 100644 tests/fixtures/k8s-rendered-values.yaml create mode 100755 tests/helm-validate.sh create mode 100644 tests/molecule/cleanup-kind.yml create mode 100644 tests/molecule/container-binary-service/converge.yml create mode 100644 tests/molecule/container-binary-service/molecule.yml create mode 100644 tests/molecule/container-binary-service/tests/test_container_binary_service.py create mode 100644 tests/molecule/k8s-basic/converge.yml create mode 100644 tests/molecule/k8s-basic/molecule.yml create mode 100644 tests/molecule/k8s-basic/tests/test_k8s_basic.py create mode 100644 tests/molecule/prepare-kind.yml diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 61a520a..ac68c27 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -18,6 +18,11 @@ jobs: pip install ansible-lint==24.12.2 yamllint==1.35.1 ansible-core==2.16.3 yamllint --config-file ./tests/yaml-lint.yml . ansible-lint --config-file ./tests/ansible-lint.yml . + + - name: Validate Helm chart + run: | + chmod +x ./tests/helm-validate.sh + ./tests/helm-validate.sh molecule: strategy: fail-fast: false @@ -28,10 +33,12 @@ jobs: - systemd-uninstall - container-basic - container-binary + - container-binary-service - container-full - container-uninstall - install-basic - install-uninstall + - k8s-basic runs-on: ubuntu-latest steps: @@ -59,4 +66,5 @@ jobs: sudo systemctl enable docker - name: Run Molecule Test + timeout-minutes: 20 run: cd tests && molecule test -s ${{ matrix.scenario }} diff --git a/AGENTS.md b/AGENTS.md index 135ab53..e1016db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,11 @@ cd tests molecule test -s container-basic # or any scenario name below ``` -**CI scenarios:** `systemd-basic`, `systemd-full`, `systemd-uninstall`, `container-basic`, `container-full`, `container-uninstall`, `install-basic`, `install-uninstall`. +**CI scenarios:** `systemd-basic`, `systemd-full`, `systemd-uninstall`, `container-basic`, `container-binary`, `container-binary-service`, `container-full`, `container-uninstall`, `install-basic`, `install-uninstall`, `k8s-basic`. + +`container-binary` exercises bind-mount mechanics with a short-lived `jq` binary. `container-binary-service` deploys a long-running daemon from a downloaded tarball (Prometheus) with config, data, ports, and readiness checks. `k8s-basic` uses the delegated driver with a local [kind](https://kind.sigs.k8s.io/) cluster to test `setup_mode: k8s` end-to-end. + +**Helm validation:** `./tests/helm-validate.sh` (also runs in the lint CI job). ### Cloud VM caveats @@ -66,4 +70,4 @@ Exercises the same stack as `setup_mode: container` (the role does not set `cgro ## Kubernetes / Helm -`setup_mode: k8s` is not in CI. Requires a cluster, `KUBECONFIG`, `KUBE_CONTEXT`, Helm, and `kubernetes.core` (see `README.md`, `helm/README.md`). +`k8s-basic` provisions kind on the CI runner (Docker required) and sets `KUBECONFIG` for the role. Chart-only validation also runs via `./tests/helm-validate.sh`. Manual deploys still need `KUBECONFIG`, `KUBE_CONTEXT`, Helm, and `kubernetes.core` (see `README.md`, `helm/README.md`). diff --git a/README.md b/README.md index dd343ae..d7cfaca 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,14 @@ export KUBE_CONTEXT= | var | description | default | | :-------------: | :--------------------------------------------------------: | :--------------: | -| _helm_chart_path_ | path to Helm chart to use for the service deployment/release | `../../helm` | -| _helm_namespace_ | Kubernetes namespace to deploy to | `default` | -| _helm_values_path_ | file to load Helm chart values (see [here](./helm/README.md) for available values) | `values.yaml` | +| _helm_chart_path_ | path to Helm chart to use for the service deployment/release | `helm` (resolved relative to the role) | +| _helm_namespace_ | Kubernetes namespace to deploy to (also rendered into chart values) | `default` | +| _helm_values_path_ | optional Helm values overlay file merged after rendered role values | `""` | +| _helm_render_values_from_role_ | map common role vars (`image`, `config`, `ports`, `cpus`, `memory`, etc.) into Helm values | `true` | +| _helm_create_namespace_ | create the target namespace during Helm install | `true` | +| _helm_wait_ / _helm_atomic_ / _helm_timeout_ | Helm install safety controls | `true` / `true` / `10m` | + +With `setup_mode: k8s`, the role renders Helm values from the same variables used by `container`, `systemd`, and `install` modes, then deploys the bundled chart. Set `helm_render_values_from_role: false` to use only `helm_values_path`. ## Containerized Apps - [O1 Containers](https://github.com/O1labs/containers) diff --git a/defaults/main.yml b/defaults/main.yml index 7ad9a82..d1157de 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -89,6 +89,26 @@ systemd: {} systemd_environment_directive: "" ### Kubernetes ### -helm_chart_path: ../../helm +helm_chart_path: helm helm_namespace: default -helm_values_path: values.yaml +helm_values_path: "" +helm_render_values_from_role: true +# helm_rendered_values_path: /tmp/-helm-values.yaml +helm_create_namespace: true +k8s_chart_create_namespace: false +helm_wait: true +helm_atomic: true +helm_timeout: 10m +k8s_common_name: "" +k8s_cluster_name: default-cluster +k8s_min_replicas: 1 +k8s_max_replicas: 1 +k8s_hpa_enabled: false +k8s_hpa_target_cpu: 50 +k8s_deploy_stateful_set: false +k8s_ingress_enabled: false +k8s_service_monitor_enabled: false +k8s_health_check_path: / +k8s_service_monitor_path: /metrics +k8s_labels: {} +k8s_annotations: {} diff --git a/handlers/main.yml b/handlers/main.yml index e251433..54825a4 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -84,4 +84,6 @@ name: "{{ name }}" state: absent release_namespace: "{{ helm_namespace }}" + kubeconfig: "{{ lookup('env', 'KUBECONFIG') | default(omit, true) }}" + context: "{{ lookup('env', 'KUBE_CONTEXT') | default(omit, true) }}" when: setup_mode == 'k8s' diff --git a/helm/README.md b/helm/README.md index 0a050d1..6aa2ab9 100644 --- a/helm/README.md +++ b/helm/README.md @@ -37,3 +37,14 @@ Helm chart that supports a basic, cloud-native containerized app in Kubernetes. | persistentVolume | persistent volume that the pvc can use | `list(object)` | `[]` | no | | storageClass | storage class that the pvc can use | `list(object)` | `[]` | no | | dockercfgOverride | Docker config secret override reference (not including `-docker` suffix) | `string` | `` | no | +| hpa.enabled | Enable HorizontalPodAutoscaler (also auto-enabled when `maxReplicas` > `minReplicas`) | `bool` | `false` | no | +| hpa.targetCPUUtilizationPercentage | CPU utilization target for autoscaling v2 | `int` | `50` | no | +| hpa.targetMemoryUtilizationPercentage | Optional memory utilization target for autoscaling v2 | `int` | | no | +| livenessProbe | Container liveness probe definition | `dict` | `{}` | no | +| readinessProbe | Container readiness probe definition | `dict` | `{}` | no | +| startupProbe | Container startup probe definition | `dict` | | no | +| ingress.enabled | Whether to create an Ingress resource | `bool` | `false` | no | +| serviceMonitor.enabled | Whether to create a Prometheus Operator ServiceMonitor | `bool` | `false` | no | +| serviceMonitor.port | Service port name scraped by ServiceMonitor | `string` | `http` | no | +| serviceMonitor.path | Metrics path scraped by ServiceMonitor | `string` | `/metrics` | no | +| podDisruptionBudget | PodDisruptionBudget settings (`minAvailable` / `maxUnavailable`) | `dict` | | no | diff --git a/helm/templates/_container.tpl b/helm/templates/_container.tpl new file mode 100644 index 0000000..0c36547 --- /dev/null +++ b/helm/templates/_container.tpl @@ -0,0 +1,125 @@ +{{/* +Container environment variables from env.config, secrets, and extraEnv. +*/}} +{{- define "basic-service.containerEnv" -}} +- name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP +{{- range $key, $value := ((.Values.env | default dict).config | default dict) }} +- name: {{ $key }} + value: {{ tpl ($value | toString) $ | quote }} +{{- end }} +{{- range $key, $value := .Values.secretEnv }} +- name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ include "basic-service.name" $ }}-env + key: {{ $key }} +{{- end }} +{{- if .Values.extraEnv }} +{{ tpl (toYaml .Values.extraEnv) . | nindent 0 }} +{{- end }} +{{- end -}} + +{{/* +Primary application container specification. +*/}} +{{- define "basic-service.mainContainer" -}} +- name: {{ template "basic-service.name" . }} + image: {{ required "A valid .Values.image is required" .Values.image }} + {{- with .Values.command }} + command: {{ toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.args }} + args: {{ toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.livenessProbe .Values.livenessProbeRPCDaemon }} + livenessProbe: + {{- toYaml (.Values.livenessProbe | default .Values.livenessProbeRPCDaemon) | nindent 4 }} + {{- end }} + {{- if or .Values.readinessProbe .Values.readinessProbeRPCDaemon }} + readinessProbe: + {{- toYaml (.Values.readinessProbe | default .Values.readinessProbeRPCDaemon) | nindent 4 }} + {{- end }} + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.extraVolumeMounts .Values.deployStatefulSet }} + volumeMounts: + {{- if .Values.deployStatefulSet }} + - mountPath: {{ .Values.statefulSetOptions.mountPath }} + name: {{ tpl .Values.statefulSetOptions.name . }} + {{- end }} + {{- range $mountName, $mountValue := .Values.extraVolumeMounts }} + {{- toYaml $mountValue | nindent 4 }} + {{- end }} + {{- end }} + env: + {{- include "basic-service.containerEnv" . | nindent 4 }} +{{- end -}} + +{{/* +Shared pod specification for Deployments and StatefulSets. +*/}} +{{- define "basic-service.podSpec" -}} +{{- if or .Values.dockercfg .Values.dockercfgOverride }} +imagePullSecrets: + - name: {{ default (include "basic-service.name" .) .Values.dockercfgOverride }}-dockercfg +{{- end }} +{{- with .Values.imagePullSecrets }} +imagePullSecrets: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if or .Values.extraVolumes .Values.volumesFromConfigMaps }} +volumes: +{{- range $volumeName, $volumeValue := .Values.extraVolumes }} + {{- toYaml $volumeValue | nindent 2 }} +{{- end }} +{{- range $volumeName, $volumeValue := .Values.volumesFromConfigMaps }} + - name: {{ $volumeName }} + configMap: + name: {{ template "basic-service.name" $ }}-{{ $volumeValue.configMapName }} +{{- end }} +{{- end }} +{{- if .Values.serviceAccountName }} +serviceAccountName: {{ .Values.serviceAccountName }} +{{- end }} +{{- with .Values.initContainers }} +initContainers: + {{- range $containerName, $containerValue := . }} + {{- toYaml $containerValue | nindent 2 }} + {{- end }} +{{- end }} +containers: + {{- include "basic-service.mainContainer" . | nindent 2 }} +{{- with .Values.extraContainers }} + {{- range $key, $value := . }} + {{- tpl (toYaml $value) $ | nindent 2 }} + {{- end }} +{{- end }} +{{- with .Values.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.terminationGracePeriodSeconds }} +terminationGracePeriodSeconds: {{ . }} +{{- end }} +{{- end -}} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index b8e4d05..56e8e85 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -1,101 +1,39 @@ -{{- if not .Values.deployStatefulSet}} +{{- if not .Values.deployStatefulSet }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{template "basic-service.name" .}}-deployment - namespace: {{.Values.namespace}} + name: {{ template "basic-service.name" . }}-deployment + namespace: {{ .Values.namespace }} labels: - {{- include "basic-service.labels" $ | indent 8 }} + app: {{ template "basic-service.name" . }} + {{- with .Values.labels }} + {{- include "basic-service.labels" $ | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} annotations: - {{- tpl (toYaml .Values.annotations) . | nindent 4 }} + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} spec: - replicas: {{int .Values.minReplicas | default 0}} + replicas: {{ int .Values.minReplicas | default 1 }} selector: matchLabels: - app: {{template "basic-service.name" .}} - {{- if .Values.deployStrategy }} - {{- toYaml .Values.deployStrategy | nindent 2 }} + {{- include "basic-service.selectorLabels" . | nindent 6 }} + {{- with .Values.deployStrategy }} + strategy: + {{- toYaml . | nindent 4 }} {{- end }} template: metadata: labels: - app: {{template "basic-service.name" .}} - {{- if .Values.labels }} - {{- include "basic-service.labels" $ | indent 8 }} + {{- include "basic-service.selectorLabels" . | nindent 8 }} + {{- with .Values.labels }} + {{- include "basic-service.labels" $ | nindent 8 }} {{- end }} annotations: checksum/secrets: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} - {{- tpl (toYaml .Values.annotations) . | nindent 4 }} - spec: - {{- if or .Values.dockercfg .Values.dockercfgOverride}} - imagePullSecrets: - - name: {{default (include "basic-service.name" .) .Values.dockercfgOverride}}-dockercfg - {{- end}} - volumes: - {{- range $volumeName, $volumeValue := .Values.extraVolumes }} - {{- toYaml $volumeValue | nindent 8}} - {{- end }} - {{- range $volumeName, $volumeValue := .Values.volumesFromConfigMaps }} - - name: {{$volumeName}} - configMap: - name: {{template "basic-service.name" $}}-{{$volumeValue.configMapName}} - {{- end }} - {{- if .Values.serviceAccountName }} - serviceAccountName: {{.Values.serviceAccountName }} - {{- end }} - {{- if .Values.initContainers }} - initContainers: - {{- range $containerName, $containerValue := .Values.initContainers }} - {{- toYaml $containerValue | nindent 6}} - {{- end }} - {{- end }} - containers: - - name: {{template "basic-service.name" .}} - image: {{.Values.image}} - {{- if .Values.command }} - command: {{ toYaml .Values.command | nindent 8 }} - {{- end }} - {{- if .Values.args }} - args: {{ toYaml .Values.args | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.containerSecurityContext | nindent 12 }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - livenessProbe: - {{- toYaml .Values.livenessProbeRPCDaemon | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbeRPCDaemon | nindent 12 }} - volumeMounts: - {{- range $mountName, $mountValue := .Values.extraVolumeMounts }} - {{- toYaml $mountValue | nindent 10}} + {{- with .Values.annotations }} + {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - {{- range $key, $value := .Values.secretEnv }} - - name: {{ $key }} - valueFrom: - secretKeyRef: - name: {{ include "basic-service.name" $ }}-env - key: {{ $key }} - {{- end }} - {{- if .Values.extraEnv }} - {{- toYaml .Values.extraEnv | nindent 12 }} - {{- end }} - {{- if .Values.extraContainers }} - {{- range $key, $value := .Values.extraContainers }} - {{- tpl (toYaml $value) $ | nindent 6}} - {{- end }} - {{- end }} - nodeSelector: - {{- toYaml .Values.nodeSelector | nindent 8 }} - affinity: - {{- toYaml .Values.affinity | nindent 8 }} - tolerations: - {{- toYaml .Values.tolerations | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - {{- end}} + spec: + {{- include "basic-service.podSpec" . | nindent 6 }} {{- end }} diff --git a/helm/templates/hpa.yml b/helm/templates/hpa.yml index 27de7f8..e04f455 100644 --- a/helm/templates/hpa.yml +++ b/helm/templates/hpa.yml @@ -1,18 +1,40 @@ -apiVersion: autoscaling/v1 +{{- $minReplicas := int (.Values.minReplicas | default 1) -}} +{{- $maxReplicas := int (.Values.maxReplicas | default 3) -}} +{{- if or .Values.hpa.enabled (gt $maxReplicas $minReplicas) }} +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: {{template "basic-service.name" .}}-hpa - namespace: {{.Values.namespace}} + name: {{ template "basic-service.name" . }}-hpa + namespace: {{ .Values.namespace }} labels: - app: {{template "basic-service.name" .}} - {{- if .Values.labels }} - {{- include "basic-service.labels" $ | indent 4}} + app: {{ template "basic-service.name" . }} + {{- with .Values.labels }} + {{- include "basic-service.labels" $ | nindent 4 }} {{- end }} spec: scaleTargetRef: apiVersion: apps/v1 - kind: Deployment - name: {{template "basic-service.name" .}}-deployment - minReplicas: {{.Values.minReplicas | default "1"}} - maxReplicas: {{.Values.maxReplicas | default "3"}} - targetCPUUtilizationPercentage: {{.Values.targetCPUUtilizationPercentage | default "50"}} + kind: {{ if .Values.deployStatefulSet }}StatefulSet{{ else }}Deployment{{ end }} + name: {{ if .Values.deployStatefulSet }}{{ template "basic-service.name" . }}-sts{{ else }}{{ template "basic-service.name" . }}-deployment{{ end }} + minReplicas: {{ $minReplicas }} + maxReplicas: {{ $maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.hpa.targetCPUUtilizationPercentage | default .Values.targetCPUUtilizationPercentage | default 50 }} + {{- $memoryTarget := .Values.hpa.targetMemoryUtilizationPercentage | default .Values.targetMemoryUtilizationPercentage }} + {{- if $memoryTarget }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ $memoryTarget }} + {{- end }} + {{- with .Values.hpa.metrics }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 4661ec9..f3bd405 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,4 +1,4 @@ -{{- if .Values.ingress.enabled -}} +{{- if and .Values.ingress .Values.ingress.enabled -}} {{- $fullName := include "basic-service.name" . -}} {{- $svcPort := .Values.servicePort -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} diff --git a/helm/templates/poddisruptionbudget.yaml b/helm/templates/poddisruptionbudget.yaml index 16d5134..c4411da 100644 --- a/helm/templates/poddisruptionbudget.yaml +++ b/helm/templates/poddisruptionbudget.yaml @@ -1,8 +1,9 @@ {{- if .Values.podDisruptionBudget }} -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: {{ include "basic-service.name" . }} + namespace: {{ .Values.namespace }} labels: {{- include "basic-service.labels" . | nindent 4 }} spec: diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index 6d06745..31beec6 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -1,11 +1,16 @@ ---- +{{- if .Values.secretEnv }} apiVersion: v1 kind: Secret metadata: name: {{ include "basic-service.name" . }}-env + namespace: {{ .Values.namespace }} labels: + app: {{ template "basic-service.name" . }} + {{- with .Values.labels }} {{- include "basic-service.labels" . | nindent 4 }} + {{- end }} data: {{- range $key, $value := .Values.secretEnv }} {{ $key }}: {{ $value | b64enc }} {{- end }} +{{- end }} diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml index f807d3b..5d485a3 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/service.yaml @@ -12,7 +12,7 @@ metadata: spec: type: {{.Values.serviceType}} ports: - - name: tcp-{{template "basic-service.name" .}} + - name: {{ if .Values.serviceMonitor.enabled }}{{ .Values.serviceMonitor.port | default "http" }}{{ else }}tcp-{{ template "basic-service.name" . }}{{ end }} port: {{.Values.servicePort}} targetPort: {{.Values.serviceTargetPort}} protocol: TCP diff --git a/helm/templates/servicemonitor.yaml b/helm/templates/servicemonitor.yaml index 9fc3271..6c81ee8 100644 --- a/helm/templates/servicemonitor.yaml +++ b/helm/templates/servicemonitor.yaml @@ -2,58 +2,65 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: {{ include "basic-service.serviceAccountName" . }} - {{- if .Values.serviceMonitor.namespace }} - namespace: {{ .Values.serviceMonitor.namespace }} - {{- end }} + name: {{ include "basic-service.name" . }} + namespace: {{ .Values.serviceMonitor.namespace | default .Values.namespace }} labels: {{- include "basic-service.labels" . | nindent 4 }} - {{- if .Values.serviceMonitor.labels }} - {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} {{- end }} - {{- if .Values.serviceMonitor.annotations }} + {{- with .Values.serviceMonitor.annotations }} annotations: - {{ toYaml .Values.serviceMonitor.annotations | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} spec: - endpoints: - - interval: {{ .Values.serviceMonitor.interval }} - {{- if .Values.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} - {{- end }} - honorLabels: true - port: metrics - path: {{ .Values.serviceMonitor.path }} - scheme: {{ .Values.serviceMonitor.scheme }} - {{- if .Values.serviceMonitor.tlsConfig }} - tlsConfig: - {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 6 }} - {{- end }} - {{- if .Values.serviceMonitor.relabelings }} - relabelings: - {{- toYaml .Values.serviceMonitor.relabelings | nindent 4 }} - {{- end }} - - interval: {{ .Values.serviceMonitor.interval }} - {{- if .Values.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} - {{- end }} - honorLabels: true - port: metrics-rpcd - path: {{ .Values.serviceMonitor.path }} - scheme: {{ .Values.serviceMonitor.scheme }} - {{- if .Values.serviceMonitor.tlsConfig }} - tlsConfig: - {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 6 }} - {{- end }} - {{- if .Values.serviceMonitor.relabelings }} - relabelings: - {{- toYaml .Values.serviceMonitor.relabelings | nindent 4 }} - {{- end }} - jobLabel: "{{ .Release.Name }}" + jobLabel: {{ .Values.serviceMonitor.jobLabel | default .Release.Name | quote }} selector: matchLabels: - {{- include "basic-service.selectorLabels" . | nindent 8 }} + {{- include "basic-service.selectorLabels" . | nindent 6 }} namespaceSelector: matchNames: - - {{ .Release.Namespace }} + - {{ .Values.namespace }} + endpoints: + {{- if .Values.serviceMonitor.endpoints }} + {{- range .Values.serviceMonitor.endpoints }} + - port: {{ .port | default "http" }} + path: {{ .path | default "/metrics" }} + interval: {{ .interval | default $.Values.serviceMonitor.interval | default "30s" }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- else }} + {{- with $.Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- end }} + honorLabels: {{ .honorLabels | default true }} + scheme: {{ .scheme | default "http" }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- else }} + - port: {{ .Values.serviceMonitor.port | default "http" }} + path: {{ .Values.serviceMonitor.path | default "/metrics" }} + interval: {{ .Values.serviceMonitor.interval | default "30s" }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + scheme: {{ .Values.serviceMonitor.scheme | default "http" }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} {{- end }} diff --git a/helm/templates/statefulset.yaml b/helm/templates/statefulset.yaml index 997f76e..0947619 100644 --- a/helm/templates/statefulset.yaml +++ b/helm/templates/statefulset.yaml @@ -1,122 +1,59 @@ -{{- if not .Values.deployStatefulSet}} +{{- if .Values.deployStatefulSet }} apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{template "basic-service.name" .}}-sts - namespace: {{.Values.namespace}} + name: {{ template "basic-service.name" . }}-sts + namespace: {{ .Values.namespace }} labels: - {{- include "basic-service.labels" $ | indent 8 }} + app: {{ template "basic-service.name" . }} + {{- with .Values.labels }} + {{- include "basic-service.labels" $ | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} annotations: - {{- tpl (toYaml .Values.annotations) . | nindent 4 }} + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} spec: - podManagementPolicy: {{ .Values.podManagementPolicy }} - replicas: {{int .Values.minReplicas | default 0}} + serviceName: {{ template "basic-service.name" . }}-headless + podManagementPolicy: {{ .Values.podManagementPolicy | default "OrderedReady" }} + replicas: {{ int .Values.minReplicas | default 1 }} selector: matchLabels: - app: {{template "basic-service.name" .}} {{- include "basic-service.selectorLabels" . | nindent 6 }} + {{- with .Values.updateStrategy }} updateStrategy: - {{- toYaml .Values.updateStrategy | nindent 4 }} + {{- toYaml . | nindent 4 }} + {{- end }} template: metadata: labels: - app: {{template "basic-service.name" .}} - {{- if .Values.labels }} - {{- include "basic-service.labels" $ | indent 8 }} + {{- include "basic-service.selectorLabels" . | nindent 8 }} + {{- with .Values.labels }} + {{- include "basic-service.labels" $ | nindent 8 }} {{- end }} annotations: checksum/secrets: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} - {{- tpl (toYaml .Values.annotations) . | nindent 4 }} - spec: - volumes: - {{- range $volumeName, $volumeValue := .Values.volumesFromConfigMaps }} - - name: {{$volumeName}} - configMap: - name: {{template "basic-service.name" $}}-{{$volumeValue.configMapName}} - {{- end }} - {{- range ., $value := .Values.extraVolumes }} - {{- toYaml $value | nindent 8}} - {{- end }} - {{- if .Values.serviceAccountName }} - serviceAccountName: {{.Values.serviceAccountName }} - {{- end }} - {{- if .Values.initContainers }} - initContainers: - {{- range ., $value := .Values.initContainers }} - {{- toYaml $value | nindent 6}} - {{- end }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{template "basic-service.name" .}} - image: {{.Values.image}} - {{- if .Values.command }} - command: {{ toYaml .Values.command | nindent 8 }} - {{- end }} - {{- if .Values.args }} - args: {{ toYaml .Values.args | nindent 8 }} + {{- with .Values.annotations }} + {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - securityContext: - {{- toYaml .Values.containerSecurityContext | nindent 12 }} + spec: + {{- include "basic-service.podSpec" . | nindent 6 }} + volumeClaimTemplates: + - metadata: + name: {{ tpl .Values.statefulSetOptions.name . }} + {{- with .Values.statefulSetOptions.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + - {{ .Values.statefulSetOptions.pvcAccessMode | default .Values.statefulSetOptions.accessMode | default "ReadWriteOnce" }} resources: - {{- toYaml .Values.resources | nindent 12 }} - livenessProbe: - {{- toYaml .Values.livenessProbeRPCDaemon | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbeRPCDaemon | nindent 12 }} - volumeMounts: - {{- if .Values.deployStatefulSet }} - - mountPath: {{.Values.statefulSetOptions.mountPath}} - name: {{ tpl .Values.statefulSetOptions.name .}} - {{- end }} - {{- range ., $value := .Values.extraVolumeMounts }} - {{- toYaml $value | nindent 10}} + requests: + storage: {{ .Values.statefulSetOptions.pvcStorageCapacity | default .Values.statefulSetOptions.storageCapacity | default "1Gi" }} + storageClassName: {{ .Values.statefulSetOptions.pvcStorageClass | default .Values.statefulSetOptions.storageClass }} + {{- with .Values.statefulSetOptions.selector }} + selector: + {{- toYaml . | nindent 10 }} {{- end }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - {{- range $key, $value := .Values.secretEnv }} - - name: {{ $key }} - valueFrom: - secretKeyRef: - name: {{ include "basic-service.name" $ }}-env - key: {{ $key }} - {{- end }} - {{- if .Values.extraEnv }} - {{- toYaml .Values.extraEnv | nindent 12 }} - {{- end }} - {{- if .Values.extraContainers }} - {{- range $key, $value := .Values.extraContainers }} - {{- tpl (toYaml $value) $ | nindent 6}} - {{- end }} - {{- end }} - nodeSelector: - {{- toYaml .Values.nodeSelector | nindent 8 }} - affinity: - {{- toYaml .Values.affinity | nindent 8 }} - tolerations: - {{- toYaml .Values.tolerations | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - {{- end}} - volumeClaimTemplates: - - metadata: - name: {{ tpl .Values.statefulSetOptions.name .}} - annotations: - {{- toYaml .Values.statefulSetOptions.annotations | nindent 8 }} - spec: - accessModes: - - {{.Values.statefulSetOptions.accessMode}} - resources: - requests: - storage: {{.Values.statefulSetOptions.storageCapacity}} - storageClassName: {{.Values.statefulSetOptions.storageClass}} - {{- if .Values.statefulSetOptions.selector }} - selector: - {{- toYaml .Values.statefulSetOptions.selector | nindent 8 }} - {{- end }} {{- end }} diff --git a/helm/templates/tests/test-connect.yaml b/helm/templates/tests/test-connect.yaml index 67e6cc8..3f24029 100644 --- a/helm/templates/tests/test-connect.yaml +++ b/helm/templates/tests/test-connect.yaml @@ -3,7 +3,7 @@ kind: Pod metadata: name: "{{ include "basic-service.fullname" . }}-test-connection" labels: - {{- include "baasic-service.labels" . | nindent 4 }} + {{- include "basic-service.labels" . | nindent 4 }} annotations: "helm.sh/hook": test spec: diff --git a/helm/values-test.yaml b/helm/values-test.yaml index c92ca25..760e812 100644 --- a/helm/values-test.yaml +++ b/helm/values-test.yaml @@ -6,7 +6,7 @@ commonName: foo-service-{{.Values.id}}.t-foo-test namespace: foo createNamespace: true -image: +image: busybox:1.36 command: ["sleep"] args: ["infinity"] @@ -93,11 +93,10 @@ pvcName: ebs pvcStorageClass: gp3-ext4 pvcStorageCapacity: 1Gi -deployStategy: - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 +deployStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 storageClass: - create: true diff --git a/helm/values.yaml b/helm/values.yaml index 407b3f3..cd93a9b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,7 +1,8 @@ name: foo-service commonName: foo-service.foo namespace: foo -image: +clusterName: foo-cluster +image: nginx:1.27 command: [] args: [] dockercfg: @@ -16,10 +17,39 @@ minReplicas: 1 maxReplicas: 3 createNamespace: true +hpa: + enabled: false + targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 80 + metrics: [] + env: config: PLACEHOLDER: placeholder +secretEnv: {} +extraEnv: [] + +resources: {} +containerSecurityContext: {} + +livenessProbe: {} +readinessProbe: {} +startupProbe: {} + +# Deprecated: use livenessProbe / readinessProbe instead. +livenessProbeRPCDaemon: {} +readinessProbeRPCDaemon: {} + +ingress: + enabled: false + +serviceMonitor: + enabled: false + port: http + path: /metrics + interval: 30s + # see: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Container initContainers: {} @@ -72,13 +102,16 @@ serviceRoles: {} # see: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ deployStatefulSet: false +podManagementPolicy: OrderedReady statefulSetOptions: + name: data + mountPath: /data pvcName: ebs pvcAccessMode: ReadWriteOnce pvcStorageClass: gp3-ext4 pvcStorageCapacity: 1Gi -deployStategy: {} +deployStrategy: {} # [Deployment] # strategy: # type: RollingUpdate @@ -87,6 +120,12 @@ deployStategy: {} # [StatefulSet] # updateStrategy: # type: RollingUpdate +updateStrategy: {} + +nodeSelector: {} +affinity: {} +tolerations: [] +terminationGracePeriodSeconds: 30 storageClass: [] @@ -125,3 +164,6 @@ persistentVolume: # driver: efs.csi.aws.com # volumeHandle: "fs-123456789" # subPath: "mount2" + +# Deprecated: use hpa.targetCPUUtilizationPercentage instead. +targetCPUUtilizationPercentage: 50 diff --git a/tasks/common/custom-facts.yml b/tasks/common/custom-facts.yml index f75014c..0d76407 100644 --- a/tasks/common/custom-facts.yml +++ b/tasks/common/custom-facts.yml @@ -9,3 +9,7 @@ - name: Load container service facts ansible.builtin.include_tasks: facts-container.yml when: setup_mode == 'container' + +- name: Load Kubernetes service facts + ansible.builtin.include_tasks: facts-k8s.yml + when: setup_mode == 'k8s' diff --git a/tasks/common/facts-k8s.yml b/tasks/common/facts-k8s.yml new file mode 100644 index 0000000..554cf0a --- /dev/null +++ b/tasks/common/facts-k8s.yml @@ -0,0 +1,71 @@ +--- +- name: Derive Kubernetes service port from role ports + ansible.builtin.set_fact: + _k8s_service_port: >- + {{ + (ports | dict2items | first).value.ingressPort + | default((ports | dict2items | first).value.servicePort) + | string + }} + _k8s_service_target_port: "{{ (ports | dict2items | first).value.servicePort | string }}" + when: ports is defined and ports | length > 0 + +- name: Derive Kubernetes CPU and memory resources + ansible.builtin.set_fact: + _k8s_cpu_millicores: "{{ (cpus | float * 1000) | int }}" + _k8s_memory: "{{ memory | regex_replace('(?i)^([0-9.]+)M$', '\\1Mi') | regex_replace('(?i)^([0-9.]+)G$', '\\1Gi') | regex_replace('(?i)^([0-9.]+)g$', '\\1Gi') }}" + when: + - cpus is defined + - memory is defined + +- name: Build Kubernetes resource requests and limits + ansible.builtin.set_fact: + k8s_resources: + requests: + cpu: "{{ _k8s_cpu_millicores }}m" + memory: "{{ _k8s_memory }}" + limits: + cpu: "{{ _k8s_cpu_millicores }}m" + memory: "{{ _k8s_memory }}" + when: + - _k8s_cpu_millicores is defined + - _k8s_memory is defined + +- name: Parse Kubernetes command string into command and args + ansible.builtin.set_fact: + k8s_command: "{{ [command.split()[0]] }}" + k8s_args: "{{ command.split()[1:] }}" + when: + - command is defined + - command is not none + - command is string + - command | length > 0 + - command.split() | length > 1 + +- name: Parse Kubernetes command string as single executable + ansible.builtin.set_fact: + k8s_command: "{{ [command] }}" + when: + - command is defined + - command is not none + - command is string + - command | length > 0 + - command.split() | length <= 1 + +- name: Use Kubernetes command list as-is + ansible.builtin.set_fact: + k8s_command: "{{ command }}" + when: + - command is defined + - command is not none + - command is sequence + - command | length > 0 + +- name: Use Kubernetes args list as-is + ansible.builtin.set_fact: + k8s_args: "{{ args }}" + when: + - args is defined + - args is not none + - args is sequence + - args | length > 0 diff --git a/tasks/k8s/render-values.yml b/tasks/k8s/render-values.yml new file mode 100644 index 0000000..7aca860 --- /dev/null +++ b/tasks/k8s/render-values.yml @@ -0,0 +1,34 @@ +--- +- name: Set rendered Helm values path + ansible.builtin.set_fact: + _helm_rendered_values_path: "{{ helm_rendered_values_path | default('/tmp/' + name + '-helm-values.yaml', true) }}" + _helm_chart_ref: "{{ role_path ~ '/helm' if helm_chart_path in ['../../helm', '../helm', 'helm'] else helm_chart_path }}" + +- name: Render Helm values from role variables + ansible.builtin.template: + src: k8s-values.yaml.j2 + dest: "{{ _helm_rendered_values_path }}" + mode: "0644" + when: helm_render_values_from_role | default(true) | bool + +- name: Resolve optional Helm values overlay path + ansible.builtin.set_fact: + _helm_values_overlay: >- + {{ + helm_values_path + if (helm_values_path is match('^/')) + else (role_path ~ '/' ~ helm_values_path) + }} + when: helm_values_path | default('', true) | length > 0 + +- name: Build Helm values file list + ansible.builtin.set_fact: + _helm_values_files: >- + {{ + ( + [_helm_rendered_values_path] + if (helm_render_values_from_role | default(true) | bool) + else [] + ) + + ([_helm_values_overlay] if (_helm_values_overlay is defined) else []) + }} diff --git a/tasks/k8s/setup.yml b/tasks/k8s/setup.yml index d4a3454..3be6850 100644 --- a/tasks/k8s/setup.yml +++ b/tasks/k8s/setup.yml @@ -30,10 +30,19 @@ args: creates: /usr/local/bin/helm +- name: Render Helm values from role variables + ansible.builtin.include_tasks: render-values.yml + - name: Deploy Helm chart kubernetes.core.helm: name: "{{ name }}" - chart_ref: "{{ helm_chart_path }}" + chart_ref: "{{ _helm_chart_ref }}" release_namespace: "{{ helm_namespace }}" - values_files: - - "{{ helm_values_path }}" + values_files: "{{ _helm_values_files }}" + wait: "{{ helm_wait | default(true) | bool }}" + atomic: "{{ helm_atomic | default(true) | bool }}" + timeout: "{{ helm_timeout | default('10m') }}" + create_namespace: "{{ helm_create_namespace | default(true) | bool }}" + kubeconfig: "{{ lookup('env', 'KUBECONFIG') | default(omit, true) }}" + context: "{{ lookup('env', 'KUBE_CONTEXT') | default(omit, true) }}" + update_repo_cache: false diff --git a/templates/k8s-values.yaml.j2 b/templates/k8s-values.yaml.j2 new file mode 100644 index 0000000..2c6f678 --- /dev/null +++ b/templates/k8s-values.yaml.j2 @@ -0,0 +1,125 @@ +# Ansible-rendered Helm values for {{ name }} +name: {{ name }} +commonName: {{ k8s_common_name | default(name) }} +namespace: {{ helm_namespace }} +clusterName: {{ k8s_cluster_name | default('default-cluster') }} +createNamespace: {{ k8s_chart_create_namespace | default(false) | bool }} + +image: {{ image | default('') | to_json }} +{% if k8s_command is defined and k8s_command | length > 0 %} +command: {{ k8s_command | to_json }} +{% endif %} +{% if k8s_args is defined and k8s_args | length > 0 %} +args: {{ k8s_args | to_json }} +{% endif %} + +minReplicas: {{ k8s_min_replicas | default(1) }} +maxReplicas: {{ k8s_max_replicas | default(1) }} + +{% if config_env is defined and config_env | length > 0 %} +env: + config: {{ config_env | to_json }} +{% endif %} + +{% if k8s_resources is defined %} +resources: {{ k8s_resources | to_json }} +{% endif %} + +{% if config is defined and config | length > 0 %} +configMaps: +{% for key, value in config.items() %} + {{ key }}: +{% if value.destinationPath is defined and value.destinationPath | regex_search('\\.[^.]+$') %} + file: {{ value.destinationPath | basename }} +{% endif %} + data: | +{{ value.data | default('') | indent(width=6, first=True) }} +{% endfor %} +{% endif %} + +{% if config is defined and config | length > 0 %} +volumesFromConfigMaps: +{% for key, value in config.items() %} + {{ key }}-volume: + configMapName: {{ key }} +{% endfor %} +{% endif %} +{% if data_dirs is defined and data_dirs | length > 0 %} +extraVolumes: +{% for key, value in data_dirs.items() %} + {{ key }}-volume: + - name: {{ key }}-data + emptyDir: {} +{% endfor %} +{% endif %} +{% if (config is defined and config | length > 0) or (data_dirs is defined and data_dirs | length > 0) %} +extraVolumeMounts: +{% if config is defined and config | length > 0 %} +{% for key, value in config.items() %} + {{ key }}-mount: + - name: {{ key }}-volume + mountPath: {{ value.destinationPath }} +{% endfor %} +{% endif %} +{% if data_dirs is defined and data_dirs | length > 0 %} +{% for key, value in data_dirs.items() %} + {{ key }}-data-mount: + - name: {{ key }}-data + mountPath: {{ value.appPath }} +{% endfor %} +{% endif %} +{% endif %} + +{% if _k8s_service_port is defined %} +serviceType: {{ k8s_service_type | default('ClusterIP') }} +servicePort: {{ _k8s_service_port }} +serviceTargetPort: {{ _k8s_service_target_port | default(_k8s_service_port) }} +{% endif %} + +{% if k8s_ingress_enabled | default(false) | bool %} +ingress: + enabled: true + className: {{ k8s_ingress_class | default('nginx') }} + hosts: + - host: {{ k8s_ingress_host | default(k8s_common_name | default(name)) }} + paths: + - path: / + pathType: Prefix +{% endif %} + +deployStatefulSet: {{ k8s_deploy_stateful_set | default(false) | bool }} +hpa: + enabled: {{ k8s_hpa_enabled | default(false) | bool }} + targetCPUUtilizationPercentage: {{ k8s_hpa_target_cpu | default(50) }} + +{% if k8s_liveness_probe is defined %} +livenessProbe: {{ k8s_liveness_probe | to_json }} +{% elif _k8s_service_target_port is defined %} +livenessProbe: + httpGet: + path: {{ k8s_health_check_path | default('/') }} + port: {{ _k8s_service_target_port }} + initialDelaySeconds: 15 + periodSeconds: 20 +{% endif %} + +{% if k8s_readiness_probe is defined %} +readinessProbe: {{ k8s_readiness_probe | to_json }} +{% elif _k8s_service_target_port is defined %} +readinessProbe: + httpGet: + path: {{ k8s_health_check_path | default('/') }} + port: {{ _k8s_service_target_port }} + initialDelaySeconds: 5 + periodSeconds: 10 +{% endif %} + +{% if k8s_service_monitor_enabled | default(false) | bool %} +serviceMonitor: + enabled: true + port: http + path: {{ k8s_service_monitor_path | default('/metrics') }} +{% endif %} + +labels: {{ k8s_labels | default({}) | to_json }} +annotations: {{ k8s_annotations | default({}) | to_json }} diff --git a/tests/fixtures/k8s-rendered-values.yaml b/tests/fixtures/k8s-rendered-values.yaml new file mode 100644 index 0000000..a36ade6 --- /dev/null +++ b/tests/fixtures/k8s-rendered-values.yaml @@ -0,0 +1,33 @@ +--- +name: fixture-service +commonName: fixture-service.local +namespace: fixture +clusterName: fixture-cluster +createNamespace: true +image: nginx:1.27 +command: ["/bin/sh", "-c"] +args: ["nginx -g 'daemon off;'"] +minReplicas: 1 +maxReplicas: 3 +env: + config: + EXAMPLE: value +serviceType: ClusterIP +servicePort: 8080 +serviceTargetPort: 80 +deployStatefulSet: false +hpa: + enabled: true + targetCPUUtilizationPercentage: 60 +livenessProbe: + httpGet: + path: / + port: 80 +readinessProbe: + httpGet: + path: / + port: 80 +serviceMonitor: + enabled: true + port: http + path: /metrics diff --git a/tests/helm-validate.sh b/tests/helm-validate.sh new file mode 100755 index 0000000..7c8bc0c --- /dev/null +++ b/tests/helm-validate.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CHART_DIR="${ROOT_DIR}/helm" +RELEASE_NAME="basic-service-ci" + +if ! command -v helm >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +fi + +helm lint "${CHART_DIR}" +helm template "${RELEASE_NAME}" "${CHART_DIR}" -f "${CHART_DIR}/values.yaml" >/dev/null +helm template "${RELEASE_NAME}" "${CHART_DIR}" -f "${CHART_DIR}/values-test.yaml" >/dev/null + +RENDERED_VALUES="${ROOT_DIR}/tests/fixtures/k8s-rendered-values.yaml" +if [[ -f "${RENDERED_VALUES}" ]]; then + helm template "${RELEASE_NAME}" "${CHART_DIR}" -f "${RENDERED_VALUES}" >/dev/null +fi + +echo "Helm chart validation succeeded." diff --git a/tests/molecule/cleanup-kind.yml b/tests/molecule/cleanup-kind.yml new file mode 100644 index 0000000..c1c9737 --- /dev/null +++ b/tests/molecule/cleanup-kind.yml @@ -0,0 +1,18 @@ +--- +- name: Cleanup kind Kubernetes cluster + hosts: localhost + connection: local + gather_facts: false + vars: + kind_cluster_name: molecule-k8s + kind_kubeconfig_path: /tmp/molecule-k8s-kubeconfig + tasks: + - name: Delete kind cluster + ansible.builtin.command: kind delete cluster --name {{ kind_cluster_name }} + failed_when: false + changed_when: false + + - name: Remove kubeconfig + ansible.builtin.file: + path: "{{ kind_kubeconfig_path }}" + state: absent diff --git a/tests/molecule/container-binary-service/converge.yml b/tests/molecule/container-binary-service/converge.yml new file mode 100644 index 0000000..7182a1e --- /dev/null +++ b/tests/molecule/container-binary-service/converge.yml @@ -0,0 +1,43 @@ +--- +- name: Converge + hosts: all + become: true + vars: + ansible_user: root + roles: + - role: basic-service + vars: + setup_mode: container + name: binary-service-container + image: debian:bookworm-slim + binary_url: https://github.com/prometheus/prometheus/releases/download/v2.47.0/prometheus-2.47.0.linux-amd64.tar.gz + binary_strip_components: 1 + binary_file_name_override: prometheus + destination_directory: /tmp/molecule-service-binary + binary_app_path: /usr/local/bin + command: > + /usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml + --storage.tsdb.path=/prometheus --web.listen-address=0.0.0.0:9090 + cpus: 0.5 + memory: 512M + network_mode: bridge + ports: + service: + ingressPort: 9090 + servicePort: 9090 + host_data_dir: /test/mnt + config: + prometheus.yml: + destinationPath: /etc/prometheus/prometheus.yml + data: | + global: + scrape_interval: 15s + scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + data_dirs: + service-data: + hostPath: /test/mnt/data + appPath: /prometheus + restart_policy: on-failure diff --git a/tests/molecule/container-binary-service/molecule.yml b/tests/molecule/container-binary-service/molecule.yml new file mode 100644 index 0000000..fa650fc --- /dev/null +++ b/tests/molecule/container-binary-service/molecule.yml @@ -0,0 +1,37 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: instance + image: docker:24.0.5-dind + privileged: true + network_mode: host + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/molecule-service-binary:/tmp/molecule-service-binary + - /test/mnt:/test/mnt +provisioner: + name: ansible + playbooks: + prepare: ../prepare-docker.yml + env: + ANSIBLE_ROLES_PATH: "../../../../" + ANSIBLE_ALLOW_BROKEN_CONDITIONALS: "True" +verifier: + name: testinfra + directory: ./tests +scenario: + name: container-binary-service + test_sequence: + - dependency + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - destroy diff --git a/tests/molecule/container-binary-service/tests/test_container_binary_service.py b/tests/molecule/container-binary-service/tests/test_container_binary_service.py new file mode 100644 index 0000000..f98deea --- /dev/null +++ b/tests/molecule/container-binary-service/tests/test_container_binary_service.py @@ -0,0 +1,54 @@ +import time + +SERVICE_CONTAINER = "binary-service-container" +SERVICE_PORT = 9090 +BINARY_STAGING_DIR = "/tmp/molecule-service-binary" +READINESS_CMD = ( + "python3 -c \"import urllib.request; " + f"urllib.request.urlopen('http://127.0.0.1:{SERVICE_PORT}/-/ready')\"" +) + + +def test_long_running_service_container_is_running(host): + container = host.docker(SERVICE_CONTAINER) + assert container.is_running + + +def test_service_binary_staged_on_host(host): + binary = host.file(f"{BINARY_STAGING_DIR}/prometheus") + assert binary.exists, "Service binary was not downloaded to the host staging directory." + assert binary.mode == 0o755, "Service binary is not executable." + + +def test_service_config_is_mounted(host): + container = host.get_host(f"docker://{SERVICE_CONTAINER}") + config_file = container.file("/etc/prometheus/prometheus.yml") + assert config_file.exists, "Service configuration file is missing in the container." + + +def test_service_data_directory_is_mounted(host): + container = host.get_host(f"docker://{SERVICE_CONTAINER}") + result = container.run("test -d /prometheus") + assert result.rc == 0, "Service data directory is not mounted in the container." + + +def test_service_publishes_port(host): + socket = host.socket(f"tcp://0.0.0.0:{SERVICE_PORT}") + assert socket.is_listening + + +def test_service_readiness_endpoint(host): + for _ in range(30): + result = host.run(READINESS_CMD) + if result.rc == 0: + return + time.sleep(1) + assert False, f"Service readiness check failed: {result.stderr}" + + +def test_service_container_restart_policy(host): + container = host.docker(SERVICE_CONTAINER) + restart_policy = container.inspect().get("HostConfig", {}).get("RestartPolicy", {}).get("Name", "") + assert restart_policy == "on-failure", ( + f"Container restart policy is not 'on-failure', it is '{restart_policy}'." + ) diff --git a/tests/molecule/k8s-basic/converge.yml b/tests/molecule/k8s-basic/converge.yml new file mode 100644 index 0000000..73cfc9a --- /dev/null +++ b/tests/molecule/k8s-basic/converge.yml @@ -0,0 +1,21 @@ +--- +- name: Converge + hosts: localhost + connection: local + gather_facts: true + roles: + - role: basic-service + vars: + setup_mode: k8s + name: test-k8s-service + image: nginx:1.27-alpine + helm_namespace: molecule-k8s + helm_create_namespace: true + k8s_common_name: test-k8s-service.local + ports: + web: + ingressPort: 80 + servicePort: 80 + cpus: 0.25 + memory: 128M + k8s_health_check_path: / diff --git a/tests/molecule/k8s-basic/molecule.yml b/tests/molecule/k8s-basic/molecule.yml new file mode 100644 index 0000000..ae43942 --- /dev/null +++ b/tests/molecule/k8s-basic/molecule.yml @@ -0,0 +1,40 @@ +--- +dependency: + name: galaxy + options: + role-file: ../requirements.yml +driver: + name: default + options: + managed: false +platforms: + - name: localhost + connection: local +provisioner: + name: ansible + connection_options: + ansible_connection: local + playbooks: + prepare: ../prepare-kind.yml + cleanup: ../cleanup-kind.yml + destroy: ../cleanup-kind.yml + env: + ANSIBLE_ROLES_PATH: "../../../../" + ANSIBLE_ALLOW_BROKEN_CONDITIONALS: "True" + KUBECONFIG: /tmp/molecule-k8s-kubeconfig +verifier: + name: testinfra + directory: ./tests +scenario: + name: k8s-basic + test_sequence: + - dependency + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - destroy diff --git a/tests/molecule/k8s-basic/tests/test_k8s_basic.py b/tests/molecule/k8s-basic/tests/test_k8s_basic.py new file mode 100644 index 0000000..af2d224 --- /dev/null +++ b/tests/molecule/k8s-basic/tests/test_k8s_basic.py @@ -0,0 +1,56 @@ +import json + +NAMESPACE = "molecule-k8s" +RELEASE = "test-k8s-service" +DEPLOYMENT = f"{RELEASE}-deployment" +SERVICE = f"{RELEASE}-svc" + + +def test_namespace_exists(host): + result = host.run(f"kubectl get namespace {NAMESPACE}") + assert result.rc == 0, f"Namespace missing: {result.stderr}" + + +def test_helm_release_is_deployed(host): + result = host.run(f"helm list -n {NAMESPACE} -o json") + assert result.rc == 0, f"helm list failed: {result.stderr}" + releases = json.loads(result.stdout) + assert any(item.get("name") == RELEASE for item in releases), "Helm release was not deployed." + + +def test_deployment_is_ready(host): + result = host.run( + f"kubectl -n {NAMESPACE} rollout status deploy/{DEPLOYMENT} --timeout=180s" + ) + assert result.rc == 0, f"Deployment not ready: {result.stderr}" + + +def test_service_has_endpoints(host): + result = host.run( + f"kubectl -n {NAMESPACE} get endpoints {SERVICE} " + "-o jsonpath='{.subsets[0].addresses[0].ip}'" + ) + assert result.rc == 0, f"Service endpoints lookup failed: {result.stderr}" + assert result.stdout.strip(), "Service has no ready endpoints." + + +def test_service_responds_in_cluster(host): + run_pod = host.run( + f"kubectl -n {NAMESPACE} run curl-test " + "--image=curlimages/curl:8.5.0 --restart=Never " + "--command -- sleep 3600" + ) + assert run_pod.rc == 0, f"Failed to start curl test pod: {run_pod.stderr}" + + wait_pod = host.run( + f"kubectl -n {NAMESPACE} wait --for=condition=ready pod/curl-test --timeout=120s" + ) + assert wait_pod.rc == 0, f"curl test pod not ready: {wait_pod.stderr}" + + curl = host.run( + f"kubectl -n {NAMESPACE} exec curl-test -- curl -sf http://{SERVICE}/" + ) + assert curl.rc == 0, f"In-cluster service request failed: {curl.stderr}" + assert "Welcome to nginx" in curl.stdout, "Service did not return the nginx welcome page." + + host.run(f"kubectl -n {NAMESPACE} delete pod curl-test --wait=true") diff --git a/tests/molecule/prepare-docker.yml b/tests/molecule/prepare-docker.yml index cca9f6d..d7d1cf6 100644 --- a/tests/molecule/prepare-docker.yml +++ b/tests/molecule/prepare-docker.yml @@ -6,8 +6,8 @@ - name: Ensure apk is updated raw: apk update - - name: Install system dependencies (sudo, Python, pip) - raw: apk add --no-cache sudo python3 py3-pip + - name: Install system dependencies (sudo, Python, pip, tar) + raw: apk add --no-cache sudo python3 py3-pip tar - name: Install python dependencies raw: pip3 install --break-system-packages --ignore-installed PyYAML pytest-testinfra molecule-docker 'requests==2.31.0' 'urllib3<2' 'docker>=6,<7' diff --git a/tests/molecule/prepare-kind.yml b/tests/molecule/prepare-kind.yml new file mode 100644 index 0000000..cbf75a3 --- /dev/null +++ b/tests/molecule/prepare-kind.yml @@ -0,0 +1,70 @@ +--- +- name: Prepare kind Kubernetes cluster + hosts: localhost + connection: local + gather_facts: true + vars: + kind_cluster_name: molecule-k8s + kind_kubeconfig_path: /tmp/molecule-k8s-kubeconfig + kind_version: v0.27.0 + tasks: + - name: Install kind + ansible.builtin.get_url: + url: "https://kind.sigs.k8s.io/dl/{{ kind_version }}/kind-linux-amd64" + dest: /usr/local/bin/kind + mode: "0755" + become: true + + - name: Lookup stable kubectl version + ansible.builtin.uri: + url: https://dl.k8s.io/release/stable.txt + return_content: true + register: kubectl_stable_version + + - name: Install kubectl + ansible.builtin.get_url: + url: >- + https://dl.k8s.io/release/{{ kubectl_stable_version.content | trim }}/bin/linux/amd64/kubectl + dest: /usr/local/bin/kubectl + mode: "0755" + become: true + + - name: Check whether Helm is installed + ansible.builtin.command: helm version + register: helm_version + failed_when: false + changed_when: false + + - name: Install Helm + ansible.builtin.shell: curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + when: helm_version.rc != 0 + args: + creates: /usr/local/bin/helm + executable: /bin/bash + become: true + + - name: Install Kubernetes Python library + ansible.builtin.pip: + name: kubernetes>=28.1.0 + extra_args: --break-system-packages + + - name: Delete existing kind cluster + ansible.builtin.command: kind delete cluster --name {{ kind_cluster_name }} + failed_when: false + changed_when: false + + - name: Create kind cluster + ansible.builtin.command: kind create cluster --name {{ kind_cluster_name }} --wait 5m + changed_when: true + + - name: Write kubeconfig for Ansible + ansible.builtin.command: + cmd: kind get kubeconfig --name {{ kind_cluster_name }} + register: kind_kubeconfig + changed_when: true + + - name: Save kubeconfig for Ansible + ansible.builtin.copy: + content: "{{ kind_kubeconfig.stdout }}" + dest: "{{ kind_kubeconfig_path }}" + mode: "0600" diff --git a/tests/molecule/requirements.yml b/tests/molecule/requirements.yml index 4cb6e1e..4dac4dd 100644 --- a/tests/molecule/requirements.yml +++ b/tests/molecule/requirements.yml @@ -3,3 +3,8 @@ roles: - name: ansible-role-systemd src: https://github.com/O1labs/ansible-role-systemd.git version: v0.3.9 + +collections: + - name: community.docker + - name: community.general + - name: kubernetes.core