From 8c636cfdd9b647677dd39b913c44ef8556e26880 Mon Sep 17 00:00:00 2001 From: Leonardo Vicentini Date: Fri, 5 Jun 2026 17:32:59 +0200 Subject: [PATCH 1/5] feat: migrate to native pg function rather than extension --- .github/workflows/test.yml | 28 + .gitignore | 2 + README.md | 9 +- TESTING.md | 155 ++++++ go.mod | 44 ++ go.sum | 119 +++++ internal/config/assets/k8s_events.schema.sql | 3 +- .../migrations/001_k8s_events_event_id.sql | 3 +- internal/pg/byo_test.go | 502 ++++++++++++++++++ internal/pg/testdata/byo_db_owner_setup.sql | 5 + .../pg/testdata/byo_schema_grants_grants.sql | 6 + .../pg/testdata/byo_schema_grants_setup.sql | 6 + 12 files changed, 878 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 TESTING.md create mode 100644 internal/pg/byo_test.go create mode 100644 internal/pg/testdata/byo_db_owner_setup.sql create mode 100644 internal/pg/testdata/byo_schema_grants_grants.sql create mode 100644 internal/pg/testdata/byo_schema_grants_setup.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f84e113 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: test + +on: + push: + branches: ['main'] + pull_request: + +jobs: + test: + name: Vet and test + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Vet + run: go vet ./... + + # Integration tests (TestBYOPostgres) spin up PostgreSQL containers + # via testcontainers; Docker is available on GitHub-hosted runners. + - name: Test + run: go test ./... -count=1 -timeout 20m diff --git a/.gitignore b/.gitignore index b89ed85..c302e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ experiments/** **/token.txt **/*.crt **/*.key + +.claude diff --git a/README.md b/README.md index cda9880..81e9895 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,9 @@ It ensures that historical event data is properly managed without manual interve ## Requirements - Kubernetes cluster (optional; service can run outside of cluster) -- PostgreSQL database +- PostgreSQL **13 or later** (uses the native `gen_random_uuid()` function; no extensions required) - Network connectivity to the database -- Appropriate database privileges for partition creation and table management +- Appropriate database privileges for partition creation and table management (DDL: tables, indexes, triggers, functions). Superuser access or `CREATE EXTENSION` privileges are **not** required. ## Configuration @@ -169,6 +169,11 @@ The first migration backfills `event_id` on existing rows with a single `UPDATE` > These endpoints are suitable for Kubernetes `livenessProbe` and `readinessProbe`. +## Testing + +See [TESTING.md](TESTING.md) for the test layers (unit, integration against real PostgreSQL, end-to-end on kind), how to run them locally, and the conventions for adding new tests. + + ## Deployment Notes - Store database credentials in Kubernetes Secrets diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..53d639f --- /dev/null +++ b/TESTING.md @@ -0,0 +1,155 @@ +# Testing deviser + +This document describes the test layers of **deviser**, how to run them locally, and the conventions to follow when adding new tests. + +## Overview + +| Layer | What it covers | Requirements | Typical runtime | +|---|---|---|---| +| Unit tests | Pure logic: migration ordering/recording, partition bound parsing, asset loading | Go only | < 1s | +| Integration tests (`TestBYOPostgres`) | Full database lifecycle against real PostgreSQL 13/17/18, including the documented Bring Your Own PostgreSQL provisioning | Go + Docker | ~40s (cached images) | +| End-to-end (`scripts/e2e.sh`) | Container image + Helm chart + live service against PostgreSQL inside a kind cluster | Docker, kind, kubectl, helm, curl | a few minutes | +| CI (`.github/workflows/test.yml`) | `go vet` + unit + integration on every PR and push to `main` | — | — | + +## Prerequisites + +- **Go** — version pinned in [`go.mod`](go.mod) (toolchain auto-downloads if needed). +- **Docker** — required only for the integration tests (they use [testcontainers-go](https://golang.testcontainers.org/) to start disposable PostgreSQL containers) and for the e2e script. Without a running Docker daemon, integration tests **skip automatically** — they never fail for that reason. + +## Running tests + +### Everything + +```sh +go vet ./... +go test ./... -count=1 +``` + +`-count=1` bypasses Go's test result cache. Use it whenever you changed SQL assets, fixtures, or anything outside `.go` files — cached results can otherwise mask a regression. + +### Unit tests only + +Either stop Docker (integration tests skip), or skip them explicitly: + +```sh +go test ./... -skip 'TestBYOPostgres' +``` + +### Integration tests only + +```sh +go test ./internal/pg -run TestBYOPostgres -count=1 -v +``` + +The first run pulls the `postgres:13-alpine`, `postgres:17-alpine` and `postgres:18-alpine` images; subsequent runs reuse them. To pre-pull: + +```sh +docker pull postgres:13-alpine postgres:17-alpine postgres:18-alpine +``` + +A single version or scenario can be selected with the subtest path: + +```sh +go test ./internal/pg -run 'TestBYOPostgres/postgres:17-alpine' -count=1 -v +go test ./internal/pg -run 'TestBYOPostgres/postgres:17-alpine/db-owner' -count=1 -v +``` + +## Test layout + +``` +internal/config/migrations_test.go embedded migration asset discovery and ordering +internal/pg/migrations_test.go migration apply/record logic (fake transaction, no DB) +internal/pg/partitions_test.go partition helpers +internal/pg/partition_manager_test.go partition bound parsing +internal/pg/byo_test.go integration: full lifecycle on real PostgreSQL +internal/pg/testdata/byo_*.sql provisioning fixtures for the BYO scenarios +``` + +## The integration test: `TestBYOPostgres` + +`internal/pg/byo_test.go` verifies that the **exact provisioning SQL documented in the Bring Your Own PostgreSQL guide** is sufficient — and minimal — for everything deviser does at runtime. It runs a 3×3 matrix: + +### PostgreSQL versions + +| Image | Why | +|---|---| +| `postgres:13-alpine` | Minimum supported version (first with native `gen_random_uuid()`) | +| `postgres:17-alpine` | Reference version for external / bring-your-own PostgreSQL | +| `postgres:18-alpine` | CNPG default version | + +One container is started per version; each scenario provisions its own role and database inside it. + +### Scenarios + +| Scenario | Provisioning | Expectation | +|---|---|---| +| `db-owner` | Verbatim guide SQL (`testdata/byo_db_owner_setup.sql`): application user **owns** the database, hyphenated identifiers | Full lifecycle passes | +| `schema-grants` | Least-privilege alternative (`testdata/byo_schema_grants_*.sql`): `CONNECT` + `USAGE, CREATE ON SCHEMA public`, no ownership | Full lifecycle passes **and** `CREATE EXTENSION` is denied — proves deviser needs neither superuser nor extension privileges | +| `grant-all-only` | Legacy model: `GRANT ALL PRIVILEGES ON DATABASE` only | Bootstrap **fails** with SQLSTATE `42501` on PostgreSQL ≥ 15 (public schema no longer world-writable); still passes on 13/14. Pins the rationale for the documented setup | + +### Full lifecycle steps + +Each passing scenario executes, **as the application role only**, mirroring `main.go` order and using the production code paths: + +1. Connect via `pgutil.ConnectionURL` + `pgutil.WaitForPostgres` (exercises URL building with hyphenated credentials) +2. Apply both schemas (`k8s_events.schema.sql`, `resources.schema.sql`) +3. Apply embedded migrations (`ApplyMigrations`, recorded in `schema_migrations`) +4. Create daily partitions via the real `CreateDailyPartitions` +5. `LISTEN events` → insert an event → assert the trigger notification carries the database-generated `event_id` and `global_uid` +6. Read/write `krateo_resources` (what the ingester/presenter components do) +7. Purge soft-deleted resources (`PurgeDeletedResources`) +8. Drop an expired partition via `PartitionManager.Maintain` +9. Re-apply schemas and migrations (restart idempotency) +10. Assert `pgcrypto` is **not** installed (guards against reintroducing the extension dependency) + +### Keeping fixtures in sync with the documentation + +`testdata/byo_db_owner_setup.sql` must stay **verbatim identical** to step 1 of the guide in the `krateo-v2-docs` repository: + +``` +docs/30-how-to-guides/50-manage-postgresql/40-bring-your-own-postgresql.md +``` + +If the guide changes, update the fixture (and vice versa). The fixture header comment carries the same pointer. + +## End-to-end: `scripts/e2e.sh` + +Builds the local image, loads it into a [kind](https://kind.sigs.k8s.io/) cluster, deploys PostgreSQL (`scripts/postgres.yaml`) and the deviser Helm chart, then asserts `/readyz` and the schema/migration state inside the database. + +```sh +POSTGRES_NAMESPACE=demo-system ./scripts/e2e.sh +``` + +> **Note:** `scripts/postgres.yaml` hardcodes the `demo-system` namespace, while the script's `POSTGRES_NAMESPACE` defaults to `test-system` — pass `POSTGRES_NAMESPACE=demo-system` explicitly (as above) until the default is aligned. + +Useful environment overrides (see the script header for the full list): `CLUSTER_NAME`, `IMAGE`, `APP_NAMESPACE`, `DB_USER`, `DB_PASS`, `DB_NAME`, `LOCAL_PORT`. + +Teardown: + +```sh +./scripts/kind-down.sh +``` + +## Continuous integration + +[`.github/workflows/test.yml`](.github/workflows/test.yml) runs `go vet ./...` and `go test ./... -count=1 -timeout 20m` on every pull request and push to `main`. GitHub-hosted runners include Docker, so the full integration matrix runs in CI as well. + +## Conventions for new tests + +- **Prefer table-driven unit tests** with the standard library only — no assertion frameworks (see `partition_manager_test.go`). +- **Fakes over mocks**: for code that talks to the database through a narrow interface, add a small in-package fake (see `fakeMigrationTx` in `migrations_test.go`) instead of spinning up a container. +- **Reach for testcontainers only when behavior depends on real PostgreSQL** (DDL, privileges, triggers, catalog queries). Reuse the helpers in `byo_test.go` (`mustConnectionURL`, `execStatements`, `listPartitionNames`, …) rather than duplicating setup. +- Integration tests must **skip, not fail, without Docker**: call `testcontainers.SkipIfProviderIsNotHealthy(t)` first. +- **Exercise production code paths** in integration tests (real `CreateDailyPartitions`, `ApplyMigrations`, `pgutil` helpers) instead of re-implementing their SQL. `*telemetry.Metrics` is nil-safe, so a `nil` metrics value is fine in tests. +- Provisioning SQL that mirrors public documentation belongs in `testdata/*.sql` with a header comment pointing at the doc — never inline it where it can drift silently. +- Multi-statement provisioning scripts are split and executed one statement at a time (`CREATE DATABASE` cannot run inside the implicit transaction of a multi-statement `Exec`); keep one statement per `;` and comments on their own lines. + +## Troubleshooting + +| Symptom | Cause / fix | +|---|---| +| `SKIP ... Docker is not running` | Start Docker Desktop (or your daemon). Integration tests skip by design without it. | +| Test reports `(cached)` and ignores your change | Re-run with `-count=1`. | +| First integration run is slow or times out | Image pulls. Pre-pull the three `postgres:*-alpine` images, or raise `-timeout`. | +| `permission denied for schema public` in a new scenario | Expected on PostgreSQL ≥ 15 unless the role owns the database or has `CREATE` on the schema — see the `grant-all-only` scenario. | +| e2e script waits forever on the PostgreSQL rollout | Namespace mismatch — run with `POSTGRES_NAMESPACE=demo-system`. | diff --git a/go.mod b/go.mod index 894eca3..70fab43 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.26.1 require ( github.com/jackc/pgx/v5 v5.9.0 github.com/krateoplatformops/plumbing v1.8.1 + github.com/testcontainers/testcontainers-go v0.40.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 go.opentelemetry.io/otel v1.42.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0 go.opentelemetry.io/otel/metric v1.42.0 @@ -13,18 +15,59 @@ require ( ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect + golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect @@ -33,4 +76,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 07d872c..0aa5248 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,55 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -26,22 +64,86 @@ github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE= github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/krateoplatformops/plumbing v1.8.1 h1:YiW6k4ZCiD5nKdgPKYbB2jYcE/PAzhV88sLyHUhHEhc= github.com/krateoplatformops/plumbing v1.8.1/go.mod h1:TE35fxU8TWaTrR0Jw+b+m48x+zkBRxyvP1uC7ZoghOQ= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 h1:s2bIayFXlbDFexo96y+htn7FzuhpXLYJNnIuglNKqOk= +github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0/go.mod h1:h+u/2KoREGTnTl9UwrQ/g+XhasAT8E6dClclAADeXoQ= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0 h1:H7O6RlGOMTizyl3R08Kn5pdM06bnH8oscSj7o11tmLA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0/go.mod h1:mBFWu/WOVDkWWsR7Tx7h6EpQB8wsv7P0Yrh0Pb7othc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= @@ -52,14 +154,27 @@ go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4Len go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= @@ -71,6 +186,10 @@ google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhH google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/internal/config/assets/k8s_events.schema.sql b/internal/config/assets/k8s_events.schema.sql index 7b3c6f5..70ad074 100644 --- a/internal/config/assets/k8s_events.schema.sql +++ b/internal/config/assets/k8s_events.schema.sql @@ -2,7 +2,8 @@ -- Table: k8s_events (partitioned by created_at) -- ============================================================ -CREATE EXTENSION IF NOT EXISTS pgcrypto; +-- gen_random_uuid() is a native core function since PostgreSQL 13: +-- no extension (pgcrypto) and no CREATE EXTENSION privilege required. CREATE TABLE IF NOT EXISTS k8s_events ( created_at TIMESTAMPTZ NOT NULL, diff --git a/internal/config/assets/migrations/001_k8s_events_event_id.sql b/internal/config/assets/migrations/001_k8s_events_event_id.sql index 1cd1f90..c729a70 100644 --- a/internal/config/assets/migrations/001_k8s_events_event_id.sql +++ b/internal/config/assets/migrations/001_k8s_events_event_id.sql @@ -1,4 +1,5 @@ -CREATE EXTENSION IF NOT EXISTS pgcrypto; +-- gen_random_uuid() is a native core function since PostgreSQL 13: +-- no extension (pgcrypto) and no CREATE EXTENSION privilege required. ALTER TABLE k8s_events ADD COLUMN IF NOT EXISTS event_id UUID DEFAULT gen_random_uuid(); diff --git a/internal/pg/byo_test.go b/internal/pg/byo_test.go new file mode 100644 index 0000000..be44584 --- /dev/null +++ b/internal/pg/byo_test.go @@ -0,0 +1,502 @@ +package pg + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "testing" + "text/template" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/krateoplatformops/deviser/internal/config" + "github.com/krateoplatformops/plumbing/pgutil" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" +) + +// Scenario "db-owner": the documented BYO setup, verbatim from the guide. +// +//go:embed testdata/byo_db_owner_setup.sql +var byoDbOwnerSetupSQL string + +// Scenario "schema-grants": least-privilege alternative, test-only evidence. +// +//go:embed testdata/byo_schema_grants_setup.sql +var byoSchemaGrantsSetupSQL string + +//go:embed testdata/byo_schema_grants_grants.sql +var byoSchemaGrantsGrantsSQL string + +// Scenario "grant-all-only": the PREVIOUSLY documented setup, kept as a +// negative case. On PostgreSQL 15+ the public schema is no longer +// world-writable, so "GRANT ALL PRIVILEGES ON DATABASE" alone is NOT enough +// for deviser's first CREATE TABLE. This scenario pins the reason why the +// guide now makes the application user the owner of the database. +const byoGrantAllOnlySetupSQL = ` +CREATE ROLE krateo_app_ga LOGIN PASSWORD 'krateo_app_ga' NOSUPERUSER NOCREATEDB NOCREATEROLE; +CREATE DATABASE krateo_db_ga; +GRANT ALL PRIVILEGES ON DATABASE krateo_db_ga TO krateo_app_ga; +` + +// TestBYOPostgres verifies the Bring Your Own PostgreSQL scenario end to end: +// the exact provisioning SQL from the guide must be sufficient (and minimal) +// for the full deviser lifecycle, executed as the application role only: +// schema bootstrap, migrations, daily partition creation, notify trigger, +// resources read/write, soft-delete purge, partition retention drop and +// restart idempotency. No superuser and no extension privileges involved. +func TestBYOPostgres(t *testing.T) { + testcontainers.SkipIfProviderIsNotHealthy(t) + + images := []string{ + "postgres:13-alpine", // minimum supported version (first with native gen_random_uuid) + "postgres:17-alpine", // reference version for external/bring-your-own PostgreSQL + "postgres:18-alpine", // CNPG default version + } + + for _, image := range images { + t.Run(image, func(t *testing.T) { + byoScenarios(t, image) + }) + } +} + +func byoScenarios(t *testing.T, image string) { + ctx := context.Background() + + container, err := tcpostgres.Run(ctx, image, + tcpostgres.WithDatabase("postgres"), + tcpostgres.WithUsername("postgres"), + tcpostgres.WithPassword("postgres"), + tcpostgres.BasicWaitStrategies(), + ) + testcontainers.CleanupContainer(t, container) + if err != nil { + t.Fatalf("start postgres container: %v", err) + } + + host, err := container.Host(ctx) + if err != nil { + t.Fatalf("container host: %v", err) + } + mappedPort, err := container.MappedPort(ctx, "5432/tcp") + if err != nil { + t.Fatalf("container mapped port: %v", err) + } + port := mappedPort.Int() + + adminPool, err := pgxpool.New(ctx, mustConnectionURL(t, "postgres", "postgres", host, port, "postgres")) + if err != nil { + t.Fatalf("connect as admin: %v", err) + } + defer adminPool.Close() + + serverMajor := serverMajorVersion(ctx, t, adminPool) + + t.Run("db-owner", func(t *testing.T) { + execStatements(ctx, t, adminPool, byoDbOwnerSetupSQL) + + appURL := mustConnectionURL(t, "krateo-db-user", "your_password", host, port, "krateo-db") + runFullLifecycle(ctx, t, appURL, lifecycleOpts{}) + }) + + t.Run("schema-grants", func(t *testing.T) { + execStatements(ctx, t, adminPool, byoSchemaGrantsSetupSQL) + + // Schema-level grants must be issued from inside the target database. + adminDbPool, err := pgxpool.New(ctx, mustConnectionURL(t, "postgres", "postgres", host, port, "krateo_db_lp")) + if err != nil { + t.Fatalf("connect as admin to krateo_db_lp: %v", err) + } + defer adminDbPool.Close() + execStatements(ctx, t, adminDbPool, byoSchemaGrantsGrantsSQL) + + appURL := mustConnectionURL(t, "krateo_app_lp", "krateo_app_lp", host, port, "krateo_db_lp") + runFullLifecycle(ctx, t, appURL, lifecycleOpts{assertNoExtensionPrivilege: true}) + }) + + t.Run("grant-all-only", func(t *testing.T) { + execStatements(ctx, t, adminPool, byoGrantAllOnlySetupSQL) + + appURL := mustConnectionURL(t, "krateo_app_ga", "krateo_app_ga", host, port, "krateo_db_ga") + runGrantAllOnlyBootstrap(ctx, t, appURL, serverMajor) + }) +} + +type lifecycleOpts struct { + // assertNoExtensionPrivilege additionally proves the role cannot run + // CREATE EXTENSION at all, so the bootstrap demonstrably requires none. + assertNoExtensionPrivilege bool +} + +// runFullLifecycle executes every database operation of the deviser lifecycle +// (plus the write/read patterns of the ingester/presenter components) as the +// application role only, mirroring the order of main.go. +func runFullLifecycle(ctx context.Context, t *testing.T, appURL string, opts lifecycleOpts) { + t.Helper() + + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{Log: log} + + // Connect exactly like main.go does. + waitCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + pool, err := pgutil.WaitForPostgres(waitCtx, log, appURL) + if err != nil { + t.Fatalf("connect as application role: %v", err) + } + defer pool.Close() + + var super bool + if err := pool.QueryRow(ctx, + `SELECT rolsuper FROM pg_roles WHERE rolname = current_user`).Scan(&super); err != nil { + t.Fatalf("check role privileges: %v", err) + } + if super { + t.Fatal("application role is superuser; test would not prove anything") + } + + if opts.assertNoExtensionPrivilege { + if _, err := pool.Exec(ctx, `CREATE EXTENSION IF NOT EXISTS pgcrypto`); err == nil { + t.Fatal("application role can install extensions; least-privilege setup is broken") + } + } + + // 1-2. Schema bootstrap + migrations (idempotent, like a service start). + bootstrap := func() { + applySchemas(ctx, t, pool, cfg) + applyTestMigrations(ctx, t, pool, cfg, log) + } + bootstrap() + + // 3. Daily partitions via the real production code path. + tpl, err := cfg.LoadSQLTemplate("partition.tpl.sql", "partition") + if err != nil { + t.Fatalf("load partition template: %v", err) + } + if err := CreateDailyPartitions(ctx, &CreateDailyPartitionsOptions{ + Pool: pool, + Log: log, + Tpl: tpl, + Days: 3, + }); err != nil { + t.Fatalf("create daily partitions as application role: %v", err) + } + + // Extra expired partition to exercise the retention drop below. + oldDay := startOfUTCDay(time.Now()).AddDate(0, 0, -10) + oldPartition := createPartitionFor(ctx, t, pool, tpl, oldDay) + + if names := listPartitionNames(ctx, t, pool); len(names) != 4 { + t.Fatalf("partitions after creation = %v, want 4 (3 daily + 1 expired)", names) + } + + // 4. The notify trigger must fire with a database-generated event_id. + listenConn, err := pgx.Connect(ctx, appURL) + if err != nil { + t.Fatalf("open LISTEN connection: %v", err) + } + defer listenConn.Close(ctx) + if _, err := listenConn.Exec(ctx, `LISTEN events`); err != nil { + t.Fatalf("LISTEN events: %v", err) + } + + var eventID string + err = pool.QueryRow(ctx, ` +INSERT INTO k8s_events ( + created_at, cluster_name, uid, global_uid, + namespace, resource_kind, resource_name, + event_type, raw, resource_version +) VALUES ( + $1, 'test-cluster', 'uid-1', 'test-cluster:uid-1', + 'default', 'Pod', 'test-pod', + 'Normal', '{}'::jsonb, '1' +) RETURNING event_id::text`, time.Now().UTC()).Scan(&eventID) + if err != nil { + t.Fatalf("insert event as application role: %v", err) + } + if len(eventID) != 36 { + t.Fatalf("event_id = %q, want database-generated UUID", eventID) + } + + notifyCtx, cancelNotify := context.WithTimeout(ctx, 10*time.Second) + defer cancelNotify() + notification, err := listenConn.WaitForNotification(notifyCtx) + if err != nil { + t.Fatalf("wait for trigger notification: %v", err) + } + var payload struct { + EventID string `json:"event_id"` + GlobalUID string `json:"global_uid"` + } + if err := json.Unmarshal([]byte(notification.Payload), &payload); err != nil { + t.Fatalf("decode notification payload %q: %v", notification.Payload, err) + } + if payload.EventID != eventID || payload.GlobalUID != "test-cluster:uid-1" { + t.Fatalf("notification payload = %+v, want event_id=%s global_uid=test-cluster:uid-1", + payload, eventID) + } + + // 5. krateo_resources read/write, as the ingester/presenter components do. + var resourceID int64 + err = pool.QueryRow(ctx, ` +INSERT INTO krateo_resources ( + cluster_name, uid, global_uid, namespace, + resource_group, resource_version, resource_kind, + resource_plural, resource_name, raw +) VALUES ( + 'test-cluster', 'res-1', 'test-cluster:res-1', 'default', + 'apps', 'v1', 'Deployment', + 'deployments', 'app-1', '{}'::jsonb +) RETURNING id`).Scan(&resourceID) + if err != nil { + t.Fatalf("insert resource: %v", err) + } + if _, err := pool.Exec(ctx, + `UPDATE krateo_resources SET status_raw = '{"ready":true}'::jsonb, updated_at = now() WHERE id = $1`, + resourceID); err != nil { + t.Fatalf("update resource: %v", err) + } + + // 6. Soft-delete purge needs DELETE privileges. + if _, err := pool.Exec(ctx, ` +INSERT INTO krateo_resources ( + cluster_name, uid, global_uid, namespace, + resource_group, resource_version, resource_kind, + resource_plural, resource_name, raw, deleted_at +) VALUES ( + 'test-cluster', 'res-2', 'test-cluster:res-2', 'default', + 'apps', 'v1', 'Deployment', + 'deployments', 'app-2', '{}'::jsonb, now() - interval '40 days' +)`); err != nil { + t.Fatalf("insert soft-deleted resource: %v", err) + } + + purged, err := PurgeDeletedResources(ctx, &PurgeDeletedResourcesOptions{ + Pool: pool, + Log: log, + RetentionDays: 30, + BatchSize: 100, + }) + if err != nil { + t.Fatalf("purge soft-deleted resources: %v", err) + } + if purged != 1 { + t.Fatalf("purged = %d, want 1", purged) + } + var remaining int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM krateo_resources`).Scan(&remaining); err != nil { + t.Fatalf("count resources: %v", err) + } + if remaining != 1 { + t.Fatalf("remaining resources = %d, want 1", remaining) + } + + // 7. Partition maintenance needs DROP TABLE on expired partitions. + pm := &PartitionManager{ + Pool: pool, + Log: log, + ParentTable: "k8s_events", + RetentionDays: 2, + } + if err := pm.Maintain(ctx); err != nil { + t.Fatalf("partition maintenance as application role: %v", err) + } + names := listPartitionNames(ctx, t, pool) + for _, name := range names { + if name == oldPartition { + t.Fatalf("expired partition %s was not dropped", oldPartition) + } + } + if len(names) != 3 { + t.Fatalf("partitions after maintenance = %v, want the 3 current daily partitions", names) + } + + // 8. Restart idempotency: re-applying schemas and migrations must succeed. + bootstrap() + + // 9. Guard against reintroducing the extension dependency: the lifecycle + // must work on a database where pgcrypto was never installed. + var pgcryptoInstalled bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pgcrypto')`).Scan(&pgcryptoInstalled); err != nil { + t.Fatalf("check pg_extension: %v", err) + } + if pgcryptoInstalled { + t.Fatal("pgcrypto extension is installed; deviser must not depend on it") + } +} + +// runGrantAllOnlyBootstrap documents that the legacy provisioning model is +// insufficient on PostgreSQL 15+ (and still works on 13/14, where the public +// schema is world-writable). +func runGrantAllOnlyBootstrap(ctx context.Context, t *testing.T, appURL string, serverMajor int) { + t.Helper() + + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{Log: log} + + pool, err := pgxpool.New(ctx, appURL) + if err != nil { + t.Fatalf("connect as application role: %v", err) + } + defer pool.Close() + + sql, err := cfg.LoadSQL("k8s_events.schema.sql") + if err != nil { + t.Fatalf("load schema: %v", err) + } + _, err = pool.Exec(ctx, sql) + + if serverMajor >= 15 { + if err == nil { + t.Fatal("schema bootstrap succeeded; GRANT ALL PRIVILEGES ON DATABASE should NOT be sufficient on PostgreSQL 15+") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42501" { // insufficient_privilege + t.Fatalf("schema bootstrap error = %v, want SQLSTATE 42501 (permission denied for schema public)", err) + } + return + } + + // PostgreSQL 13/14: public schema is world-writable, legacy grants are + // still enough to complete the bootstrap. + if err != nil { + t.Fatalf("apply schema on PostgreSQL <15: %v", err) + } + applySchemas(ctx, t, pool, cfg) + applyTestMigrations(ctx, t, pool, cfg, log) +} + +func applySchemas(ctx context.Context, t *testing.T, pool *pgxpool.Pool, cfg *config.Config) { + t.Helper() + for _, schema := range []string{"k8s_events.schema.sql", "resources.schema.sql"} { + sql, err := cfg.LoadSQL(schema) + if err != nil { + t.Fatalf("load schema %s: %v", schema, err) + } + if _, err := pool.Exec(ctx, sql); err != nil { + t.Fatalf("apply schema %s as application role: %v", schema, err) + } + } +} + +func applyTestMigrations(ctx context.Context, t *testing.T, pool *pgxpool.Pool, cfg *config.Config, log *slog.Logger) { + t.Helper() + assets, err := cfg.LoadMigrations() + if err != nil { + t.Fatalf("load migrations: %v", err) + } + migrations := make([]Migration, 0, len(assets)) + for _, asset := range assets { + migrations = append(migrations, Migration{ + Version: asset.Version, + SQL: asset.SQL, + }) + } + if err := ApplyMigrations(ctx, pool, log, migrations); err != nil { + t.Fatalf("apply migrations as application role: %v", err) + } +} + +func createPartitionFor(ctx context.Context, t *testing.T, pool *pgxpool.Pool, tpl *template.Template, day time.Time) string { + t.Helper() + name := fmt.Sprintf("k8s_events_%04d_%02d_%02d", day.Year(), day.Month(), day.Day()) + var sb strings.Builder + if err := tpl.Execute(&sb, map[string]string{ + "PartitionName": name, + "StartDate": day.Format(partitionTimestampLayout), + "EndDate": day.AddDate(0, 0, 1).Format(partitionTimestampLayout), + }); err != nil { + t.Fatalf("render partition template: %v", err) + } + if _, err := pool.Exec(ctx, sb.String()); err != nil { + t.Fatalf("create partition %s: %v", name, err) + } + return name +} + +func listPartitionNames(ctx context.Context, t *testing.T, pool *pgxpool.Pool) []string { + t.Helper() + rows, err := pool.Query(ctx, ` +SELECT c.relname +FROM pg_class c +JOIN pg_inherits i ON i.inhrelid = c.oid +WHERE i.inhparent = 'k8s_events'::regclass`) + if err != nil { + t.Fatalf("list partitions: %v", err) + } + defer rows.Close() + + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan partition name: %v", err) + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate partitions: %v", err) + } + return names +} + +func mustConnectionURL(t *testing.T, user, pass, host string, port int, dbname string) string { + t.Helper() + // Build the URL with the same production code used by deviser's config. + u, err := pgutil.ConnectionURL(user, pass, host, port, dbname, map[string]string{"sslmode": "disable"}) + if err != nil { + t.Fatalf("build connection URL: %v", err) + } + return u +} + +func serverMajorVersion(ctx context.Context, t *testing.T, pool *pgxpool.Pool) int { + t.Helper() + var n int + if err := pool.QueryRow(ctx, `SELECT current_setting('server_version_num')::int`).Scan(&n); err != nil { + t.Fatalf("read server version: %v", err) + } + return n / 10000 +} + +func execStatements(ctx context.Context, t *testing.T, pool *pgxpool.Pool, script string) { + t.Helper() + for _, stmt := range sqlStatements(script) { + if _, err := pool.Exec(ctx, stmt); err != nil { + t.Fatalf("setup statement %q: %v", stmt, err) + } + } +} + +// sqlStatements splits a provisioning script into single statements: +// CREATE DATABASE cannot run inside the implicit transaction that wraps a +// multi-statement Exec, so each statement is executed on its own. +// Comment lines are stripped first, so they may freely contain semicolons. +func sqlStatements(script string) []string { + var sb strings.Builder + for _, line := range strings.Split(script, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "--") { + continue + } + sb.WriteString(trimmed) + sb.WriteString("\n") + } + + var stmts []string + for _, chunk := range strings.Split(sb.String(), ";") { + if stmt := strings.TrimSpace(chunk); stmt != "" { + stmts = append(stmts, stmt) + } + } + return stmts +} diff --git a/internal/pg/testdata/byo_db_owner_setup.sql b/internal/pg/testdata/byo_db_owner_setup.sql new file mode 100644 index 0000000..f1ed18b --- /dev/null +++ b/internal/pg/testdata/byo_db_owner_setup.sql @@ -0,0 +1,5 @@ +-- Scenario "db-owner": the documented Bring Your Own PostgreSQL setup. +-- Keep VERBATIM in sync with the krateo-v2-docs guide: +-- docs/30-how-to-guides/50-manage-postgresql/40-bring-your-own-postgresql.md (step 1) +CREATE USER "krateo-db-user" WITH ENCRYPTED PASSWORD 'your_password'; +CREATE DATABASE "krateo-db" OWNER "krateo-db-user"; diff --git a/internal/pg/testdata/byo_schema_grants_grants.sql b/internal/pg/testdata/byo_schema_grants_grants.sql new file mode 100644 index 0000000..5230ec2 --- /dev/null +++ b/internal/pg/testdata/byo_schema_grants_grants.sql @@ -0,0 +1,6 @@ +-- Scenario "schema-grants": grants for the application role. +-- Run INSIDE the target database (krateo_db_lp) as an administrative user. +-- CONNECT plus USAGE/CREATE on the public schema is enough for all deviser +-- DDL (tables, indexes, functions, triggers) but not for CREATE EXTENSION. +GRANT CONNECT ON DATABASE krateo_db_lp TO krateo_app_lp; +GRANT USAGE, CREATE ON SCHEMA public TO krateo_app_lp; diff --git a/internal/pg/testdata/byo_schema_grants_setup.sql b/internal/pg/testdata/byo_schema_grants_setup.sql new file mode 100644 index 0000000..09e24a7 --- /dev/null +++ b/internal/pg/testdata/byo_schema_grants_setup.sql @@ -0,0 +1,6 @@ +-- Scenario "schema-grants": least-privilege alternative to database ownership. +-- Not part of the public guide; kept as engineering evidence that deviser +-- needs neither superuser nor CREATE EXTENSION privileges. +-- Run on the maintenance database as an administrative user. +CREATE ROLE krateo_app_lp LOGIN PASSWORD 'krateo_app_lp' NOSUPERUSER NOCREATEDB NOCREATEROLE; +CREATE DATABASE krateo_db_lp; From b84f1cea08be47bde6c66b55ba7be04a7bcb042a Mon Sep 17 00:00:00 2001 From: Leonardo Vicentini Date: Mon, 8 Jun 2026 10:41:40 +0200 Subject: [PATCH 2/5] feat: added latest non-working version guard + e2e added in CI --- .github/workflows/test.yml | 27 ++++++++++++++ TESTING.md | 21 +++++++---- internal/pg/byo_test.go | 74 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f84e113..bd3f0df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,3 +26,30 @@ jobs: # via testcontainers; Docker is available on GitHub-hosted runners. - name: Test run: go test ./... -count=1 -timeout 20m + + e2e: + name: End-to-end (kind) + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + # docker, kubectl, helm and curl are preinstalled on the runner; + # only kind needs installing. install_only skips creating a cluster + # so scripts/e2e.sh can create its own via scripts/kind-up.sh. + - name: Install kind + uses: helm/kind-action@v1 + with: + install_only: true + + # POSTGRES_NAMESPACE must match scripts/postgres.yaml (demo-system). + - name: End-to-end + env: + POSTGRES_NAMESPACE: demo-system + run: ./scripts/e2e.sh diff --git a/TESTING.md b/TESTING.md index 53d639f..a2dd701 100644 --- a/TESTING.md +++ b/TESTING.md @@ -41,10 +41,10 @@ go test ./... -skip 'TestBYOPostgres' go test ./internal/pg -run TestBYOPostgres -count=1 -v ``` -The first run pulls the `postgres:13-alpine`, `postgres:17-alpine` and `postgres:18-alpine` images; subsequent runs reuse them. To pre-pull: +The first run pulls the `postgres:12-alpine`, `postgres:13-alpine`, `postgres:17.2-alpine`, `postgres:17-alpine` and `postgres:18-alpine` images; subsequent runs reuse them. To pre-pull: ```sh -docker pull postgres:13-alpine postgres:17-alpine postgres:18-alpine +docker pull postgres:12-alpine postgres:13-alpine postgres:17.2-alpine postgres:17-alpine postgres:18-alpine ``` A single version or scenario can be selected with the subtest path: @@ -61,20 +61,21 @@ internal/config/migrations_test.go embedded migration asset discovery and or internal/pg/migrations_test.go migration apply/record logic (fake transaction, no DB) internal/pg/partitions_test.go partition helpers internal/pg/partition_manager_test.go partition bound parsing -internal/pg/byo_test.go integration: full lifecycle on real PostgreSQL +internal/pg/byo_test.go integration: full lifecycle on real PostgreSQL + minimum-version guard internal/pg/testdata/byo_*.sql provisioning fixtures for the BYO scenarios ``` ## The integration test: `TestBYOPostgres` -`internal/pg/byo_test.go` verifies that the **exact provisioning SQL documented in the Bring Your Own PostgreSQL guide** is sufficient — and minimal — for everything deviser does at runtime. It runs a 3×3 matrix: +`internal/pg/byo_test.go` verifies that the **exact provisioning SQL documented in the Bring Your Own PostgreSQL guide** is sufficient — and minimal — for everything deviser does at runtime. It runs a 4×3 matrix (versions × scenarios): ### PostgreSQL versions | Image | Why | |---|---| | `postgres:13-alpine` | Minimum supported version (first with native `gen_random_uuid()`) | -| `postgres:17-alpine` | Reference version for external / bring-your-own PostgreSQL | +| `postgres:17.2-alpine` | Pinned patch release, exact-version coverage | +| `postgres:17-alpine` | Latest 17.x patch release | | `postgres:18-alpine` | CNPG default version | One container is started per version; each scenario provisions its own role and database inside it. @@ -102,6 +103,14 @@ Each passing scenario executes, **as the application role only**, mirroring `mai 9. Re-apply schemas and migrations (restart idempotency) 10. Assert `pgcrypto` is **not** installed (guards against reintroducing the extension dependency) +### Below-minimum version guard (must fail) + +`TestBYOPostgresBelowMinimumVersion` runs the schema bootstrap on `postgres:12-alpine` (the latest and final 12.x patch release, EOL) and asserts it **fails** with SQLSTATE `42883`: `gen_random_uuid()` is not a core function before PostgreSQL 13. It provisions with the documented db-owner setup on purpose, so the failure is attributable to the missing function and not to privileges. If this test ever starts passing, the documented minimum version claim must be revisited. + +```sh +go test ./internal/pg -run TestBYOPostgresBelowMinimumVersion -count=1 -v +``` + ### Keeping fixtures in sync with the documentation `testdata/byo_db_owner_setup.sql` must stay **verbatim identical** to step 1 of the guide in the `krateo-v2-docs` repository: @@ -150,6 +159,6 @@ Teardown: |---|---| | `SKIP ... Docker is not running` | Start Docker Desktop (or your daemon). Integration tests skip by design without it. | | Test reports `(cached)` and ignores your change | Re-run with `-count=1`. | -| First integration run is slow or times out | Image pulls. Pre-pull the three `postgres:*-alpine` images, or raise `-timeout`. | +| First integration run is slow or times out | Image pulls. Pre-pull the five `postgres:*-alpine` images, or raise `-timeout`. | | `permission denied for schema public` in a new scenario | Expected on PostgreSQL ≥ 15 unless the role owns the database or has `CREATE` on the schema — see the `grant-all-only` scenario. | | e2e script waits forever on the PostgreSQL rollout | Namespace mismatch — run with `POSTGRES_NAMESPACE=demo-system`. | diff --git a/internal/pg/byo_test.go b/internal/pg/byo_test.go index be44584..610e771 100644 --- a/internal/pg/byo_test.go +++ b/internal/pg/byo_test.go @@ -56,9 +56,10 @@ func TestBYOPostgres(t *testing.T) { testcontainers.SkipIfProviderIsNotHealthy(t) images := []string{ - "postgres:13-alpine", // minimum supported version (first with native gen_random_uuid) - "postgres:17-alpine", // reference version for external/bring-your-own PostgreSQL - "postgres:18-alpine", // CNPG default version + "postgres:13-alpine", // minimum supported version (first with native gen_random_uuid) + "postgres:17.2-alpine", // pinned patch release, exact-version coverage + "postgres:17-alpine", // latest 17.x patch release + "postgres:18-alpine", // CNPG default version } for _, image := range images { @@ -68,6 +69,73 @@ func TestBYOPostgres(t *testing.T) { } } +// TestBYOPostgresBelowMinimumVersion documents the PostgreSQL 13 floor: +// on 12.x gen_random_uuid() is not a core function, so the schema bootstrap +// must fail even with full database ownership. The provisioning uses the +// documented db-owner setup on purpose, so the failure is attributable to +// the missing function and not to privileges. +func TestBYOPostgresBelowMinimumVersion(t *testing.T) { + testcontainers.SkipIfProviderIsNotHealthy(t) + + ctx := context.Background() + + // Latest (final) 12.x patch release; PostgreSQL 12 is EOL. + container, err := tcpostgres.Run(ctx, "postgres:12-alpine", + tcpostgres.WithDatabase("postgres"), + tcpostgres.WithUsername("postgres"), + tcpostgres.WithPassword("postgres"), + tcpostgres.BasicWaitStrategies(), + ) + testcontainers.CleanupContainer(t, container) + if err != nil { + t.Fatalf("start postgres container: %v", err) + } + + host, err := container.Host(ctx) + if err != nil { + t.Fatalf("container host: %v", err) + } + mappedPort, err := container.MappedPort(ctx, "5432/tcp") + if err != nil { + t.Fatalf("container mapped port: %v", err) + } + port := mappedPort.Int() + + adminPool, err := pgxpool.New(ctx, mustConnectionURL(t, "postgres", "postgres", host, port, "postgres")) + if err != nil { + t.Fatalf("connect as admin: %v", err) + } + defer adminPool.Close() + + execStatements(ctx, t, adminPool, byoDbOwnerSetupSQL) + + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{Log: log} + + pool, err := pgxpool.New(ctx, mustConnectionURL(t, "krateo-db-user", "your_password", host, port, "krateo-db")) + if err != nil { + t.Fatalf("connect as application role: %v", err) + } + defer pool.Close() + + sql, err := cfg.LoadSQL("k8s_events.schema.sql") + if err != nil { + t.Fatalf("load schema: %v", err) + } + + _, err = pool.Exec(ctx, sql) + if err == nil { + t.Fatal("schema bootstrap succeeded on PostgreSQL 12; the documented minimum (13) would be wrong") + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "42883" { // undefined_function + t.Fatalf("schema bootstrap error = %v, want SQLSTATE 42883 (gen_random_uuid does not exist)", err) + } + if !strings.Contains(pgErr.Message, "gen_random_uuid") { + t.Fatalf("error message %q does not mention gen_random_uuid", pgErr.Message) + } +} + func byoScenarios(t *testing.T, image string) { ctx := context.Background() From d7e27c09514f0187b546fc26a39a6bc97bf71930 Mon Sep 17 00:00:00 2001 From: Leonardo Vicentini Date: Mon, 8 Jun 2026 12:22:09 +0200 Subject: [PATCH 3/5] feat: added dev krateo profile override --- krateo-overrides.deviser-dev.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 krateo-overrides.deviser-dev.yaml diff --git a/krateo-overrides.deviser-dev.yaml b/krateo-overrides.deviser-dev.yaml new file mode 100644 index 0000000..cecc380 --- /dev/null +++ b/krateo-overrides.deviser-dev.yaml @@ -0,0 +1,18 @@ +components: + db-maintenance: + stepConfig: + install-deviser: + with: + values: + image: + repository: leovice/deviser # Point to your dev image registry + pullPolicy: Always + tag: "latest" + events-stack: + stepConfig: + install-events-ingester: + with: + version: 0.5.3 + install-events-presenter: + with: + version: 0.9.3 From 430059de069e721eca4df4b4c4418ecfb8dde9f8 Mon Sep 17 00:00:00 2001 From: Leonardo Vicentini Date: Mon, 8 Jun 2026 14:06:52 +0200 Subject: [PATCH 4/5] fix: improved logs for purge operation --- internal/pg/purge.go | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/pg/purge.go b/internal/pg/purge.go index dc19637..3398d97 100644 --- a/internal/pg/purge.go +++ b/internal/pg/purge.go @@ -9,6 +9,12 @@ import ( "github.com/krateoplatformops/deviser/internal/telemetry" ) +// defaultPurgeBatchSize is the fallback batch size used when a caller does not +// supply a positive BatchSize. In production main.go always passes the +// configured value (see config.defaultSoftDeletePurgeBatchSize); this guards +// direct and test callers. +const defaultPurgeBatchSize = 1000 + type PurgeDeletedResourcesOptions struct { Pool *pgxpool.Pool Log *slog.Logger @@ -34,11 +40,12 @@ func PurgeDeletedResources(ctx context.Context, opts *PurgeDeletedResourcesOptio batchSize := opts.BatchSize if batchSize <= 0 { - batchSize = 1000 + batchSize = defaultPurgeBatchSize } cutoff := time.Now().UTC().AddDate(0, 0, -opts.RetentionDays) + batches := 0 for { result, err := opts.Pool.Exec(ctx, ` DELETE FROM krateo_resources @@ -57,10 +64,14 @@ func PurgeDeletedResources(ctx context.Context, opts *PurgeDeletedResourcesOptio if err != nil { opts.Log.Error("failed to purge soft-deleted resources", slog.Any("err", err), - slog.Time("cutoff", cutoff)) + slog.Time("cutoff", cutoff), + slog.Int64("rows_deleted", totalDeleted), + slog.Int("batches", batches), + slog.Int64("duration_ms", time.Since(started).Milliseconds())) return totalDeleted, err } + batches++ deleted := result.RowsAffected() totalDeleted += deleted @@ -69,11 +80,26 @@ func PurgeDeletedResources(ctx context.Context, opts *PurgeDeletedResourcesOptio } } + elapsed := time.Since(started) + + // Nothing was eligible for hard deletion this cycle. This is the steady + // state (it runs hourly), so keep it at debug to avoid implying a purge + // happened when no rows were removed. + if totalDeleted == 0 { + opts.Log.Debug("soft-delete purge ran, no resources past retention", + slog.Int("retention_days", opts.RetentionDays), + slog.Time("cutoff", cutoff), + slog.Int64("duration_ms", elapsed.Milliseconds())) + return 0, nil + } + opts.Log.Info("purged soft-deleted resources", - slog.Int64("rows", totalDeleted), + slog.Int64("rows_deleted", totalDeleted), + slog.Int("batches", batches), slog.Int("retention_days", opts.RetentionDays), slog.Int("batch_size", batchSize), - slog.Time("cutoff", cutoff)) + slog.Time("cutoff", cutoff), + slog.Int64("duration_ms", elapsed.Milliseconds())) return totalDeleted, nil } From 6b7f56d51aa859facbca481effc5350f5ecbbc38 Mon Sep 17 00:00:00 2001 From: Leonardo Vicentini Date: Wed, 10 Jun 2026 16:36:08 +0200 Subject: [PATCH 5/5] chore: changed dev values --- krateo-overrides.deviser-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/krateo-overrides.deviser-dev.yaml b/krateo-overrides.deviser-dev.yaml index cecc380..9f3de33 100644 --- a/krateo-overrides.deviser-dev.yaml +++ b/krateo-overrides.deviser-dev.yaml @@ -5,7 +5,7 @@ components: with: values: image: - repository: leovice/deviser # Point to your dev image registry + repository: # Point to your dev image registry pullPolicy: Always tag: "latest" events-stack: