diff --git a/helm/common/templates/_db_setup_job.tpl b/helm/common/templates/_db_setup_job.tpl index 12cbccb34..9c2e7668d 100644 --- a/helm/common/templates/_db_setup_job.tpl +++ b/helm/common/templates/_db_setup_job.tpl @@ -135,32 +135,39 @@ spec: echo "SERVICE_PGDB=$SERVICE_PGDB" echo "SERVICE_PGUSER=$SERVICE_PGUSER" - until pg_isready -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -d template1 + until pg_isready -h $PGHOST -p $PGPORT -U $PGUSER -d template1 do >&2 echo "Postgres is unavailable - sleeping" sleep 5 done >&2 echo "Postgres is up - executing command" + printf '%s\n' \ + "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'service_user', :'service_pass')" \ + "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'service_user')\\gexec" \ + "ALTER ROLE :\"service_user\" WITH LOGIN PASSWORD :'service_pass';" \ + "SELECT format('CREATE DATABASE %I OWNER %I', :'service_db', :'service_user')" \ + "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'service_db')\\gexec" \ + "GRANT ALL ON DATABASE :\"service_db\" TO :\"service_user\" WITH GRANT OPTION;" \ + | psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d postgres \ + -v service_user="$SERVICE_PGUSER" \ + -v service_db="$SERVICE_PGDB" \ + -v service_pass="$SERVICE_PGPASS" \ + -f - - if psql -lqt | cut -d \| -f 1 | grep -qw $SERVICE_PGDB; then - gen3_log_info "Database exists" - PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" + printf '%s\n' \ + "CREATE EXTENSION IF NOT EXISTS ltree;" \ + "ALTER ROLE :\"service_user\" WITH LOGIN;" \ + "GRANT ALL ON SCHEMA public TO :\"service_user\";" \ + "ALTER SCHEMA public OWNER TO :\"service_user\";" \ + | psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$SERVICE_PGDB" \ + -v service_user="$SERVICE_PGUSER" \ + -f - - # Update secret to signal that db is ready, and services can start - kubectl patch secret/{{ .Chart.Name }}-dbcreds -p '{"data":{"dbcreated":"dHJ1ZQo="}}' - else - echo "database does not exist" - psql -tc "SELECT 1 FROM pg_database WHERE datname = '$SERVICE_PGDB'" | grep -q 1 || psql -c "CREATE DATABASE \"$SERVICE_PGDB\";" - gen3_log_info psql -tc "SELECT 1 FROM pg_user WHERE usename = '$SERVICE_PGUSER'" | grep -q 1 || psql -c "CREATE USER \"$SERVICE_PGUSER\" WITH PASSWORD '$SERVICE_PGPASS';" - psql -tc "SELECT 1 FROM pg_user WHERE usename = '$SERVICE_PGUSER'" | grep -q 1 || psql -c "CREATE USER \"$SERVICE_PGUSER\" WITH PASSWORD '$SERVICE_PGPASS';" - psql -c "GRANT ALL ON DATABASE \"$SERVICE_PGDB\" TO \"$SERVICE_PGUSER\" WITH GRANT OPTION;" - psql -d $SERVICE_PGDB -c "CREATE EXTENSION ltree; ALTER ROLE \"$SERVICE_PGUSER\" WITH LOGIN" - PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" + PGPASSWORD=$SERVICE_PGPASS psql -d "$SERVICE_PGDB" -h "$PGHOST" -p "$PGPORT" -U "$SERVICE_PGUSER" -c "\conninfo" - # Update secret to signal that db has been created, and services can start - kubectl patch secret/{{ .Chart.Name }}-dbcreds -p '{"data":{"dbcreated":"dHJ1ZQo="}}' - fi + # Update secret to signal that db has been created, and services can start + kubectl patch secret/{{ .Chart.Name }}-dbcreds -p '{"data":{"dbcreated":"dHJ1ZQo="}}' {{- end}} {{- end }} diff --git a/helm/fence/templates/presigned-url-fence.yaml b/helm/fence/templates/presigned-url-fence.yaml deleted file mode 100644 index 6006a6ad9..000000000 --- a/helm/fence/templates/presigned-url-fence.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: presigned-url-fence-deployment - labels: - app: presigned-url-fence -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - app: presigned-url-fence - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: presigned-url-fence - spec: - serviceAccountName: {{ include "fence.serviceAccountName" . }} - volumes: - {{- toYaml .Values.volumes | nindent 8 }} - containers: - - name: presigned-url-fence - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: Always - ports: - - name: http - containerPort: 80 - protocol: TCP - - name: https - containerPort: 443 - protocol: TCP - - name: container - containerPort: 6567 - protocol: TCP - livenessProbe: - httpGet: - path: /_status - port: http - initialDelaySeconds: 30 - periodSeconds: 60 - timeoutSeconds: 30 - readinessProbe: - httpGet: - path: /_status - port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - command: ["/bin/bash"] - args: - - "-c" - - | - set -euo pipefail - echo "${FENCE_PUBLIC_CONFIG:-""}" > /var/www/fence/fence-config-public.yaml - - if [[ -f /var/www/fence/yaml_merge.py ]]; then - python /var/www/fence/yaml_merge.py \ - /var/www/fence/fence-config-public.yaml \ - /var/run/fence-secrets/fence-config-secret.yaml \ - > /var/www/fence/fence-config.yaml - else - # If yaml_merge.py doesn't exist, just use the secret config - cp /var/run/fence-secrets/fence-config-secret.yaml /var/www/fence/fence-config.yaml - fi - - if [[ -f /var/run/fence-secrets/jwt_private_key.pem ]]; then - mkdir -p /fence/keys/key - cp /var/run/fence-secrets/jwt_private_key.pem /fence/keys/key/jwt_private_key.pem - chmod 600 /fence/keys/key/jwt_private_key.pem - openssl rsa -in /fence/keys/key/jwt_private_key.pem -pubout > /fence/keys/key/jwt_public_key.pem - fi - - bash /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then bash /dockerrun.sh; fi - env: - {{- toYaml .Values.env | nindent 12 }} - volumeMounts: - {{- toYaml .Values.volumeMounts | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/fence/templates/service.yaml b/helm/fence/templates/service.yaml index e58877089..e64635e54 100644 --- a/helm/fence/templates/service.yaml +++ b/helm/fence/templates/service.yaml @@ -13,18 +13,3 @@ spec: name: http selector: {{- include "fence.selectorLabels" . | nindent 4 }} ---- -apiVersion: v1 -kind: Service -metadata: - name: presigned-url-fence-service -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - app: presigned-url-fence - diff --git a/helm/gecko/files/init-data/nav.json b/helm/gecko/files/init-data/nav.json index 3237da1ca..dc71508e0 100644 --- a/helm/gecko/files/init-data/nav.json +++ b/helm/gecko/files/init-data/nav.json @@ -52,6 +52,13 @@ "href": "/Apps", "perms": null }, + { + "title": "Upload", + "description": "Upload local files into authorized storage.", + "icon": "/icons/apps/Upload.svg", + "href": "/upload", + "perms": "" + }, { "title": "Directory Structure", "description": "Search for files via a tree based interactive search", @@ -74,9 +81,9 @@ "perms": null }, { - "title": "GraphQL Query", + "title": "Query Editor", "description": "Query graph databases via a web interface", - "icon": "/icons/query.svg", + "icon": "/icons/Search.svg", "href": "/Query", "perms": null } diff --git a/helm/gecko/templates/deployment.yaml b/helm/gecko/templates/deployment.yaml index d423e705e..15b065d06 100644 --- a/helm/gecko/templates/deployment.yaml +++ b/helm/gecko/templates/deployment.yaml @@ -74,16 +74,18 @@ spec: key: serviceName - name: GRIP_PORT value: "8202" + {{- if .Values.qdrant.enabled }} - name: QDRANT_HOST value: {{ printf "%s-qdrant" .Release.Name | quote }} - name: QDRANT_PORT - value: "6334" + value: {{ .Values.qdrant.port | quote }} - name: QDRANT_API_KEY valueFrom: secretKeyRef: - name: {{ "qdrant-api-key-secret" }} - key: {{ "api-key" }} + name: {{ .Values.qdrant.apiKeySecretName | quote }} + key: {{ .Values.qdrant.apiKeySecretKey | quote }} optional: false + {{- end }} - name: PGPASSWORD valueFrom: secretKeyRef: @@ -138,4 +140,4 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/helm/gecko/templates/qdrant-pv.yaml b/helm/gecko/templates/qdrant-pv.yaml index 6f60f79f8..a893a6b60 100644 --- a/helm/gecko/templates/qdrant-pv.yaml +++ b/helm/gecko/templates/qdrant-pv.yaml @@ -1,3 +1,4 @@ +{{- if and .Values.qdrant.enabled .Values.qdrant.persistence.enabled }} apiVersion: v1 kind: PersistentVolume metadata: @@ -9,10 +10,11 @@ metadata: meta.helm.sh/release-namespace: "default" spec: capacity: - storage: 26Gi + storage: {{ .Values.qdrant.persistence.size | quote }} accessModes: - ReadWriteOnce - storageClassName: "qdrant-manual-storage" + storageClassName: {{ .Values.qdrant.persistence.storageClass | quote }} persistentVolumeReclaimPolicy: Retain hostPath: - path: "/mnt/data/qdrant-local" + path: {{ .Values.qdrant.persistence.hostPath | quote }} +{{- end }} diff --git a/helm/gecko/values.yaml b/helm/gecko/values.yaml index cf1af9049..4e3bc0d89 100644 --- a/helm/gecko/values.yaml +++ b/helm/gecko/values.yaml @@ -102,6 +102,25 @@ postgresql: # -- (bool) Option to persist the dbs data. enabled: false +qdrant: + # -- (bool) Whether Gecko should connect to Qdrant. + enabled: false + # -- (string) Kubernetes Secret containing the Qdrant API key. + apiKeySecretName: qdrant-api-key-secret + # -- (string) Key in the Qdrant API key Secret. + apiKeySecretKey: api-key + # -- (string) Qdrant gRPC port. + port: "6334" + persistence: + # -- (bool) Whether to create the local Qdrant PersistentVolume. + enabled: false + # -- (string) StorageClass used by the local Qdrant PersistentVolume. + storageClass: qdrant-manual-storage + # -- (string) Local path used by the local Qdrant PersistentVolume. + hostPath: /mnt/data/qdrant-local + # -- (string) Size of the local Qdrant PersistentVolume. + size: 26Gi + # -- (int) Number of replicas for the deployment. replicaCount: 1 diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 0febf36ad..f7a75c5ad 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -114,6 +114,11 @@ dependencies: - name: qdrant version: 1.15.4 repository: "https://qdrant.github.io/qdrant-helm" + condition: qdrant.enabled +- name: syfon + version: 0.1.0 + repository: "file://../syfon" + condition: syfon.enabled # A chart can be either an 'application' or a 'library' chart. # diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 7c21aad93..3a6d3d9c4 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -284,6 +284,15 @@ wts: # -- (bool) Whether to deploy the wts subchart. enabled: true +syfon: + # -- (bool) Whether to deploy the syfon subchart. + enabled: false + +gecko: + qdrant: + # -- (bool) Whether Gecko should connect to Qdrant. + enabled: false + # Disable persistence by default so we can spin up and down ephemeral environments postgresql: primary: @@ -299,7 +308,7 @@ qdrant: secretKeyRef: name: qdrant-api-key-secret # Name of the Kubernetes Secret from Step 1 key: api-key - enabled: true + enabled: false replicaCount: 1 resources: limits: diff --git a/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf b/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf deleted file mode 100644 index 522fad15f..000000000 --- a/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf +++ /dev/null @@ -1,10 +0,0 @@ -location ~ \/ga4gh\/drs\/v1\/objects\/(.*)\/access { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - - set $proxy_service "presigned-url-fence"; - set $upstream http://presigned-url-fence-service$des_domain; - rewrite ^/user/(.*) /$1 break; - proxy_pass $upstream; -} diff --git a/helm/revproxy/gen3.nginx.conf/fence-service.conf b/helm/revproxy/gen3.nginx.conf/fence-service.conf index dccbfa83e..37400d81b 100644 --- a/helm/revproxy/gen3.nginx.conf/fence-service.conf +++ b/helm/revproxy/gen3.nginx.conf/fence-service.conf @@ -1,7 +1,4 @@ -# AuthN-proxy uses fence to provide authentication to downstream services -# that don't implement our auth i.e. shiny, jupyter. -# Fence also sets the REMOTE_USER header to the username -# of the logged in user for later use +# AuthN-proxy (unchanged) location /authn-proxy { internal; set $proxy_service "fence"; @@ -12,57 +9,41 @@ location /authn-proxy { proxy_set_header Content-Length ""; proxy_set_header X-Forwarded-For "$realip"; proxy_set_header X-UserId "$userid"; - proxy_set_header X-ReqId "$request_id"; - proxy_set_header X-SessionId "$session_id"; - proxy_set_header X-VisitorId "$visitor_id"; - - # nginx bug that it checks even if request_body off + proxy_set_header X-ReqId "$request_id"; + proxy_set_header X-SessionId "$session_id"; + proxy_set_header X-VisitorId "$visitor_id"; client_max_body_size 0; } - -location /user/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - - set $proxy_service "fence"; +# -------------------------------------------- +# FENCE OWNS AUTH/USER PROFILE FLOWS +# -------------------------------------------- +location /user/register { + # no CSRF check at revproxy layer (as you already do) + set $proxy_service "fence"; set $upstream http://fence-service$des_domain; rewrite ^/user/(.*) /$1 break; proxy_pass $upstream; } -location /user/register { - # Like /user/ but without CSRF check. Registration form submission is - # incompatible with revproxy-level cookie-to-header CSRF check. - # Fence enforces its own CSRF protection here so this is OK. - set $proxy_service "fence"; - set $upstream http://fence-service$des_domain; - rewrite ^/user/(.*) /$1 break; - proxy_pass $upstream; +location /user/metrics { + deny all; } -location /user/data/download { +# Catch-all for all other /user/* -> fence +location ^~ /user/ { if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; } - set $proxy_service "presigned-url-fence"; - set $upstream http://presigned-url-fence-service$des_domain; + set $proxy_service "fence"; + set $upstream http://fence-service$des_domain; rewrite ^/user/(.*) /$1 break; proxy_pass $upstream; } -location /user/metrics { - deny all; -} - -# OpenID Connect Discovery Endpoints -location /.well-known/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - - set $proxy_service "fence"; +# OpenID discovery -> fence +location ^~ /.well-known/ { + set $proxy_service "fence"; set $upstream http://fence-service$des_domain; proxy_pass $upstream; } diff --git a/helm/revproxy/gen3.nginx.conf/indexd-service.conf b/helm/revproxy/gen3.nginx.conf/indexd-service.conf index 20f9414a7..aa85adb34 100644 --- a/helm/revproxy/gen3.nginx.conf/indexd-service.conf +++ b/helm/revproxy/gen3.nginx.conf/indexd-service.conf @@ -1,26 +1,26 @@ -location /ga4gh/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } +#location /ga4gh/ { +# if ($csrf_check !~ ^ok-\S.+$) { +# return 403 "failed csrf check"; +# } +# +# set $proxy_service "indexd"; +# set $upstream http://indexd-service$des_domain; +# +# proxy_pass $upstream; +# proxy_redirect http://$host/ https://$host/; +#} - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/; -} - -location /index/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - - set $proxy_service "indexd"; - set $upstream http://indexd-service$des_domain; - - rewrite ^/index/(.*) /$1 break; - - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/index/; -} +#location /index/ { +# if ($csrf_check !~ ^ok-\S.+$) { +# return 403 "failed csrf check"; +# } +# +# set $proxy_service "indexd"; +# set $upstream http://indexd-service$des_domain; +# +# rewrite ^/index/(.*) /$1 break; +# +# proxy_pass $upstream; +# proxy_redirect http://$host/ https://$host/index/; +#} diff --git a/helm/revproxy/gen3.nginx.conf/syfon-service.conf b/helm/revproxy/gen3.nginx.conf/syfon-service.conf new file mode 100644 index 000000000..d69ee6d5c --- /dev/null +++ b/helm/revproxy/gen3.nginx.conf/syfon-service.conf @@ -0,0 +1,94 @@ +# Shared upstream +set $drs_upstream http://syfon$des_domain:8080; + +# GA4GH DRS +location ^~ /ga4gh/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "syfon"; + proxy_pass $drs_upstream; + proxy_redirect http://$host/ https://$host/; +} + +# Index API (exact + subtree) +location /index { + set $proxy_service "syfon"; + proxy_pass $drs_upstream; + proxy_redirect http://$host/ https://$host/; +} + +location ^~ /index/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "syfon"; + proxy_pass $drs_upstream; + proxy_redirect http://$host/ https://$host/; +} + +# Upload / download canonical routes +location ^~ /upload { + + set $proxy_service "syfon"; + proxy_pass $drs_upstream; + proxy_redirect http://$host/ https://$host/; + + # multipart helpers + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; +} + +location ^~ /download/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "syfon"; + proxy_pass $drs_upstream; + proxy_redirect http://$host/ https://$host/; +} + +# Git LFS transport +location ^~ /info/lfs/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "syfon"; + proxy_pass $drs_upstream; + proxy_redirect http://$host/ https://$host/; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; +} + +location ^~ /data/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "syfon"; + proxy_pass http://syfon$des_domain:8080; + proxy_redirect http://$host/ https://$host/; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + +} + +location /healthz { + set $proxy_service "syfon"; + proxy_pass http://syfon$des_domain:8080; +} diff --git a/helm/syfon/Chart.yaml b/helm/syfon/Chart.yaml new file mode 100644 index 000000000..3a7b9ef4e --- /dev/null +++ b/helm/syfon/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: syfon +description: Helm chart for syfon (GA4GH DRS + Gen3 compatibility) +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/helm/syfon/README.md b/helm/syfon/README.md new file mode 100644 index 000000000..0bd1c63a8 --- /dev/null +++ b/helm/syfon/README.md @@ -0,0 +1,114 @@ +# syfon Helm Chart + +This chart deploys `syfon` with: + +- Syfon config mounted into the pod at `/etc/drs/config.yaml` from `config` + using a Kubernetes Secret +- DB credentials injected via secret env vars (`DRS_DB_*`) +- Optional PostgreSQL init job that mirrors indexd-style setup: + - creates app DB user + - creates app database + - applies DRS schema tables + +## Syfon Config + +`config` is rendered directly as Syfon's server config. Use the same keys that +`syfon serve --config` accepts. + +In `gen3` auth mode, the chart fills `config.auth.fence_url` from +`global.hostname` when it is omitted, rendering it as +`https:///user`. Set `config.auth.fence_url` explicitly only +when Syfon should trust a different public Fence endpoint. + +Example: + +```yaml +config: + port: 8080 + auth: + mode: gen3 + # Optional; defaults to https:///user + fence_url: https://gen3.example.org/user + routes: + docs: true + ga4gh: true + internal: true + lfs: true + metrics: true + signing: + default_expiry_seconds: 900 + credential_encryption: + master_key: base64-or-hex-or-32-byte-key + s3_credentials: + - bucket: cbds + provider: s3 + region: us-east-1 + endpoint: https://s3.example.org + access_key: access-key + secret_key: secret-key + resources: + - organization: cbds + project: training + org_path: programs/cbds + project_path: projects/training + bucket_scopes: + - organization: cbds + project_id: training + bucket: cbds + org_path: programs/cbds + project_path: projects/training +``` + +Configured `s3_credentials` are loaded by the Syfon server on startup. Syfon +requires a credential encryption key for non-empty bucket credentials; set +`credential_encryption.master_key` in the same config block. The key may be a +32-byte raw string, a 64-character hex string, or base64-encoded 32-byte key. + +Configured `bucket_scopes` are loaded on startup too. `organization` and +`project_id` are the Gen3 authz labels. `organization_sub_path` / +`project_sub_path` are storage-layout prefixes, and the chart also accepts the +shorter aliases `org_path` / `project_path` and normalizes them in the rendered +config. You can define these scopes either as top-level `bucket_scopes` or +inline under each `s3_credentials[*].resources` entry. Inline resource entries +use `organization`, `project`, `org_path`, and `project_path`; the chart +attaches the parent bucket and renders the final server config as normal +`bucket_scopes`. Syfon stores the full scope prefix and prepends that prefix +when signing imported record URLs that are relative to the bucket root. You can +also set a complete `path` or explicit `bucket` plus `path_prefix`. + +## Key Compatibility Notes + +- Secret keys mirror indexd credentials naming (`db_host`, `db_username`, `db_password`, `db_database`) with additional `db_port` and `db_sslmode`. +- In `gen3` mode, `syfon` requires PostgreSQL. +- The rendered Syfon config is stored as a Kubernetes Secret because it can + contain bucket credentials and `credential_encryption.master_key`. +- Database connection values are still supplied from Kubernetes secrets via + `DRS_DB_*` env vars. + +## Install + +```bash +helm upgrade --install syfon ./helm/syfon +``` + +## Existing Secrets + +To reuse existing DB secrets: + +- Set `postgres.app.existingSecret` +- Set `postgres.admin.existingSecret` (if `postgres.initJob.enabled=true`) + +## Health Probes + +The chart configures both readiness and liveness probes against `GET /healthz` on the container `http` port. + +Tune probe behavior via: + +- `probes.liveness.*` +- `probes.readiness.*` + +## PostgreSQL Source of Truth + +By default this chart now inherits PostgreSQL host/port/admin credentials from `global.postgres.master.*` (the same pattern used by other Gen3 charts). + +Service-specific values under `postgres.app.*` and `postgres.admin.*` still override global values when set. diff --git a/helm/syfon/templates/_helpers.tpl b/helm/syfon/templates/_helpers.tpl new file mode 100644 index 000000000..8c1ffb930 --- /dev/null +++ b/helm/syfon/templates/_helpers.tpl @@ -0,0 +1,61 @@ +{{- define "syfon.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "syfon.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := include "syfon.name" . -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "syfon.labels" -}} +app.kubernetes.io/name: {{ include "syfon.name" . }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "syfon.selectorLabels" -}} +app.kubernetes.io/name: {{ include "syfon.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "syfon.appDbSecretName" -}} +{{- if .Values.postgres.app.existingSecret -}} +{{- .Values.postgres.app.existingSecret -}} +{{- else -}} +{{- .Values.postgres.app.secretName -}} +{{- end -}} +{{- end -}} + +{{- define "syfon.adminDbSecretName" -}} +{{- if .Values.postgres.admin.existingSecret -}} +{{- .Values.postgres.admin.existingSecret -}} +{{- else -}} +{{- .Values.postgres.admin.secretName -}} +{{- end -}} +{{- end -}} + +{{- define "syfon.fenceURL" -}} +{{- $cfg := .Values.config | default dict -}} +{{- $auth := get $cfg "auth" | default dict -}} +{{- $configured := get $auth "fence_url" | default "" | toString | trim -}} +{{- if $configured -}} +{{- $configured -}} +{{- else -}} +{{- $global := .Values.global | default dict -}} +{{- $hostname := get $global "hostname" | default "" | toString | trim -}} +{{- if $hostname -}} +{{- $host := trimSuffix "/" (trimPrefix "https://" (trimPrefix "http://" $hostname)) -}} +{{- printf "https://%s/user" $host -}} +{{- end -}} +{{- end -}} +{{- end -}} + diff --git a/helm/syfon/templates/config-secret.yaml b/helm/syfon/templates/config-secret.yaml new file mode 100644 index 000000000..1164bd200 --- /dev/null +++ b/helm/syfon/templates/config-secret.yaml @@ -0,0 +1,25 @@ +{{- $cfg := deepCopy (.Values.config | default dict) -}} +{{- $inputBuckets := default (list) (index $cfg "buckets") -}} +{{- if and (eq (len $inputBuckets) 0) (hasKey $cfg "s3_credentials") -}} + {{- $inputBuckets = default (list) (index $cfg "s3_credentials") -}} +{{- end -}} +{{- $_ := set $cfg "buckets" $inputBuckets -}} +{{- $_ := unset $cfg "s3_credentials" -}} +{{- $auth := deepCopy (get $cfg "auth" | default dict) -}} +{{- if and (eq (get $auth "mode" | default "" | toString) "gen3") (not (get $auth "fence_url")) -}} + {{- $fenceURL := include "syfon.fenceURL" . -}} + {{- if $fenceURL -}} + {{- $_ := set $auth "fence_url" $fenceURL -}} + {{- $_ := set $cfg "auth" $auth -}} + {{- end -}} +{{- end -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "syfon.fullname" . }}-config + labels: + {{- include "syfon.labels" . | nindent 4 }} +type: Opaque +stringData: + config.yaml: | +{{ toYaml $cfg | nindent 4 }} diff --git a/helm/syfon/templates/deployment.yaml b/helm/syfon/templates/deployment.yaml new file mode 100644 index 000000000..165d4a9bd --- /dev/null +++ b/helm/syfon/templates/deployment.yaml @@ -0,0 +1,94 @@ +{{- $cfg := (.Values.config | default dict) -}} +{{- $configPort := (get $cfg "port" | default 8080) -}} +{{- $fenceURL := include "syfon.fenceURL" . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "syfon.fullname" . }} + labels: + {{- include "syfon.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "syfon.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "syfon.selectorLabels" . | nindent 8 }} + spec: + containers: + - name: syfon + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["serve", "--config", "/etc/drs/config.yaml"] + ports: + - name: http + containerPort: {{ $configPort }} + protocol: TCP + livenessProbe: + httpGet: + path: {{ .Values.probes.liveness.path | quote }} + port: http + initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} + readinessProbe: + httpGet: + path: {{ .Values.probes.readiness.path | quote }} + port: http + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + env: + - name: DRS_PORT + value: {{ $configPort | quote }} + {{- if $fenceURL }} + - name: DRS_FENCE_URL + value: {{ $fenceURL | quote }} + {{- end }} + - name: DRS_DB_HOST + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_host + - name: DRS_DB_PORT + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_port + - name: DRS_DB_USER + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_username + - name: DRS_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_password + - name: DRS_DB_DATABASE + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_database + - name: DRS_DB_SSLMODE + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_sslmode + {{- with .Values.extraEnv }} +{{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/drs + readOnly: true + resources: +{{ toYaml .Values.resources | indent 12 }} + volumes: + - name: config + secret: + secretName: {{ include "syfon.fullname" . }}-config diff --git a/helm/syfon/templates/postgres-init-job.yaml b/helm/syfon/templates/postgres-init-job.yaml new file mode 100644 index 000000000..6ef6f4d1b --- /dev/null +++ b/helm/syfon/templates/postgres-init-job.yaml @@ -0,0 +1,122 @@ +{{- if .Values.postgres.initJob.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "syfon.fullname" . }}-postgres-init + labels: + {{- include "syfon.labels" . | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "syfon.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: OnFailure + containers: + - name: postgres-init + image: {{ .Values.postgres.initJob.image }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + set -eu + export PGPASSWORD="${PG_ADMIN_PASSWORD}" + + max_wait_seconds="${PG_WAIT_TIMEOUT_SECONDS:-180}" + waited_seconds=0 + until pg_isready -h "${PG_ADMIN_HOST}" -p "${PG_ADMIN_PORT}" -U "${PG_ADMIN_USER}"; do + if [ "${waited_seconds}" -ge "${max_wait_seconds}" ]; then + echo "Timed out after ${max_wait_seconds}s waiting for PostgreSQL at ${PG_ADMIN_HOST}:${PG_ADMIN_PORT}" + exit 1 + fi + echo "Waiting for PostgreSQL at ${PG_ADMIN_HOST}:${PG_ADMIN_PORT}... (${waited_seconds}s/${max_wait_seconds}s)" + sleep 2 + waited_seconds=$((waited_seconds + 2)) + done + + psql -h "${PG_ADMIN_HOST}" -p "${PG_ADMIN_PORT}" -U "${PG_ADMIN_USER}" -d "${PG_ADMIN_DB}" -v ON_ERROR_STOP=1 \ + -c "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='${DRS_DB_USER}') THEN CREATE ROLE \"${DRS_DB_USER}\" LOGIN PASSWORD '${DRS_DB_PASSWORD}'; ELSE ALTER ROLE \"${DRS_DB_USER}\" WITH LOGIN PASSWORD '${DRS_DB_PASSWORD}'; END IF; END \$\$;" + + if ! psql -h "${PG_ADMIN_HOST}" -p "${PG_ADMIN_PORT}" -U "${PG_ADMIN_USER}" -d "${PG_ADMIN_DB}" -tAc "SELECT 1 FROM pg_database WHERE datname='${DRS_DB_NAME}'" | grep -q 1; then + psql -h "${PG_ADMIN_HOST}" -p "${PG_ADMIN_PORT}" -U "${PG_ADMIN_USER}" -d "${PG_ADMIN_DB}" -v ON_ERROR_STOP=1 \ + -c "CREATE DATABASE \"${DRS_DB_NAME}\" OWNER \"${DRS_DB_USER}\";" + fi + + psql -h "${PG_ADMIN_HOST}" -p "${PG_ADMIN_PORT}" -U "${PG_ADMIN_USER}" -d "${PG_ADMIN_DB}" -v ON_ERROR_STOP=1 \ + -c "GRANT ALL PRIVILEGES ON DATABASE \"${DRS_DB_NAME}\" TO \"${DRS_DB_USER}\";" + + export PGPASSWORD="${DRS_DB_PASSWORD}" + psql -h "${DRS_DB_HOST}" -p "${DRS_DB_PORT}" -U "${DRS_DB_USER}" -d "${DRS_DB_NAME}" -v ON_ERROR_STOP=1 -f /sql/drs_schema.sql + env: + - name: PG_WAIT_TIMEOUT_SECONDS + value: {{ .Values.postgres.initJob.waitTimeoutSeconds | quote }} + - name: DRS_DB_HOST + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_host + - name: DRS_DB_PORT + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_port + - name: DRS_DB_USER + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_username + - name: DRS_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_password + - name: DRS_DB_NAME + valueFrom: + secretKeyRef: + name: {{ include "syfon.appDbSecretName" . }} + key: db_database + - name: PG_ADMIN_HOST + valueFrom: + secretKeyRef: + name: {{ include "syfon.adminDbSecretName" . }} + key: host + - name: PG_ADMIN_PORT + valueFrom: + secretKeyRef: + name: {{ include "syfon.adminDbSecretName" . }} + key: port + - name: PG_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ include "syfon.adminDbSecretName" . }} + key: username + - name: PG_ADMIN_PASSWORD + {{- $global := (.Values.global | default dict) -}} + {{- $pg := (get $global "postgres" | default dict) -}} + {{- $master := (get $pg "master" | default dict) -}} + {{- $globalMasterPass := (get $master "password" | default "") -}} + {{- if and (not .Values.postgres.admin.existingSecret) (get $global "dev") (not .Values.postgres.admin.password) (not $globalMasterPass) }} + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-postgresql + key: postgres-password + {{- else }} + valueFrom: + secretKeyRef: + name: {{ include "syfon.adminDbSecretName" . }} + key: password + {{- end }} + - name: PG_ADMIN_DB + valueFrom: + secretKeyRef: + name: {{ include "syfon.adminDbSecretName" . }} + key: database + volumeMounts: + - name: schema + mountPath: /sql + volumes: + - name: schema + configMap: + name: {{ include "syfon.fullname" . }}-postgres-schema +{{- end }} diff --git a/helm/syfon/templates/postgres-schema-configmap.yaml b/helm/syfon/templates/postgres-schema-configmap.yaml new file mode 100644 index 000000000..731e921ae --- /dev/null +++ b/helm/syfon/templates/postgres-schema-configmap.yaml @@ -0,0 +1,282 @@ +{{- if .Values.postgres.initJob.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "syfon.fullname" . }}-postgres-schema + labels: + {{- include "syfon.labels" . | nindent 4 }} +data: + drs_schema.sql: | + CREATE TABLE IF NOT EXISTS drs_object ( + id TEXT PRIMARY KEY, + size BIGINT, + created_time TIMESTAMPTZ, + updated_time TIMESTAMPTZ, + name TEXT, + version TEXT, + description TEXT + ); + + CREATE TABLE IF NOT EXISTS drs_object_access_method ( + object_id TEXT NOT NULL, + url TEXT NOT NULL, + type TEXT NOT NULL, + org TEXT NOT NULL DEFAULT '', + project TEXT NOT NULL DEFAULT '', + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS drs_object_checksum ( + object_id TEXT NOT NULL, + type TEXT NOT NULL, + checksum TEXT NOT NULL, + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS drs_object_alias ( + alias_id TEXT PRIMARY KEY, + object_id TEXT NOT NULL, + FOREIGN KEY(object_id) REFERENCES drs_object(id) ON DELETE CASCADE + ); + + ALTER TABLE drs_object_access_method + ADD COLUMN IF NOT EXISTS org TEXT NOT NULL DEFAULT ''; + + ALTER TABLE drs_object_access_method + ADD COLUMN IF NOT EXISTS project TEXT NOT NULL DEFAULT ''; + + CREATE TABLE IF NOT EXISTS s3_credential ( + bucket TEXT PRIMARY KEY, + provider TEXT NOT NULL DEFAULT 's3', + region TEXT, + access_key TEXT, + secret_key TEXT, + endpoint TEXT + ); + + ALTER TABLE s3_credential + ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 's3'; + + + CREATE TABLE IF NOT EXISTS bucket_scope ( + organization TEXT NOT NULL, + project_id TEXT NOT NULL, + bucket TEXT NOT NULL, + path_prefix TEXT NULL, + PRIMARY KEY (organization, project_id) + ); + + CREATE TABLE IF NOT EXISTS lfs_pending_metadata ( + oid TEXT PRIMARY KEY, + candidate_json JSONB NOT NULL, + created_time TIMESTAMPTZ NOT NULL, + expires_time TIMESTAMPTZ NOT NULL + ); + + CREATE TABLE IF NOT EXISTS object_usage ( + object_id TEXT PRIMARY KEY REFERENCES drs_object(id) ON DELETE CASCADE, + upload_count BIGINT NOT NULL DEFAULT 0, + download_count BIGINT NOT NULL DEFAULT 0, + last_upload_time TIMESTAMPTZ NULL, + last_download_time TIMESTAMPTZ NULL, + updated_time TIMESTAMPTZ NOT NULL + ); + + CREATE TABLE IF NOT EXISTS object_usage_event ( + id BIGSERIAL PRIMARY KEY, + object_id TEXT NOT NULL, + event_type TEXT NOT NULL CHECK (event_type IN ('upload','download')), + event_time TIMESTAMPTZ NOT NULL + ); + + CREATE TABLE IF NOT EXISTS transfer_attribution_event ( + event_id TEXT PRIMARY KEY, + access_grant_id TEXT NOT NULL DEFAULT '', + event_type TEXT NOT NULL CHECK (event_type IN ('access_issued')), + direction TEXT NOT NULL DEFAULT 'download' CHECK (direction IN ('download','upload')), + event_time TIMESTAMPTZ NOT NULL, + request_id TEXT NOT NULL DEFAULT '', + object_id TEXT NOT NULL DEFAULT '', + sha256 TEXT NOT NULL DEFAULT '', + object_size BIGINT NOT NULL DEFAULT 0, + organization TEXT NOT NULL DEFAULT '', + project TEXT NOT NULL DEFAULT '', + access_id TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + bucket TEXT NOT NULL DEFAULT '', + storage_url TEXT NOT NULL DEFAULT '', + range_start BIGINT NULL, + range_end BIGINT NULL, + bytes_requested BIGINT NOT NULL DEFAULT 0, + bytes_completed BIGINT NOT NULL DEFAULT 0, + actor_email TEXT NOT NULL DEFAULT '', + actor_subject TEXT NOT NULL DEFAULT '', + auth_mode TEXT NOT NULL DEFAULT '', + client_name TEXT NOT NULL DEFAULT '', + client_version TEXT NOT NULL DEFAULT '', + transfer_session_id TEXT NOT NULL DEFAULT '' + ); + + CREATE TABLE IF NOT EXISTS access_grant ( + access_grant_id TEXT PRIMARY KEY, + first_issued_at TIMESTAMPTZ NOT NULL, + last_issued_at TIMESTAMPTZ NOT NULL, + issue_count BIGINT NOT NULL DEFAULT 0, + object_id TEXT NOT NULL DEFAULT '', + sha256 TEXT NOT NULL DEFAULT '', + object_size BIGINT NOT NULL DEFAULT 0, + organization TEXT NOT NULL DEFAULT '', + project TEXT NOT NULL DEFAULT '', + access_id TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + bucket TEXT NOT NULL DEFAULT '', + storage_url TEXT NOT NULL DEFAULT '', + actor_email TEXT NOT NULL DEFAULT '', + actor_subject TEXT NOT NULL DEFAULT '', + auth_mode TEXT NOT NULL DEFAULT '' + ); + + CREATE TABLE IF NOT EXISTS provider_transfer_event ( + provider_event_id TEXT PRIMARY KEY, + access_grant_id TEXT NOT NULL DEFAULT '', + direction TEXT NOT NULL CHECK (direction IN ('download','upload')), + event_time TIMESTAMPTZ NOT NULL, + request_id TEXT NOT NULL DEFAULT '', + provider_request_id TEXT NOT NULL DEFAULT '', + object_id TEXT NOT NULL DEFAULT '', + sha256 TEXT NOT NULL DEFAULT '', + object_size BIGINT NOT NULL DEFAULT 0, + organization TEXT NOT NULL DEFAULT '', + project TEXT NOT NULL DEFAULT '', + access_id TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + bucket TEXT NOT NULL DEFAULT '', + object_key TEXT NOT NULL DEFAULT '', + storage_url TEXT NOT NULL DEFAULT '', + range_start BIGINT NULL, + range_end BIGINT NULL, + bytes_transferred BIGINT NOT NULL DEFAULT 0, + http_method TEXT NOT NULL DEFAULT '', + http_status INTEGER NOT NULL DEFAULT 0, + requester_principal TEXT NOT NULL DEFAULT '', + source_ip TEXT NOT NULL DEFAULT '', + user_agent TEXT NOT NULL DEFAULT '', + raw_event_ref TEXT NOT NULL DEFAULT '', + actor_email TEXT NOT NULL DEFAULT '', + actor_subject TEXT NOT NULL DEFAULT '', + auth_mode TEXT NOT NULL DEFAULT '', + reconciliation_status TEXT NOT NULL DEFAULT 'unmatched' CHECK (reconciliation_status IN ('matched','ambiguous','unmatched')) + ); + + CREATE TABLE IF NOT EXISTS provider_transfer_sync_run ( + sync_id TEXT PRIMARY KEY, + provider TEXT NOT NULL DEFAULT '', + bucket TEXT NOT NULL DEFAULT '', + organization TEXT NOT NULL DEFAULT '', + project TEXT NOT NULL DEFAULT '', + from_time TIMESTAMPTZ NOT NULL, + to_time TIMESTAMPTZ NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending','completed','failed')), + requested_at TIMESTAMPTZ NOT NULL, + started_at TIMESTAMPTZ NULL, + completed_at TIMESTAMPTZ NULL, + imported_events BIGINT NOT NULL DEFAULT 0, + matched_events BIGINT NOT NULL DEFAULT 0, + ambiguous_events BIGINT NOT NULL DEFAULT 0, + unmatched_events BIGINT NOT NULL DEFAULT 0, + error_message TEXT NOT NULL DEFAULT '' + ); + + ALTER TABLE transfer_attribution_event + ADD COLUMN IF NOT EXISTS access_grant_id TEXT NOT NULL DEFAULT ''; + + ALTER TABLE transfer_attribution_event + ADD COLUMN IF NOT EXISTS direction TEXT NOT NULL DEFAULT 'download'; + + CREATE INDEX IF NOT EXISTS drs_object_access_method_object_id_idx + ON drs_object_access_method(object_id); + + CREATE INDEX IF NOT EXISTS drs_object_checksum_object_id_idx + ON drs_object_checksum(object_id); + + CREATE INDEX IF NOT EXISTS drs_object_checksum_checksum_idx + ON drs_object_checksum(checksum); + + CREATE INDEX IF NOT EXISTS drs_object_access_method_scope_idx + ON drs_object_access_method(org, project); + + CREATE INDEX IF NOT EXISTS drs_object_alias_object_id_idx + ON drs_object_alias(object_id); + + CREATE INDEX IF NOT EXISTS idx_bucket_scope_bucket + ON bucket_scope(bucket); + + CREATE INDEX IF NOT EXISTS idx_lfs_pending_metadata_expires + ON lfs_pending_metadata(expires_time); + + CREATE INDEX IF NOT EXISTS idx_lfs_pending_metadata_created + ON lfs_pending_metadata(created_time); + + CREATE INDEX IF NOT EXISTS idx_object_usage_last_download_time + ON object_usage(last_download_time); + + CREATE INDEX IF NOT EXISTS idx_object_usage_last_upload_time + ON object_usage(last_upload_time); + + CREATE INDEX IF NOT EXISTS idx_object_usage_event_object_id + ON object_usage_event(object_id); + + CREATE INDEX IF NOT EXISTS idx_object_usage_event_event_time + ON object_usage_event(event_time); + + CREATE INDEX IF NOT EXISTS idx_transfer_attr_scope_time + ON transfer_attribution_event(organization, project, event_type, event_time); + + CREATE INDEX IF NOT EXISTS idx_transfer_attr_direction_time + ON transfer_attribution_event(direction, event_time); + + CREATE INDEX IF NOT EXISTS idx_transfer_attr_actor_time + ON transfer_attribution_event(actor_email, actor_subject, event_time); + + CREATE INDEX IF NOT EXISTS idx_transfer_attr_provider_time + ON transfer_attribution_event(provider, bucket, event_time); + + CREATE INDEX IF NOT EXISTS idx_transfer_attr_sha_time + ON transfer_attribution_event(sha256, event_time); + + CREATE INDEX IF NOT EXISTS idx_transfer_attr_session + ON transfer_attribution_event(transfer_session_id); + + CREATE INDEX IF NOT EXISTS idx_access_grant_storage_time + ON access_grant(provider, bucket, storage_url, last_issued_at); + + CREATE INDEX IF NOT EXISTS idx_access_grant_scope_time + ON access_grant(organization, project, last_issued_at); + + CREATE INDEX IF NOT EXISTS idx_access_grant_sha_time + ON access_grant(sha256, last_issued_at); + + CREATE INDEX IF NOT EXISTS idx_provider_transfer_scope_time + ON provider_transfer_event(organization, project, direction, event_time); + + CREATE INDEX IF NOT EXISTS idx_provider_transfer_actor_time + ON provider_transfer_event(actor_email, actor_subject, event_time); + + CREATE INDEX IF NOT EXISTS idx_provider_transfer_provider_time + ON provider_transfer_event(provider, bucket, event_time); + + CREATE INDEX IF NOT EXISTS idx_provider_transfer_sha_time + ON provider_transfer_event(sha256, event_time); + + CREATE INDEX IF NOT EXISTS idx_provider_transfer_status + ON provider_transfer_event(reconciliation_status, event_time); + + CREATE INDEX IF NOT EXISTS idx_provider_transfer_grant + ON provider_transfer_event(access_grant_id); + + CREATE INDEX IF NOT EXISTS idx_provider_sync_bucket_time + ON provider_transfer_sync_run(provider, bucket, requested_at); + + CREATE INDEX IF NOT EXISTS idx_provider_sync_scope_time + ON provider_transfer_sync_run(organization, project, requested_at); +{{- end }} diff --git a/helm/syfon/templates/secret-admin-db.yaml b/helm/syfon/templates/secret-admin-db.yaml new file mode 100644 index 000000000..b6e73f2f2 --- /dev/null +++ b/helm/syfon/templates/secret-admin-db.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.postgres.initJob.enabled (not .Values.postgres.admin.existingSecret) }} +{{- $global := (.Values.global | default dict) -}} +{{- $pg := (get $global "postgres" | default dict) -}} +{{- $master := (get $pg "master" | default dict) -}} +{{- $globalMasterHost := (get $master "host" | default "") -}} +{{- $globalMasterPort := (get $master "port" | default "5432") -}} +{{- $globalMasterUser := (get $master "username" | default "postgres") -}} +{{- $globalMasterPass := (get $master "password" | default "") -}} +{{- $releasePostgresHost := printf "%s-postgresql" .Release.Name -}} +{{- $resolvedHost := (coalesce .Values.postgres.admin.host $globalMasterHost $releasePostgresHost) -}} +{{- $resolvedPort := (coalesce .Values.postgres.admin.port $globalMasterPort "5432") -}} +{{- $resolvedUser := (coalesce .Values.postgres.admin.username $globalMasterUser "postgres") -}} +{{- $resolvedPass := (coalesce .Values.postgres.admin.password $globalMasterPass "") -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.postgres.admin.secretName }} + labels: + {{- include "syfon.labels" . | nindent 4 }} +type: Opaque +stringData: + host: {{ $resolvedHost | quote }} + port: {{ $resolvedPort | quote }} + username: {{ $resolvedUser | quote }} + password: {{ $resolvedPass | quote }} + database: {{ .Values.postgres.admin.database | quote }} +{{- end }} diff --git a/helm/syfon/templates/secret-app-db.yaml b/helm/syfon/templates/secret-app-db.yaml new file mode 100644 index 000000000..f3ea17328 --- /dev/null +++ b/helm/syfon/templates/secret-app-db.yaml @@ -0,0 +1,24 @@ +{{- if not .Values.postgres.app.existingSecret }} +{{- $global := (.Values.global | default dict) -}} +{{- $pg := (get $global "postgres" | default dict) -}} +{{- $master := (get $pg "master" | default dict) -}} +{{- $globalMasterHost := (get $master "host" | default "") -}} +{{- $globalMasterPort := (get $master "port" | default "5432") -}} +{{- $releasePostgresHost := printf "%s-postgresql" .Release.Name -}} +{{- $resolvedHost := (coalesce .Values.postgres.app.db_host $globalMasterHost $releasePostgresHost) -}} +{{- $resolvedPort := (coalesce .Values.postgres.app.db_port $globalMasterPort "5432") -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.postgres.app.secretName }} + labels: + {{- include "syfon.labels" . | nindent 4 }} +type: Opaque +stringData: + db_host: {{ $resolvedHost | quote }} + db_port: {{ $resolvedPort | quote }} + db_username: {{ .Values.postgres.app.db_username | quote }} + db_password: {{ .Values.postgres.app.db_password | quote }} + db_database: {{ .Values.postgres.app.db_database | quote }} + db_sslmode: {{ .Values.postgres.app.db_sslmode | quote }} +{{- end }} diff --git a/helm/syfon/templates/service.yaml b/helm/syfon/templates/service.yaml new file mode 100644 index 000000000..dd4dd3baf --- /dev/null +++ b/helm/syfon/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "syfon.fullname" . }} + labels: + {{- include "syfon.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "syfon.selectorLabels" . | nindent 4 }} + diff --git a/helm/syfon/values.yaml b/helm/syfon/values.yaml new file mode 100644 index 000000000..09457ef14 --- /dev/null +++ b/helm/syfon/values.yaml @@ -0,0 +1,77 @@ +fullnameOverride: syfon + +replicaCount: 1 + +image: + repository: quay.io/ohsu-comp-bio/syfon + tag: latest + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 8080 + +config: + port: 8080 + auth: + mode: gen3 + # Public Fence endpoint used to validate Gen3 token issuers and fetch /user/user privileges. + # Defaults to https:///user when omitted. + fence_url: "" + routes: + docs: true + ga4gh: true + internal: true + lfs: true + metrics: true + signing: + default_expiry_seconds: 900 + credential_encryption: {} + # Preferred Syfon config shape. Each bucket may carry nested org/project routing + # resources, which Syfon flattens into runtime bucket scope records on startup. + buckets: [] + # Legacy inputs still accepted by Syfon, but the chart no longer synthesizes + # bucket_scopes from credential resources. + s3_credentials: [] + bucket_scopes: [] + +postgres: + app: + existingSecret: "" + secretName: syfon-db + db_host: "" + db_port: "" + db_username: syfon_user + db_password: syfon_pass + db_database: syfon_db + db_sslmode: disable + admin: + existingSecret: "" + secretName: syfon-db-admin + host: "" + port: "" + username: "" + password: "" + database: postgres + initJob: + enabled: true + image: postgres:16 + waitTimeoutSeconds: 180 + +resources: {} + +extraEnv: [] + +probes: + liveness: + path: /healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + readiness: + path: /healthz + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3