diff --git a/.github/labeler.yml b/.github/labeler.yml index 83d472b..84a80b9 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -57,3 +57,8 @@ github-config: - ".github/CODEOWNERS" - ".releaserc.yml" - ".gitignore" + +# Changes to the canonical deployment matrix consumed by gitops-update.yml +deployment-matrix: + - changed-files: + - any-glob-to-any-file: "config/deployment-matrix.yml" diff --git a/.github/labels.yml b/.github/labels.yml index 8035f58..50b29d5 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -95,3 +95,7 @@ - name: validate color: "1d76db" description: Changes to PR validation composite actions (src/validate/) + +- name: deployment-matrix + color: "5319e7" + description: Changes to the canonical deployment matrix (config/deployment-matrix.yml) diff --git a/.github/workflows/gitops-update.yml b/.github/workflows/gitops-update.yml index 5f3acb0..4488965 100644 --- a/.github/workflows/gitops-update.yml +++ b/.github/workflows/gitops-update.yml @@ -16,13 +16,25 @@ on: type: string required: false deploy_in_firmino: - description: 'Deploy to Firmino server' + description: 'Force-off override for Firmino. Set to false to suppress deployment even if the manifest includes this app on Firmino.' type: boolean default: true deploy_in_clotilde: - description: 'Deploy to Clotilde server' + description: 'Force-off override for Clotilde. Set to false to suppress deployment even if the manifest includes this app on Clotilde.' type: boolean default: true + deploy_in_anacleto: + description: 'Force-off override for Anacleto. Set to false to suppress deployment even if the manifest includes this app on Anacleto.' + type: boolean + default: true + deployment_matrix_file: + description: 'Path (within the shared-workflows checkout) to the deployment matrix manifest. Override only if you maintain a fork/alternative manifest.' + type: string + default: 'config/deployment-matrix.yml' + deployment_matrix_ref: + description: 'Git ref of LerianStudio/github-actions-shared-workflows to read the deployment matrix from. Defaults to main (always latest). Override only when testing a branch.' + type: string + default: 'main' artifact_pattern: description: 'Pattern to download artifacts (defaults to "gitops-tags--*" if not provided)' type: string @@ -80,6 +92,11 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKERHUB_IMAGE_PULL_TOKEN }} + # Trust model: this is a `workflow_call` reusable workflow, NOT triggered by + # untrusted PRs. `inputs.gitops_repository` is supplied by trusted caller + # workflows (default: LerianStudio/midaz-firmino-gitops) and the MANAGE_TOKEN + # is required for the subsequent commit/push step. CodeQL flags this as + # `actions/untrusted-checkout` defensively but the call surface is internal-only. - name: Checkout GitOps Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -88,6 +105,19 @@ jobs: path: gitops fetch-depth: 0 + - name: Checkout deployment matrix manifest + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + repository: LerianStudio/github-actions-shared-workflows + ref: ${{ inputs.deployment_matrix_ref }} + path: shared-workflows + sparse-checkout: | + ${{ inputs.deployment_matrix_file }} + sparse-checkout-cone-mode: false + # Read-only sparse checkout of a single manifest file — no credentials + # needed, and we never execute code from this checkout. + persist-credentials: false + - name: Setup application name and paths id: setup shell: bash @@ -101,15 +131,6 @@ jobs: echo "app_name=$APP_NAME" >> "$GITHUB_OUTPUT" echo "Application name: $APP_NAME" - # Determine which servers to deploy to - DEPLOY_FIRMINO="${{ inputs.deploy_in_firmino }}" - DEPLOY_CLOTILDE="${{ inputs.deploy_in_clotilde }}" - - echo "deploy_firmino=$DEPLOY_FIRMINO" >> "$GITHUB_OUTPUT" - echo "deploy_clotilde=$DEPLOY_CLOTILDE" >> "$GITHUB_OUTPUT" - echo "Deploy to Firmino: $DEPLOY_FIRMINO" - echo "Deploy to Clotilde: $DEPLOY_CLOTILDE" - # Generate commit message prefix if not provided if [[ -z "${{ inputs.commit_message_prefix }}" ]]; then COMMIT_PREFIX="$APP_NAME" @@ -140,6 +161,98 @@ jobs: sudo chmod +x /usr/local/bin/yq yq --version + - name: Resolve target clusters from deployment matrix + id: resolve_clusters + shell: bash + env: + APP_NAME: ${{ steps.setup.outputs.app_name }} + MANIFEST: shared-workflows/${{ inputs.deployment_matrix_file }} + DEPLOY_FIRMINO: ${{ inputs.deploy_in_firmino }} + DEPLOY_CLOTILDE: ${{ inputs.deploy_in_clotilde }} + DEPLOY_ANACLETO: ${{ inputs.deploy_in_anacleto }} + run: | + set -euo pipefail + + if [[ ! -f "$MANIFEST" ]]; then + echo "::error::Deployment matrix manifest not found: $MANIFEST" + exit 1 + fi + + echo "Reading deployment matrix: $MANIFEST" + echo "App name: $APP_NAME" + + # Resolve clusters whose `apps:` list contains $APP_NAME. + # Convert manifest to JSON via yq, then query with jq (more portable than yq's + # native env() — and jq is preinstalled on the runner anyway). + # + # NOTE: We use `index($app)` instead of `contains([$app])` because jq's + # contains() does substring matching on strings — "plugin-br-bank-transfer" + # would falsely match "plugin-br-bank-transfer-jd". `index()` is exact equality. + # Output one cluster per line, sorted for determinism. + # Do not swallow yq/jq errors with `|| true` — a corrupt manifest or + # broken query should fail fast, not be confused with "app not registered". + # Pipefail ensures intermediate failures surface; an empty RESOLVED from + # a *successful* query is the legitimate "no matching clusters" case and + # is handled below. + RESOLVED=$(yq -o=json '.' "$MANIFEST" \ + | jq -r --arg app "$APP_NAME" \ + '.clusters | to_entries[] | select(.value.apps // [] | index($app)) | .key' \ + | sort -u) + + if [[ -z "$RESOLVED" ]]; then + echo "::warning::App '$APP_NAME' is not registered in any cluster of the deployment matrix." + echo "If this is a new app, add it to $MANIFEST in shared-workflows." + echo "clusters=" >> "$GITHUB_OUTPUT" + echo "has_clusters=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Manifest resolution for '$APP_NAME':" + while IFS= read -r line; do + echo " - $line" + done <<< "$RESOLVED" + + # Apply force-off overrides from inputs. + # An input set to "false" removes that cluster from the resolved set, + # even if the manifest includes the app on it. + FILTERED="" + while IFS= read -r cluster; do + [[ -z "$cluster" ]] && continue + case "$cluster" in + firmino) + if [[ "$DEPLOY_FIRMINO" == "false" ]]; then + echo " ✗ firmino — suppressed by deploy_in_firmino=false" + continue + fi + ;; + clotilde) + if [[ "$DEPLOY_CLOTILDE" == "false" ]]; then + echo " ✗ clotilde — suppressed by deploy_in_clotilde=false" + continue + fi + ;; + anacleto) + if [[ "$DEPLOY_ANACLETO" == "false" ]]; then + echo " ✗ anacleto — suppressed by deploy_in_anacleto=false" + continue + fi + ;; + esac + FILTERED="${FILTERED:+$FILTERED }$cluster" + done <<< "$RESOLVED" + + if [[ -z "$FILTERED" ]]; then + echo "::warning::All clusters resolved from the manifest were suppressed by force-off inputs." + echo "clusters=" >> "$GITHUB_OUTPUT" + echo "has_clusters=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "" + echo "Final cluster set: $FILTERED" + echo "clusters=$FILTERED" >> "$GITHUB_OUTPUT" + echo "has_clusters=true" >> "$GITHUB_OUTPUT" + - name: Git pull before update shell: bash run: | @@ -205,50 +318,47 @@ jobs: - name: Apply tags to values.yaml (multi-server) id: apply_tags shell: bash + env: + # Pass through env to avoid `${{ ... }}` interpolation directly into the + # script body (CodeQL: actions/code-injection). IS_* are inherited from job env. + APP_NAME: ${{ steps.setup.outputs.app_name }} + HAS_CLUSTERS: ${{ steps.resolve_clusters.outputs.has_clusters }} + RESOLVED_SERVERS: ${{ steps.resolve_clusters.outputs.clusters }} + MAPPINGS: ${{ inputs.yaml_key_mappings }} + CONFIGMAP_MAPPINGS: ${{ inputs.configmap_updates }} run: | set -euo pipefail - # Get app name - APP_NAME="${{ steps.setup.outputs.app_name }}" - - # Determine environments to update based on tag type - if [[ "${{ env.IS_BETA }}" == "true" ]]; then + # Determine environments to update based on tag type (IS_* from job env) + if [[ "$IS_BETA" == "true" ]]; then ENVIRONMENTS="dev" ENV_LABEL="beta/dev" - elif [[ "${{ env.IS_RC }}" == "true" ]]; then + elif [[ "$IS_RC" == "true" ]]; then ENVIRONMENTS="stg" ENV_LABEL="rc/stg" - elif [[ "${{ env.IS_PRODUCTION }}" == "true" ]]; then + elif [[ "$IS_PRODUCTION" == "true" ]]; then ENVIRONMENTS="dev stg prd sandbox" ENV_LABEL="production" - elif [[ "${{ env.IS_SANDBOX }}" == "true" ]]; then + elif [[ "$IS_SANDBOX" == "true" ]]; then ENVIRONMENTS="sandbox" ENV_LABEL="sandbox" else - echo "Unable to detect environment from tag: ${{ github.ref }}" + echo "Unable to detect environment from tag: ${GITHUB_REF}" exit 1 fi echo "env_label=$ENV_LABEL" >> "$GITHUB_OUTPUT" echo "Detected tag type: $ENV_LABEL" echo "Environments to update: $ENVIRONMENTS" - # Determine servers to deploy to - SERVERS="" - if [[ "${{ steps.setup.outputs.deploy_firmino }}" == "true" ]]; then - SERVERS="firmino" - fi - if [[ "${{ steps.setup.outputs.deploy_clotilde }}" == "true" ]]; then - if [[ -n "$SERVERS" ]]; then - SERVERS="$SERVERS clotilde" - else - SERVERS="clotilde" - fi - fi - - if [[ -z "$SERVERS" ]]; then - echo "No servers selected for deployment. Enable deploy_in_firmino or deploy_in_clotilde." - exit 1 + # Servers come from the deployment matrix manifest, filtered by force-off inputs. + # See step `resolve_clusters` for resolution logic. + if [[ "$HAS_CLUSTERS" != "true" ]]; then + echo "No clusters selected for deployment (manifest empty for this app, or all clusters suppressed)." + echo "sync_matrix=[]" >> "$GITHUB_OUTPUT" + echo "has_sync_targets=false" >> "$GITHUB_OUTPUT" + exit 0 fi + SERVERS="$RESOLVED_SERVERS" echo "Servers to deploy to: $SERVERS" # First, check which artifacts actually exist @@ -314,7 +424,7 @@ jobs: FILE_CHANGED=false # Apply mappings from inputs - only if artifact exists - MAPPINGS='${{ inputs.yaml_key_mappings }}' + # MAPPINGS is set in step env: (avoids code-injection from inputs) while IFS='|' read -r artifact_key yaml_key; do ARTIFACT_FILE=".gitops-tags/${artifact_key}" if [[ -f "$ARTIFACT_FILE" ]]; then @@ -336,8 +446,8 @@ jobs: done < <(echo "$MAPPINGS" | jq -r 'to_entries[] | "\(.key)|\(.value)"') # Apply configmap updates if configured - only if artifact exists - if [[ -n "${{ inputs.configmap_updates }}" ]]; then - CONFIGMAP_MAPPINGS='${{ inputs.configmap_updates }}' + # CONFIGMAP_MAPPINGS is set in step env: (avoids code-injection from inputs) + if [[ -n "$CONFIGMAP_MAPPINGS" ]]; then while IFS='|' read -r artifact_key configmap_key; do ARTIFACT_FILE=".gitops-tags/${artifact_key}" if [[ -f "$ARTIFACT_FILE" ]]; then @@ -475,14 +585,54 @@ jobs: matrix: target: ${{ fromJson(needs.update_gitops.outputs.sync_matrix) }} steps: - - name: Execute ArgoCD Sync - uses: LerianStudio/github-actions-argocd-sync@main - with: - app-name: ${{ matrix.target.server }}-${{ needs.update_gitops.outputs.app_name }} - argo-cd-token: ${{ secrets.ARGOCD_GHUSER_TOKEN }} - argo-cd-url: ${{ secrets.ARGOCD_URL }} - env-prefix: ${{ matrix.target.env }} - skip-if-not-exists: 'true' + - name: Install ArgoCD CLI + shell: bash + run: | + set -euo pipefail + curl -sSL -o /tmp/argocd https://github.com/argoproj/argo-cd/releases/download/v3.0.6/argocd-linux-amd64 + sudo install -m 755 /tmp/argocd /usr/local/bin/argocd + argocd version --client + + - name: Sync ArgoCD application + shell: bash + env: + ARGOCD_URL: ${{ secrets.ARGOCD_URL }} + ARGOCD_TOKEN: ${{ secrets.ARGOCD_GHUSER_TOKEN }} + APP_NAME: ${{ matrix.target.server }}-${{ needs.update_gitops.outputs.app_name }}-${{ matrix.target.env }} + run: | + set -uo pipefail + + echo "::group::argocd app get $APP_NAME" + if ! argocd app get "$APP_NAME" --server "$ARGOCD_URL" --auth-token "$ARGOCD_TOKEN" --grpc-web; then + rc=$? + echo "::endgroup::" + echo "::warning::Failed to get ArgoCD app '$APP_NAME' (exit $rc). Skipping sync — check whether the app exists, auth is valid, and the user has 'applications, get' permission." + exit 0 + fi + echo "::endgroup::" + + echo "::group::argocd app sync $APP_NAME" + for attempt in 1 2 3 4 5; do + if argocd app sync "$APP_NAME" --server "$ARGOCD_URL" --auth-token "$ARGOCD_TOKEN" --grpc-web; then + break + fi + if [[ "$attempt" == "5" ]]; then + echo "::endgroup::" + echo "::error::Sync failed for $APP_NAME after 5 attempts" + exit 1 + fi + echo "Sync attempt $attempt failed, retrying in 5s..." + sleep 5 + done + echo "::endgroup::" + + echo "::group::argocd app wait $APP_NAME" + if ! argocd app wait "$APP_NAME" --server "$ARGOCD_URL" --auth-token "$ARGOCD_TOKEN" --grpc-web; then + echo "::endgroup::" + echo "::error::Timeout waiting for sync completion of $APP_NAME" + exit 1 + fi + echo "::endgroup::" # Slack notification notify: diff --git a/.github/workflows/go-pr-analysis.yml b/.github/workflows/go-pr-analysis.yml index f016b67..571d48e 100644 --- a/.github/workflows/go-pr-analysis.yml +++ b/.github/workflows/go-pr-analysis.yml @@ -102,7 +102,7 @@ jobs: has_changes: ${{ steps.set-matrix.outputs.has_changes }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 @@ -114,17 +114,16 @@ jobs: # For PRs, compare base and head BASE_SHA="${{ github.event.pull_request.base.sha }}" HEAD_SHA="${{ github.event.pull_request.head.sha }}" - git fetch origin $BASE_SHA --depth=1 2>/dev/null || true - FILES=$(git diff --name-only $BASE_SHA $HEAD_SHA 2>/dev/null || git diff --name-only origin/${{ github.base_ref }}...HEAD) + git fetch origin "$BASE_SHA" --depth=1 2>/dev/null || true + FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || git diff --name-only "origin/${{ github.base_ref }}...HEAD") elif [[ "${{ github.event.before }}" == "0000000000000000000000000000000000000000" ]] || [[ -z "${{ github.event.before }}" ]]; then - PREV_COMMIT=$(git rev-parse HEAD^) - if [[ $? -eq 0 ]]; then - FILES=$(git diff --name-only $PREV_COMMIT HEAD) + if PREV_COMMIT=$(git rev-parse HEAD^ 2>/dev/null); then + FILES=$(git diff --name-only "$PREV_COMMIT" HEAD) else FILES=$(git ls-tree -r --name-only HEAD) fi else - FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }}) + FILES=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}") fi printf "files<> "$GITHUB_OUTPUT" @@ -232,10 +231,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -248,9 +247,12 @@ jobs: GOPRIVATE: ${{ inputs.go_private_modules }} - name: Install golangci-lint + env: + GOLANGCI_LINT_VERSION: ${{ inputs.golangci_lint_version }} run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${{ inputs.golangci_lint_version }} - echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + GOPATH_BIN="$(go env GOPATH)/bin" + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$GOPATH_BIN" "$GOLANGCI_LINT_VERSION" + echo "$GOPATH_BIN" >> "$GITHUB_PATH" - name: Detect Makefile lint target id: detect-make @@ -259,14 +261,14 @@ jobs: if [[ -f "Makefile" ]] || [[ -f "makefile" ]] || [[ -f "GNUmakefile" ]]; then if make -n lint >/dev/null 2>&1; then echo "Makefile with 'lint' target detected" - echo "use_make=true" >> $GITHUB_OUTPUT + echo "use_make=true" >> "$GITHUB_OUTPUT" else echo "Makefile exists but no 'lint' target found" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi else echo "No Makefile found" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi - name: Run GolangCI-Lint (make) @@ -293,10 +295,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -315,14 +317,14 @@ jobs: if [[ -f "Makefile" ]] || [[ -f "makefile" ]] || [[ -f "GNUmakefile" ]]; then if make -n sec >/dev/null 2>&1; then echo "Makefile with 'sec' target detected" - echo "use_make=true" >> $GITHUB_OUTPUT + echo "use_make=true" >> "$GITHUB_OUTPUT" else echo "Makefile exists but no 'sec' target found" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi else echo "No Makefile found" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi - name: Run Security (make) @@ -333,13 +335,13 @@ jobs: - name: Run Gosec for SARIF id: gosec-sarif - uses: securego/gosec@v2.25.0 + uses: securego/gosec@223e19b8856e00f02cc67804499a83f77e208f3c # v2.25.0 with: args: -no-fail -fmt sarif -out gosec-${{ matrix.app.name }}.sarif ./${{ matrix.app.working_dir }}/... - name: Upload Gosec SARIF if: always() - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4 with: sarif_file: gosec-${{ matrix.app.name }}.sarif category: gosec-${{ matrix.app.name }} @@ -367,10 +369,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -391,24 +393,24 @@ jobs: if [[ -f "Makefile" ]] || [[ -f "makefile" ]] || [[ -f "GNUmakefile" ]]; then if make -n coverage-unit >/dev/null 2>&1; then echo "Makefile with 'coverage-unit' target detected" - echo "use_make=true" >> $GITHUB_OUTPUT - echo "make_target=coverage-unit" >> $GITHUB_OUTPUT + echo "use_make=true" >> "$GITHUB_OUTPUT" + echo "make_target=coverage-unit" >> "$GITHUB_OUTPUT" elif make -n cover-html >/dev/null 2>&1; then echo "Makefile with 'cover-html' target detected" - echo "use_make=true" >> $GITHUB_OUTPUT - echo "make_target=cover-html" >> $GITHUB_OUTPUT + echo "use_make=true" >> "$GITHUB_OUTPUT" + echo "make_target=cover-html" >> "$GITHUB_OUTPUT" elif make -n test >/dev/null 2>&1; then echo "Makefile with 'test' target detected (no coverage)" - echo "use_make=true" >> $GITHUB_OUTPUT - echo "make_target=test" >> $GITHUB_OUTPUT - echo "no_coverage=true" >> $GITHUB_OUTPUT + echo "use_make=true" >> "$GITHUB_OUTPUT" + echo "make_target=test" >> "$GITHUB_OUTPUT" + echo "no_coverage=true" >> "$GITHUB_OUTPUT" else echo "No test targets found in Makefile, using go test" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi else echo "No Makefile found, using go test" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi - name: Download dependencies @@ -446,10 +448,10 @@ jobs: run: | # make test ran but doesn't generate coverage, run go test for coverage only echo "Generating coverage with go test (make test doesn't generate coverage)" - + # Get package list, excluding /tests/ and /api/ directories (standard exclusions) PACKAGES=$(go list ./... | awk '!/\/tests($|\/)/' | awk '!/\/api($|\/)/') - + if [[ -n "$PACKAGES" ]]; then echo "$PACKAGES" | xargs go test -coverprofile=coverage.txt -covermode=atomic > /dev/null 2>&1 || true else @@ -465,18 +467,18 @@ jobs: # Get package list, excluding /tests/ and /api/ directories (standard exclusions) # .ignorecoverunit patterns are applied to the coverage REPORT, not to test execution PACKAGES=$(go list ./... | awk '!/\/tests($|\/)/' | awk '!/\/api($|\/)/') - + if [[ -z "$PACKAGES" ]]; then echo "No packages found after filtering /tests/ and /api/" exit 0 fi - + PACKAGE_COUNT=$(echo "$PACKAGES" | wc -l | tr -d ' ') echo "Running tests on $PACKAGE_COUNT packages (excluded /tests/, /api/)" echo "$PACKAGES" | xargs go test -v -race -coverprofile=coverage.txt -covermode=atomic - name: Upload coverage artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: coverage-${{ matrix.app.name }} path: ${{ matrix.app.working_dir }}/coverage.txt @@ -496,10 +498,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -512,7 +514,7 @@ jobs: GOPRIVATE: ${{ inputs.go_private_modules }} - name: Download coverage artifact - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: coverage-${{ matrix.app.name }} path: ${{ matrix.app.working_dir }} @@ -539,29 +541,29 @@ jobs: fi echo "Filtering coverage with .ignorecoverunit patterns..." - + # Read patterns, filter comments and empty lines, convert to pipe-separated regex PATTERNS=$(grep -v '^#' "$IGNORE_FILE" | grep -v '^[[:space:]]*$' | tr '\n' '|' | sed 's/|$//') - + if [[ -z "$PATTERNS" ]]; then echo "No patterns found in .ignorecoverunit" exit 0 fi - + echo "Patterns: $PATTERNS" - + # Convert glob patterns to regex (escape dots, convert * to .*) REGEX_PATTERNS=$(echo "$PATTERNS" | sed 's/\./\\./g' | sed 's/\*/.*/g') echo "Regex patterns: $REGEX_PATTERNS" - + # Keep header line, filter body head -1 coverage.txt > coverage_filtered.txt tail -n +2 coverage.txt | grep -vE "$REGEX_PATTERNS" >> coverage_filtered.txt || true - + BEFORE=$(wc -l < coverage.txt | tr -d ' ') AFTER=$(wc -l < coverage_filtered.txt | tr -d ' ') echo "Filtered coverage: $BEFORE lines -> $AFTER lines" - + mv coverage_filtered.txt coverage.txt - name: Calculate coverage @@ -570,7 +572,7 @@ jobs: run: | if [[ -f coverage.txt ]]; then COVERAGE=$(go tool cover -func=coverage.txt | grep total | awk '{print $3}' | sed 's/%//') - echo "coverage=$COVERAGE" >> $GITHUB_OUTPUT + echo "coverage=$COVERAGE" >> "$GITHUB_OUTPUT" echo "Total coverage: $COVERAGE%" # Generate coverage by package table @@ -608,7 +610,7 @@ jobs: } }' | sort >> coverage-report.md else - echo "coverage=0" >> $GITHUB_OUTPUT + echo "coverage=0" >> "$GITHUB_OUTPUT" echo "No coverage file found" echo "No coverage data available" > coverage-report.md fi @@ -621,7 +623,7 @@ jobs: fi - name: Upload coverage report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: coverage-report-${{ matrix.app.name }} path: | @@ -674,7 +676,7 @@ jobs: issue_number: context.issue.number, }); - const botComment = comments.find(comment => + const botComment = comments.find(comment => comment.body.includes(`Unit Test Coverage Report: \`${appName}\``) ); @@ -711,18 +713,20 @@ jobs: - name: Coverage summary working-directory: ${{ matrix.app.working_dir }} run: | - echo "## Coverage Summary: ${{ matrix.app.name }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Total Coverage**: ${{ steps.coverage.outputs.coverage }}%" >> $GITHUB_STEP_SUMMARY - echo "- **Threshold**: ${{ inputs.coverage_threshold }}%" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - if [[ -f coverage.txt ]]; then - echo "### Coverage by Function" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - go tool cover -func=coverage.txt >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - fi + { + echo "## Coverage Summary: ${{ matrix.app.name }}" + echo "" + echo "- **Total Coverage**: ${{ steps.coverage.outputs.coverage }}%" + echo "- **Threshold**: ${{ inputs.coverage_threshold }}%" + echo "" + if [[ -f coverage.txt ]]; then + echo "### Coverage by Function" + echo "" + echo '```' + go tool cover -func=coverage.txt + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" # ============================================ # BUILD VERIFICATION @@ -738,10 +742,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -760,14 +764,14 @@ jobs: if [[ -f "Makefile" ]] || [[ -f "makefile" ]] || [[ -f "GNUmakefile" ]]; then if make -n build >/dev/null 2>&1; then echo "Makefile with 'build' target detected" - echo "use_make=true" >> $GITHUB_OUTPUT + echo "use_make=true" >> "$GITHUB_OUTPUT" else echo "Makefile exists but no 'build' target found" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi else echo "No Makefile found" - echo "use_make=false" >> $GITHUB_OUTPUT + echo "use_make=false" >> "$GITHUB_OUTPUT" fi - name: Download dependencies @@ -799,10 +803,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -832,10 +836,10 @@ jobs: app: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version: ${{ inputs.go_version }} cache: true @@ -862,7 +866,7 @@ jobs: for i in $(seq 1 ${{ inputs.test_determinism_runs }}); do echo "Run $i/${{ inputs.test_determinism_runs }}" - if ! echo $PACKAGES | xargs go test -count=1 -shuffle=on; then + if ! echo "$PACKAGES" | xargs go test -count=1 -shuffle=on; then echo "Tests failed on run $i" exit 1 fi diff --git a/.github/workflows/gptchangelog.yml b/.github/workflows/gptchangelog.yml index 6da450d..f7bfffe 100644 --- a/.github/workflows/gptchangelog.yml +++ b/.github/workflows/gptchangelog.yml @@ -255,7 +255,7 @@ jobs: uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3 id: app-token with: - app-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} + client-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} private-key: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_PRIVATE_KEY }} - name: Checkout repository @@ -393,32 +393,31 @@ jobs: echo "📝 Commits for $APP_NAME:" echo "$COMMITS_TEXT" - # Get unique contributors (GitHub usernames) for this app - # Try to extract GitHub username from email (format: user@users.noreply.github.com or id+username@users.noreply.github.com) + # Resolve contributors via GitHub's commit->author.login mapping. + # Guessing handles from email local-parts is unreliable and has produced + # false @mentions of unrelated third parties when authors commit with a + # corporate/personal email instead of @users.noreply.github.com. if [ "$WORKING_DIR" != "." ]; then - RAW_EMAILS=$(git log "$SINCE".."$LAST_TAG" --format='%ae' -- "$WORKING_DIR" 2>/dev/null | sort -u) + SHAS=$(git log "$SINCE".."$LAST_TAG" --format='%H' -- "$WORKING_DIR" 2>/dev/null) else - RAW_EMAILS=$(git log "$SINCE".."$LAST_TAG" --format='%ae' 2>/dev/null | sort -u) + SHAS=$(git log "$SINCE".."$LAST_TAG" --format='%H' 2>/dev/null) fi - # Collect unique usernames (same user may have multiple emails) USERNAMES_FILE=$(mktemp) - for EMAIL in $RAW_EMAILS; do - # Skip service accounts used for automated commits - if [[ "$EMAIL" == *"srv.iam"* ]] || [[ "$EMAIL" == *"noreply"* && "$EMAIL" != *"users.noreply.github.com" ]]; then - continue + for SHA in $SHAS; do + LOGIN=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${SHA}" \ + --jq '.author.login // empty' 2>/dev/null || true) + # Skip bots and unresolved authors (null login). + if [[ -n "$LOGIN" && "$LOGIN" != *"[bot]" ]]; then + echo "$LOGIN" >> "$USERNAMES_FILE" fi - if [[ "$EMAIL" == *"@users.noreply.github.com" ]]; then - # Extract username from GitHub noreply email - USERNAME=$(echo "$EMAIL" | sed 's/@users.noreply.github.com//' | sed 's/.*+//') - else - # Use email prefix as fallback - USERNAME=$(echo "$EMAIL" | cut -d@ -f1) - fi - echo "$USERNAME" >> "$USERNAMES_FILE" done - # Deduplicate usernames and format as @username - CONTRIBUTORS=$(sort -u "$USERNAMES_FILE" | sed 's/^/@/' | tr '\n' ', ' | sed 's/, $//') + + if [ -s "$USERNAMES_FILE" ]; then + CONTRIBUTORS=$(sort -u "$USERNAMES_FILE" | sed 's/^/@/' | tr '\n' ', ' | sed 's/, $//') + else + CONTRIBUTORS="" + fi rm -f "$USERNAMES_FILE" echo "👥 Contributors: $CONTRIBUTORS" diff --git a/.github/workflows/helm-update-chart.yml b/.github/workflows/helm-update-chart.yml index 561b7df..f16325f 100644 --- a/.github/workflows/helm-update-chart.yml +++ b/.github/workflows/helm-update-chart.yml @@ -102,7 +102,7 @@ jobs: id: app-token uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 with: - app-id: ${{ secrets.APP_ID }} + client-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Extract payload diff --git a/.github/workflows/release-notification.yml b/.github/workflows/release-notification.yml index a36cd34..87dba2b 100644 --- a/.github/workflows/release-notification.yml +++ b/.github/workflows/release-notification.yml @@ -117,7 +117,7 @@ jobs: uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3 id: app-token with: - app-id: ${{ secrets.APP_ID }} + client-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Checkout diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b2de026..f3bd734 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,7 +106,7 @@ jobs: - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 id: app-token with: - app-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} + client-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} private-key: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_PRIVATE_KEY }} - name: Checkout repository diff --git a/.github/workflows/self-pr-validation.yml b/.github/workflows/self-pr-validation.yml index 5fb3566..267bdca 100644 --- a/.github/workflows/self-pr-validation.yml +++ b/.github/workflows/self-pr-validation.yml @@ -47,7 +47,7 @@ jobs: all_files: ${{ steps.detect.outputs.all-files }} steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Detect changed files id: detect @@ -63,7 +63,7 @@ jobs: if: needs.changed-files.outputs.yaml_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: YAML Lint uses: ./src/lint/yamllint @@ -78,7 +78,7 @@ jobs: if: needs.changed-files.outputs.workflow_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Action Lint uses: ./src/lint/actionlint @@ -93,7 +93,7 @@ jobs: if: needs.changed-files.outputs.action_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Pinned Actions Check uses: ./src/lint/pinned-actions @@ -108,7 +108,7 @@ jobs: if: needs.changed-files.outputs.markdown_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Markdown Link Check uses: ./src/lint/markdown-link-check @@ -123,7 +123,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Spelling Check uses: ./src/lint/typos @@ -138,7 +138,7 @@ jobs: if: needs.changed-files.outputs.action_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Shell Check uses: ./src/lint/shellcheck @@ -153,7 +153,7 @@ jobs: if: needs.changed-files.outputs.action_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: README Check uses: ./src/lint/readme-check @@ -168,13 +168,28 @@ jobs: if: needs.changed-files.outputs.composite_files != '' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Composite Schema Lint uses: ./src/lint/composite-schema with: files: ${{ needs.changed-files.outputs.composite_files }} + # ----------------- Deployment Matrix Lint ----------------- + deployment-matrix: + name: Deployment Matrix Lint + runs-on: blacksmith-4vcpu-ubuntu-2404 + needs: changed-files + if: contains(needs.changed-files.outputs.all_files, 'config/deployment-matrix.yml') + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Deployment Matrix Lint + uses: ./src/lint/deployment-matrix + with: + manifest-file: config/deployment-matrix.yml + # ----------------- CodeQL Analysis ----------------- codeql: name: CodeQL Analysis @@ -188,7 +203,7 @@ jobs: actions: read steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Generate CodeQL config for changed files id: codeql-config @@ -227,11 +242,11 @@ jobs: pull-requests: write issues: write checks: read - needs: [changed-files, yamllint, actionlint, pinned-actions, markdown-link-check, typos, shellcheck, readme-check, composite-schema] + needs: [changed-files, yamllint, actionlint, pinned-actions, markdown-link-check, typos, shellcheck, readme-check, composite-schema, deployment-matrix] if: always() && github.event_name == 'pull_request' && needs.changed-files.result == 'success' steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Post Lint Report uses: ./src/notify/pr-lint-reporter @@ -253,3 +268,5 @@ jobs: readme-files: ${{ needs.changed-files.outputs.action_files }} composite-schema-result: ${{ needs.composite-schema.result }} composite-schema-files: ${{ needs.changed-files.outputs.composite_files }} + deployment-matrix-result: ${{ needs.deployment-matrix.result }} + deployment-matrix-files: ${{ contains(needs.changed-files.outputs.all_files, 'config/deployment-matrix.yml') && 'config/deployment-matrix.yml' || '' }} diff --git a/.github/workflows/self-release.yml b/.github/workflows/self-release.yml index c87daac..0e32d93 100644 --- a/.github/workflows/self-release.yml +++ b/.github/workflows/self-release.yml @@ -14,6 +14,9 @@ on: - '*.md' - '**/*.txt' - '*.txt' + # The deployment matrix is resolved from main at runtime by callers, so + # matrix-only changes propagate without a new release tag. + - 'config/deployment-matrix.yml' tags-ignore: - '**' @@ -29,3 +32,50 @@ jobs: with: runner_type: ubuntu-latest secrets: inherit + + # Only run after a stable release on main; develop/release-candidate produce + # beta/rc tags that the reusable skips anyway via stable_releases_only. + generate-changelog: + needs: publish-release + if: github.ref == 'refs/heads/main' + uses: ./.github/workflows/gptchangelog.yml + with: + runner_type: ubuntu-latest + stable_releases_only: true + secrets: inherit + + # Force-update the floating major version tag (e.g. v1) to point at the + # latest stable release. Enables downstream composites to pin to @v1 and + # always resolve to the latest stable v1.x.x release. Stable-only: gated to + # main so beta/rc releases from develop/release-candidate don't move v1. + update-major-tag: + needs: publish-release + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + id: app-token + with: + client-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} + private-key: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_PRIVATE_KEY }} + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 + with: + gpg_private_key: ${{ secrets.LERIAN_CI_CD_USER_GPG_KEY }} + passphrase: ${{ secrets.LERIAN_CI_CD_USER_GPG_KEY_PASSWORD }} + git_committer_name: ${{ secrets.LERIAN_CI_CD_USER_NAME }} + git_committer_email: ${{ secrets.LERIAN_CI_CD_USER_EMAIL }} + git_config_global: true + git_user_signingkey: true + git_tag_gpgsign: true + + - name: Update floating major version tag + uses: ./src/config/update-major-tag diff --git a/.github/workflows/typescript-release.yml b/.github/workflows/typescript-release.yml index 511efab..0617952 100644 --- a/.github/workflows/typescript-release.yml +++ b/.github/workflows/typescript-release.yml @@ -117,7 +117,7 @@ jobs: - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 id: app-token with: - app-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} + client-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} private-key: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_PRIVATE_KEY }} - name: Checkout repository diff --git a/config/deployment-matrix.yml b/config/deployment-matrix.yml new file mode 100644 index 0000000..059fa8c --- /dev/null +++ b/config/deployment-matrix.yml @@ -0,0 +1,109 @@ +# Deployment Matrix +# +# Source of truth for which apps deploy to which Kubernetes clusters via the +# `gitops-update.yml` reusable workflow. +# +# Cluster-centric layout — mirrors how clusters are operated (lifecycle, +# capacity, incident response). To answer "what runs on cluster X?", read +# the cluster block. To answer "where does app X deploy?", run the helper +# script described in docs/gitops-update-workflow.md (or grep across blocks). +# +# Schema: +# version integer, required, currently 1 +# apps.registry list of every app name allowed to appear in any cluster +# clusters. cluster block with explicit `apps:` list (no `all_apps` +# shortcut today — every cluster has its own subset) +# +# Resolution at runtime: +# 1. Workflow reads this file at the pinned ref of the shared-workflows repo. +# 2. For the caller's `app_name`, the loader collects every cluster whose +# `apps:` list contains it. +# 3. The `deploy_in_` workflow inputs act as **force-off** overrides: +# setting any of them to `false` removes that cluster from the resolved +# set even if the manifest includes the app. +# +# Out of scope (managed manually or by other tooling — do NOT add here): +# - underwriter, jd-mock-api, mock-btg-server, control-plane, +# platform-console (no CI commit history in gitops repo) +# - ledger, dockerhub-secret (no values.yaml — kustomize / k8s Secret) +# - hortencia-apps (legacy umbrella, not workflow-managed) +# - environments/production/* (platform infra: addons, observability) + +version: 1 + +apps: + registry: + # Core platform + - midaz + - fetcher + - flowker + - matcher + - reporter + - tracer + - product-console + + # Plugins + - plugin-access-manager # caller repo may push as plugin-identity / plugin-auth / casdoor + - plugin-fees + - plugin-br-pix-direct-jd + - plugin-br-pix-indirect-btg + - plugin-br-bank-transfer # generic build (firmino + clotilde) + - plugin-br-bank-transfer-jd # JD-specific build (anacleto only) + + # Clotilde-exclusive Lerian platform suite + - backoffice-console + - cs-platform + - forge + - lerian-map + - tenant-manager + +clusters: + firmino: + apps: + - midaz + - fetcher + - flowker + - matcher + - reporter + - tracer + - product-console + - plugin-access-manager + - plugin-fees + - plugin-br-pix-direct-jd + - plugin-br-pix-indirect-btg + - plugin-br-bank-transfer + + clotilde: + apps: + - midaz + - fetcher + - flowker + - matcher + - reporter + - tracer + - product-console + - plugin-access-manager + - plugin-fees + - plugin-br-pix-direct-jd + - plugin-br-pix-indirect-btg + - plugin-br-bank-transfer + - backoffice-console + - cs-platform + - forge + - lerian-map + - tenant-manager + + anacleto: + apps: + - midaz + - fetcher + - flowker + - matcher + - reporter + - tracer + - product-console + - plugin-access-manager + - plugin-fees + - plugin-br-pix-direct-jd + - plugin-br-pix-indirect-btg + - plugin-br-bank-transfer-jd diff --git a/docs/gitops-update-workflow.md b/docs/gitops-update-workflow.md index 80d5d6c..ce96728 100644 --- a/docs/gitops-update-workflow.md +++ b/docs/gitops-update-workflow.md @@ -4,7 +4,9 @@ Reusable workflow for updating GitOps repository with new image tags across mult ## Features -- **Multi-server deployment**: Deploy to Firmino and/or Clotilde servers with dynamic path generation +- **Manifest-driven topology**: Cluster membership per app is declared in [`config/deployment-matrix.yml`](../config/deployment-matrix.yml) — no caller-side configuration required to add a cluster to an existing app +- **Multi-server deployment**: Deploy to Firmino, Clotilde and/or Anacleto with dynamic path generation +- **Force-off overrides**: `deploy_in_` inputs can suppress a cluster declared in the manifest, useful for emergency containment without editing the manifest - **Convention-based configuration**: Auto-generates paths, names, and patterns from repository name - **Multi-environment support**: dev (beta), stg (rc), prd (production), sandbox - **Production sync**: Production releases automatically update all environments (dev, stg, prd, sandbox) on all servers @@ -18,7 +20,7 @@ Reusable workflow for updating GitOps repository with new image tags across mult ## Usage -### Minimal Example (Convention-Based, Both Servers) +### Minimal Example (Manifest-Driven) ```yaml update_gitops: @@ -32,30 +34,19 @@ update_gitops: > **Required Secrets**: `MANAGE_TOKEN`, `LERIAN_CI_CD_USER_NAME`, `LERIAN_CI_CD_USER_EMAIL`, `ARGOCD_GHUSER_TOKEN`, `ARGOCD_URL`, `DOCKER_USERNAME`, `DOCKER_PASSWORD` +The workflow reads `config/deployment-matrix.yml` from the shared-workflows repo (by default from `main`, override via `deployment_matrix_ref`) and resolves the cluster set automatically based on `app_name`. No `deploy_in_*` inputs are required for the common case. + **Auto-generated values** (for repo `my-backend-service`): -- App name: `my-backend-service` +- App name: `my-backend-service` (must be present in the deployment matrix) - Artifact pattern: `gitops-tags-my-backend-service-*` -- GitOps paths: - - Firmino: `gitops/environments/firmino/helmfile/applications/{env}/my-backend-service/values.yaml` - - Clotilde: `gitops/environments/clotilde/helmfile/applications/{env}/my-backend-service/values.yaml` -- ArgoCD apps: `firmino-my-backend-service-{env}`, `clotilde-my-backend-service-{env}` +- GitOps paths (one per cluster declared in the manifest): + - `gitops/environments//helmfile/applications/{env}/my-backend-service/values.yaml` +- ArgoCD apps: `-my-backend-service-{env}` for every resolved cluster - Commit prefix: `my-backend-service` -### Single Server Example (Firmino Only) - -```yaml -update_gitops: - needs: build_backend - if: needs.build_backend.result == 'success' - uses: LerianStudio/github-actions-shared-workflows/.github/workflows/gitops-update.yml@v1.0.0 - with: - deploy_in_firmino: true - deploy_in_clotilde: false - yaml_key_mappings: '{"backend.tag": ".auth.image.tag"}' - secrets: inherit -``` +### Force-Off Example (Skip Anacleto for One Run) -### Single Server Example (Clotilde Only) +Useful when you need to ship a hotfix to Firmino and Clotilde but skip Anacleto temporarily (e.g., maintenance window) without touching the manifest: ```yaml update_gitops: @@ -63,12 +54,13 @@ update_gitops: if: needs.build_backend.result == 'success' uses: LerianStudio/github-actions-shared-workflows/.github/workflows/gitops-update.yml@v1.0.0 with: - deploy_in_firmino: false - deploy_in_clotilde: true + deploy_in_anacleto: false yaml_key_mappings: '{"backend.tag": ".auth.image.tag"}' secrets: inherit ``` +`deploy_in_` inputs only **subtract** clusters from the resolved set — they cannot add a cluster the manifest does not list. + ### Multi-Component Example (Midaz) ```yaml @@ -98,8 +90,11 @@ update_gitops: |-------|------|---------|-------------| | `gitops_repository` | string | `LerianStudio/midaz-firmino-gitops` | GitOps repository to update | | `app_name` | string | (repo name) | Application name (auto-detected from repository) | -| `deploy_in_firmino` | boolean | `true` | Deploy to Firmino server | -| `deploy_in_clotilde` | boolean | `true` | Deploy to Clotilde server | +| `deploy_in_firmino` | boolean | `true` | Force-off override for Firmino (`false` = subtract from manifest-resolved set) | +| `deploy_in_clotilde` | boolean | `true` | Force-off override for Clotilde (`false` = subtract from manifest-resolved set) | +| `deploy_in_anacleto` | boolean | `true` | Force-off override for Anacleto (`false` = subtract from manifest-resolved set) | +| `deployment_matrix_file` | string | `config/deployment-matrix.yml` | Path to the deployment matrix manifest within the shared-workflows checkout | +| `deployment_matrix_ref` | string | `main` | Git ref of `LerianStudio/github-actions-shared-workflows` to read the deployment matrix from. Default `main` ensures all callers see manifest updates immediately, regardless of the workflow ref they pin. Override only when testing a branch. | | `artifact_pattern` | string | `gitops-tags-{app}-*` | Pattern to download artifacts (auto-generated) | | `commit_message_prefix` | string | (repo name) | Prefix for commit message (auto-generated) | | `runner_type` | string | `blacksmith-4vcpu-ubuntu-2404` | GitHub runner type | @@ -135,6 +130,73 @@ update_gitops: | `DOCKER_USERNAME` | Docker Hub username (to avoid rate limits) | | `DOCKER_PASSWORD` | Docker Hub password | +## Deployment Matrix + +The workflow's cluster topology is declared in [`config/deployment-matrix.yml`](../config/deployment-matrix.yml) — a single source of truth maintained in this repo. + +### How it works + +1. The caller invokes the workflow at a pinned ref (e.g. `@v1.24.0`). +2. The workflow checks out the deployment matrix from `main` (or from the ref supplied via `deployment_matrix_ref`) — sparse checkout of the manifest file only. This decoupling lets manifest updates propagate to every caller without bumping the pinned workflow tag. +3. For the caller's `app_name`, the workflow collects every cluster whose `apps:` list contains it. +4. `deploy_in_` inputs are applied as **force-off** overrides on the resolved set. +5. The remaining cluster set drives both the GitOps file updates and the ArgoCD sync matrix. + +### Anatomy of the manifest + +```yaml +version: 1 + +apps: + registry: + - midaz + - plugin-fees + # ... every app that uses this workflow + +clusters: + firmino: + apps: [midaz, plugin-fees, ...] + clotilde: + apps: [midaz, plugin-fees, ...] + anacleto: + apps: [midaz, ...] +``` + +- `apps.registry` is the set of legal app names — typo gate. +- Each `clusters..apps` is an explicit list of which apps this cluster hosts. +- A cluster is added by appending one block. A cluster is removed by deleting it. Affects only this repo — caller workflows are untouched. + +### Adding a new app to a cluster + +1. Open a PR in this repo editing `config/deployment-matrix.yml`: + - Add the app name to `apps.registry` (if new). + - Add the app name to `clusters..apps`. +2. The `deployment-matrix` lint job validates schema, integrity, and duplicates on the PR. +3. Once merged, callers consuming the new ref (via Renovate/Dependabot or manual bump) automatically include the cluster on their next release — zero change required in caller repos. + +### Adding a new cluster + +1. Create `environments//...` in the GitOps repo (with at least the app `values.yaml` files you want to populate). +2. In this repo, add a `clusters.:` block listing the apps that should deploy to it. +3. (Optional) Add a `deploy_in_` input to `gitops-update.yml` if you want callers to be able to force-off the new cluster individually. + +### Force-off semantics + +`deploy_in_` inputs default to `true` and only **subtract** from the manifest-resolved set: + +| Manifest says | Input value | Result | +|---|---|---| +| App included in cluster | `true` (default) | Deploys to cluster | +| App included in cluster | `false` | **Suppressed** — does not deploy | +| App NOT included in cluster | `true` (default) | Does not deploy | +| App NOT included in cluster | `false` | Does not deploy | + +Inputs cannot **add** a cluster that the manifest does not list — that prevents accidental cross-cluster spillover. + +### Apps not in the manifest + +If `app_name` is not found in any cluster, the workflow logs a warning and exits cleanly (no failure). This is the expected behavior for apps managed manually or by other tooling. + ## Multi-Server Path Generation The workflow dynamically generates paths for each server and environment combination: @@ -144,7 +206,7 @@ gitops/environments//helmfile/applications///values.yaml ``` Where: -- ``: `firmino` or `clotilde` (controlled by `deploy_in_firmino` and `deploy_in_clotilde` inputs) +- ``: any cluster resolved from the deployment matrix (current set: `firmino`, `clotilde`, `anacleto`), minus those force-off via `deploy_in_: false` - ``: `dev`, `stg`, `prd`, or `sandbox` (determined by tag type) - ``: from `inputs.app_name` or auto-detected from repository name @@ -168,22 +230,13 @@ This allows for partial deployments where not all server/environment combination ### Example: Production Release -When a production tag (e.g., `v1.2.3`) is pushed with both servers enabled, the workflow will: +When a production tag (e.g., `v1.2.3`) is pushed for an app declared in all three clusters, the workflow will: -1. Generate paths for Firmino: - - `gitops/environments/firmino/helmfile/applications/dev/my-app/values.yaml` - - `gitops/environments/firmino/helmfile/applications/stg/my-app/values.yaml` - - `gitops/environments/firmino/helmfile/applications/prd/my-app/values.yaml` - - `gitops/environments/firmino/helmfile/applications/sandbox/my-app/values.yaml` - -2. Generate paths for Clotilde: - - `gitops/environments/clotilde/helmfile/applications/dev/my-app/values.yaml` - - `gitops/environments/clotilde/helmfile/applications/stg/my-app/values.yaml` - - `gitops/environments/clotilde/helmfile/applications/prd/my-app/values.yaml` - - `gitops/environments/clotilde/helmfile/applications/sandbox/my-app/values.yaml` - -3. Apply tags to all existing files (skip missing ones with warning) -4. Sync ArgoCD apps for each server/environment where files were updated +1. Resolve cluster set from manifest: `firmino`, `clotilde`, `anacleto`. +2. For each cluster, generate paths for every production environment (`dev`, `stg`, `prd`, `sandbox`): + - `gitops/environments//helmfile/applications//my-app/values.yaml` +3. Apply tags to all existing files (skip missing ones with warning). +4. Sync ArgoCD apps for each cluster/environment where files were updated. ## ArgoCD Multi-Server Sync @@ -194,11 +247,9 @@ When `enable_argocd_sync` is `true`, the workflow syncs ArgoCD applications for ArgoCD apps are named using the pattern: `--` Examples: -- `firmino-midaz-dev` -- `firmino-midaz-stg` -- `firmino-midaz-prd` -- `clotilde-midaz-dev` -- `clotilde-midaz-stg` +- `firmino-midaz-dev`, `firmino-midaz-stg`, `firmino-midaz-prd` +- `clotilde-midaz-dev`, `clotilde-midaz-stg`, `clotilde-midaz-sandbox` +- `anacleto-midaz-dev` ### Sync Behavior @@ -268,24 +319,36 @@ update_gitops: ### Key Changes 1. **Removed inputs:** - - `gitops_server` - No longer needed; use `deploy_in_firmino` and `deploy_in_clotilde` instead + - `gitops_server` - No longer needed; cluster topology is declared in the deployment matrix - `gitops_file_dev`, `gitops_file_stg`, `gitops_file_prd`, `gitops_file_sandbox` - Paths are now auto-generated - `argocd_app_name` - Now auto-generated based on server/app/env pattern - `environment_detection`, `manual_environment` - Simplified to automatic detection only -2. **New inputs:** - - `deploy_in_firmino` (default: `true`) - Enable deployment to Firmino server - - `deploy_in_clotilde` (default: `true`) - Enable deployment to Clotilde server +2. **Inputs that became force-off overrides:** + - `deploy_in_firmino`, `deploy_in_clotilde`, `deploy_in_anacleto` (all default `true`) — only **subtract** clusters from the manifest-resolved set; cannot add a cluster the manifest does not list + +3. **New inputs:** + - `deployment_matrix_file` (default: `config/deployment-matrix.yml`) — alternative manifest path for forks/testing -3. **Path generation:** - - Paths are automatically generated based on server and environment - - Pattern: `gitops/environments//helmfile/applications///values.yaml` +4. **Path generation:** + - Paths are automatically generated based on cluster (from manifest) and environment (from tag) + - Pattern: `gitops/environments//helmfile/applications///values.yaml` -4. **ArgoCD sync:** - - Now syncs apps for each server/environment combination where files were updated - - Pattern: `--` +5. **ArgoCD sync:** + - Syncs apps for each cluster/environment combination where files were updated + - Pattern: `--` - Checks if app exists before attempting sync +### Migrating an existing caller to manifest-driven topology + +> ⚠️ **Semantic change to `deploy_in_*` inputs** — callers that previously relied on `deploy_in_firmino: true` (etc.) to **include** a cluster will now silently deploy nowhere if their app is not listed in the manifest. The inputs only **subtract** from the manifest-resolved set; they never add. The prerequisite for any deployment is a manifest entry. Workflow logs a warning when `app_name` is missing from every cluster, so these cases surface quickly — but add your app to the manifest before merging this bump if you haven't already. + +If your caller currently passes `deploy_in_firmino: true, deploy_in_clotilde: true` explicitly: + +1. Add your `app_name` to `apps.registry` and to the appropriate `clusters..apps` lists in [`config/deployment-matrix.yml`](../config/deployment-matrix.yml) (single PR in this repo). +2. Once merged and the caller bumps to the new shared-workflows ref (Renovate/Dependabot), the explicit `deploy_in_*: true` inputs become redundant and can be removed from the caller. +3. Keep `deploy_in_: false` only where you want to force-off a cluster the manifest declares. + ## Troubleshooting ### No changes to commit @@ -306,6 +369,16 @@ Ensure the artifact pattern matches your uploaded artifacts: - Pattern: `gitops-tags-*` matches `gitops-tags-backend`, `gitops-tags-frontend`, etc. - Check artifact names in the build job +### App is not registered in any cluster of the deployment matrix + +The workflow logs this warning and exits cleanly when `app_name` is missing from the manifest. Either: +- Add the app to `config/deployment-matrix.yml` in this repo (and bump the caller's pinned ref), or +- Confirm the app is intentionally managed outside this workflow (manual edits, kustomize, separate tooling). + +### All clusters resolved from the manifest were suppressed + +You explicitly set every `deploy_in_: false`. Either remove one of the overrides, or confirm this run is intentionally a no-op. + ### YAML key not updated Verify the YAML key path in your mappings: @@ -315,8 +388,10 @@ Verify the YAML key path in your mappings: ## Best Practices -1. **Start with both servers enabled** - the workflow gracefully handles missing files -2. **Use specific artifact patterns** to avoid conflicts -3. **Test with beta tags first** before deploying to production -4. **Monitor ArgoCD sync results** in workflow logs -5. **Keep YAML key mappings simple** and consistent across environments +1. **Add new apps/clusters via the deployment matrix**, not via per-caller `deploy_in_*` flags — single source of truth wins +2. **Reserve `deploy_in_: false`** for emergency containment or temporary suppression, not for permanent topology decisions +3. **Use specific artifact patterns** to avoid conflicts +4. **Test with beta tags first** before deploying to production +5. **Monitor ArgoCD sync results** in workflow logs +6. **Keep YAML key mappings simple** and consistent across environments +7. **Pin via Renovate/Dependabot** so manifest updates propagate automatically as new ref bumps diff --git a/src/config/update-major-tag/README.md b/src/config/update-major-tag/README.md new file mode 100644 index 0000000..b66b548 --- /dev/null +++ b/src/config/update-major-tag/README.md @@ -0,0 +1,75 @@ + + + + + +
Lerian

update-major-tag

+ +Force-update the floating major version tag (e.g. `v1`) to point at the latest stable `vX.Y.Z` tag in the repository. Intended to run after a successful stable release so callers can pin composite actions to `@v1` and always resolve to the latest stable release. + +### Behavior + +1. Fetches all tags from the remote. +2. Finds the greatest stable tag matching `^v[0-9]+\.[0-9]+\.[0-9]+$` (pre-release tags like `-beta.N` / `-rc.N` are ignored). +3. Derives the major prefix (`v1.26.0 → v1`). +4. If the major tag already points at the resolved commit, exits with a notice — idempotent. +5. Otherwise, creates/moves the major tag as an annotated tag and force-pushes it. + +### Assumptions + +- The caller has already checked out the repository with `fetch-depth: 0` (so all tags are reachable). +- The checkout was authenticated with a token that has permission to push tags (typically via `actions/checkout@... with.token:`). +- For signed tags, the caller has imported a GPG key and enabled `git_tag_gpgsign` (`git tag -a` will auto-sign when `tag.gpgSign=true` is set globally). + +## Inputs + +_None._ All behavior is derived from the repository's tag list. + +## Outputs + +| Output | Description | +|---|---| +| `skip` | `true` when no tag update was performed — either no stable tag was found, or the major tag already pointed at the latest stable commit | +| `tag-updated` | `true` when the floating major tag was force-pushed to a new commit | +| `major-tag` | The major tag name that was considered (e.g. `v1`). Empty when no stable tag was found | +| `latest-tag` | The latest stable tag the major tag was aligned with (e.g. `v1.26.0`). Empty when no stable tag was found | + +## Usage + +```yaml +jobs: + update-major-tag: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/create-github-app-token@ # v3.1.1 + id: app-token + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - uses: actions/checkout@ # v6 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - uses: crazy-max/ghaction-import-gpg@ # v7 + with: + gpg_private_key: ${{ secrets.GPG_KEY }} + passphrase: ${{ secrets.GPG_PASSPHRASE }} + git_committer_name: ${{ secrets.CI_USER_NAME }} + git_committer_email: ${{ secrets.CI_USER_EMAIL }} + git_config_global: true + git_user_signingkey: true + git_tag_gpgsign: true + + - uses: LerianStudio/github-actions-shared-workflows/src/config/update-major-tag@v1 +``` + +## Required permissions + +```yaml +permissions: + contents: write +``` diff --git a/src/config/update-major-tag/action.yml b/src/config/update-major-tag/action.yml new file mode 100644 index 0000000..c384062 --- /dev/null +++ b/src/config/update-major-tag/action.yml @@ -0,0 +1,74 @@ +name: Update Major Version Tag +description: Force-update the floating major version tag (e.g. v1) to point at the latest stable semver tag (vX.Y.Z) found in the repository. + +outputs: + skip: + description: 'true when no tag update was performed (no stable tag found, or major already points at the latest stable commit)' + value: ${{ steps.update.outputs.skip }} + tag-updated: + description: 'true when the floating major tag was force-pushed to a new commit' + value: ${{ steps.update.outputs.tag-updated }} + major-tag: + description: 'The major tag name that was considered (e.g. v1). Empty when no stable tag was found.' + value: ${{ steps.update.outputs.major-tag }} + latest-tag: + description: 'The latest stable tag the major tag was aligned with (e.g. v1.26.0). Empty when no stable tag was found.' + value: ${{ steps.update.outputs.latest-tag }} + +runs: + using: composite + steps: + - id: update + name: Resolve and push major version tag + shell: bash + run: | + set -euo pipefail + + git fetch --tags --force --prune + + LATEST=$(git tag --list \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | tail -n1 || true) + + if [ -z "$LATEST" ]; then + echo "::notice::No stable release tag found — skipping major tag update." + { + echo "skip=true" + echo "tag-updated=false" + echo "major-tag=" + echo "latest-tag=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + VERSION="${LATEST#v}" + MAJOR="v${VERSION%%.*}" + SHA=$(git rev-list -n1 "refs/tags/$LATEST") + + CURRENT_SHA="" + if git rev-parse --verify --quiet "refs/tags/$MAJOR" >/dev/null; then + CURRENT_SHA=$(git rev-list -n1 "refs/tags/$MAJOR") + fi + + if [ "$CURRENT_SHA" = "$SHA" ]; then + echo "::notice::$MAJOR already points at $LATEST ($SHA) — nothing to do." + { + echo "skip=true" + echo "tag-updated=false" + echo "major-tag=$MAJOR" + echo "latest-tag=$LATEST" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Moving $MAJOR → $LATEST ($SHA)" + git tag -f -a "$MAJOR" "$SHA" -m "Release $MAJOR ($LATEST)" + git push origin "refs/tags/$MAJOR:refs/tags/$MAJOR" --force + + { + echo "skip=false" + echo "tag-updated=true" + echo "major-tag=$MAJOR" + echo "latest-tag=$LATEST" + } >> "$GITHUB_OUTPUT" diff --git a/src/lint/deployment-matrix/README.md b/src/lint/deployment-matrix/README.md new file mode 100644 index 0000000..3ba0ed1 --- /dev/null +++ b/src/lint/deployment-matrix/README.md @@ -0,0 +1,53 @@ + + + + + +
Lerian

deployment-matrix

+ +Validate the deployment matrix manifest at `config/deployment-matrix.yml` (or any custom path). This manifest is the source of truth consumed by the `gitops-update.yml` reusable workflow to decide which apps deploy to which Kubernetes clusters. + +Checks performed: + +**Schema** +- `version` is an integer equal to `1` +- `apps.registry` is a list of non-empty strings +- `clusters` is a mapping of `` → cluster spec +- Each `clusters..apps` is a list of non-empty strings + +**Integrity** +- Every app listed in any `clusters..apps` is declared in `apps.registry` (typo gate) +- No duplicates inside `apps.registry` +- No duplicates inside any `clusters..apps` + +**Hygiene (warnings, not errors)** +- Apps in `apps.registry` not referenced by any cluster are flagged — likely pre-onboarding entries, but worth reviewing to avoid dead registrations + +## Inputs + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `manifest-file` | Path to the deployment matrix YAML manifest | No | `config/deployment-matrix.yml` | + +## Usage as composite step + +```yaml +jobs: + deployment-matrix: + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Deployment Matrix Lint + uses: LerianStudio/github-actions-shared-workflows/src/lint/deployment-matrix@v1.x.x + with: + manifest-file: config/deployment-matrix.yml +``` + +## Required permissions + +```yaml +permissions: + contents: read +``` diff --git a/src/lint/deployment-matrix/action.yml b/src/lint/deployment-matrix/action.yml new file mode 100644 index 0000000..f33f544 --- /dev/null +++ b/src/lint/deployment-matrix/action.yml @@ -0,0 +1,149 @@ +name: Deployment Matrix Lint +description: Validate the deployment matrix manifest schema, app/cluster integrity, and detect duplicates or orphans. + +inputs: + manifest-file: + description: Path to the deployment matrix YAML manifest + required: false + default: "config/deployment-matrix.yml" + +runs: + using: composite + steps: + # ----------------- Setup ----------------- + - name: Install dependencies + shell: bash + run: | + if ! python3 -c "import yaml" 2>/dev/null; then + sudo apt-get install -y --no-install-recommends python3-yaml + fi + + # ----------------- Log ----------------- + - name: Log manifest under analysis + shell: bash + env: + MANIFEST: ${{ inputs.manifest-file }} + run: | + echo "::group::Deployment matrix file" + echo " - $MANIFEST" + echo "::endgroup::" + + # ----------------- Check ----------------- + - name: Validate deployment matrix + shell: bash + env: + MANIFEST: ${{ inputs.manifest-file }} + run: | + python3 - <<'PYEOF' + import os, sys, yaml + + path = os.environ.get('MANIFEST', '') + violations = 0 + warnings = 0 + + def err(msg, line=None): + global violations + loc = f',line={line}' if line else '' + print(f'::error file={path}{loc}::{msg}') + violations += 1 + + def warn(msg): + global warnings + print(f'::warning file={path}::{msg}') + warnings += 1 + + if not path or not os.path.isfile(path): + print(f'::error::Manifest file not found: {path!r}') + sys.exit(1) + + try: + with open(path) as f: + data = yaml.safe_load(f) + except Exception as e: + err(f'Could not parse YAML: {e}') + sys.exit(1) + + if not isinstance(data, dict): + err('Manifest root must be a YAML mapping.') + sys.exit(1) + + # ── version ── + version = data.get('version') + if not isinstance(version, int): + err('"version" must be an integer.') + elif version != 1: + err(f'Unsupported manifest version {version}; this composite only knows version 1.') + + # ── apps.registry ── + apps_block = data.get('apps') + if not isinstance(apps_block, dict): + err('"apps" must be a mapping containing a "registry" list.') + registry = [] + else: + registry = apps_block.get('registry') + if not isinstance(registry, list): + err('"apps.registry" must be a list of app names.') + registry = [] + else: + for i, item in enumerate(registry): + if not isinstance(item, str) or not item.strip(): + err(f'apps.registry[{i}] must be a non-empty string.') + seen = set() + for item in registry: + if isinstance(item, str): + if item in seen: + err(f'Duplicate app in apps.registry: "{item}".') + seen.add(item) + + registry_set = {a for a in registry if isinstance(a, str)} + + # ── clusters ── + clusters = data.get('clusters') + if not isinstance(clusters, dict): + err('"clusters" must be a mapping of → cluster spec.') + clusters = {} + + referenced_apps = set() + + for cluster_name, cluster_spec in clusters.items(): + if not isinstance(cluster_name, str) or not cluster_name.strip(): + err(f'Cluster name must be a non-empty string (got {cluster_name!r}).') + continue + + if not isinstance(cluster_spec, dict): + err(f'clusters.{cluster_name} must be a mapping.') + continue + + cluster_apps = cluster_spec.get('apps') + if not isinstance(cluster_apps, list): + err(f'clusters.{cluster_name}.apps must be a list of app names.') + continue + + seen = set() + for i, app in enumerate(cluster_apps): + if not isinstance(app, str) or not app.strip(): + err(f'clusters.{cluster_name}.apps[{i}] must be a non-empty string.') + continue + if app in seen: + err(f'Duplicate app "{app}" in clusters.{cluster_name}.apps.') + seen.add(app) + if app not in registry_set: + err(f'clusters.{cluster_name}.apps lists "{app}" which is missing from apps.registry.') + referenced_apps.add(app) + + # ── orphans (warning only) ── + orphans = registry_set - referenced_apps + for app in sorted(orphans): + warn(f'App "{app}" is in apps.registry but not referenced by any cluster — pre-onboarding entry?') + + # ── summary ── + if violations > 0: + print(f'::error::Deployment matrix has {violations} violation(s) and {warnings} warning(s).') + sys.exit(1) + + msg = f'Deployment matrix is valid ({len(registry_set)} apps registered, {len(clusters)} clusters defined' + if warnings: + msg += f', {warnings} warning(s)' + msg += ').' + print(msg) + PYEOF diff --git a/src/lint/pinned-actions/README.md b/src/lint/pinned-actions/README.md index 02578de..41bba25 100644 --- a/src/lint/pinned-actions/README.md +++ b/src/lint/pinned-actions/README.md @@ -5,35 +5,62 @@ -Ensure all third-party GitHub Action references use pinned versions (`@vX.Y.Z` or `@sha`), not mutable refs like `@main` or `@master`. +Enforces the repository pinning policy on every `uses:` reference in workflow and composite files. External actions must be pinned by commit SHA; internal (`LerianStudio/`) references are validated against the composite-vs-reusable policy below. + +## Policy + +| Ref kind | Detection | Required pin | Severity on violation | +|---|---|---|---| +| **External action** (anything outside the org) | `uses:` path does not match any `warn-patterns` prefix | Full commit SHA (40–64 hex chars) | ❌ **error** — fails the step | +| **Internal composite** | `uses:` path contains `/src/` | Floating major tag (`@v1`, `@v2`, …) or testing branch (`@develop` / `@main`) | ⚠️ **warning** | +| **Internal reusable workflow** | `uses:` path contains `/.github/workflows/` | Exact version tag (`@v1.2.3`, `@v1.2.3-beta.1`) or testing branch (`@develop` / `@main`) | ⚠️ **warning** | +| **Internal, unknown shape** | matches `warn-patterns` but neither path pattern above | Any semver-like tag or testing branch (legacy tolerance) | ⚠️ **warning** | + +**Rationale** + +- Composites are small, low-risk building blocks. Floating `@v1` lets callers track the latest stable major without bumping per patch — the repo release pipeline moves `@v1` atomically on each stable release in `main`. +- Reusable workflows orchestrate entire pipelines with broader blast radius. Exact pinning (`@v1.2.3`) forces explicit, auditable caller updates. +- External actions use SHA pins because upstream tags are mutable — Dependabot keeps the SHAs fresh automatically. ## Inputs | Input | Description | Required | Default | |-------|-------------|----------|---------| -| `files` | Space-separated list of workflow/composite files to check | No | `` | -| `ignore-patterns` | Pipe-separated org/owner prefixes to skip (e.g. internal actions) | No | `LerianStudio/` | +| `files` | Comma-separated list of workflow/composite files to check (empty = skip) | No | `""` | +| `warn-patterns` | Pipe-separated org/owner prefixes treated as internal (warn-only) | No | `LerianStudio/` | -## Usage as composite step +## Usage ```yaml -- name: Checkout - uses: actions/checkout@v4 - - name: Pinned Actions Check - uses: LerianStudio/github-actions-shared-workflows/src/lint/pinned-actions@v1.2.3 + uses: LerianStudio/github-actions-shared-workflows/src/lint/pinned-actions@v1 with: - files: ".github/workflows/ci.yml .github/workflows/deploy.yml" - ignore-patterns: "LerianStudio/|my-org/" + files: ".github/workflows/ci.yml,.github/workflows/deploy.yml" ``` -## Usage via reusable workflow +## Examples ```yaml -jobs: - lint: - uses: LerianStudio/github-actions-shared-workflows/.github/workflows/pr-lint.yml@v1.2.3 - secrets: inherit +# ✅ External action — SHA pin (required) +uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + +# ❌ External action — tag pin (error) +uses: actions/checkout@v6 + +# ✅ Internal composite — floating major +uses: LerianStudio/github-actions-shared-workflows/src/security/trivy-fs-scan@v1 + +# ⚠️ Internal composite — exact version (warning, should be @v1) +uses: LerianStudio/github-actions-shared-workflows/src/security/trivy-fs-scan@v1.24.1 + +# ✅ Internal reusable workflow — exact version +uses: LerianStudio/github-actions-shared-workflows/.github/workflows/release.yml@v1.26.0 + +# ⚠️ Internal reusable workflow — floating major (warning, should be @vX.Y.Z) +uses: LerianStudio/github-actions-shared-workflows/.github/workflows/release.yml@v1 + +# ✅ Either kind — develop/main for testing a change +uses: LerianStudio/github-actions-shared-workflows/src/config/labels-sync@develop ``` ## Required permissions diff --git a/src/lint/pinned-actions/action.yml b/src/lint/pinned-actions/action.yml index b95030b..b3d8bdc 100644 --- a/src/lint/pinned-actions/action.yml +++ b/src/lint/pinned-actions/action.yml @@ -57,12 +57,32 @@ runs: done if [ "$is_internal" = true ]; then - # Internal: any semver ref passes — vX, vX.Y.Z, vX.Y.Z-beta.N, vX.Y.Z-rc.N, branches like develop/main - if printf '%s\n' "$ref" | grep -Eq '^v[0-9]+(\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?)?$|^(develop|main)$'; then - continue + # Internal pinning policy: + # - Composite actions (path contains /src/) → floating major tag @vN + # - Reusable workflows (path contains /.github/workflows/) → exact version tag @vN.M.P[-pre] + # - Testing branches (develop|main) are always accepted for both + # - Unknown internal ref shapes fall back to any semver ref (legacy tolerance) + target=${normalized%@*} + + if printf '%s\n' "$target" | grep -Fq '/src/'; then + if printf '%s\n' "$ref" | grep -Eq '^v[0-9]+$|^(develop|main)$'; then + continue + fi + echo "::warning file=${file},line=${line_num}::Internal composite must use floating major tag (e.g. @v1) or develop/main for testing: $normalized" + warnings=$((warnings + 1)) + elif printf '%s\n' "$target" | grep -Fq '/.github/workflows/'; then + if printf '%s\n' "$ref" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$|^(develop|main)$'; then + continue + fi + echo "::warning file=${file},line=${line_num}::Internal reusable workflow must use exact version tag (e.g. @v1.2.3) or develop/main for testing: $normalized" + warnings=$((warnings + 1)) + else + if printf '%s\n' "$ref" | grep -Eq '^v[0-9]+(\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?)?$|^(develop|main)$'; then + continue + fi + echo "::warning file=${file},line=${line_num}::Internal action not pinned to a version: $normalized" + warnings=$((warnings + 1)) fi - echo "::warning file=${file},line=${line_num}::Internal action not pinned to a version: $normalized" - warnings=$((warnings + 1)) else # External: must be a commit SHA (40 or 64 hex chars) if printf '%s\n' "$ref" | grep -Eiq '^[0-9a-f]{40,64}$'; then diff --git a/src/notify/pr-lint-reporter/README.md b/src/notify/pr-lint-reporter/README.md index acf061e..3e20826 100644 --- a/src/notify/pr-lint-reporter/README.md +++ b/src/notify/pr-lint-reporter/README.md @@ -28,6 +28,8 @@ Posts a formatted lint analysis summary as a PR comment, aggregating results fro | `readme-files` | Comma-separated list of files checked for README presence | No | `` | | `composite-schema-result` | Result of the composite-schema job | No | `skipped` | | `composite-schema-files` | Comma-separated list of action files validated by composite-schema | No | `` | +| `deployment-matrix-result` | Result of the deployment-matrix job | No | `skipped` | +| `deployment-matrix-files` | Comma-separated list of deployment matrix manifest files validated | No | `` | ## Usage as composite step @@ -35,7 +37,7 @@ Posts a formatted lint analysis summary as a PR comment, aggregating results fro jobs: lint-report: runs-on: blacksmith-4vcpu-ubuntu-2404 - needs: [changed-files, yamllint, actionlint, pinned-actions, markdown-link-check, typos, shellcheck, readme-check, composite-schema] + needs: [changed-files, yamllint, actionlint, pinned-actions, markdown-link-check, typos, shellcheck, readme-check, composite-schema, deployment-matrix] if: always() && github.event_name == 'pull_request' && needs.changed-files.result == 'success' steps: - name: Checkout @@ -61,6 +63,8 @@ jobs: readme-files: ${{ needs.changed-files.outputs.action_files }} composite-schema-result: ${{ needs.composite-schema.result }} composite-schema-files: ${{ needs.changed-files.outputs.composite_files }} + deployment-matrix-result: ${{ needs.deployment-matrix.result }} + deployment-matrix-files: ${{ contains(needs.changed-files.outputs.all_files, 'config/deployment-matrix.yml') && 'config/deployment-matrix.yml' || '' }} ``` ## Required permissions diff --git a/src/notify/pr-lint-reporter/action.yml b/src/notify/pr-lint-reporter/action.yml index a2a215e..80a8b0c 100644 --- a/src/notify/pr-lint-reporter/action.yml +++ b/src/notify/pr-lint-reporter/action.yml @@ -69,6 +69,14 @@ inputs: description: Comma-separated list of action files validated by composite-schema required: false default: "" + deployment-matrix-result: + description: Result of the deployment-matrix job (success/failure/skipped/cancelled) + required: false + default: "skipped" + deployment-matrix-files: + description: Comma-separated list of deployment matrix manifest files validated + required: false + default: "" runs: using: composite @@ -85,7 +93,8 @@ runs: const typosFiles = ${{ toJSON(inputs['typos-files']) }}; const shellcheckFiles = ${{ toJSON(inputs['shellcheck-files']) }}; const readmeFiles = ${{ toJSON(inputs['readme-files']) }}; - const compositeSchemaFiles = ${{ toJSON(inputs['composite-schema-files']) }}; + const compositeSchemaFiles = ${{ toJSON(inputs['composite-schema-files']) }}; + const deploymentMatrixFiles = ${{ toJSON(inputs['deployment-matrix-files']) }}; const checks = [ { @@ -137,6 +146,12 @@ runs: result: '${{ inputs.composite-schema-result }}', files: compositeSchemaFiles.trim().split(',').filter(Boolean), }, + { + jobName: 'Deployment Matrix Lint', + label: 'Deployment Matrix', + result: '${{ inputs.deployment-matrix-result }}', + files: deploymentMatrixFiles.trim().split(',').filter(Boolean), + }, ]; const icon = (r) => ({ success: '✅', failure: '❌', skipped: '⏭️' }[r] ?? '⚠️'); diff --git a/src/security/codeql-reporter/README.md b/src/security/codeql-reporter/README.md index c042875..9eb3a98 100644 --- a/src/security/codeql-reporter/README.md +++ b/src/security/codeql-reporter/README.md @@ -7,6 +7,8 @@ Composite action that reads CodeQL SARIF output and posts a formatted security report as a PR comment. Uses an idempotent comment strategy (updates existing comment on re-runs). Designed to run after [`codeql-analyze`](../codeql-analyze/). +Findings parsed from SARIF are cross-referenced with the Code Scanning REST API (`listAlertsForRepo` on `refs/pull//merge`) so alerts that were **dismissed** via the Security tab or are already **fixed** on the merge ref do not show up in the PR comment. A footer notes the number of hidden findings so reviewers know the comment omitted some. If the API call fails (e.g. missing permissions), the action degrades gracefully to the SARIF-only view and emits a warning. + ## Inputs | Input | Description | Required | Default | @@ -54,7 +56,9 @@ steps: ```yaml permissions: contents: read - security-events: write + security-events: write # required for alert-state enrichment; `read` is enough on its own pull-requests: write actions: read ``` + +> `security-events: write` (already required by [`codeql-analyze`](../codeql-analyze/)) covers the `listAlertsForRepo` call used to filter dismissed/fixed alerts. If only `read` is granted the enrichment still works; if the permission is missing entirely the action falls back to the raw SARIF view with a warning. diff --git a/src/security/codeql-reporter/action.yml b/src/security/codeql-reporter/action.yml index 9c76614..da87a60 100644 --- a/src/security/codeql-reporter/action.yml +++ b/src/security/codeql-reporter/action.yml @@ -73,6 +73,18 @@ runs: const truncate = (str, max) => str.length > max ? str.substring(0, max - 3) + '...' : str; + // Normalize file paths to repo-relative form. SARIF can emit absolute + // file:// URIs (e.g. file:///github/workspace/src/foo.js) or leading + // ./ prefixes, while the Code Scanning API always returns clean + // repo-relative paths — normalizing both sides keeps the (rule, path, + // line) lookup key aligned. + const normalizePath = (p) => + String(p ?? '') + .replace(/^file:\/\/\/github\/workspace\//, '') + .replace(/^file:\/\//, '') + .replace(/^\.\//, '') + .replace(/^\//, ''); + // ── Read SARIF files ── function readSarifFiles() { const findings = []; @@ -109,7 +121,7 @@ runs: rule: result.ruleId || 'unknown', severity: result.level || 'warning', message: result.message?.text || rule.shortDescription?.text || 'No description', - file: location?.artifactLocation?.uri || 'unknown', + file: normalizePath(location?.artifactLocation?.uri) || 'unknown', line: location?.region?.startLine || 0, description: rule.fullDescription?.text || rule.shortDescription?.text || '', tags: rule.properties?.tags || [], @@ -129,8 +141,72 @@ runs: // unpinned-tag is handled by our own pinned-actions lint check with org-aware logic const SUPPRESSED_RULES = ['actions/unpinned-tag']; + // ── Enrich findings with Code Scanning alert states ── + // SARIF is structural: it re-reports the same locations on every run, even after + // a reviewer dismisses the alert via the Security tab. Cross-reference the REST + // API so dismissed/fixed alerts stop showing up in the PR comment. + async function filterByAlertStates(rawFindings) { + const prNumber = context.payload.pull_request?.number || context.issue?.number; + if (!prNumber) { + return { findings: rawFindings, hidden: 0, enriched: false }; + } + + let alerts; + try { + alerts = await github.paginate( + github.rest.codeScanning.listAlertsForRepo, + { + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/pull/${prNumber}/merge`, + per_page: 100, + } + ); + } catch (e) { + core.warning( + `Could not fetch Code Scanning alert states (falling back to SARIF-only view): ${e.message}` + ); + return { findings: rawFindings, hidden: 0, enriched: false }; + } + + // Bucket alert states by (ruleId, path, startLine). A single location can carry + // multiple alert rows across history; if any are currently open, we must not + // suppress it — only suppress when every alert at that location is dismissed/fixed. + const statesByKey = new Map(); + for (const alert of alerts) { + const ruleId = alert.rule?.id; + const loc = alert.most_recent_instance?.location; + const alertPath = normalizePath(loc?.path); + if (!ruleId || !alertPath) continue; + const key = `${ruleId}\u0000${alertPath}\u0000${loc?.start_line ?? 0}`; + if (!statesByKey.has(key)) statesByKey.set(key, []); + statesByKey.get(key).push(alert.state); + } + + const suppressedKeys = new Set(); + for (const [key, states] of statesByKey) { + if (states.includes('open')) continue; + if (states.some(s => s === 'dismissed' || s === 'fixed')) { + suppressedKeys.add(key); + } + } + + const kept = []; + let hidden = 0; + for (const f of rawFindings) { + const key = `${f.rule}\u0000${f.file}\u0000${f.line || 0}`; + if (suppressedKeys.has(key)) { + hidden++; + } else { + kept.push(f); + } + } + return { findings: kept, hidden, enriched: true }; + } + // ── Build Report ── - const findings = readSarifFiles().filter(f => !SUPPRESSED_RULES.includes(f.rule)); + const rawFindings = readSarifFiles().filter(f => !SUPPRESSED_RULES.includes(f.rule)); + const { findings, hidden: hiddenCount, enriched } = await filterByAlertStates(rawFindings); findingsCount = findings.length; body += `## \u{1F6E1}\uFE0F CodeQL Analysis Results\n\n`; @@ -142,6 +218,9 @@ runs: body += `\u26A0\uFE0F Some SARIF files could not be parsed and no findings were recovered. Check the analysis step logs.\n\n`; } else if (findings.length === 0) { body += `\u2705 No security issues found.\n\n`; + if (enriched && hiddenCount > 0) { + body += `_${hiddenCount} finding(s) hidden (dismissed or fixed). See the Security tab for the full list._\n\n`; + } } else { hasFindings = true; @@ -179,6 +258,10 @@ runs: } body += '\n'; + + if (enriched && hiddenCount > 0) { + body += `_${hiddenCount} finding(s) hidden (dismissed or fixed). See the Security tab for the full list._\n\n`; + } } // ── Useful Links ──