diff --git a/.github/scripts/prepare-backport-checkout.sh b/.github/scripts/prepare-backport-checkout.sh new file mode 100644 index 00000000000..5286275516a --- /dev/null +++ b/.github/scripts/prepare-backport-checkout.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +target_branch="${1:?target branch is required}" +commit_range="${2:?commit range is required}" +workspace_branch="ci-backport-${target_branch//\//-}" + +git fetch --no-tags origin "${target_branch}" +git config user.name "github-actions[bot]" +git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + +if [[ "${commit_range}" != *..* ]]; then + echo "Invalid commit range: ${commit_range}" >&2 + exit 1 +fi +start_sha="${commit_range%..*}" +end_sha="${commit_range##*..}" + +if [[ -z "$(git rev-list -n 1 "${commit_range}")" ]]; then + echo "No commits found in range ${commit_range}" >&2 + exit 1 +fi + +# Build a single squash commit whose parent is the range start and whose tree +# matches the range end. Cherry-picking this squash onto the release branch +# applies the cumulative diff in one 3-way merge, which avoids spurious +# conflicts when intermediate commits in the range happen to overlap with +# changes already present (under different SHAs) on the release branch. +end_tree="$(git rev-parse "${end_sha}^{tree}")" +squash_sha="$(git commit-tree -p "${start_sha}" -m "ci: squashed backport of ${commit_range}" "${end_tree}")" + +git checkout -B "${workspace_branch}" "origin/${target_branch}" +git cherry-pick -x "${squash_sha}" diff --git a/.github/workflows/direct-backport-push.yml b/.github/workflows/direct-backport-push.yml new file mode 100644 index 00000000000..149a5cbd0ec --- /dev/null +++ b/.github/workflows/direct-backport-push.yml @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Direct Backport Push + +on: + push: + branches: + - main + +permissions: + actions: read + contents: write + pull-requests: read + +jobs: + discover: + name: Discover direct backport targets + runs-on: ubuntu-latest + outputs: + pr_number: ${{ steps.discover.outputs.pr_number }} + targets: ${{ steps.discover.outputs.targets }} + has_targets: ${{ steps.discover.outputs.has_targets }} + steps: + - name: Resolve merged PR and green targets + id: discover + uses: actions/github-script@v8 + with: + script: | + const sha = context.sha; + const { owner, repo } = context.repo; + + const response = await github.request( + "GET /repos/{owner}/{repo}/commits/{commit_sha}/pulls", + { + owner, + repo, + commit_sha: sha, + } + ); + + const pullRequest = response.data.find((pr) => pr.merge_commit_sha === sha) ?? response.data[0]; + if (!pullRequest) { + core.info(`No merged pull request is associated with ${sha}.`); + core.setOutput("pr_number", ""); + core.setOutput("targets", "[]"); + core.setOutput("has_targets", "false"); + return; + } + + const requestedTargets = [...new Set( + pullRequest.labels + .map((label) => label.name) + .filter((name) => /^release\/.+$/.test(name)) + )].sort(); + + if (requestedTargets.length === 0) { + core.info(`PR #${pullRequest.number} does not request any backports.`); + core.setOutput("pr_number", String(pullRequest.number)); + core.setOutput("targets", "[]"); + core.setOutput("has_targets", "false"); + return; + } + + const buildRuns = await github.paginate( + github.rest.actions.listWorkflowRuns, + { + owner, + repo, + workflow_id: "github-action-build.yml", + head_sha: pullRequest.head.sha, + per_page: 100, + } + ); + + let greenTargets = []; + if (buildRuns.length === 0) { + core.warning(`No Build workflow runs found for ${pullRequest.head.sha}.`); + } else { + const allJobs = []; + for (const run of buildRuns) { + const jobs = await github.paginate( + github.rest.actions.listJobsForWorkflowRun, + { + owner, + repo, + run_id: run.id, + per_page: 100, + } + ); + allJobs.push(...jobs); + } + + greenTargets = requestedTargets.filter((target) => { + const prefix = `backport (${target}) / `; + const targetJobs = allJobs.filter((job) => job.name.startsWith(prefix)); + return targetJobs.length > 0 && targetJobs.every((job) => job.conclusion === "success"); + }); + } + + const skippedTargets = requestedTargets.filter((target) => !greenTargets.includes(target)); + if (skippedTargets.length > 0) { + core.warning(`Skipping targets without a successful Backport run: ${skippedTargets.join(", ")}`); + } + + core.setOutput("pr_number", String(pullRequest.number)); + core.setOutput("targets", JSON.stringify(greenTargets)); + core.setOutput("has_targets", greenTargets.length > 0 ? "true" : "false"); + + push-backports: + needs: discover + if: ${{ needs.discover.outputs.has_targets == 'true' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.discover.outputs.targets) }} + steps: + - name: Checkout main + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Cherry-pick merge commit onto target branch + env: + MERGE_SHA: ${{ github.sha }} + TARGET_BRANCH: ${{ matrix.target }} + PR_NUMBER: ${{ needs.discover.outputs.pr_number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + parent_count=$(git rev-list --parents -n 1 "${MERGE_SHA}" | awk '{print NF-1}') + if [[ "${parent_count}" -ne 1 ]]; then + echo "Direct backport expects a squash-merged commit on main. ${MERGE_SHA} has ${parent_count} parents." >&2 + exit 1 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + pr_title=$(gh pr view "${PR_NUMBER}" --json title --jq .title) + pr_body=$(gh pr view "${PR_NUMBER}" --json body --jq .body) + + git fetch --no-tags origin "${TARGET_BRANCH}" + git checkout -B "${TARGET_BRANCH}" "origin/${TARGET_BRANCH}" + git cherry-pick --no-commit "${MERGE_SHA}" + + { + printf '%s (#%s)\n\n' "${pr_title}" "${PR_NUMBER}" + if [[ -n "${pr_body}" ]]; then + printf '%s\n\n' "${pr_body}" + fi + printf '(cherry picked from commit %s)\n' "${MERGE_SHA}" + } | git commit -F - + + git push origin "HEAD:${TARGET_BRANCH}" diff --git a/.github/workflows/github-action-build.yml b/.github/workflows/github-action-build.yml index 4d790e306c1..47e74f52cbe 100644 --- a/.github/workflows/github-action-build.yml +++ b/.github/workflows/github-action-build.yml @@ -27,14 +27,105 @@ on: - 'main' - 'release/**' pull_request: + types: + - opened + - reopened + - synchronize + - labeled + - unlabeled workflow_dispatch: +permissions: + checks: write + contents: read + pull-requests: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + # Precheck decides which downstream jobs run for this event: + # - run_frontend / run_scala / run_python / run_agent_service: gate the + # main build stacks. All true today; placeholder for future path- or + # label-based selection. + # - backport_targets: JSON array of release/* labels currently on the PR. + # Drives the backport matrix; empty array means no backport runs. + precheck: + name: Precheck + runs-on: ubuntu-latest + outputs: + run_frontend: ${{ steps.decide.outputs.run_frontend }} + run_scala: ${{ steps.decide.outputs.run_scala }} + run_python: ${{ steps.decide.outputs.run_python }} + run_agent_service: ${{ steps.decide.outputs.run_agent_service }} + backport_targets: ${{ steps.decide.outputs.backport_targets }} + steps: + - name: Decide which jobs to run + id: decide + uses: actions/github-script@v8 + with: + script: | + const eventName = context.eventName; + + // Main build stacks: always run. + const stacks = ["run_frontend", "run_scala", "run_python", "run_agent_service"]; + for (const key of stacks) { + core.setOutput(key, "true"); + } + + // Backport targets: all current release/* labels on the PR. + let targets = []; + if (eventName === "pull_request") { + const labels = context.payload.pull_request.labels.map((l) => l.name); + targets = [...new Set(labels.filter((n) => /^release\/.+$/.test(n)))].sort(); + } + + if (targets.length === 0) { + core.info(`No backport targets on PR.`); + } else { + core.info(`Backport targets: ${targets.join(", ")}`); + } + core.setOutput("backport_targets", JSON.stringify(targets)); + + cleanup-stale-backport: + if: ${{ github.event_name == 'pull_request' && github.event.action == 'unlabeled' && startsWith(github.event.label.name, 'release/') }} + runs-on: ubuntu-latest + steps: + - name: Cancel obsolete backport check_runs for the removed target + uses: actions/github-script@v8 + with: + script: | + const { owner, repo } = context.repo; + const target = context.payload.label.name; + const headSha = context.payload.pull_request.head.sha; + const prefix = `backport (${target}) `; + + const checks = await github.paginate( + github.rest.checks.listForRef, + { owner, repo, ref: headSha, per_page: 100 } + ); + + for (const check of checks) { + if (!check.name.startsWith(prefix)) continue; + if (check.status === "completed" && check.conclusion === "cancelled") continue; + try { + await github.rest.checks.update({ + owner, + repo, + check_run_id: check.id, + status: "completed", + conclusion: "cancelled", + }); + core.info(`Cancelled check ${check.name}`); + } catch (e) { + core.warning(`Failed to update check ${check.id} (${check.name}): ${e.message}`); + } + } + frontend: + needs: precheck + if: ${{ needs.precheck.outputs.run_frontend == 'true' }} name: frontend (${{ matrix.os }}, 18) runs-on: ${{ matrix.os }} strategy: @@ -83,6 +174,8 @@ jobs: run: yarn --cwd frontend run test:ci scala: + needs: precheck + if: ${{ needs.precheck.outputs.run_scala == 'true' }} strategy: matrix: os: [ ubuntu-22.04 ] @@ -189,6 +282,8 @@ jobs: run: sbt test python: + needs: precheck + if: ${{ needs.precheck.outputs.run_python == 'true' }} strategy: matrix: os: [ ubuntu-latest ] @@ -228,6 +323,8 @@ jobs: cd amber/src/main/python && pytest -sv agent-service: + needs: precheck + if: ${{ needs.precheck.outputs.run_agent_service == 'true' }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -262,3 +359,22 @@ jobs: run: bun run typecheck - name: Run unit tests run: bun test + + backport: + needs: precheck + if: ${{ needs.precheck.outputs.backport_targets != '[]' }} + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.precheck.outputs.backport_targets) }} + uses: ./.github/workflows/reusable-build.yml + with: + checkout_ref: refs/pull/${{ github.event.pull_request.number }}/head + backport_target_branch: ${{ matrix.target }} + backport_commit_range: ${{ format('{0}..{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) }} + job_name_suffix: "" + run_frontend: true + run_scala: true + run_python: true + run_agent_service: true + secrets: inherit diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 00000000000..3e34206f90f --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Reusable Build + +on: + workflow_call: + inputs: + checkout_ref: + required: false + type: string + default: "" + backport_target_branch: + required: false + type: string + default: "" + backport_commit_range: + required: false + type: string + default: "" + job_name_suffix: + required: false + type: string + default: "" + run_frontend: + required: false + type: boolean + default: true + run_scala: + required: false + type: boolean + default: true + run_python: + required: false + type: boolean + default: true + run_agent_service: + required: false + type: boolean + default: true + +env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + +jobs: + frontend: + if: ${{ inputs.run_frontend }} + name: ${{ format('frontend{0} ({1}, 18)', inputs.job_name_suffix, matrix.os) }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: macos-latest + arch: arm64 + - os: ubuntu-latest + arch: x64 + - os: windows-latest + arch: x64 + node-version: + - 20.19.0 + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ inputs.checkout_ref || github.sha }} + fetch-depth: 0 + - name: Prepare backport workspace + if: ${{ inputs.backport_target_branch != '' }} + working-directory: ${{ github.workspace }} + run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}" + - name: Setup node + uses: actions/setup-node@v5 + with: + node-version: ${{ matrix.node-version }} + architecture: ${{ matrix.arch }} + - uses: actions/cache@v4 + with: + path: frontend/.yarn/cache + key: ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v4-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v4- + - name: Prepare Yarn 4.14.1 + run: corepack enable && corepack prepare yarn@4.14.1 --activate + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install dependency + timeout-minutes: 20 + run: yarn --cwd frontend install --immutable --inline-builds --network-timeout=100000 + - name: Lint with Prettier & ESLint + run: yarn --cwd frontend format:ci + - name: Run frontend unit tests + run: yarn --cwd frontend run test:ci + - name: Prod build + run: yarn --cwd frontend run build:ci + + scala: + if: ${{ inputs.run_scala }} + name: ${{ format('scala{0} ({1}, 11)', inputs.job_name_suffix, matrix.os) }} + strategy: + matrix: + os: [ubuntu-22.04] + java-version: [11] + runs-on: ${{ matrix.os }} + env: + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + ref: ${{ inputs.checkout_ref || github.sha }} + fetch-depth: 0 + - name: Prepare backport workspace + if: ${{ inputs.backport_target_branch != '' }} + working-directory: ${{ github.workspace }} + run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}" + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: "temurin" + java-version: 11 + - name: Setup Python for Scala tests + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Show Python + run: python --version || python3 --version + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f amber/requirements.txt ]; then pip install -r amber/requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then pip install -r amber/operator-requirements.txt; fi + - name: Setup sbt launcher + uses: sbt/setup-sbt@508b753e53cb6095967669e0911487d2b9bc9f41 # v1.1.22 + - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 # v8.1.0 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + - name: Lint with scalafmt + run: sbt scalafmtCheckAll + - name: Create Databases + run: | + psql -h localhost -U postgres -f sql/texera_ddl.sql + psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql + psql -h localhost -U postgres -f sql/texera_lakefs.sql + env: + PGPASSWORD: postgres + - name: Create texera_db_for_test_cases + run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases -f sql/texera_ddl.sql + env: + PGPASSWORD: postgres + - name: Compile with sbt + run: sbt clean package + - name: Lint with scalafix + run: sbt "scalafixAll --check" + - name: Set docker-java API version + run: | + echo "api.version=1.52" >> ~/.docker-java.properties + cat ~/.docker-java.properties + - name: Run backend tests + run: sbt test + + python: + if: ${{ inputs.run_python }} + name: ${{ format('python{0} ({1}, {2})', inputs.job_name_suffix, matrix.os, matrix.python-version) }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12", "3.13"] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ inputs.checkout_ref || github.sha }} + fetch-depth: 0 + - name: Prepare backport workspace + if: ${{ inputs.backport_target_branch != '' }} + run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}" + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f amber/requirements.txt ]; then pip install -r amber/requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then pip install -r amber/operator-requirements.txt; fi + - name: Install PostgreSQL + run: sudo apt-get update && sudo apt-get install -y postgresql + - name: Start PostgreSQL Service + run: sudo systemctl start postgresql + - name: Create Database and User + run: | + cd sql && sudo -u postgres psql -f iceberg_postgres_catalog.sql + - name: Lint with Ruff + run: | + cd amber/src/main/python && ruff check . && ruff format --check . + - name: Test with pytest + run: | + cd amber/src/main/python && pytest -sv + + agent-service: + if: ${{ inputs.run_agent_service }} + name: ${{ format('agent-service{0} ({1})', inputs.job_name_suffix, matrix.os) }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + bun-version: ["1.3.3"] + defaults: + run: + working-directory: agent-service + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ inputs.checkout_ref || github.sha }} + fetch-depth: 0 + - name: Prepare backport workspace + if: ${{ inputs.backport_target_branch != '' }} + working-directory: ${{ github.workspace }} + run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}" + - name: Setup Bun + run: | + curl -fsSL https://bun.sh/install | bash -s -- bun-v${{ matrix.bun-version }} + echo "$HOME/.bun/bin" >> $GITHUB_PATH + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Lint with Prettier + run: bun run format:check + - name: Typecheck + run: bun run typecheck + - name: Run unit tests + run: bun test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3b0dfdd5e8..f31b0052e09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,7 @@ yarn format:fix ### 4. PR Review - [ ] Ask a Texera Committer (by commenting on the PR) to triage your PR, i.e., request a reviewer, and assign the PR to you. - [ ] Add appropriate labels such as `fix`, `enhancement`, `docs`, etc. +- [ ] If the change should also land in a release branch, add the matching `release/` label (e.g. `release/v1.1.0-incubating`); the change will be backported to that branch automatically. - [ ] Ensure that all CI checks pass (see [GitHub Actions](https://github.com/Texera/texera/actions)). - [ ] Fully test your changes locally.