build.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Build

on:
  workflow_call:
    inputs:
      checkout_ref:
        required: false
        type: string
        default: ""
      backport_target_branch:
        required: false
        type: string
        default: ""
      backport_commit_range:
        required: false
        type: string
        default: ""
      job_name_suffix:
        required: false
        type: string
        default: ""
      run_frontend:
        required: false
        type: boolean
        default: true
      run_amber:
        required: false
        type: boolean
        default: true
      run_amber_integration:
        required: false
        type: boolean
        default: true
      run_platform:
        required: false
        type: boolean
        default: true
      run_python:
        required: false
        type: boolean
        default: true
      run_agent_service:
        required: false
        type: boolean
        default: true
      mode:
        # PR (default) | nightly | release. Only "PR" passes
        # --ignore-transitive-version to check_binary_deps.py.
        required: false
        type: string
        default: PR

env:
  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}

jobs:
  frontend:
    if: ${{ inputs.run_frontend }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        include:
          - os: macos-latest
            arch: arm64
          - os: ubuntu-latest
            arch: x64
          - os: windows-latest
            arch: x64
        node-version:
          - 24.10.0
    steps:
      - name: Checkout Texera
        uses: actions/checkout@v5
        with:
          ref: ${{ inputs.checkout_ref || github.sha }}
          fetch-depth: 0
      - name: Prepare backport workspace
        if: ${{ inputs.backport_target_branch != '' }}
        working-directory: ${{ github.workspace }}
        run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}"
      - name: Setup node
        uses: actions/setup-node@v5
        with:
          node-version: ${{ matrix.node-version }}
          architecture: ${{ matrix.arch }}
      - uses: actions/cache@v5
        with:
          path: frontend/.yarn/cache
          key: ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v4-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v4-
      - name: Prepare Yarn 4.14.1
        run: corepack enable && corepack prepare yarn@4.14.1 --activate
      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      - name: Install dependency
        timeout-minutes: 20
        run: yarn --cwd frontend install --immutable --inline-builds --network-timeout=100000
      - name: Lint with Prettier & ESLint
        run: yarn --cwd frontend format:ci
      - name: Prod build
        run: yarn --cwd frontend run build:ci
      - name: Check bundled npm packages against per-module LICENSE-binary files
        if: matrix.os == 'ubuntu-latest'
        run: ./bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} npm frontend/dist/3rdpartylicenses.json
      - name: Run frontend unit tests
        run: yarn --cwd frontend run test:ci
      - name: Upload frontend coverage to Codecov
        if: matrix.os == 'ubuntu-latest' && always()
        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./frontend/coverage/**/lcov.info
          flags: frontend
          fail_ci_if_error: false

  amber:
    # The amber job runs the cross-cutting Scala lints (scalafmtCheckAll,
    # scalafixAll --check) once on behalf of every Scala module, then builds
    # and tests just the WorkflowExecutionService dist. Per-service builds
    # and tests for the platform services live in the `platform` matrix
    # below. License-binary checks are scoped to the amber dist.
    if: ${{ inputs.run_amber }}
    strategy:
      matrix:
        os: [ubuntu-22.04]
        java-version: [11]
    runs-on: ${{ matrix.os }}
    env:
      JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
      JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U postgres"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          ref: ${{ inputs.checkout_ref || github.sha }}
          fetch-depth: 0
      - name: Prepare backport workspace
        if: ${{ inputs.backport_target_branch != '' }}
        working-directory: ${{ github.workspace }}
        run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}"
      - name: Setup JDK
        uses: actions/setup-java@v5
        with:
          distribution: "temurin"
          java-version: 11
      - name: Create Databases
        # Must run before any sbt compile step: the build's JOOQ source
        # generators connect to texera_db while compiling.
        run: |
          psql -h localhost -U postgres -f sql/texera_ddl.sql
          psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql
          psql -h localhost -U postgres -f sql/texera_lakefs.sql
        env:
          PGPASSWORD: postgres
      - name: Setup sbt launcher
        uses: sbt/setup-sbt@508b753e53cb6095967669e0911487d2b9bc9f41 # v1.1.22
      - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 # v8.1.0
        with:
          extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]'
      - name: Lint and build amber distributable bundle
        # Single sbt invocation: scalafmt -> scalafix -> amber dist.
        # scalafmtCheckAll and scalafixAll cover every Scala module, so the
        # platform matrix below skips them. scalafix triggers compile (and
        # JOOQ codegen), which the dist command then reuses incrementally.
        run: |
          sbt scalafmtCheckAll \
              "scalafixAll --check" \
              WorkflowExecutionService/dist
      - name: Unzip amber dist and check binary licenses
        # Per-module LICENSE-binary files live at the repo root after #4668;
        # the amber JVM dist is checked against amber/LICENSE-binary-java.
        # The audit always runs (mirroring the previous 'if: always()' on its
        # own step) and never fails the step; the binding check's exit code
        # drives it.
        run: |
          set -euo pipefail
          mkdir -p /tmp/dists
          unzip -q amber/target/universal/amber-*.zip -d /tmp/dists/

          check_exit=0
          ./bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} jar \
            --license-binary amber/LICENSE-binary-java \
            /tmp/dists/amber-*/lib || check_exit=$?
          ./bin/licensing/audit_jar_licenses.py /tmp/dists/amber-*/lib || true
          exit "$check_exit"
      - name: Create texera_db_for_test_cases
        run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases -f sql/texera_ddl.sql
        env:
          PGPASSWORD: postgres
      - name: Set docker-java API version
        run: |
          echo "api.version=1.52" >> ~/.docker-java.properties
          cat ~/.docker-java.properties
      - name: Run amber and common module tests with coverage
        # 'jacoco' runs tests under sbt-jacoco's JVM agent and emits per-
        # module jacoco.xml that the codecov upload step picks up.
        # `WorkflowExecutionService/jacoco` only runs that project's
        # Test config (sbt's `test` task does not transit dependsOn),
        # so common modules' tests are listed explicitly here. Modules
        # with no tests (Auth, Config) are skipped.
        #
        # AMBER_TEST_FILTER=skip-integration tells amber/build.sbt to
        # exclude @org.apache.texera.amber.tags.IntegrationTest specs;
        # those run in the amber-integration job below.
        env:
          AMBER_TEST_FILTER: skip-integration
        run: |
          sbt "DAO/jacoco" \
              "PyBuilder/jacoco" \
              "WorkflowCore/jacoco" \
              "WorkflowOperator/jacoco" \
              "WorkflowExecutionService/jacoco"
      - name: Upload amber and common coverage to Codecov
        if: always()
        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./**/target/scala-2.13/jacoco/report/jacoco.xml
          flags: amber
          fail_ci_if_error: false

  amber-integration:
    # Runs Scala tests tagged @org.apache.texera.amber.tags.IntegrationTest —
    # currently the e2e specs that spawn Python UDF workers. Provisions
    # Python deps that the lighter `amber` job no longer installs. Cross-
    # cutting lints (scalafmt / scalafix) and the amber dist + binary
    # license check stay in `amber`; this job is tests-only.
    if: ${{ inputs.run_amber_integration }}
    strategy:
      matrix:
        os: [ubuntu-22.04]
        java-version: [11]
    runs-on: ${{ matrix.os }}
    env:
      JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
      JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U postgres"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          ref: ${{ inputs.checkout_ref || github.sha }}
          fetch-depth: 0
      - name: Prepare backport workspace
        if: ${{ inputs.backport_target_branch != '' }}
        working-directory: ${{ github.workspace }}
        run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}"
      - name: Setup JDK
        uses: actions/setup-java@v5
        with:
          distribution: "temurin"
          java-version: 11
      - name: Setup Python for Scala-Python integration tests
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"
      - name: Show Python
        run: python --version || python3 --version
      - name: Install Python dependencies
        # The integration tests spawn Python UDF workers; install
        # everything they need on the host. uv for speed; no licensing
        # concerns because no dist is built here.
        # --index-strategy unsafe-best-match makes uv consider every
        # version on every index (pip's default) rather than stopping at
        # the first index that lists a package. operator-requirements.txt
        # adds the pytorch CPU index as an --extra-index-url, which
        # mirrors a subset of common deps (e.g. pillow); without this
        # flag a dependabot bump to a version not yet mirrored there
        # fails to resolve even though PyPI has it.
        run: |
          python -m pip install uv
          if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi
          if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi
      - name: Create Databases
        run: |
          psql -h localhost -U postgres -f sql/texera_ddl.sql
          psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql
          psql -h localhost -U postgres -f sql/texera_lakefs.sql
        env:
          PGPASSWORD: postgres
      - name: Setup sbt launcher
        uses: sbt/setup-sbt@508b753e53cb6095967669e0911487d2b9bc9f41 # v1.1.22
      - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 # v8.1.0
        with:
          extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]'
      - name: Create texera_db_for_test_cases
        run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases -f sql/texera_ddl.sql
        env:
          PGPASSWORD: postgres
      - name: Lint and run amber integration tests
        # AMBER_TEST_FILTER=integration-only tells amber/build.sbt to
        # keep only @org.apache.texera.amber.tags.IntegrationTest
        # specs. The Java @TagAnnotation makes the marker visible to
        # ScalaTest's reflection, so `-n TAG` correctly narrows the
        # run.
        #
        # scalafmtCheckAll + scalafixAll --check are run here as well
        # because an integration-only PR fires only the
        # `amber-integration` label; the amber job's own cross-cutting
        # lint would not run, and the change would otherwise land
        # unlinted. Costs ~30s when amber also runs, which is fine.
        # No jacoco — these specs exercise code paths already covered
        # by amber's unit-test coverage.
        env:
          AMBER_TEST_FILTER: integration-only
        run: |
          sbt scalafmtCheckAll \
              "scalafixAll --check" \
              "WorkflowExecutionService/test"

  platform:
    # Per-service build, test, and license check for the non-amber Scala
    # services. Each matrix entry runs its own dist + test in isolation
    # against per-module LICENSE-binary (#4668). scalafmt / scalafix already
    # cover every module in the amber job above, so this matrix skips them.
    if: ${{ inputs.run_platform }}
    name: ${{ format('platform{0} ({1})', inputs.job_name_suffix, matrix.service) }}
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include:
          - service: config-service
            sbt_project: ConfigService
          - service: access-control-service
            sbt_project: AccessControlService
          - service: file-service
            sbt_project: FileService
          - service: computing-unit-managing-service
            sbt_project: ComputingUnitManagingService
          - service: workflow-compiling-service
            sbt_project: WorkflowCompilingService
    env:
      JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
      JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
    services:
      # Each platform service transitively depends on DAO, which runs JOOQ
      # code generation at compile time and needs the live texera schema.
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U postgres"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          ref: ${{ inputs.checkout_ref || github.sha }}
          fetch-depth: 0
      - name: Prepare backport workspace
        if: ${{ inputs.backport_target_branch != '' }}
        working-directory: ${{ github.workspace }}
        run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}"
      - name: Setup JDK
        uses: actions/setup-java@v5
        with:
          distribution: "temurin"
          java-version: 11
      - name: Setup sbt launcher
        uses: sbt/setup-sbt@508b753e53cb6095967669e0911487d2b9bc9f41 # v1.1.22
      - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 # v8.1.0
        with:
          extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]'
      - name: Create Databases
        run: |
          psql -h localhost -U postgres -f sql/texera_ddl.sql
          psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql
          psql -h localhost -U postgres -f sql/texera_lakefs.sql
        env:
          PGPASSWORD: postgres
      - name: Build dist and run ${{ matrix.service }} tests with coverage
        # Single sbt invocation so dist + test share compiled state. Use
        # `jacoco` so the codecov upload step has a report to pick up.
        run: sbt "${{ matrix.sbt_project }}/dist" "${{ matrix.sbt_project }}/jacoco"
      - name: Unzip ${{ matrix.service }} dist and check binary licenses
        # Each platform service has its own LICENSE-binary at the repo root
        # after #4668; check this service's dist against just its own file.
        run: |
          set -euo pipefail
          mkdir -p /tmp/dists
          unzip -q ${{ matrix.service }}/target/universal/${{ matrix.service }}-*.zip -d /tmp/dists/

          check_exit=0
          ./bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} jar \
            --license-binary ${{ matrix.service }}/LICENSE-binary \
            /tmp/dists/${{ matrix.service }}-*/lib || check_exit=$?
          ./bin/licensing/audit_jar_licenses.py /tmp/dists/${{ matrix.service }}-*/lib || true
          exit "$check_exit"
      - name: Upload ${{ matrix.service }} coverage to Codecov
        # Per-service flag so each matrix entry has its own Codecov view
        # rather than being merged into one umbrella `platform` flag.
        if: always()
        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./${{ matrix.service }}/target/scala-2.13/jacoco/report/jacoco.xml
          flags: ${{ matrix.service }}
          fail_ci_if_error: false

  python:
    if: ${{ inputs.run_python }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10", "3.11", "3.12", "3.13"]
    runs-on: ${{ matrix.os }}
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U postgres"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5
    steps:
      - name: Checkout Texera
        uses: actions/checkout@v5
        with:
          ref: ${{ inputs.checkout_ref || github.sha }}
          fetch-depth: 0
      - name: Prepare backport workspace
        if: ${{ inputs.backport_target_branch != '' }}
        run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Unit-test licensing scripts
        # Stdlib only, no install needed. Runs on every matrix row (3.10 →
        # 3.13) so the script's behavior is guarded across all supported
        # Python versions before the license check itself runs (3.12 only).
        run: python3 -m unittest discover -s bin/licensing -p "test_*.py" -v
      - name: Install dependencies
        # 3.12 is the only leg that drives the binary-license check via
        # pip-licenses. Keep stock pip there so the resolved versions
        # match amber/LICENSE-binary-python (also generated with pip,
        # tracking what the production image installs). Other legs use
        # uv purely for install-speed.
        run: |
          if [ "${{ matrix.python-version }}" = "3.12" ]; then
            python -m pip install --upgrade pip pip-licenses
            install="pip install"
          else
            python -m pip install uv
            # See amber-integration job for why --index-strategy is set.
            install="uv pip install --system --index-strategy unsafe-best-match"
          fi
          if [ -f amber/requirements.txt ]; then $install -r amber/requirements.txt; fi
          if [ -f amber/operator-requirements.txt ]; then $install -r amber/operator-requirements.txt; fi
      - name: Generate pip-licenses manifest
        if: matrix.python-version == '3.12'
        run: pip-licenses --format=csv --ignore-packages pip-licenses prettytable wcwidth > /tmp/pip-licenses.csv
      - name: Check installed Python packages against per-module LICENSE-binary files
        if: matrix.python-version == '3.12'
        run: ./bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} python /tmp/pip-licenses.csv
      - name: Create iceberg catalog database
        run: psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql
        env:
          PGPASSWORD: postgres
      - name: Lint with Ruff
        run: |
          cd amber && ruff check src/main/python src/test/python && ruff format --check src/main/python src/test/python
      - name: Install dev dependencies
        # Test-only deps live in amber/dev-requirements.txt and are
        # installed after the LICENSE-binary snapshot above so they never
        # appear in pip-licenses output. Packaging skips this file. uv
        # is safe here regardless of leg because it runs post-snapshot.
        run: |
          python -m pip install uv
          if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi
      - name: Test with pytest
        run: |
          cd amber && pytest --cov=src/main/python --cov-report=xml -sv
      - name: Upload python coverage to Codecov
        if: matrix.python-version == '3.12' && always()
        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./amber/coverage.xml
          flags: python
          fail_ci_if_error: false

  agent-service:
    if: ${{ inputs.run_agent_service }}
    name: ${{ format('agent-service{0} ({1})', inputs.job_name_suffix, matrix.os) }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
        bun-version: ["1.3.3"]
    defaults:
      run:
        working-directory: agent-service
    steps:
      - name: Checkout Texera
        uses: actions/checkout@v5
        with:
          ref: ${{ inputs.checkout_ref || github.sha }}
          fetch-depth: 0
      - name: Prepare backport workspace
        if: ${{ inputs.backport_target_branch != '' }}
        working-directory: ${{ github.workspace }}
        run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}"
      - name: Setup Bun
        run: |
          curl -fsSL https://bun.sh/install | bash -s -- bun-v${{ matrix.bun-version }}
          echo "$HOME/.bun/bin" >> $GITHUB_PATH
      - name: Install production dependencies
        run: bun install --production --frozen-lockfile
      - name: Generate agent-service license manifest
        if: matrix.os == 'ubuntu-latest'
        run: |
          mkdir -p dist
          bun run bin/collect-licenses.ts > dist/3rdpartylicenses.json
      - name: Check bundled agent-service packages against per-module LICENSE-binary files
        if: matrix.os == 'ubuntu-latest'
        run: ../bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} agent-npm dist/3rdpartylicenses.json
      - name: Install development dependencies
        run: bun install --frozen-lockfile
      - name: Lint with Prettier
        run: bun run format:check
      - name: Typecheck
        run: bun run typecheck
      - name: Run unit tests
        run: bun test --coverage --coverage-reporter=lcov
      - name: Upload agent-service coverage to Codecov
        if: matrix.os == 'ubuntu-latest' && always()
        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./agent-service/coverage/lcov.info
          flags: agent-service
          fail_ci_if_error: false