.github/workflows/maven_test.yml - master - githubmirror/spark

.github/workflows/maven_test.yml
380 строк20 KB
Kousuke Saruta
[SPARK-57897][BUILD][CORE] Create `connector/credential-aws` module with CI/release integration
01 авг 2026, 12:22
01 авг 2026, 12:22cc19d86
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Build and test using Maven

on:
  workflow_call:
    inputs:
      java:
        required: false
        type: string
        default: 17
      branch:
        description: Branch to run the build against
        required: false
        type: string
        default: master
      hadoop:
        description: Hadoop version to run with. HADOOP_PROFILE environment variable should accept it.
        required: false
        type: string
        default: hadoop3
      os:
        description: OS to run this build.
        required: false
        type: string
        default: ubuntu-latest
      arch:
        description: The target architecture (x86, x64, arm64) of the Python interpreter.
        required: false
        type: string
        default: x64
      envs:
        description: Additional environment variables to set when running the tests. Should be in JSON format.
        required: false
        type: string
        default: '{}'
jobs:
  # Precompile Spark with Maven once and publish target/ + ~/.m2/.../spark as
  # an artifact for the matrix entries below to consume. Optional: any failure
  # here degrades the matrix to its original local `clean install` path.
  precompile-maven:
    name: "Precompile Spark with Maven"
    runs-on: ${{ inputs.os }}
    # If this job fails or is cancelled, the matrix entries fall back to
    # running `mvn clean install` locally as before.
    continue-on-error: true
    outputs:
      # Pinned so the build job checks out the same snapshot, even if the branch advances mid-run.
      head_sha: ${{ steps.resolve-sha.outputs.head_sha }}
    env:
      HADOOP_PROFILE: ${{ inputs.hadoop }}
      HIVE_PROFILE: hive2.3
      SPARK_LOCAL_IP: localhost
      GITHUB_PREV_SHA: ${{ github.event.before }}
    steps:
      - name: Checkout Spark repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          repository: apache/spark
          ref: ${{ inputs.branch }}
      - name: Resolve apache/spark HEAD SHA
        id: resolve-sha
        run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
      - name: Sync the current branch with the latest in Apache Spark
        if: github.repository != 'apache/spark'
        run: |
          echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
          git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
      - name: Cache SBT and Maven
        # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
        if: ${{ runner.os != 'macOS' }}
        uses: actions/cache@v5
        with:
          path: |
            build/apache-maven-*
            build/*.jar
            ~/.sbt
          key: build-${{ runner.os }}-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
          restore-keys: |
            build-${{ runner.os }}-
      - name: Cache Maven local repository
        # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
        if: ${{ runner.os != 'macOS' }}
        uses: actions/cache@v5
        with:
          path: ~/.m2/repository
          key: maven-${{ runner.os }}-java${{ inputs.java }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-${{ runner.os }}-java${{ inputs.java }}-
      - name: Install Java ${{ inputs.java }}
        uses: actions/setup-java@v5
        with:
          distribution: zulu
          java-version: ${{ inputs.java }}
      - name: Build Spark with Maven
        shell: |
          bash -c "if script -qec true 2>/dev/null; then script -qec bash\ {0}; else script -qe /dev/null bash {0}; fi"
        run: |
          set -e
          export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
          export MAVEN_CLI_OPTS="--no-transfer-progress"
          export JAVA_VERSION=${{ inputs.java }}
          ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} clean install
      - name: Package compile output
        run: |
          # Exclude assembly/ from the artifact: 11 of 12 matrix entries wipe it
          # right after extraction (SPARK-51628 regression-test-for-SPARK-51600),
          # and the connect entry rebuilds it via `mvn install -pl assembly`.
          find . \( -path './build' -o -path './.git' -o -path './assembly' \) -prune \
            -o -type d -name target -print0 \
            | tar --null -cf - -T - | zstd -c -T0 > compile-target.tar.zst
          if [ -d "$HOME/.m2/repository/org/apache/spark" ]; then
            tar -C "$HOME/.m2/repository/org/apache" -cf - spark | zstd -c -T0 > compile-m2-spark.tar.zst
          fi
          ls -lh compile-target.tar.zst compile-m2-spark.tar.zst
      - name: Upload compile artifact
        uses: actions/upload-artifact@v7
        with:
          name: spark-maven-compile-${{ inputs.branch }}-java${{ inputs.java }}-${{ github.run_id }}
          path: |
            compile-target.tar.zst
            compile-m2-spark.tar.zst
          retention-days: 1
          if-no-files-found: error

  # Build: build Spark and run the tests for specified modules using maven
  build:
    name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }}"
    needs: precompile-maven
    if: (!cancelled())
    runs-on: ${{ inputs.os }}
    # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
    # timeout-minutes: 150
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        java:
          - ${{ inputs.java }}
        hadoop:
          - ${{ inputs.hadoop }}
        hive:
          - hive2.3
        modules:
          - >-
            core,launcher,common#unsafe,common#kvstore,common#network-common,common#network-shuffle,common#sketch,common#utils,common#utils-java,common#variant
          - >-
            graphx,streaming,hadoop-cloud
          - >-
            mllib-local,mllib,sql#pipelines
          - >-
            repl,sql#hive-thriftserver
          - >-
            connector#kafka-0-10,connector#kafka-0-10-sql,connector#kafka-0-10-token-provider,connector#spark-ganglia-lgpl,connector#protobuf,connector#avro,connector#kinesis-asl,connector#credential-aws
          - >-
            sql#api,sql#catalyst,resource-managers#yarn,resource-managers#kubernetes#core
          - >-
            connect
        # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
        included-tags: [ "" ]
        excluded-tags: [ "" ]
        comment: [ "" ]
        include:
          # Hive tests
          - modules: sql#hive
            java: ${{ inputs.java }}
            hadoop: ${{ inputs.hadoop }}
            hive: hive2.3
            included-tags: org.apache.spark.tags.SlowHiveTest
            comment: "- slow tests"
          - modules: sql#hive
            java: ${{ inputs.java }}
            hadoop: ${{ inputs.hadoop }}
            hive: hive2.3
            excluded-tags: org.apache.spark.tags.SlowHiveTest
            comment: "- other tests"
          # SQL tests
          - modules: sql#core
            java: ${{ inputs.java }}
            hadoop: ${{ inputs.hadoop }}
            hive: hive2.3
            included-tags: org.apache.spark.tags.ExtendedSQLTest
            comment: "- extended tests"
          - modules: sql#core
            java: ${{ inputs.java }}
            hadoop: ${{ inputs.hadoop }}
            hive: hive2.3
            included-tags: org.apache.spark.tags.SlowSQLTest
            comment: "- slow tests"
          - modules: sql#core
            java: ${{ inputs.java }}
            hadoop: ${{ inputs.hadoop }}
            hive: hive2.3
            excluded-tags: org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowSQLTest
            comment: "- other tests"
    env:
      MODULES_TO_TEST: ${{ matrix.modules }}
      EXCLUDED_TAGS: ${{ matrix.excluded-tags }}
      INCLUDED_TAGS: ${{ matrix.included-tags }}
      HADOOP_PROFILE: ${{ matrix.hadoop }}
      HIVE_PROFILE: ${{ matrix.hive }}
      SPARK_LOCAL_IP: localhost
      GITHUB_PREV_SHA: ${{ github.event.before }}
    steps:
      - name: Checkout Spark repository
        uses: actions/checkout@v6
        # In order to fetch changed files
        with:
          fetch-depth: 0
          repository: apache/spark
          # Fall back to the branch when the precompile job failed before resolving the SHA.
          ref: ${{ needs.precompile-maven.outputs.head_sha || inputs.branch }}
      - name: Sync the current branch with the latest in Apache Spark
        if: github.repository != 'apache/spark'
        run: |
          echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
          git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
      # Cache local repositories. Note that GitHub Actions cache has a 10G limit.
      - name: Cache SBT and Maven
        # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
        if: ${{ runner.os != 'macOS' }}
        uses: actions/cache@v5
        with:
          path: |
            build/apache-maven-*
            build/*.jar
            ~/.sbt
          key: build-${{ runner.os }}-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
          restore-keys: |
            build-${{ runner.os }}-
      - name: Cache Maven local repository
        # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
        if: ${{ runner.os != 'macOS' }}
        uses: actions/cache@v5
        with:
          path: ~/.m2/repository
          key: maven-${{ runner.os }}-java${{ matrix.java }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-${{ runner.os }}-java${{ matrix.java }}-
      - name: Install Java ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
      - name: Install Python 3.12
        uses: actions/setup-python@v6
        # We should install one Python that is higher than 3+ for SQL and Yarn because:
        # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
        # - Yarn has a Python specific test too, for example, YarnClusterSuite.
        if: contains(matrix.modules, 'resource-managers#yarn') || (contains(matrix.modules, 'sql#core')) || contains(matrix.modules, 'connect')
        with:
          python-version: '3.12'
          architecture: ${{ inputs.arch }}
      - name: Install Python packages (Python 3.12)
        if: contains(matrix.modules, 'resource-managers#yarn') || (contains(matrix.modules, 'sql#core')) || contains(matrix.modules, 'connect')
        run: |
          python3.12 -m pip install 'numpy>=1.23.2' pyarrow 'pandas==2.3.3' pyyaml scipy unittest-xml-reporting 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'zstandard==0.25.0'
          python3.12 -m pip list
      - name: Download precompiled artifact
        id: download-precompiled
        if: needs.precompile-maven.result == 'success'
        continue-on-error: true
        uses: actions/download-artifact@v8
        with:
          name: spark-maven-compile-${{ inputs.branch }}-java${{ matrix.java }}-${{ github.run_id }}
      - name: Extract precompiled artifact
        id: extract-precompiled
        if: steps.download-precompiled.outcome == 'success'
        continue-on-error: true
        run: |
          zstd -dc compile-target.tar.zst | tar -xf -
          rm compile-target.tar.zst
          if [ -f compile-m2-spark.tar.zst ]; then
            mkdir -p "$HOME/.m2/repository/org/apache"
            zstd -dc compile-m2-spark.tar.zst | tar -C "$HOME/.m2/repository/org/apache" -xf -
            rm compile-m2-spark.tar.zst
          fi
      # Run the tests using script command.
      # BSD's script command doesn't support -c option, and the usage is different from Linux's one.
      # The kind of script command is tested by `script -qec true`.
      - name: Run tests
        env: ${{ fromJSON(inputs.envs) }}
        shell: |
          bash -c "if script -qec true 2>/dev/null; then script -qec bash\ {0}; else script -qe /dev/null bash {0}; fi"
        run: |
          # Fix for TTY related issues when launching the Ammonite REPL in tests.
          export TERM=vt100
          # `set -e` to make the exit status as expected due to use script command to run the commands
          set -e
          export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
          export MAVEN_CLI_OPTS="--no-transfer-progress"
          export JAVA_VERSION=${{ matrix.java }}
          export INPUT_BRANCH=${{ inputs.branch }}
          export ENABLE_KINESIS_TESTS=0
          # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10
          export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"`
          if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
            echo "Reusing precompiled artifact, skipping local Maven clean install."
            # SPARK-51628 regression coverage is naturally preserved on the reuse
            # path: the precompile artifact excludes assembly/, so non-connect
            # tests already run with the assembly module's jars dir missing.
            # Connect tests strongly depend on a built assembly module; rebuild
            # it here.
            if [ "$MODULES_TO_TEST" = "connect" ]; then
              echo "Building assembly module for connect tests."
              ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} -pl assembly install
            fi
          else
            ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} clean install
            # SPARK-51628: wipe the assembly module so tests exercise the
            # SPARK-51600 prepend fallback path. Connect tests strongly depend
            # on a built assembly module, so they are excluded.
            if [ "$MODULES_TO_TEST" != "connect" ]; then
              echo "Clean up the assembly module before maven testing"
              ./build/mvn $MAVEN_CLI_OPTS clean -pl assembly
            fi
          fi

          if [[ "$INCLUDED_TAGS" != "" ]]; then
            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} -Dtest.include.tags="$INCLUDED_TAGS" test -fae
          elif [[ "$MODULES_TO_TEST" == "connect" && "$INPUT_BRANCH" == "branch-4.0" ]]; then
            # SPARK-53914: Remove sql/connect/client/jdbc from `-pl` for branch-4.0, this branch can be deleted after the EOL of branch-4.0.
            ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae
          elif [[ "$MODULES_TO_TEST" == "connect" ]]; then
            ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jdbc,sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae
          elif [[ "$EXCLUDED_TAGS" != "" ]]; then
            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} -Dtest.exclude.tags="$EXCLUDED_TAGS" test -fae
          elif [[ "$MODULES_TO_TEST" == *"sql#hive-thriftserver"* ]]; then
            # To avoid a compilation loop, for the `sql/hive-thriftserver` module, run `clean install` instead
            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} clean install -fae
          elif [[ "$MODULES_TO_TEST" == *"sql#pipelines"* && "$INPUT_BRANCH" == "branch-4.0" ]]; then
            # SPARK-52441: Remove sql/pipelines from TEST_MODULES for branch-4.0, this branch can be deleted after the EOL of branch-4.0.
            TEST_MODULES=${TEST_MODULES/,sql\/pipelines/}
            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} test -fae
          elif [[ "$MODULES_TO_TEST" == *"common#utils-java"* && "$INPUT_BRANCH" == "branch-4.0" ]]; then
            # SPARK-53138: Remove common/utils-java from TEST_MODULES for branch-4.0, this branch can be deleted after the EOL of branch-4.0.
            TEST_MODULES=${TEST_MODULES/,common\/utils-java/}
            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} test -fae
          else
            ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} test -fae
          fi
      - name: Clean up local Maven repository
        run: |  
          rm -rf ~/.m2/repository/org/apache/spark
      - name: Upload test results to report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
          path: |
            **/target/test-reports/*.xml
            **/target/surefire-reports/*.xml
      - name: Upload unit tests log files
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
          path: "**/target/unit-tests.log"