build/mvn - master - githubmirror/spark

build/mvn
184 строки7 KB
Hyukjin Kwon
[SPARK-58025][INFRA] Redirect build/mvn fallback message to stderr to avoid corrupting version captures
08 июл 2026, 07:11
08 июл 2026, 07:11421067d
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

SELF=$(cd $(dirname $0) && pwd)
. "$SELF/util.sh"

if [ -z "${JAVA_HOME}" -a "$(command -v javac)" ]; then
  if [ "$(uname -s)" = "Darwin" ]; then
    # For some versions of macOS, "/usr/bin/javac" is a real file instead of a symbolic link,
    # so the JAVA_HOME may be set to path "/usr" improperly.
    # The following command is an appropriate way of setting JAVA_HOME on macOS.
    export JAVA_HOME="$(/usr/libexec/java_home)"
  else
    export JAVA_HOME="$(dirname $(dirname $(realpath $(command -v javac))))"
  fi
fi

# Determine the current working directory
_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Preserve the calling directory
_CALLING_DIR="$(pwd)"
# Options used during compilation
_COMPILE_JVM_OPTS="-Xss128m -Xmx4g -XX:ReservedCodeCacheSize=128m"

# Installs any application tarball given a URL, the expected tarball name,
# and, optionally, a checkable binary path to determine if the binary has
# already been installed. Arguments:
# 1 - Mirror host
# 2 - URL path on host
# 3 - URL query string
# 4 - checksum suffix
# 5 - Tarball Name
# 6 - Checkable Binary
install_app() {
  local mirror_host="$1"
  local url_path="$2"
  local url_query="$3"
  local checksum_suffix="$4"
  local local_tarball="${_DIR}/$5"
  local binary="${_DIR}/$6"
  local remote_tarball="${mirror_host}/${url_path}${url_query}"
  local local_checksum="${local_tarball}.${checksum_suffix}"
  local remote_checksum="${mirror_host}/${url_path}.${checksum_suffix}${url_query}"

  local curl_opts="--retry 3 --silent --show-error -L"
  local wget_opts="--no-verbose"

  if [ ! -f "$binary" ]; then
    # check if we already have the tarball
    # check if we have curl installed
    # download application
    if [ ! -f "${local_tarball}" -a "$(command -v curl)" ]; then
      echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2
      curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
      if [ ! -z "${checksum_suffix}" ]; then
        echo "exec: curl ${curl_opts} ${remote_checksum}" 1>&2
        curl ${curl_opts} "${remote_checksum}" > "${local_checksum}"
      fi
    fi
    # if the file still doesn't exist, lets try `wget` and cross our fingers
    if [ ! -f "${local_tarball}" -a "$(command -v wget)" ]; then
      echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2
      wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
      if [ ! -z "${checksum_suffix}" ]; then
        echo "exec: wget ${wget_opts} ${remote_checksum}" 1>&2
        wget ${wget_opts} -O "${local_checksum}" "${remote_checksum}"
      fi
    fi
    # if both were unsuccessful, exit
    if [ ! -f "${local_tarball}" ]; then
      echo -n "ERROR: Cannot download ${remote_tarball} with cURL or wget; please install manually and try again." 1>&2
      exit 2
    fi
    # Checksum may not have been specified; don't check if doesn't exist
    if [ "$(command -v shasum)" ]; then
      if [ -f "${local_checksum}" ]; then
        echo "  ${local_tarball}" >> ${local_checksum} # two spaces + file are important!
        # Assuming SHA512 here for now
        echo "Verifying checksum from ${local_checksum}" 1>&2
        if ! shasum -a 512 -c "${local_checksum}" > /dev/null ; then
          echo "Bad checksum from ${remote_checksum}" 1>&2
          exit 2
        fi
      fi
    else
      echo "Skipping checksum because shasum is not installed." 1>&2
    fi

    cd "${_DIR}" && tar -xzf "${local_tarball}"
    rm -rf "${local_tarball}"
    rm -f "${local_checksum}"
  fi
}

# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }

# Determine the Maven version from the root pom.xml file and
# install maven under the build/ folder if needed.
install_mvn() {
  local MVN_VERSION=`grep "<maven.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
  MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
  if [ -f "$MVN_BIN" ]; then
    return
  fi
  MVN_BIN="$(command -v mvn)"
  if [ "$MVN_BIN" ]; then
    local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
  fi
  if [ $(version $MVN_DETECTED_VERSION) -ne $(version $MVN_VERSION) ]; then
    local MVN_TARBALL="apache-maven-${MVN_VERSION}-bin.tar.gz"
    local FILE_PATH="org/apache/maven/apache-maven/${MVN_VERSION}/${MVN_TARBALL}"
    local MIRROR_BASE=${DEFAULT_ARTIFACT_REPOSITORY:-https://maven-central.storage-download.googleapis.com/maven2/}
    local MIRROR_URL_QUERY=""

    if [ $(command -v curl) ]; then
      if ! curl -L --output /dev/null --silent --head --fail "${MIRROR_BASE%/}/${FILE_PATH}${MIRROR_URL_QUERY}" ; then
        # Fall back to archive.apache.org for older Maven
        echo "Falling back to archive.apache.org to download Maven" 1>&2
        FILE_PATH="maven/maven-3/${MVN_VERSION}/binaries/${MVN_TARBALL}"
        MIRROR_BASE="https://archive.apache.org/dist"
        MIRROR_URL_QUERY=""
      fi
    fi

    install_app \
      "${MIRROR_BASE%/}" \
      "${FILE_PATH}" \
      "${MIRROR_URL_QUERY}" \
      "sha512" \
      "${MVN_TARBALL}" \
      "apache-maven-${MVN_VERSION}/bin/mvn"

    MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
  fi
}

install_mvn

# Reset the current working directory
cd "${_CALLING_DIR}"

# Set any `mvn` options if not already present
export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}

# Some Maven plugins (e.g. the mvn-scalafmt plugin used by `dev/lint-scala`) resolve their own
# dependencies through Coursier, which ignores MAVEN_MIRROR_URL and the POM repositories. Bridge
# MAVEN_MIRROR_URL into COURSIER_REPOSITORIES so mirror-only environments work out of the box.
# Respect an existing COURSIER_REPOSITORIES if the caller already set one.
if [ -n "${MAVEN_MIRROR_URL}" ] && [ -z "${COURSIER_REPOSITORIES}" ]; then
  export COURSIER_REPOSITORIES="ivy2Local|${MAVEN_MIRROR_URL}"
fi

echo "Using \`mvn\` from path: $MVN_BIN" 1>&2

if [ ! -z "${SPARK_LOCAL_HOSTNAME}" ]; then
  echo "Using SPARK_LOCAL_HOSTNAME=$SPARK_LOCAL_HOSTNAME" 1>&2
fi
if [ ! -z "${SPARK_LOCAL_IP}" ]; then
  echo "Using SPARK_LOCAL_IP=$SPARK_LOCAL_IP" 1>&2
fi

# call the `mvn` command as usual
# SPARK-25854
"${MVN_BIN}" "$@"
MVN_RETCODE=$?

exit $MVN_RETCODE