bin/pyspark - master - githubmirror/spark

bin/pyspark
102 строки4 KB
Takuya Ueshin
[SPARK-51212][PYTHON] Add a separated PySpark package for Spark Connect by default
17 фев 2025, 05:45
17 фев 2025, 05:45f798e7a
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

if [ -z "${SPARK_HOME}" ]; then
  source "$(dirname "$0")"/find-spark-home
fi

source "${SPARK_HOME}"/bin/load-spark-env.sh
export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]"

# In Spark 2.0, IPYTHON and IPYTHON_OPTS are removed and pyspark fails to launch if either option
# is set in the user's environment. Instead, users should set PYSPARK_DRIVER_PYTHON=ipython
# to use IPython and set PYSPARK_DRIVER_PYTHON_OPTS to pass options when starting the Python driver
# (e.g. PYSPARK_DRIVER_PYTHON_OPTS='notebook').  This supports full customization of the IPython
# and executor Python executables.

# Fail noisily if removed options are set
if [[ -n "$IPYTHON" || -n "$IPYTHON_OPTS" ]]; then
  echo "Error in pyspark startup:"
  echo "IPYTHON and IPYTHON_OPTS are removed in Spark 2.0+. Remove these from the environment and set PYSPARK_DRIVER_PYTHON and PYSPARK_DRIVER_PYTHON_OPTS instead."
  exit 1
fi

# Default to standard python3 interpreter unless told otherwise
if [[ -z "$PYSPARK_PYTHON" ]]; then
  PYSPARK_PYTHON=python3
fi
if [[ -z "$PYSPARK_DRIVER_PYTHON" ]]; then
  PYSPARK_DRIVER_PYTHON=$PYSPARK_PYTHON
fi
export PYSPARK_PYTHON
export PYSPARK_DRIVER_PYTHON
export PYSPARK_DRIVER_PYTHON_OPTS

# Attempt to find JAVA_HOME.
# If JAVA_HOME not set, install JDK 17 and set JAVA_HOME using a temp dir, and adding the
# temp dir to the PYTHONPATH.
if [ -n "${JAVA_HOME}" ]; then
  RUNNER="${JAVA_HOME}/bin/java"
else
  if [ "$(command -v java)" ]; then
    RUNNER="java"
  else
    echo -n "JAVA_HOME is not set. Would you like to install JDK 17 and set JAVA_HOME? (Y/N) " >&2

    read -r input

    if [[ "${input,,}" == "y" ]]; then
        TEMP_DIR=$(mktemp -d)
        $PYSPARK_DRIVER_PYTHON -m pip install --target="$TEMP_DIR" install-jdk
        export JAVA_HOME=$(PYTHONPATH="$TEMP_DIR" $PYSPARK_DRIVER_PYTHON -c 'import jdk; print(jdk.install("17"))')
        RUNNER="${JAVA_HOME}/bin/java"
        echo "JDK was installed to the path \"$JAVA_HOME\""
        echo "You can avoid needing to re-install JDK by setting your JAVA_HOME environment variable to \"$JAVA_HOME\""
    else
        echo "JDK installation skipped. You can manually install JDK (17 or later) and set JAVA_HOME in your environment."
        exit 1
    fi
  fi
fi

# Add the PySpark classes to the Python path:
export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9.9-src.zip:$PYTHONPATH"

# Load the PySpark shell.py script when ./pyspark is used interactively:
export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"

export SPARK_CONNECT_MODE=$($PYSPARK_DRIVER_PYTHON -c "from pyspark.util import spark_connect_mode; print(spark_connect_mode())")

if [[ "$SPARK_CONNECT_MODE" != "0" && "$SPARK_CONNECT_MODE" != "1" ]]; then
  echo "The environment variable SPARK_CONNECT_MODE has unknown value or pyspark.util package is not available: $SPARK_CONNECT_MODE"
  unset SPARK_CONNECT_MODE
fi

# For pyspark tests
if [[ -n "$SPARK_TESTING" ]]; then
  unset YARN_CONF_DIR
  unset HADOOP_CONF_DIR
  export PYTHONHASHSEED=0
  exec "$PYSPARK_DRIVER_PYTHON" -m "$@"
  exit
fi

exec "${SPARK_HOME}"/bin/spark-submit pyspark-shell-main --name "PySparkShell" "$@"