scripts/docker - master - githubmirror/commcare-hq

scripts/docker
366 строк14 KB
Daniel Miller
Re-enable "Docker setup" group
28 май 2025, 15:57
28 май 2025, 15:5793fdd9d
#! /bin/bash
# CommCare HQ docker helper

function usage() {
    case $1 in
        test)
            echo "Run tests"
            echo ""
            echo "$0 test [OPTIONS]"
            echo ""
            echo "OPTIONS are passed to the selected test runner."
            echo ""
            echo "Set TEST environment variable to select a test runner:"
            echo "  TEST=python|python-sharded|javascript|python-sharded-and-javascript"
            echo ""
            echo "The default is python."
            echo ""
            echo "Tests are run with a project name of 'hqtest', so the"
            echo "containers and volumes it creates will not interfere with"
            echo "'hqservice' containers managed by other commands."
            echo ""
            ;;
        bash)
            echo "Run bash in container"
            echo ""
            echo "$0 bash [SERVICE_NAME]"
            echo ""
            echo "Run bash in the currently running container if it is up."
            echo "Otherwise create a new container and run bash inside it."
            echo "See docker/hq-compose.yml for applicable service names."
            echo "The default is 'web'."
            echo ""
            ;;
        runserver)
            echo "Run Django web server on port 8000"
            echo ""
            echo "$0 runserver [-d|--bootstrap|COMMAND]"
            echo ""
            echo "Options:"
            echo ""
            echo "  --bootstrap     Migrate new database and setup demo user."
            echo "  -d              Run in background."
            echo ""
            ;;
        shell)
            echo "Run Django shell in web container"
            echo ""
            ;;
        hqtest)
            echo "Manage test containers"
            echo ""
            echo "$0 hqtest [COMMAND]"
            echo ""
            echo "OPTIONS are passed to the selected test runner."
            echo ""
            echo "Examples:"
            echo ""
            echo "  # list test containers"
            echo "  $0 hqtest ps"
            echo ""
            echo "  # stop test containers"
            echo "  $0 hqtest stop"
            echo ""
            echo "  # remove all test containers and volumes"
            echo "  $0 hqtest teardown"
            echo ""
            ;;
        rebuild)
            echo "Stop, remove, and rebuild image(s)"
            echo ""
            echo "$0 rebuild [SERVICES...]"
            echo ""
            echo "Existing local images are removed before rebuilding."
            echo ""
            ;;
        teardown)
            echo "Remove all containers, images and volumes"
            echo ""
            ;;
        *)
            echo "Manage docker services for CommCare HQ"
            echo ""
            echo "$0 COMMAND [OPTIONS]"
            echo ""
            echo "$0 help COMMAND"
            echo ""
            echo "Commands:"
            echo "  bash        Run bash in container"
            echo "  help        Get help on a command"
            echo "  hqtest      Manage test containers"
            echo "  rebuild     Stop, remove, and rebuild image(s)"
            echo "  runserver   Run Django web server on port 8000"
            echo "  shell       Run Django shell in web container"
            echo "  teardown    Remove all containers, images and volumes"
            echo "  test        Run tests"
            echo ""
            echo "All other commands are passed directly to docker compose."
            echo ""
            echo "Examples:"
            echo ""
            echo "  # list service containers"
            echo "  $0 ps"
            echo ""
            echo "  # stop service containers"
            echo "  $0 stop"
            echo ""
            echo "  # run services in background"
            echo "  $0 up -d"
            echo ""
            ;;
    esac
    exit
}

compose_os_prefix='hq-compose-os-'
if [ -n "$COMPOSE_OS_FILE" ]; then
    # Allow overriding in the local shell env to bypass OS detection logic.
    here=$(dirname "$0")
    dockerdir=$(python -c 'import os, sys; print(os.path.abspath(sys.argv[1]))' "${here}/../docker")
    if ! [ -f "$COMPOSE_OS_FILE" ] && ! [ -f "${dockerdir}/${COMPOSE_OS_FILE}" ]; then
        echo "ERROR: invalid compose OS file: $COMPOSE_OS_FILE"
        echo "Please specify a valid YAML file, or consider using an existing one:"
        ls "${dockerdir}/${compose_os_prefix}"*.yml | sed -E 's,^.*/docker/,  - docker/,'
        exit 1
    fi
else
    # This exists because of M1 Macs
    os_info=$(uname -s)-$(uname -m)-$(uname -r | cut -d. -f1)
    os_suffix='default'
    case "$os_info" in
        Darwin-arm64-20)
            os_suffix='macos-m1-11'
            ;;
        Darwin-arm64-21)
            os_suffix='macos-m1-12'
            ;;
        Darwin-arm64-*)
            os_suffix='macos-m1-12'
            echo "WARNING: You are entering new territory with your new M1 Mac. Congratulations!"
            echo "Defaulting to most recently known M1 setup: $os_suffix"
            echo ""
            echo "To remove this message in the future, create a new 'base OS'"
            echo "(or copy an existing, working) docker compose configuration"
            echo "for your OS. For example:"
            echo ""
            macos_version=$(sw_vers -productVersion | cut -d. -f1)
            suggested_file="${compose_os_prefix}macos-m1-${macos_version}.yml"
            echo "$ cp -pv ${compose_os_prefix}${os_suffix}.yml $suggested_file"
            echo ""
            ;;
    esac
    export COMPOSE_OS_FILE="${compose_os_prefix}${os_suffix}.yml"
fi

CMD="$1"
shift
if [ "$CMD" == "test" -o "$CMD" == "hqtest" ]; then
    if [ "$CMD" == "hqtest" ]; then
        CMD="$1"
        shift
        if [ -z "$CMD" ]; then
            CMD=help
        fi
    fi
    export COMPOSE_FILE="docker/hq-compose.yml"
    export COMPOSE_PROJECT_NAME=hqtest
    if [ -n "$TRAVIS_BUILD_DIR" ]; then
        export VOLUME_PREFIX="$TRAVIS_BUILD_DIR/docker-volumes/"
    else
        # Since we don't want to persist tests data beyond the lifecycle of the container
        # we don't want to specify a volume path. Prior to v2 of docker compse this wasn't
        # an issue since not specifying a path resulted in using named volumes which docker
        # would dynamically create. From >v2 docker compose requires declaring named volumes
        # in the compose file which means we can't use the same compose file using named volumes
        # in one instance and unnamed in another.
        # The solution was to specify a blank path in tests rather than a volume name. Docker compose
        # will then create the volume as before.
        # Making this work relies on a exploiting variable substitution:
        #     ${VARIABLE-default} evaluates to default only if VARIABLE is unset in the environment.
        # In normal operations we set VOLUME_PREFIX and leave BLANK_IF_TESTS unset which will cause
        # this "${BLANK_IF_TESTS-vol1:}" to evaluate to "vol1:" (note the ':'). During tests we set "BLANK_IF_TESTS"
        # to an empty string and the result of the above expression is an empty string.
        # Note: I also tried using the long syntax for volume specification but wasn't able to get it to work.
        # See https://docs.docker.com/compose/compose-file/compose-file-v2/#volumes
        export VOLUME_PREFIX=""
        export BLANK_IF_TESTS=""
    fi
else
    if [[ "$CMD" =~ ^(runserver|bash|shell|ps)$ ]]; then
        SERVICES="runserver"
        if [ "$CMD" == "runserver" ]; then
            CMD="$1"
            shift
            if [ "$CMD" == "--bootstrap" ]; then
                CMD=bootstrap
            elif [ "$CMD" == "-d" ]; then
                CMD="up -d"
            elif [ -z "$CMD" ]; then
                CMD="up"
            fi
        fi
    else
        SERVICES="services"
        if [ -z "$CMD" ]; then
            CMD="help"
        fi
    fi

    XDG_DATA_HOME=${XDG_DATA_HOME:-$HOME/.local/share}
    if [ -n "$DOCKER_MACHINE_NAME" ]; then
        # docker toolbox: create dockerhq dir in virtualbox VM, but not on tmpfs
        DOCKER_DATA_HOME=/mnt/sda1/var/lib/dockerhq
        docker-machine ssh $DOCKER_MACHINE_NAME sudo mkdir -p $DOCKER_DATA_HOME
        KAFKA_ADVERTISED_HOST_NAME=$(docker-machine ip $DOCKER_MACHINE_NAME)
    else
        # docker on linux or docker for mac beta
        DOCKER_DATA_HOME=$XDG_DATA_HOME/dockerhq
        mkdir -p $DOCKER_DATA_HOME
        KAFKA_ADVERTISED_HOST_NAME=localhost
    fi

    export ES_CLUSTER_NAME="${ES_CLUSTER_NAME:-$(hostname)}"
    export COMPOSE_FILE=docker/hq-compose-${SERVICES}.yml
    export COMPOSE_PROJECT_NAME=hqservice
    export VOLUME_PREFIX="$DOCKER_DATA_HOME/"
    export KAFKA_ADVERTISED_HOST_NAME="$KAFKA_ADVERTISED_HOST_NAME"
fi
# export variables for compose file (they must be set or it complains)
export CI="$CI"
export CODECOV_TOKEN="$CODECOV_TOKEN"
export DATADOG_API_KEY="$DATADOG_API_KEY"
export DATADOG_APP_KEY="$DATADOG_APP_KEY"
export DOCKERFILE="${DOCKERFILE:-Dockerfile}"
export DOCKER_HQ_OVERLAYFS_METACOPY="${DOCKER_HQ_OVERLAYFS_METACOPY:-off}"
export DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock}
export GITHUB_ACTIONS="$GITHUB_ACTIONS"
export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME"
export JS_SETUP="${JS_SETUP:-no}"
export JS_TEST_EXTENSIONS="$JS_TEST_EXTENSIONS"
export DIVIDED_WE_RUN="$DIVIDED_WE_RUN"
export REUSE_DB="$REUSE_DB"
export STRIPE_PRIVATE_KEY="$STRIPE_PRIVATE_KEY"
export TRAVIS="$TRAVIS"
export TRAVIS_BRANCH="$TRAVIS_BRANCH"
export TRAVIS_BUILD_ID="$TRAVIS_BUILD_ID"
export TRAVIS_BUILD_NUMBER="$TRAVIS_BUILD_NUMBER"
export TRAVIS_COMMIT="$TRAVIS_COMMIT"
export TRAVIS_EVENT_TYPE="$TRAVIS_EVENT_TYPE"
export TRAVIS_JOB_ID="$TRAVIS_JOB_ID"
export TRAVIS_JOB_NUMBER="$TRAVIS_JOB_NUMBER"
export TRAVIS_PULL_REQUEST="$TRAVIS_PULL_REQUEST"
export TRAVIS_PULL_REQUEST_BRANCH="$TRAVIS_PULL_REQUEST_BRANCH"
export TRAVIS_PULL_REQUEST_SHA="$TRAVIS_PULL_REQUEST_SHA"
export TRAVIS_REPO_SLUG="$TRAVIS_REPO_SLUG"

# Check pinned service versions
pg_version=14
export DOCKER_HQ_POSTGRES_VERSION="${DOCKER_HQ_POSTGRES_VERSION:-$pg_version}"
if (( "$DOCKER_HQ_POSTGRES_VERSION" < "$pg_version" )); then
    echo "WARNING postgres is pinned with DOCKER_HQ_POSTGRES_VERSION=$DOCKER_HQ_POSTGRES_VERSION"
    echo "Version $pg_version is recommended."
    echo
    echo "Upgrade with https://gist.github.com/millerdev/547bac773483402554797e578fbd238f"
    echo "and then unset DOCKER_HQ_POSTGRES_VERSION in your shell environment."
fi

if [ "$CI" == "true" -o "$DOCKER_HQ_OVERLAY" == "none" ]; then
    # don't mount /mnt/commcare-hq-ro volume read-only on CI
    # overwrite of localsettings.py, etc. doesn't matter here
    export RO=""
    export DOCKER_HQ_OVERLAY="${DOCKER_HQ_OVERLAY:-none}"
else
    # TODO: use overlayfs and drop support for aufs
    export RO=":ro"
    export DOCKER_HQ_OVERLAY="${DOCKER_HQ_OVERLAY:-aufs}"
fi

if [ "$DOCKER_HQ_OVERLAY" == "overlayfs" ]; then
    export DOCKER_HQ_OVERLAYFS_CHMOD="${DOCKER_HQ_OVERLAYFS_CHMOD:-yes}"
else
    export DOCKER_HQ_OVERLAYFS_CHMOD="${DOCKER_HQ_OVERLAYFS_CHMOD:-no}"
fi

case $CMD in
    -h | --help | help | services | "")
        if [ "$COMPOSE_PROJECT_NAME" == "hqtest" ]; then
            usage hqtest
        else
            usage $@
        fi
        ;;
    teardown)
        TEARDOWN=yes
        shift
        CMD="down --rmi local -v"
        ;;
esac

if [ "$CMD" == "test" ]; then
    # Disable group if https://github.com/github/feedback/discussions/8848 resurfaces
    [ -n "$GITHUB_ACTIONS" ] && echo "::group::Docker setup"  # ends in docker/run.sh
    echo "Pulling docker containers..."
    docker compose pull --quiet

    # Print service versions
    docker compose run --no-deps --rm minio minio --version
    docker compose run --no-deps --rm postgres postgres --version
    docker compose run --no-deps --rm redis redis-server --version
    # Cannot get Couch version easily before it is running.
    #
    # The major version of elasticsearch is hard-coded in the container name
    # so it is not convenient to execute a command on the container without
    # hard-coding the major version here.
    #
    # It is not convenient to get versions of kafka or zookeeper.
    #
    # Some version information is encoded in docker image names
    docker images

    docker compose run --rm web run_tests "${TEST:-python}" "$@"
elif [ "$CMD" == "shell" ]; then
    docker compose run --rm web ./manage.py $CMD "$@"
elif [ "$CMD" == "bootstrap" ]; then
    docker compose run --rm web bootstrap
elif [ "$CMD" == "bash" ]; then
    SERVICE_NAME="${1:-web}"
    shift
    CONTAINER="${COMPOSE_PROJECT_NAME}_${SERVICE_NAME}_1"
    if [ "$(docker inspect -f {{.State.Running}} $CONTAINER 2> /dev/null)" == "true" ]; then
        docker exec -it $CONTAINER $CMD "$@"
    else
        docker compose run --rm $SERVICE_NAME $CMD "$@"
    fi
elif [ "$CMD" == "rebuild" ]; then
    if [ -z "$1" ]; then
        docker compose down --rmi local
        docker compose build
    else
        docker compose stop "$@"
        docker compose rm "$@"
        docker compose build "$@"
    fi
else
    docker compose $CMD "$@"
    if [ "$TEARDOWN" == "yes" ]; then
        if [ "$COMPOSE_PROJECT_NAME" == "hqservice" ]; then
            echo "THIS WILL DELETE ALL SERVICE DATA"
            read -p "Delete volumes? [yN] " -n 1 -r
            echo
            if [[ ! $REPLY =~ ^[Yy]$ ]]; then
                echo "Volumes not deleted."
                exit
            fi
        fi
        docker volume rm \
            ${VOLUME_PREFIX}couchdb \
            ${VOLUME_PREFIX}elasticsearch6 \
            ${VOLUME_PREFIX}kafka \
            ${VOLUME_PREFIX}lib \
            ${VOLUME_PREFIX}postgresql \
            ${VOLUME_PREFIX}redis \
            ${VOLUME_PREFIX}minio-conf \
            ${VOLUME_PREFIX}minio-data \
            ${VOLUME_PREFIX}zookeeper
    fi
fi