.github/workflows/_docs.yml - main - vshmidt/pytorch

.github/workflows/_docs.yml
212 строк8 KB
Thanh Ha
Switch docs build from c5 to c7i (#165082)
11 окт 2025, 17:59
11 окт 2025, 17:5979a33e2
name: build docs

on:
  workflow_call:
    inputs:
      build-environment:
        required: true
        type: string
        description: Top-level label for what's being built/tested.
      docker-image:
        required: true
        type: string
        description: Docker image to run in.
      push:
        required: false
        type: boolean
        default: false
        description: If set, push the docs to the docs website.
      run-doxygen:
        required: false
        type: boolean
        default: false
        description: If set, will enable C++ API doc generation using doxygen / breathe / exhale.
      sync-tag:
        required: false
        type: string
        default: ""
        description: |
          If this is set, our linter will use this to make sure that every other
          job with the same `sync-tag` is identical.
      s3-bucket:
        description: S3 bucket to download artifact
        required: false
        type: string
        default: "gha-artifacts"
      aws-role-to-assume:
        description: role to assume for downloading artifacts
        required: false
        type: string
        default: ""
      upload-aws-role-to-assume:
        description: role to assume for downloading artifacts
        required: false
        type: string
        default: ""
      runner_prefix:
        description: prefix for runner label
        type: string
        default: ""
    secrets:
      GH_PYTORCHBOT_TOKEN:
        required: false
        description: Permissions for pushing to the docs site.

jobs:
  build-docs:
    # Don't run on forked repos.
    if: github.repository_owner == 'pytorch'
    runs-on: ${{ matrix.runner }}
    environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'pytorchbot-env' || '' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - docs_type: cpp
            # We recently seeing lots of exit code 137 running this in Docker indicating
            # an OOM issue when running the job, so this upgrades the runner from 4xlarge
            # to the next available tier of 12xlarge. So much memory just to generate cpp
            # doc
            runner: ${{ inputs.runner_prefix }}linux.12xlarge.memory
            # TODO: Nightly cpp docs take longer and longer to finish (more than 3h now)
            # Let's try to figure out how this can be improved
            timeout-minutes: 360
          - docs_type: python
            runner: ${{ inputs.runner_prefix }}linux.c7i.2xlarge
            # It takes less than 30m to finish python docs unless there are issues
            timeout-minutes: 30
    # Set a fixed name for this job instead of using the current matrix-generated name, i.e. build-docs (cpp, linux.12xlarge, 180)
    # The current name requires updating the database last docs push query from test-infra every time the matrix is updated
    name: build-docs-${{ matrix.docs_type }}-${{ inputs.push }}
    steps:
      - name: Setup SSH (Click me for login details)
        uses: pytorch/test-infra/.github/actions/setup-ssh@main
        with:
          github-secret: ${{ secrets.GITHUB_TOKEN }}
          instructions: |
            All builds are done inside the container, to start an interactive session run:
              docker exec -it $(docker container ps --format '{{.ID}}') bash
            To start Python docs build type:
              cd docs && make html && make coverage

      # [see note: pytorch repo ref]
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@main

      - name: Setup Linux
        uses: ./.github/actions/setup-linux

      - name: configure aws credentials
        if : ${{ inputs.aws-role-to-assume != '' }}
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
        with:
          role-to-assume: ${{ inputs.aws-role-to-assume }}
          role-session-name: gha-linux-test
          aws-region: us-east-1

      - name: Calculate docker image
        id: calculate-docker-image
        uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
        with:
          docker-image-name: ${{ inputs.docker-image }}

      - name: Pull docker image
        uses: pytorch/test-infra/.github/actions/pull-docker-image@main
        with:
          docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}

      - name: Download build artifacts
        uses: ./.github/actions/download-build-artifacts
        with:
          name: ${{ inputs.build-environment }}
          s3-bucket: ${{ inputs.s3-bucket }}

      - name: Generate netrc (only for docs-push)
        if: inputs.push
        env:
          GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
        run: |
          # sometimes .netrc exists as a directory even though this is the temp folder
          rm -rf "${RUNNER_TEMP}/.netrc"
          # set credentials for https pushing
          echo "machine github.com" > "${RUNNER_TEMP}/.netrc"
          echo "login pytorchbot" >> "${RUNNER_TEMP}/.netrc"
          echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${RUNNER_TEMP}/.netrc"

      - name: Build ${{ matrix.docs_type }} docs
        timeout-minutes: ${{ matrix.timeout-minutes }}
        id: build-docs
        env:
          # After https://github.com/pytorch/pytorch/pull/88373, pull workflow can now be run periodically,
          # so using a schedule event to determine if the docs should be pushed or not doesn't hold true
          # anymore
          WITH_PUSH: ${{ inputs.push }}
          DOCKER_IMAGE: ${{ inputs.docker-image }}
          DOCS_TYPE: ${{ matrix.docs_type }}
          RUN_DOXYGEN: ${{ inputs.run-doxygen }}
          BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
        run: |
          set -ex
          # Convert refs/tags/v1.12.0rc3 into 1.12
          if [[ "${GITHUB_REF}" =~ ^refs/tags/v([0-9]+\.[0-9]+)\.* ]]; then
            target="${BASH_REMATCH[1]}"
          else
            target="main"
          fi
          # detached container should get cleaned up by teardown_ec2_linux
          container_name=$(docker run \
            -e BUILD_ENVIRONMENT \
            -e MAX_JOBS="$(nproc --ignore=2)" \
            -e SHA1="$GITHUB_SHA" \
            -e DOCS_VERSION="${target}" \
            -e DOCS_TYPE \
            -e RUN_DOXYGEN \
            -e WITH_PUSH \
            --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
            --security-opt seccomp=unconfined \
            --cap-add=SYS_PTRACE \
            --tty \
            --detach \
            --user jenkins \
            -v "${RUNNER_TEMP}/.netrc":/var/lib/jenkins/.netrc \
            -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
            -w /var/lib/jenkins/workspace \
            "${DOCKER_IMAGE}"
          )
          docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install $(echo dist/*.whl)[opt-einsum] && ./.ci/pytorch/${DOCS_TYPE}_doc_push_script.sh"

      - name: Chown workspace
        uses: ./.github/actions/chown-workspace
        if: always()

      - name: configure aws credentials
        if : ${{ inputs.upload-aws-role-to-assume != '' }}
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
        with:
          role-to-assume: ${{ inputs.upload-aws-role-to-assume }}
          role-session-name: gha-linux-test
          aws-region: us-east-1

      - name: Upload Python Docs Preview
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' && steps.build-docs.outcome == 'success' }}
        with:
          retention-days: 14
          s3-bucket: doc-previews
          if-no-files-found: error
          path: pytorch_docs/main/
          s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}

      - name: Upload C++ Docs Preview
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        with:
          retention-days: 14
          if-no-files-found: error
          s3-bucket: doc-previews
          path: cppdocs/
          s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/cppdocs

      - name: Teardown Linux
        uses: pytorch/test-infra/.github/actions/teardown-linux@main
        if: always()