.github/workflows/python-tests.yml - develop - vshmidt/streamlit

.github/workflows/python-tests.yml
414 строк14 KB
Lukas Masuch
Increase coverage change threshold to 0.05% (#13821)
04 фев 2026, 19:02
04 фев 2026, 19:0286edbeb
name: Python Unit Tests

on:
  push:
    branches:
      - "develop"
  pull_request:
    types: [opened, synchronize, reopened]

  # Allows workflow to be called from other workflows
  workflow_call:
    inputs:
      ref:
        required: true
        type: string
    secrets:
      SNOWFLAKE_ACCOUNT:
        description: "Snowflake account passed from caller workflows for snowflake integration tests"
        required: true
      SNOWFLAKE_PASSWORD:
        description: "Snowflake account password passed from caller workflows for snowflake integration tests"
        required: true

# Avoid duplicate workflows on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-python
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  FORCE_COLOR: "1"

jobs:
  build_info:
    runs-on: ubuntu-latest

    name: "Build info"

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Set Python version vars
        id: build_info
        uses: ./.github/actions/build_info

    outputs:
      PYTHON_VERSIONS: ${{ steps.build_info.outputs.PYTHON_VERSIONS }}
      PYTHON_MIN_VERSION: ${{ steps.build_info.outputs.PYTHON_MIN_VERSION }}
      PYTHON_MAX_VERSION: ${{ steps.build_info.outputs.PYTHON_MAX_VERSION }}

  py-unit-tests:
    needs:
      - build_info

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        python_version: "${{ fromJson(needs.build_info.outputs.PYTHON_VERSIONS) }}"

    env:
      PYTHON_VERSION: >-
        ${{
          (
            matrix.python_version == 'min' && needs.build_info.outputs.PYTHON_MIN_VERSION ||
            (matrix.python_version == 'max' && needs.build_info.outputs.PYTHON_MAX_VERSION || matrix.python_version)
          )
        }}
      COVERAGE_FILE: .coverage.${{ matrix.python_version }}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"

      - name: Setup virtual env
        uses: ./.github/actions/make_init
        with:
          python_version: ${{ env.PYTHON_VERSION }}
      - name: Run Linters
        run: make python-lint
        env:
          RUFF_OUTPUT_FORMAT: github
      - name: Run Type Checkers
        run: make python-types
      - name: Run Python Tests
        run: make python-tests
      - name: CLI Smoke Tests
        run: make cli-smoke-tests
      - name: Upload coverage data
        uses: actions/upload-artifact@v6
        with:
          name: coverage_data_${{ matrix.python_version }}
          path: ${{ env.COVERAGE_FILE }}
          include-hidden-files: true
          if-no-files-found: ignore

  py-integration-tests:
    needs:
      - build_info

    runs-on: ubuntu-latest

    env:
      COVERAGE_FILE: .coverage.integration

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
        with:
          python_version: "3.12"
          # Deactivate usage of cached venv to avoid interfering with other Python
          # version jobs. The key generation for the cache could resolve to the same key,
          # which might cause some unexpected issues with dependencies.
          use_cached_venv: false
      - name: Install integration dependencies
        run: uv sync --python python --group integration
      - name: Run Python integration tests
        run: make python-integration-tests
        env:
          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
      - name: Upload coverage data
        uses: actions/upload-artifact@v6
        with:
          name: coverage_data_integration
          path: ${{ env.COVERAGE_FILE }}
          include-hidden-files: true
          if-no-files-found: ignore

  py-min-deps-test:
    needs:
      - build_info

    # The min pillow version we support in Streamlit cannot be installed on
    # Ubuntu 24 -> thats why we use Ubuntu 22.04 here instead.
    runs-on: ubuntu-22.04

    strategy:
      fail-fast: false

    env:
      COVERAGE_FILE: .coverage.min_deps

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Setup virtual env
        uses: ./.github/actions/make_init
        with:
          python_version: ${{ needs.build_info.outputs.PYTHON_MIN_VERSION }}
          use_cached_venv: false
      - name: Validate min-constraints-gen
        run: |
          make update-min-deps

          git_status=$(git status --porcelain -- scripts/assets/min-constraints-gen.txt)
          if [[ -n $git_status ]]; then
            echo "::error::The min constraints file is out of date! Please run \`make update-min-deps\` and commit the result."
            echo "::group::git diff scripts/assets/min-constraints-gen.txt"
            git diff scripts/assets/min-constraints-gen.txt
            echo "::endgroup::"
            exit 1
          else
            echo "min constraints file is up to date."
          fi
      - name: Install min dependencies (force reinstall)
        run: uv pip install -r scripts/assets/min-constraints-gen.txt --force-reinstall
      - name: Make local modules visible
        run: uv pip install --editable ./lib --no-deps
      - name: Run Python Tests
        run: make python-tests
      - name: CLI Smoke Tests
        run: make cli-smoke-tests
      - name: Upload coverage data
        uses: actions/upload-artifact@v6
        with:
          name: coverage_data_min_deps
          path: ${{ env.COVERAGE_FILE }}
          include-hidden-files: true
          if-no-files-found: ignore

  py-coverage-report:
    needs:
      - py-unit-tests
      - py-integration-tests
      - py-min-deps-test
    runs-on: ubuntu-latest
    permissions:
      # Required for downloading artifacts
      actions: read
      # Required for creating/updating PR comments
      pull-requests: write

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"

      - name: Set up Python 3.12
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Download all coverage files
        uses: actions/download-artifact@v7
        with:
          pattern: coverage_data_*
          merge-multiple: true
          path: lib

      - name: Combine coverage & generate report
        run: |
          python -m pip install --upgrade coverage[toml]

          # Combine all coverage files
          python -m coverage combine

          # Generate JSON report
          python -m coverage json -o coverage.json

          # Generate HTML report
          python -m coverage html --directory=htmlcov

          # Save report to a file first
          python -m coverage report --format=markdown > coverage-report.md

          # Set as environment variable with proper delimiter syntax for multiline content
          echo "COVERAGE_REPORT<<EOF" >> $GITHUB_ENV
          cat coverage-report.md >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

          # Also write to summary
          cat coverage-report.md >> $GITHUB_STEP_SUMMARY
        working-directory: lib

      - name: Upload combined coverage data
        uses: actions/upload-artifact@v6
        with:
          name: combined_coverage_data
          path: lib/.coverage
          include-hidden-files: true
      - name: Upload combined coverage JSON
        uses: actions/upload-artifact@v6
        with:
          name: combined_coverage_json
          path: lib/coverage.json
      - name: Upload combined coverage HTML report
        uses: actions/upload-artifact@v6
        with:
          name: combined_coverage_report
          path: lib/htmlcov
          retention-days: 14

      - name: Get latest develop run ID
        if: github.event_name == 'pull_request'
        id: get-latest-run
        uses: actions/github-script@v8
        with:
          script: |
            const { data: runs } = await github.rest.actions.listWorkflowRuns({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'python-tests.yml',
              branch: 'develop',
              status: 'success',
              per_page: 1
            });

            if (runs.workflow_runs.length === 0) {
              throw new Error('No successful workflow runs found on develop branch');
            }

            core.setOutput('run_id', runs.workflow_runs[0].id);

      - name: Download develop coverage
        if: github.event_name == 'pull_request'
        uses: actions/download-artifact@v7
        with:
          repository: streamlit/streamlit
          run-id: ${{ steps.get-latest-run.outputs.run_id }}
          name: combined_coverage_json
          path: /tmp/develop_coverage
          # Needs a token to download the artifact from another
          # workflow run:
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Check coverage changes and comment
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v8
        with:
          script: |
            const COVERAGE_CHANGE_THRESHOLD = 0.05; // 0.05%

            try {
              // Read current coverage from the existing file
              const currentCoverage = require('./lib/coverage.json');

              // Read develop coverage from downloaded artifact
              const developCoverage = require('/tmp/develop_coverage/coverage.json');

              // Calculate total coverage
              const currentTotal = currentCoverage.totals.percent_covered;
              const developTotal = developCoverage.totals.percent_covered;

              // Calculate percentage difference
              const diffPercent = currentTotal - developTotal;
              const absDiffPercent = Math.abs(diffPercent);

              console.log(`Current coverage: ${currentTotal.toFixed(4)}%`);
              console.log(`Develop coverage: ${developTotal.toFixed(4)}%`);
              console.log(`Coverage difference: ${diffPercent.toFixed(4)}%`);

            // Check if this PR is from a fork - fork PRs have limited GITHUB_TOKEN permissions
            // and cannot post comments to the PR
            const isForkPR = context.payload.pull_request?.head?.repo?.fork === true;
            if (isForkPR) {
              console.log('PR is from a fork - skipping coverage comment (insufficient permissions)');
              console.log('Coverage information is available in the job summary above.');
              return;
            }

              const changeType = diffPercent > 0 ? 'increased' : 'decreased';
              const emoji = diffPercent > 0 ? '📈' : '📉';

              const commentBody = `### ${emoji} Python coverage change detected

              The Python unit test coverage has **${changeType} by ${Math.abs(diffPercent).toFixed(4)}%**

              - Current PR: ${currentTotal.toFixed(4)}% (${currentCoverage.totals.num_statements} statements, ${currentCoverage.totals.missing_lines} missed)
              - Latest develop: ${developTotal.toFixed(4)}% (${developCoverage.totals.num_statements} statements, ${developCoverage.totals.missing_lines} missed)

              ${absDiffPercent >= COVERAGE_CHANGE_THRESHOLD
                ? (diffPercent > 0
                    ? '🎉 Great job on improving test coverage!'
                    : '💡 Consider adding more unit tests to maintain or improve coverage.'
                  )
                : '✅ Coverage change is within normal range.'
              }

              <details>
              <summary>Coverage by files</summary>

              ${process.env.COVERAGE_REPORT}
              </details>

              📊 [View detailed coverage comparison](https://issues.streamlit.app/Test_Coverage_(Python)?pr=${context.issue.number})`;


              // Unique identifier for our comment
              const commentIdentifier = '<!-- STREAMLIT-PYTHON-COVERAGE-CHECK -->';
              const fullCommentBody = `${commentIdentifier}\n${commentBody}`;

              // Get all comments on the PR
              const { data: comments } = await github.rest.issues.listComments({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
              });

              // Look for our previous coverage comment
              const coverageComment = comments.find(comment => comment.body.includes(commentIdentifier));

              if (coverageComment) {
                // Update existing comment
                await github.rest.issues.updateComment({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  comment_id: coverageComment.id,
                  body: fullCommentBody
                });
                console.log('Updated existing coverage comment');
              } else if (absDiffPercent >= COVERAGE_CHANGE_THRESHOLD) {
                // Only create new comment for significant changes
                await github.rest.issues.createComment({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: context.issue.number,
                  body: fullCommentBody
                });
                console.log('Created new coverage comment');
              } else {
                console.log('Coverage change is not significant and no existing comment found, skipping comment');
              }
            } catch (error) {
              console.error(`Error checking coverage changes: ${error.message}`);
              throw error;
            }