pytorch-lightning

1
# Python package
2
# Create and test a Python package on multiple Python versions.
3
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
4
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
5

6
trigger:
7
  tags:
8
    include: ["*"]
9
  branches:
10
    include:
11
      - "master"
12
      - "release/*"
13
      - "refs/tags/*"
14

15
pr:
16
  branches:
17
    include:
18
      - "master"
19
      - "release/*"
20
  paths:
21
    include:
22
      - ".azure/gpu-benchmarks.yml"
23
      - "requirements/fabric/**"
24
      - "requirements/pytorch/**"
25
      - "src/lightning/fabric/**"
26
      - "src/lightning/pytorch/**"
27
      - "tests/parity_fabric/**"
28
      - "tests/parity_pytorch/**"
29
    exclude:
30
      - "requirements/*/docs.txt"
31
      - "*.md"
32
      - "**/*.md"
33

34
schedules:
35
  - cron: "0 0 * * *" # At the end of every day
36
    displayName: Daily midnight benchmark
37
    branches:
38
      include:
39
        - "master"
40

41
jobs:
42
  - job: benchmarks
43
    timeoutInMinutes: "90"
44
    cancelTimeoutInMinutes: "2"
45
    pool: lit-rtx-3090
46
    variables:
47
      DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
48
    container:
49
      # TODO: Upgrade to Python 3.11
50
      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
51
      options: "--gpus=all --shm-size=32g"
52
    strategy:
53
      matrix:
54
        "pkg: Fabric":
55
          PACKAGE_NAME: "fabric"
56
        "pkg: Pytorch":
57
          PACKAGE_NAME: "pytorch"
58
    workspace:
59
      clean: all
60

61
    steps:
62
      - bash: |
63
          echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
64
          cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
65
          echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
66
        displayName: "set env. vars"
67

68
      - bash: |
69
          echo $CUDA_VISIBLE_DEVICES
70
          echo $TORCH_URL
71
          whereis nvidia
72
          nvidia-smi
73
          which python && which pip
74
          python --version
75
          pip --version
76
          pip list
77
        displayName: "Image info & NVIDIA"
78

79
      - bash: pip install -e .[dev] --find-links ${TORCH_URL}
80
        env:
81
          FREEZE_REQUIREMENTS: "1"
82
        displayName: "Install package"
83

84
      - bash: |
85
          set -e
86
          python requirements/collect_env_details.py
87
          python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
88
        displayName: "Env details"
89

90
      - bash: |
91
          pip install -q -r .actions/requirements.txt
92
          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
93
            --source_import="lightning.fabric,lightning.pytorch" \
94
            --target_import="lightning_fabric,pytorch_lightning"
95
        displayName: "Adjust tests"
96

97
      - bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
98
        env:
99
          PL_RUNNING_BENCHMARKS: "1"
100
          PL_RUN_CUDA_TESTS: "1"
101
        workingDirectory: tests/
102
        displayName: "Testing: benchmarks"
103

104
      - bash: bash run_standalone_tasks.sh
105
        workingDirectory: tests/parity_fabric
106
        # without succeeded this could run even if the job has already failed
107
        condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
108
        env:
109
          PL_RUN_CUDA_TESTS: "1"
110
        displayName: "Testing: fabric standalone tasks"
111
        timeoutInMinutes: "10"
112