pytorch-lightning

1
# Python package
2
# Create and test a Python package on multiple Python versions.
3
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
4
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
5

6
trigger:
7
  tags:
8
    include:
9
      - "*"
10
  branches:
11
    include:
12
      - "master"
13
      - "release/*"
14
      - "refs/tags/*"
15

16
schedules:
17
  - cron: "0 0 * * *" # At the end of every day
18
    displayName: Daily midnight testing
19
    branches:
20
      include:
21
        - "master"
22

23
pr:
24
  branches:
25
    include:
26
      - "master"
27
      - "release/*"
28
  paths:
29
    include:
30
      - ".actions/*"
31
      - ".azure/app-cloud-e2e.yml"
32
      - "src/lightning/__about__.py"
33
      - "src/lightning/__init__.py"
34
      - "src/lightning/__main__.py"
35
      - "src/lightning/__setup__.py"
36
      - "src/lightning/__version__.py"
37
      - "src/lightning/app/**"
38
      - "src/lightning_app/*"
39
      - "examples/app/**"
40
      - "requirements/app/**"
41
      - "tests/integrations_app/**"
42
      - "setup.py"
43
    exclude:
44
      - "!tests/integrations_app/flagship/**"
45
      - "requirements/*/docs.txt"
46
      - "*.md"
47
      - "**/*.md"
48

49
# variables are automatically exported as environment variables so this will override pip's default cache dir
50
variables:
51
  - name: pip_cache_dir
52
    value: $(Pipeline.Workspace)/.pip
53
  - name: local_id
54
    value: $(Build.BuildId)
55
  - name: video_artifact_dir
56
    value: ./videos
57

58
jobs:
59
  - job: test_e2e
60
    pool: "azure-cpus"
61
    container:
62
      # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags
63
      image: mcr.microsoft.com/playwright/python:v1.38.0-focal
64
      options: "--shm-size=4gb"
65
    strategy:
66
      matrix:
67
        "App: v0_app":
68
          name: "v0_app"
69
          dir: "public"
70
        "App: boring_app":
71
          name: "boring_app"
72
          dir: "public"
73
        "App: template_streamlit_ui":
74
          name: "template_streamlit_ui"
75
          dir: "public"
76
        "App: template_react_ui":
77
          name: "template_react_ui"
78
          dir: "public"
79
        # 'App: template_jupyterlab':  # TODO: clarify where these files lives
80
        #   name: "template_jupyterlab"
81
        "App: installation_commands_app":
82
          name: "installation_commands_app"
83
          dir: "public"
84
        "App: drive":
85
          name: "drive"
86
          dir: "public"
87
        "App: payload":
88
          name: "payload"
89
          dir: "public"
90
        "App: commands_and_api":
91
          name: "commands_and_api"
92
          dir: "public"
93
        "App: quick_start":
94
          name: "quick_start"
95
          dir: "public"
96
        "App: idle_timeout":
97
          name: "idle_timeout"
98
          dir: "local"
99
        "App: collect_failures":
100
          name: "collect_failures"
101
          dir: "local"
102
        "App: custom_work_dependencies":
103
          name: "custom_work_dependencies"
104
          dir: "local"
105
    timeoutInMinutes: "15"
106
    cancelTimeoutInMinutes: "1"
107
    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
108
    workspace:
109
      clean: all
110
    variables:
111
      FREEZE_REQUIREMENTS: "1"
112
      HEADLESS: "1"
113
      PACKAGE_LIGHTNING: "1"
114
      CLOUD: "1"
115
      VIDEO_LOCATION: $(video_artifact_dir)
116
      PR_NUMBER: $(local_id)
117
      TEST_APP_NAME: $(name)
118
      TEST_APP_FOLDER: $(dir)
119
      HAR_LOCATION: "./artifacts/hars"
120
      SLOW_MO: "50"
121
      LIGHTNING_DEBUG: "1"
122
    steps:
123
      - script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
124
        displayName: "Set id for this PR"
125
        condition: eq(variables['Build.Reason'], 'PullRequest')
126

127
      - bash: |
128
          whoami
129
          mkdir -p "$(video_artifact_dir)/$(name)"
130
          printf "local id: $(local_id)\n"
131
          python --version
132
          pip --version
133
          echo "allow fail: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}"
134
        displayName: "Info"
135

136
      # TODO: we are testing it as `lightning`, so add also version for `lightning_app`
137
      - bash: |
138
          pip install -e .[app-dev] \
139
            -f https://download.pytorch.org/whl/cpu/torch_stable.html
140
        displayName: "Install Lightning & dependencies"
141

142
      - bash: python -m playwright install # --with-deps
143
        displayName: "Install Playwright system dependencies"
144

145
      # The magic happens here it doesn't need to install the quick start dependencies.
146
      # This test is very important to test the main user story of lightning app.
147
      # It also e2e tests running on cloud without installing dependencies.
148
      - bash: |
149
          git clone https://github.com/Lightning-AI/lightning-quick-start examples/app/quick-start
150
        # without succeeded this could run even if the job has already failed
151
        condition: and(succeeded(), eq(variables['name'], 'quick_start'))
152
        displayName: "Clone Quick start Repo"
153
      - bash: |
154
          git clone https://github.com/Lightning-AI/lightning-template-react examples/app/template_react_ui
155
        # without succeeded this could run even if the job has already failed
156
        condition: and(succeeded(), eq(variables['name'], 'template_react_ui'))
157
        displayName: "Clone Template React UI Repo"
158

159
      # Replace imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
160
      - bash: |
161
          pip install -q -r .actions/requirements.txt
162
          python .actions/assistant.py copy_replace_imports \
163
            --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"
164
        displayName: "Adjust examples"
165

166
      - bash: pip --version && pip list
167
        displayName: "List pip dependency"
168

169
      - bash: |
170
          ls -l examples/app/$(TEST_APP_NAME)
171
          echo ${TEST_FILE}
172
          python -m pytest ${TEST_FILE}::test_$(TEST_APP_NAME)_example_cloud \
173
            --timeout=360 --capture=no -v --color=yes
174
        env:
175
          TEST_FILE: tests/integrations_app/$(TEST_APP_FOLDER)/test_$(TEST_APP_NAME).py
176
          #LAI_USER: $(LAI_USER)  # for STAGING
177
          #LAI_PASS: $(LAI_PASS)  # for STAGING
178
          LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
179
          LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
180
          LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
181
          LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
182
        # Todo: investigate why these apps are failing
183
        continueOnError: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}
184
        displayName: "Run the tests"
185

186
      - task: PublishPipelineArtifact@1
187
        condition: failed()
188
        inputs:
189
          path: "$(video_artifact_dir)/$(name)"
190
          artifactName: $(name)
191
          publishLocation: "pipeline"
192
        displayName: "Publish videos"
193

194
      - bash: |
195
          time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
196
        condition: always()
197
        continueOnError: "true"
198
        timeoutInMinutes: "3"
199
        env:
200
          #LAI_USER: $(LAI_USER)  # for STAGING
201
          #LAI_PASS: $(LAI_PASS)  # for STAGING
202
          LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
203
          LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
204
          LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
205
          LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
206
        displayName: "Clean Previous Apps"
207