colossalai

1
name: Test Example on Schedule
2
on:
3
  # run at 00:00 of every Sunday(singapore time) so here is UTC time Saturday 16:00
4
  schedule:
5
    - cron:  '0 16 * * 6'
6
  workflow_dispatch:
7

8
jobs:
9
  # This is for all files' weekly check. Specifically, this job is to find all the directories.
10
  matrix_preparation:
11
    if: github.repository == 'hpcaitech/ColossalAI'
12
    name: Prepare matrix for weekly check
13
    runs-on: ubuntu-latest
14
    outputs:
15
      matrix: ${{ steps.setup-matrix.outputs.matrix }}
16
    steps:
17
    - name: 📚 Checkout
18
      uses: actions/checkout@v3
19

20
    - name: setup matrix
21
      id: setup-matrix
22
      run: |
23
        res=`python .github/workflows/scripts/example_checks/check_example_weekly.py`
24
        all_loc=$( IFS=',' ; echo "${res[*]}" )
25
        echo "Found the examples: $all_loc"
26
        echo "matrix={\"directory\":$(echo "$all_loc")}" >> $GITHUB_OUTPUT
27

28
  weekly_check:
29
    if: github.repository == 'hpcaitech/ColossalAI'
30
    name: Weekly check all examples
31
    needs: matrix_preparation
32
    runs-on: [self-hosted, gpu]
33
    strategy:
34
      fail-fast: false
35
      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
36
    container:
37
      image: hpcaitech/pytorch-cuda:2.1.0-12.1.0
38
    timeout-minutes: 10
39
    steps:
40
      - name: 📚 Checkout
41
        uses: actions/checkout@v3
42

43
      - name: Install Colossal-AI
44
        run: |
45
          BUILD_EXT=1 pip install -v .
46

47
      - name: Traverse all files
48
        run: |
49
          example_dir=${{ matrix.directory }}
50
          echo "Testing ${example_dir} now"
51
          cd "${PWD}/examples/${example_dir}"
52
          bash test_ci.sh
53
        env:
54
          NCCL_SHM_DISABLE: 1
55

56
      - name: Notify Lark
57
        id: message-preparation
58
        if: ${{ failure() }}
59
        run: |
60
          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
61
          msg="Example tests failed for $EXAMPLE_DIR, please visit $url for details"
62
          echo $msg
63
          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
64
        env:
65
          SERVER_URL: ${{github.server_url }}
66
          REPO: ${{ github.repository }}
67
          RUN_ID: ${{ github.run_id }}
68
          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
69
          EXAMPLE_DIR: ${{ matrix.directory }}
70