peft

1
name: BNB from source self-hosted runner with slow tests (scheduled)
2

3
on:
4
  workflow_dispatch:
5
  schedule:
6
    - cron: "0 2 * * *"
7

8
env:
9
  RUN_SLOW: "yes"
10
  IS_GITHUB_CI: "1"
11
  # To be able to run tests on CUDA 12.2
12
  NVIDIA_DISABLE_REQUIRE: "1"
13
  SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
14

15

16
jobs:
17
  run_all_tests_single_gpu:
18
    strategy:
19
      fail-fast: false
20
      matrix:
21
          docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest"]
22
    runs-on: [self-hosted, single-gpu, nvidia-gpu, t4, ci]
23
    env:
24
      CUDA_VISIBLE_DEVICES: "0"
25
      TEST_TYPE: "single_gpu_${{ matrix.docker-image-name }}"
26
    container:
27
      image: ${{ matrix.docker-image-name }}
28
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
29
    defaults:
30
      run:
31
        shell: bash
32
    steps:
33
      - uses: actions/checkout@v3
34
      - name: Pip install
35
        run: |
36
          source activate peft
37
          pip install -e . --no-deps
38
          pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
39
          pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26
40
          mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone # rename to transformers clone to avoid modules conflict
41
          if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
42
            cd transformers-clone
43
            transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
44
            echo "Checking out tag for Transformers version: v$transformers_version"
45
            git fetch --tags
46
            git checkout tags/v$transformers_version
47
            cd .. 
48
          fi
49
      - name: Run examples on single GPU
50
        if: always()
51
        run: |
52
          source activate peft
53
          make tests_examples_single_gpu_bnb
54
      
55
      - name: Run core tests on single GPU
56
        if: always()
57
        run: |
58
          source activate peft
59
          make tests_core_single_gpu_bnb
60

61
      - name: Run transformers tests on single GPU
62
        if: always()
63
        run: |
64
          source activate peft
65
          make transformers_tests
66
          
67
      - name: Generate Report
68
        if: always()
69
        run: |
70
          pip install slack_sdk tabulate
71
          python scripts/log_reports.py --slack_channel_name bnb-daily-ci >> $GITHUB_STEP_SUMMARY
72

73
  run_all_tests_multi_gpu:
74
    strategy:
75
      fail-fast: false
76
      matrix:
77
        docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest"]
78
    runs-on: [self-hosted, multi-gpu, nvidia-gpu, t4, ci]
79
    env:
80
      CUDA_VISIBLE_DEVICES: "0,1"
81
      TEST_TYPE: "multi_gpu_${{ matrix.docker-image-name }}"
82
    container:
83
      image: ${{ matrix.docker-image-name }}
84
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
85
    defaults:
86
      run:
87
        shell: bash
88
    steps:
89
      - uses: actions/checkout@v3
90
      - name: Pip install
91
        run: |
92
          source activate peft
93
          pip install -e . --no-deps
94
          pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
95
          pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26
96
          mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone
97
          if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
98
            cd transformers-clone
99
            transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
100
            echo "Checking out tag for Transformers version: v$transformers_version"
101
            git fetch --tags
102
            git checkout tags/v$transformers_version
103
            cd ..
104
          fi 
105

106
      - name: Run core GPU tests on multi-gpu
107
        if: always()
108
        run: |
109
          source activate peft
110
        
111
      - name: Run examples on multi GPU
112
        if: always()
113
        run: |
114
          source activate peft
115
          make tests_examples_multi_gpu_bnb
116
      
117
      - name: Run core tests on multi GPU
118
        if: always()
119
        run: |
120
          source activate peft
121
          make tests_core_multi_gpu_bnb
122

123
      - name: Run transformers tests on multi GPU
124
        if: always()
125
        run: |
126
          source activate peft
127
          make transformers_tests
128
          
129
      - name: Generate Report
130
        if: always()
131
        run: |
132
          pip install slack_sdk tabulate
133
          python scripts/log_reports.py --slack_channel_name bnb-daily-ci >> $GITHUB_STEP_SUMMARY
134