cilium

1
name: Conformance AKS (ci-aks)
2

3
# Any change in triggers needs to be reflected in the concurrency group.
4
on:
5
  workflow_dispatch:
6
    inputs:
7
      PR-number:
8
        description: "Pull request number."
9
        required: true
10
      context-ref:
11
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
12
        required: true
13
      SHA:
14
        description: "SHA under test (head of the PR branch)."
15
        required: true
16
      extra-args:
17
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
18
        required: false
19
        default: '{}'
20
  # Run every 6 hours
21
  schedule:
22
    - cron:  '0 0/6 * * *'
23

24
# By specifying the access of one of the scopes, all of those that are not
25
# specified are set to 'none'.
26
permissions:
27
  # To be able to access the repository with actions/checkout
28
  contents: read
29
  # To allow retrieving information from the PR API
30
  pull-requests: read
31
  # To be able to set commit status
32
  statuses: write
33
  # Required to generate OIDC tokens for `az` authentication
34
  id-token: write
35

36
concurrency:
37
  # Structure:
38
  # - Workflow name
39
  # - Event type
40
  # - A unique identifier depending on event type:
41
  #   - schedule: SHA
42
  #   - workflow_dispatch: PR number
43
  #
44
  # This structure ensures a unique concurrency group name is generated for each
45
  # type of testing, such that re-runs will cancel the previous run.
46
  group: |
47
    ${{ github.workflow }}
48
    ${{ github.event_name }}
49
    ${{
50
      (github.event_name == 'schedule' && github.sha) ||
51
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
52
    }}
53
  cancel-in-progress: true
54

55
env:
56
  name: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
57
  cost_reduction: --node-vm-size Standard_B2s --node-osdisk-size 30
58
  cilium_cli_ci_version:
59
  CILIUM_CLI_MODE: helm
60
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
61

62
jobs:
63
  commit-status-start:
64
    name: Commit Status Start
65
    runs-on: ubuntu-latest
66
    steps:
67
      - name: Set initial commit status
68
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
69
        with:
70
          sha: ${{ inputs.SHA || github.sha }}
71

72
  generate-matrix:
73
    name: Generate Matrix
74
    runs-on: ubuntu-latest
75
    outputs:
76
      matrix: ${{ steps.set-matrix.outputs.matrix }}
77
    steps:
78
      - name: Checkout context ref (trusted)
79
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
80
        with:
81
          ref: ${{ inputs.context-ref || github.sha }}
82
          persist-credentials: false
83

84
      - name: Convert YAML to JSON
85
        run: |
86
          work_dir=".github/actions/azure"
87
          destination_directory="/tmp/generated/azure"
88
          mkdir -p "${destination_directory}"
89

90
          yq -o=json "${work_dir}/k8s-versions.yaml" | jq . > "${destination_directory}/azure.json"
91

92
      - name: Generate Matrix
93
        id: set-matrix
94
        run: |
95
          cd /tmp/generated/azure
96

97
          # Use complete matrix in case of scheduled run
98
          # main -> event_name = schedule
99
          # other stable branches -> PR-number starting with v (e.g. v1.14)
100
          if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.PR-number }}" == v* ]];then
101
            jq '{ "include": [ .include[] | select(.disabled==null) ] }' azure.json > /tmp/matrix.json
102
          else
103
            jq '{ "include": [ .include[] | select(.default) ] }' azure.json > /tmp/matrix.json
104
          fi
105

106
          echo "Generated matrix:"
107
          cat /tmp/matrix.json
108
          echo "matrix=$(jq -c . < /tmp/matrix.json)" >> $GITHUB_OUTPUT
109

110
  installation-and-connectivity:
111
    name: Installation and Connectivity Test
112
    needs: generate-matrix
113
    runs-on: ubuntu-latest
114
    timeout-minutes: 90
115
    env:
116
      job_name: "Installation and Connectivity Test"
117
    strategy:
118
      fail-fast: false
119
      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
120

121
    steps:
122
      - name: Checkout context ref (trusted)
123
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
124
        with:
125
          ref: ${{ inputs.context-ref || github.sha }}
126
          persist-credentials: false
127

128
      - name: Set Environment Variables
129
        uses: ./.github/actions/set-env-variables
130

131
      - name: Override cluster name
132
        run: |
133
          # Extend default name with matrix index to avoid cluster name conflicts
134
          NAME=${{ env.name }}-${{ matrix.index }}
135
          echo "name=${NAME}" >> "$GITHUB_ENV"
136

137
      - name: Get Cilium's default values
138
        id: default_vars
139
        uses: ./.github/actions/helm-default
140
        with:
141
          image-tag: ${{ inputs.SHA }}
142
          chart-dir: ./untrusted/install/kubernetes/cilium
143

144
      - name: Set up job variables
145
        id: vars
146
        run: |
147
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
148
            OWNER="${{ inputs.PR-number }}"
149
          else
150
            OWNER="${{ github.ref_name }}"
151
            OWNER="${OWNER/./-}"
152
          fi
153

154
          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
155
            --cluster-name=${{ env.name }} \
156
            --helm-set loadBalancer.l7.backend=envoy \
157
            --helm-set tls.secretsBackend=k8s \
158
            --helm-set=azure.resourceGroup=${{ env.name }}"
159
          CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled \
160
            --hubble=false --collect-sysdump-on-failure --external-target bing.com --external-cidr 8.0.0.0/8 --external-ip 8.8.4.4 --external-other-ip 8.8.8.8"
161
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
162
          echo connectivity_test_defaults=${CONNECTIVITY_TEST_DEFAULTS} >> $GITHUB_OUTPUT
163
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT
164
          echo owner=${OWNER} >> $GITHUB_OUTPUT
165

166
      - name: Install Cilium CLI
167
        uses: cilium/cilium-cli@7306e3cdc6caee738157f08e3e1ba26179f104e5 # v0.15.23
168
        with:
169
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
170
          release-version: ${{ env.CILIUM_CLI_VERSION }}
171
          ci-version: ${{ env.cilium_cli_ci_version }}
172

173
      - name: Login to Azure
174
        uses: azure/login@cb79c773a3cfa27f31f25eb3f677781210c9ce3d # v1.6.1
175
        with:
176
          creds: ${{ secrets.AZURE_PR_SP_CREDS }}
177

178
      - name: Install aks-preview CLI extension
179
        run: |
180
            az extension add --name aks-preview
181
            az extension update --name aks-preview
182
            az version
183

184
      - name: Create AKS cluster
185
        run: |
186
          # Create group
187
          az group create \
188
            --name ${{ env.name }} \
189
            --location ${{ matrix.location }} \
190
            --tags usage=${{ github.repository_owner }}-${{ github.event.repository.name }} owner=${{ steps.vars.outputs.owner }}
191

192
          # Create AKS cluster
193
          az aks create \
194
            --resource-group ${{ env.name }} \
195
            --name ${{ env.name }} \
196
            --location ${{ matrix.location }} \
197
            --kubernetes-version ${{ matrix.version }} \
198
            --network-plugin none \
199
            --node-count 2 \
200
            ${{ env.cost_reduction }} \
201
            --generate-ssh-keys
202

203
      - name: Get cluster credentials
204
        run: |
205
          az aks get-credentials \
206
            --resource-group ${{ env.name }} \
207
            --name ${{ env.name }}
208

209
      - name: Wait for images to be available
210
        timeout-minutes: 30
211
        shell: bash
212
        run: |
213
          for image in cilium-ci operator-azure-ci hubble-relay-ci ; do
214
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
215
          done
216

217
      # Warning: since this is a privileged workflow, subsequent workflow job
218
      # steps must take care not to execute untrusted code.
219
      - name: Checkout pull request branch (NOT TRUSTED)
220
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
221
        with:
222
          ref: ${{ steps.vars.outputs.sha }}
223
          persist-credentials: false
224
          path: untrusted
225
          sparse-checkout: |
226
            install/kubernetes/cilium
227

228
      - name: Install Cilium
229
        id: install-cilium
230
        run: |
231
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}
232

233
      - name: Enable Relay
234
        run: |
235
          cilium hubble enable
236

237
      - name: Wait for Cilium status to be ready
238
        run: |
239
          cilium status --wait --wait-duration=10m
240

241
      - name: Port forward Relay
242
        run: |
243
          cilium hubble port-forward&
244
          sleep 10s
245
          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
246

247
      - name: Make JUnit report directory
248
        run: |
249
          mkdir -p cilium-junits
250

251
      - name: Run connectivity test (${{ join(matrix.*, ', ') }})
252
        run: |
253
          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
254
          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - 1.xml" \
255
          --junit-property github_job_step="Run connectivity test (${{ join(matrix.*, ', ') }})"
256

257
      - name: Clean up Cilium
258
        run: |
259
          pkill -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" || test $? -eq 1
260
          cilium uninstall --wait
261

262
      - name: Create custom IPsec secret
263
        run: |
264
          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="15 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
265

266
      - name: Install Cilium with encryption
267
        run: |
268
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
269
            --helm-set encryption.enabled=true \
270
            --helm-set encryption.type=ipsec
271

272
      - name: Enable Relay
273
        run: |
274
          cilium hubble enable
275

276
      - name: Wait for Cilium status to be ready
277
        run: |
278
          cilium status --wait --wait-duration=10m
279

280
      - name: Port forward Relay
281
        run: |
282
          cilium hubble port-forward&
283
          sleep 10s
284
          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
285

286
      - name: Run connectivity test with IPSec (${{ join(matrix.*, ', ') }})
287
        run: |
288
          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} --force-deploy \
289
          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - 2.xml" \
290
          --junit-property github_job_step="Run connectivity test with IPSec (${{ join(matrix.*, ', ') }})"
291

292
      - name: Post-test information gathering
293
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
294
        run: |
295
          kubectl get pods --all-namespaces -o wide
296
          cilium status
297
          cilium sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
298
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
299

300
      - name: Clean up AKS
301
        if: ${{ always() }}
302
        run: |
303
          az group delete --name ${{ env.name }} --yes --no-wait
304
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
305

306
      - name: Upload artifacts
307
        if: ${{ !success() }}
308
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
309
        with:
310
          name: cilium-sysdumps-${{ matrix.index }}
311
          path: cilium-sysdump-*.zip
312

313
      - name: Upload JUnits [junit]
314
        if: ${{ always() }}
315
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
316
        with:
317
          name: cilium-junits-${{ matrix.index }}
318
          path: cilium-junits/*.xml
319

320
      - name: Publish Test Results As GitHub Summary
321
        if: ${{ always() }}
322
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
323
        with:
324
          junit-directory: "cilium-junits"
325

326
  merge-upload:
327
    if: ${{ always() }}
328
    name: Merge and Upload Artifacts
329
    runs-on: ubuntu-latest
330
    needs: installation-and-connectivity
331
    steps:
332
      - name: Merge Sysdumps
333
        if: ${{ needs.installation-and-connectivity.result == 'failure' }}
334
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
335
        with:
336
          name: cilium-sysdumps
337
          pattern: cilium-sysdumps-*
338
          retention-days: 5
339
          delete-merged: true
340
        continue-on-error: true
341
      - name: Merge JUnits
342
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
343
        with:
344
          name: cilium-junits
345
          pattern: cilium-junits-*
346
          retention-days: 5
347
          delete-merged: true
348

349
  commit-status-final:
350
    if: ${{ always() }}
351
    name: Commit Status Final
352
    needs: installation-and-connectivity
353
    runs-on: ubuntu-latest
354
    steps:
355
      - name: Set final commit status
356
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
357
        with:
358
          sha: ${{ inputs.SHA || github.sha }}
359
          status: ${{ needs.installation-and-connectivity.result }}
360