cilium

1
name: Conformance External Workloads (ci-external-workloads)
2

3
# Any change in triggers needs to be reflected in the concurrency group.
4
on:
5
  workflow_dispatch:
6
    inputs:
7
      PR-number:
8
        description: "Pull request number."
9
        required: true
10
      context-ref:
11
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
12
        required: true
13
      SHA:
14
        description: "SHA under test (head of the PR branch)."
15
        required: true
16
      extra-args:
17
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
18
        required: false
19
        default: '{}'
20
  # Run every 6 hours
21
  schedule:
22
    - cron:  '0 4/6 * * *'
23

24
# By specifying the access of one of the scopes, all of those that are not
25
# specified are set to 'none'.
26
permissions:
27
  # To be able to access the repository with actions/checkout
28
  contents: read
29
  # To allow retrieving information from the PR API
30
  pull-requests: read
31
  # To be able to set commit status
32
  statuses: write
33
  # To be able to request the JWT from GitHub's OIDC provider
34
  id-token: write
35

36
concurrency:
37
  # Structure:
38
  # - Workflow name
39
  # - Event type
40
  # - A unique identifier depending on event type:
41
  #   - schedule: SHA
42
  #   - workflow_dispatch: PR number
43
  #
44
  # This structure ensures a unique concurrency group name is generated for each
45
  # type of testing, such that re-runs will cancel the previous run.
46
  group: |
47
    ${{ github.workflow }}
48
    ${{ github.event_name }}
49
    ${{
50
      (github.event_name == 'schedule' && github.sha) ||
51
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
52
    }}
53
  cancel-in-progress: true
54

55
env:
56
  clusterName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}-vm
57
  vmName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}-vm
58
  vmStartupScript: .github/gcp-vm-startup.sh
59
  cilium_cli_ci_version:
60
  CILIUM_CLI_MODE: helm
61
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
62
  USE_GKE_GCLOUD_AUTH_PLUGIN: True
63

64
jobs:
65
  commit-status-start:
66
    name: Commit Status Start
67
    runs-on: ubuntu-latest
68
    steps:
69
      - name: Set initial commit status
70
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
71
        with:
72
          sha: ${{ inputs.SHA || github.sha }}
73

74
  generate-matrix:
75
    name: Generate Matrix
76
    runs-on: ubuntu-latest
77
    outputs:
78
      matrix: ${{ steps.set-matrix.outputs.matrix }}
79
    steps:
80
      - name: Checkout context ref (trusted)
81
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
82
        with:
83
          ref: ${{ inputs.context-ref || github.sha }}
84
          persist-credentials: false
85

86
      - name: Convert YAML to JSON
87
        run: |
88
          work_dir=".github/actions/gke"
89
          destination_directory="/tmp/generated/gke"
90
          mkdir -p "${destination_directory}"
91

92
          yq -o=json ${work_dir}/k8s-versions.yaml | jq . > "${destination_directory}/gke.json"
93

94
      - name: Generate Matrix
95
        id: set-matrix
96
        run: |
97
          cd /tmp/generated/gke
98

99
          # Use complete matrix in case of scheduled run
100
          # main -> event_name = schedule
101
          # other stable branches -> PR-number starting with v (e.g. v1.14)
102
          if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.PR-number }}" == v* ]];then
103
            jq '{ "include": [ .k8s[] ] }' gke.json > /tmp/matrix.json
104
          else
105
            jq '{ "include": [ .k8s[] | select(.default) ] }' gke.json > /tmp/matrix.json
106
          fi
107

108
          echo "Generated matrix:"
109
          cat /tmp/matrix.json
110
          echo "matrix=$(jq -c . < /tmp/matrix.json)" >> $GITHUB_OUTPUT
111

112
  installation-and-connectivity:
113
    name: Installation and Connectivity Test
114
    needs: generate-matrix
115
    runs-on: ubuntu-latest
116
    timeout-minutes: 45
117
    env:
118
      job_name: "Installation and Connectivity Test"
119
      preemptible: ${{ github.event_name != 'schedule' && '--preemptible' || '' }}
120
    strategy:
121
      fail-fast: false
122
      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
123

124
    steps:
125
      - name: Checkout context ref (trusted)
126
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
127
        with:
128
          ref: ${{ inputs.context-ref || github.sha }}
129
          persist-credentials: false
130

131
      - name: Set Environment Variables
132
        uses: ./.github/actions/set-env-variables
133

134
      - name: Get Cilium's default values
135
        id: default_vars
136
        uses: ./.github/actions/helm-default
137
        with:
138
          image-tag: ${{ inputs.SHA }}
139
          chart-dir: ./untrusted/install/kubernetes/cilium
140

141
      - name: Set up job variables
142
        id: vars
143
        run: |
144
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
145
            OWNER="${{ inputs.PR-number }}"
146
          else
147
            OWNER="${{ github.ref_name }}"
148
            OWNER="${OWNER/./-}"
149
          fi
150

151
          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
152
            --cluster-name=${{ env.clusterName }} \
153
            --datapath-mode=tunnel \
154
            --helm-set kubeProxyReplacement=true"
155
          CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled --hubble=false --collect-sysdump-on-failure \
156
            --external-target google.com --external-cidr 8.0.0.0/8 --external-ip 8.8.4.4 --external-other-ip 8.8.8.8"
157
          # Explicitly specify LoadBalancer service type since the default type is NodePort in Helm mode.
158
          # Ref: https://github.com/cilium/cilium-cli/pull/1527#discussion_r1177244379
159
          #
160
          # In Helm mode, externalWorkloads.enabled is set to false by default. You need to pass
161
          # --enable-external-workloads flag to enable it.
162
          # Ref: https://github.com/cilium/cilium/pull/25259
163
          CLUSTERMESH_ENABLE_DEFAULTS="--service-type LoadBalancer --enable-external-workloads"
164
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
165
          echo connectivity_test_defaults=${CONNECTIVITY_TEST_DEFAULTS} >> $GITHUB_OUTPUT
166
          echo clustermesh_enable_defaults=${CLUSTERMESH_ENABLE_DEFAULTS} >> $GITHUB_OUTPUT
167
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT
168
          echo owner=${OWNER} >> $GITHUB_OUTPUT
169

170
      - name: Install Cilium CLI
171
        uses: cilium/cilium-cli@7306e3cdc6caee738157f08e3e1ba26179f104e5 # v0.15.23
172
        with:
173
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
174
          release-version: ${{ env.CILIUM_CLI_VERSION }}
175
          ci-version: ${{ env.cilium_cli_ci_version }}
176

177
      - name: Set up gcloud credentials
178
        id: 'auth'
179
        uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
180
        with:
181
          workload_identity_provider: ${{ secrets.GCP_PR_WORKLOAD_IDENTITY_PROVIDER }}
182
          service_account: ${{ secrets.GCP_PR_SA }}
183
          create_credentials_file: true
184
          export_environment_variables: true
185

186
      - name: Set up gcloud CLI
187
        uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
188
        with:
189
          project_id: ${{ secrets.GCP_PROJECT_ID }}
190
          version: "405.0.0"
191

192
      - name: Install gke-gcloud-auth-plugin
193
        run: |
194
          gcloud components install gke-gcloud-auth-plugin
195

196
      - name: Display gcloud CLI info
197
        run: |
198
          gcloud info
199

200
      - name: Create GCP VM
201
        uses: nick-invision/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
202
        with:
203
          retry_on: error
204
          timeout_minutes: 1
205
          max_attempts: 10
206
          command: |
207
            gcloud compute instances create ${{ env.vmName }}-${{ matrix.vmIndex }} \
208
              --labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
209
              --zone ${{ matrix.zone }} \
210
              --machine-type e2-custom-2-4096 \
211
              --boot-disk-type pd-standard \
212
              --boot-disk-size 10GB \
213
              ${{ env.preemptible }} \
214
              --image-project ubuntu-os-cloud \
215
              --image-family ubuntu-2004-lts \
216
              --metadata hostname=${{ env.vmName }}-${{ matrix.vmIndex }} \
217
              --metadata-from-file startup-script=${{ env.vmStartupScript}}
218

219
      - name: Create GKE cluster
220
        run: |
221
          gcloud container clusters create ${{ env.clusterName }} \
222
            --labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
223
            --zone ${{ matrix.zone }} \
224
            --cluster-version ${{ matrix.version }} \
225
            --enable-ip-alias \
226
            --node-taints node.cilium.io/agent-not-ready=true:NoExecute \
227
            --cluster-ipv4-cidr="/21" \
228
            --services-ipv4-cidr="/24" \
229
            --image-type COS_CONTAINERD \
230
            --num-nodes 2 \
231
            --machine-type e2-custom-2-4096 \
232
            --disk-type pd-standard \
233
            --disk-size 20GB \
234
            ${{ env.preemptible }}
235

236
      - name: Get cluster credentials
237
        run: |
238
          gcloud container clusters get-credentials ${{ env.clusterName }} --zone ${{ matrix.zone }}
239

240
      # Warning: since this is a privileged workflow, subsequent workflow job
241
      # steps must take care not to execute untrusted code.
242
      - name: Checkout pull request branch (NOT TRUSTED)
243
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
244
        with:
245
          ref: ${{ steps.vars.outputs.sha }}
246
          persist-credentials: false
247
          path: untrusted
248
          sparse-checkout: |
249
            install/kubernetes/cilium
250

251
      - name: Wait for images to be available
252
        timeout-minutes: 30
253
        shell: bash
254
        run: |
255
          for image in cilium-ci operator-generic-ci hubble-relay-ci clustermesh-apiserver-ci ; do
256
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
257
          done
258

259
      - name: Install Cilium in cluster
260
        id: install-cilium
261
        run: |
262
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}
263

264
      - name: Enable cluster mesh
265
        run: |
266
          cilium clustermesh enable ${{ steps.vars.outputs.clustermesh_enable_defaults }}
267

268
      - name: Wait for cluster mesh status to be ready
269
        run: |
270
          cilium clustermesh status --wait
271

272
      - name: Add VM to cluster mesh
273
        run: |
274
          cilium clustermesh vm create ${{ env.vmName }}-${{ matrix.vmIndex }} -n default --ipv4-alloc-cidr 10.192.1.0/30
275
          cilium clustermesh vm status
276

277
      - name: Install Cilium on VM
278
        run: |
279
          cilium clustermesh vm install install-external-workload.sh --config debug
280
          gcloud compute scp install-external-workload.sh ${{ env.vmName }}-${{ matrix.vmIndex }}:~/ --zone ${{ matrix.zone }}
281
          gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
282
            --command "~/install-external-workload.sh"
283
          sleep 5s
284
          gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
285
            --command "sudo cilium-dbg status"
286

287
      - name: Verify cluster DNS on VM
288
        # Limit nslookup to the first (global) DNS server setting
289
        run: |
290
          gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
291
            --command "nslookup -d2 -retry=10 -timeout=5 -norecurse clustermesh-apiserver.kube-system.svc.cluster.local \$(systemd-resolve --status | grep -m 1 \"Current DNS Server:\" | cut -d':' -f2)"
292

293
      - name: Ping clustermesh-apiserver from VM
294
        run: |
295
          gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} \
296
            --command "ping -c 3 \$(sudo cilium-dbg service list get -o jsonpath='{[?(@.spec.flags.name==\"clustermesh-apiserver\")].spec.backend-addresses[0].ip}')"
297

298
      - name: Make JUnit report directory
299
        run: |
300
          mkdir -p cilium-junits
301

302
      - name: Run connectivity test (${{ join(matrix.*, ', ') }})
303
        run: |
304
          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
305
          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
306
          --junit-property github_job_step="Run connectivity test (${{ join(matrix.*, ', ') }})"
307

308
      - name: Post-test information gathering
309
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
310
        run: |
311
          kubectl get pods --all-namespaces -o wide
312
          kubectl get cew --all-namespaces -o wide
313
          kubectl get cep --all-namespaces -o wide
314
          cilium status
315
          cilium clustermesh status
316
          cilium clustermesh vm status
317
          gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} --command "sudo cilium status"
318
          gcloud compute ssh ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} --command "sudo docker logs cilium --timestamps"
319
          cilium sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
320
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
321

322
      - name: Clean up GKE cluster and VM
323
        if: ${{ always() }}
324
        run: |
325
          while [ "$(gcloud container operations list --zone ${{ matrix.zone }} --filter="status=RUNNING AND targetLink~${{ env.clusterName }}" --format="value(name)")" ];do
326
            echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
327
          done
328
          gcloud container clusters delete ${{ env.clusterName }} --zone ${{ matrix.zone }} --quiet --async
329
          gcloud compute instances delete ${{ env.vmName }}-${{ matrix.vmIndex }} --zone ${{ matrix.zone }} --quiet
330
        shell: bash {0} # Disable default fail-fast behavior so that all commands run independently
331

332
      - name: Upload artifacts
333
        if: ${{ !success() }}
334
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
335
        with:
336
          name: cilium-sysdumps-${{ matrix.vmIndex }}
337
          path: cilium-sysdump-*.zip
338

339
      - name: Upload JUnits [junit]
340
        if: ${{ always() }}
341
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
342
        with:
343
          name: cilium-junits-${{ matrix.vmIndex }}
344
          path: cilium-junits/*.xml
345

346
      - name: Publish Test Results As GitHub Summary
347
        if: ${{ always() }}
348
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
349
        with:
350
          junit-directory: "cilium-junits"
351

352
  merge-upload:
353
    if: ${{ always() }}
354
    name: Merge and Upload Artifacts
355
    runs-on: ubuntu-latest
356
    needs: installation-and-connectivity
357
    steps:
358
      - name: Merge Sysdumps
359
        if: ${{ needs.installation-and-connectivity.result == 'failure' }}
360
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
361
        with:
362
          name: cilium-sysdumps
363
          pattern: cilium-sysdumps-*
364
          retention-days: 5
365
          delete-merged: true
366
        continue-on-error: true
367
      - name: Merge JUnits
368
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
369
        with:
370
          name: cilium-junits
371
          pattern: cilium-junits-*
372
          retention-days: 5
373
          delete-merged: true
374

375
  commit-status-final:
376
    if: ${{ always() }}
377
    name: Commit Status Final
378
    needs: installation-and-connectivity
379
    runs-on: ubuntu-latest
380
    steps:
381
      - name: Set final commit status
382
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
383
        with:
384
          sha: ${{ inputs.SHA || github.sha }}
385
          status: ${{ needs.installation-and-connectivity.result }}
386