cilium

1
name: Conformance GKE (ci-gke)
2

3
# Any change in triggers needs to be reflected in the concurrency group.
4
on:
5
  workflow_dispatch:
6
    inputs:
7
      PR-number:
8
        description: "Pull request number."
9
        required: true
10
      context-ref:
11
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
12
        required: true
13
      SHA:
14
        description: "SHA under test (head of the PR branch)."
15
        required: true
16
      extra-args:
17
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
18
        required: false
19
        default: '{}'
20
  # Run every 6 hours
21
  schedule:
22
    - cron:  '0 2/6 * * *'
23

24
# By specifying the access of one of the scopes, all of those that are not
25
# specified are set to 'none'.
26
permissions:
27
  # To be able to access the repository with actions/checkout
28
  contents: read
29
  # To allow retrieving information from the PR API
30
  pull-requests: read
31
  # To be able to set commit status
32
  statuses: write
33
  # To be able to request the JWT from GitHub's OIDC provider
34
  id-token: write
35

36
concurrency:
37
  # Structure:
38
  # - Workflow name
39
  # - Event type
40
  # - A unique identifier depending on event type:
41
  #   - schedule: SHA
42
  #   - workflow_dispatch: PR number
43
  #
44
  # This structure ensures a unique concurrency group name is generated for each
45
  # type of testing, such that re-runs will cancel the previous run.
46
  group: |
47
    ${{ github.workflow }}
48
    ${{ github.event_name }}
49
    ${{
50
      (github.event_name == 'schedule' && github.sha) ||
51
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
52
    }}
53
  cancel-in-progress: true
54

55
env:
56
  clusterName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
57
  cilium_cli_ci_version:
58
  CILIUM_CLI_MODE: helm
59
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
60
  USE_GKE_GCLOUD_AUTH_PLUGIN: True
61

62
jobs:
63
  commit-status-start:
64
    name: Commit Status Start
65
    runs-on: ubuntu-latest
66
    steps:
67
      - name: Set initial commit status
68
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
69
        with:
70
          sha: ${{ inputs.SHA || github.sha }}
71

72
  generate-matrix:
73
    name: Generate Matrix
74
    runs-on: ubuntu-latest
75
    outputs:
76
      matrix: ${{ steps.set-matrix.outputs.matrix }}
77
    steps:
78
      - name: Checkout context ref (trusted)
79
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
80
        with:
81
          ref: ${{ inputs.context-ref || github.sha }}
82
          persist-credentials: false
83

84
      - name: Convert YAML to JSON
85
        run: |
86
          work_dir=".github/actions/gke"
87
          destination_directory="/tmp/generated/gke"
88
          mkdir -p "${destination_directory}"
89

90
          ls ${work_dir}/*.yaml | grep -v 'schema\|classic' | while read file;do
91
            filename=$(basename "$file")
92
            new_filename="${filename%.yaml}.json"
93
            yq -o=json "${file}" | jq . > "${destination_directory}/${new_filename}"
94
          done
95
          
96
          # Merge 2 files into one
97
          jq -s "add" ${destination_directory}/*.json > "${destination_directory}/gke.json"
98

99
      - name: Generate Matrix
100
        id: set-matrix
101
        run: |
102
          cd /tmp/generated/gke
103

104
          # Use complete matrix in case of scheduled run
105
          # main -> event_name = schedule
106
          # other stable branches -> PR-number starting with v (e.g. v1.14)
107
          if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.PR-number }}" == v* ]];then
108
            cp gke.json /tmp/matrix.json
109
          else
110
            jq '{ "k8s": [ .k8s[] | select(.default) ], "config": .config}' gke.json > /tmp/matrix.json
111
          fi
112

113
          echo "Generated matrix:"
114
          cat /tmp/matrix.json
115
          echo "matrix=$(jq -c . < /tmp/matrix.json)" >> $GITHUB_OUTPUT
116

117
  installation-and-connectivity:
118
    name: Installation and Connectivity Test
119
    needs: generate-matrix
120
    runs-on: ubuntu-latest
121
    timeout-minutes: 75
122
    env:
123
      job_name: "Installation and Connectivity Test"
124
      preemptible: ${{ github.event_name != 'schedule' && '--preemptible' || '' }}
125
    strategy:
126
      fail-fast: false
127
      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
128

129
    steps:
130
      - name: Checkout context ref (trusted)
131
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
132
        with:
133
          ref: ${{ inputs.context-ref || github.sha }}
134
          persist-credentials: false
135

136
      - name: Set Environment Variables
137
        uses: ./.github/actions/set-env-variables
138

139
      - name: Get Cilium's default values
140
        id: default_vars
141
        uses: ./.github/actions/helm-default
142
        with:
143
          image-tag: ${{ inputs.SHA }}
144
          chart-dir: ./untrusted/install/kubernetes/cilium
145

146
      - name: Set up job variables
147
        id: vars
148
        run: |
149
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
150
            OWNER="${{ inputs.PR-number }}"
151
          else
152
            OWNER="${{ github.ref_name }}"
153
            OWNER="${OWNER/./-}"
154
          fi
155

156
          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
157
            --cluster-name=${{ env.clusterName }}-${{ matrix.config.index }} \
158
            --helm-set=hubble.relay.enabled=true \
159
            --helm-set=agentNotReadyTaintKey=ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready \
160
            --helm-set loadBalancer.l7.backend=envoy \
161
            --helm-set tls.secretsBackend=k8s \
162
            --wait=false"
163

164
          CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled --hubble=false --collect-sysdump-on-failure \
165
            --external-target google.com --external-cidr 8.0.0.0/8 --external-ip 8.8.8.8 --external-other-ip 8.8.4.4"
166
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
167
          echo hubble_enable_defaults=${HUBBLE_ENABLE_DEFAULTS} >> $GITHUB_OUTPUT
168
          echo connectivity_test_defaults=${CONNECTIVITY_TEST_DEFAULTS} >> $GITHUB_OUTPUT
169
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT
170
          echo owner=${OWNER} >> $GITHUB_OUTPUT
171

172
      - name: Install Cilium CLI
173
        uses: cilium/cilium-cli@7306e3cdc6caee738157f08e3e1ba26179f104e5 # v0.15.23
174
        with:
175
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
176
          release-version: ${{ env.CILIUM_CLI_VERSION }}
177
          ci-version: ${{ env.cilium_cli_ci_version }}
178

179
      - name: Set up gcloud credentials
180
        id: 'auth'
181
        uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
182
        with:
183
          workload_identity_provider: ${{ secrets.GCP_PR_WORKLOAD_IDENTITY_PROVIDER }}
184
          service_account: ${{ secrets.GCP_PR_SA }}
185
          create_credentials_file: true
186
          export_environment_variables: true
187

188
      - name: Set up gcloud CLI
189
        uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
190
        with:
191
          project_id: ${{ secrets.GCP_PROJECT_ID }}
192
          version: "405.0.0"
193

194
      - name: Install gke-gcloud-auth-plugin
195
        run: |
196
          gcloud components install gke-gcloud-auth-plugin
197

198
      - name: Display gcloud CLI info
199
        run: |
200
          gcloud info
201

202
      - name: Create GKE cluster
203
        run: |
204
          gcloud container clusters create ${{ env.clusterName }}-${{ matrix.config.index }} \
205
            --labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
206
            --zone ${{ matrix.k8s.zone }} \
207
            --cluster-version ${{ matrix.k8s.version }} \
208
            --enable-ip-alias \
209
            --create-subnetwork="range=/26" \
210
            --cluster-ipv4-cidr="/21" \
211
            --services-ipv4-cidr="/24" \
212
            --image-type COS_CONTAINERD \
213
            --num-nodes 2 \
214
            --machine-type e2-custom-2-4096 \
215
            --disk-type pd-standard \
216
            --disk-size 20GB \
217
            --node-taints ignore-taint.cluster-autoscaler.kubernetes.io/cilium-agent-not-ready=true:NoExecute \
218
            ${{ env.preemptible }}
219

220
      - name: Get cluster credentials
221
        run: |
222
          gcloud container clusters get-credentials ${{ env.clusterName }}-${{ matrix.config.index }} --zone ${{ matrix.k8s.zone }}
223

224
      # Warning: since this is a privileged workflow, subsequent workflow job
225
      # steps must take care not to execute untrusted code.
226
      - name: Checkout pull request branch (NOT TRUSTED)
227
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
228
        with:
229
          ref: ${{ steps.vars.outputs.sha }}
230
          persist-credentials: false
231
          path: untrusted
232
          sparse-checkout: |
233
            install/kubernetes/cilium
234

235
      - name: Wait for images to be available
236
        timeout-minutes: 30
237
        shell: bash
238
        run: |
239
          for image in cilium-ci operator-generic-ci hubble-relay-ci ; do
240
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
241
          done
242

243
      - name: Create custom IPsec secret
244
        if: ${{ matrix.config.type == 'ipsec' || matrix.config.type == 'tunnel-ipsec' }}
245
        run: |
246
          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="15 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
247

248
      - name: Install Cilium
249
        id: install-cilium
250
        run: |
251
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} ${{ matrix.config.cilium-install-opts }}
252

253
      - name: Wait for Cilium to be ready
254
        run: |
255
          cilium status --wait --wait-duration=10m
256
          kubectl get pods -n kube-system
257

258
      - name: Port forward Relay
259
        run: |
260
          cilium hubble port-forward&
261
          sleep 10s
262
          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
263

264
      - name: Make JUnit report directory
265
        run: |
266
          mkdir -p cilium-junits
267

268
      - name: Run connectivity test (${{ matrix.k8s.version }}, ${{ matrix.config.index }}, ${{ matrix.config.type }})
269
        run: |
270
          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
271
          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.k8s.*, ', ') }}, ${{ join(matrix.config.*, ', ') }}).xml" \
272
          --junit-property github_job_step="Run connectivity test (${{ matrix.k8s.version }}, ${{ matrix.config.index }}, ${{ matrix.config.type }})"
273

274
      - name: Post-test information gathering
275
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
276
        run: |
277
          kubectl get pods --all-namespaces -o wide
278
          cilium status
279
          cilium sysdump --output-filename cilium-sysdump-final-${{ matrix.k8s.version }}-${{ matrix.config.index }}-${{ matrix.config.type }}
280
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
281

282
      - name: Clean up GKE
283
        if: ${{ always() }}
284
        run: |
285
          while [ "$(gcloud container operations list --zone ${{ matrix.k8s.zone }} --filter="status=RUNNING AND targetLink~${{ env.clusterName }}-${{ matrix.config.index }}" --format="value(name)")" ];do
286
            echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
287
          done
288
          gcloud container clusters delete ${{ env.clusterName }}-${{ matrix.config.index }} --zone ${{ matrix.k8s.zone }} --quiet --async
289
        shell: bash {0} # Disable default fail-fast behavior so that all commands run independently
290

291
      - name: Upload artifacts
292
        if: ${{ !success() }}
293
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
294
        with:
295
          name: cilium-sysdumps-${{ matrix.config.index }}-${{ matrix.k8s.vmIndex }}
296
          path: cilium-sysdump-*.zip
297

298
      - name: Upload JUnits [junit]
299
        if: ${{ always() }}
300
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
301
        with:
302
          name: cilium-junits-${{ matrix.config.index }}-${{ matrix.k8s.vmIndex }}
303
          path: cilium-junits/*.xml
304

305
      - name: Publish Test Results As GitHub Summary
306
        if: ${{ always() }}
307
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
308
        with:
309
          junit-directory: "cilium-junits"
310

311
  merge-upload:
312
    if: ${{ always() }}
313
    name: Merge and Upload Artifacts
314
    runs-on: ubuntu-latest
315
    needs: installation-and-connectivity
316
    steps:
317
      - name: Merge Sysdumps
318
        if: ${{ needs.installation-and-connectivity.result == 'failure' }}
319
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
320
        with:
321
          name: cilium-sysdumps
322
          pattern: cilium-sysdumps-*
323
          retention-days: 5
324
          delete-merged: true
325
        continue-on-error: true
326
      - name: Merge JUnits
327
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
328
        with:
329
          name: cilium-junits
330
          pattern: cilium-junits-*
331
          retention-days: 5
332
          delete-merged: true
333

334
  commit-status-final:
335
    if: ${{ always() }}
336
    name: Commit Status Final
337
    needs: installation-and-connectivity
338
    runs-on: ubuntu-latest
339
    steps:
340
      - name: Set final commit status
341
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
342
        with:
343
          sha: ${{ inputs.SHA || github.sha }}
344
          status: ${{ needs.installation-and-connectivity.result }}
345