cilium

1
name: Conformance AWS-CNI (ci-awscni)
2

3
# Any change in triggers needs to be reflected in the concurrency group.
4
on:
5
  workflow_dispatch:
6
    inputs:
7
      PR-number:
8
        description: "Pull request number."
9
        required: true
10
      context-ref:
11
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
12
        required: true
13
      SHA:
14
        description: "SHA under test (head of the PR branch)."
15
        required: true
16
      extra-args:
17
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
18
        required: false
19
        default: '{}'
20
  # Run every 6 hours
21
  schedule:
22
    - cron:  '30 0/6 * * *'
23

24
# By specifying the access of one of the scopes, all of those that are not
25
# specified are set to 'none'.
26
permissions:
27
  # To be able to access the repository with actions/checkout
28
  contents: read
29
  # To allow retrieving information from the PR API
30
  pull-requests: read
31
  # To be able to set commit status
32
  statuses: write
33
  # To be able to request the JWT from GitHub's OIDC provider
34
  id-token: write
35

36
concurrency:
37
  # Structure:
38
  # - Workflow name
39
  # - Event type
40
  # - A unique identifier depending on event type:
41
  #   - schedule: SHA
42
  #   - workflow_dispatch: PR number
43
  #
44
  # This structure ensures a unique concurrency group name is generated for each
45
  # type of testing, such that re-runs will cancel the previous run.
46
  group: |
47
    ${{ github.workflow }}
48
    ${{ github.event_name }}
49
    ${{
50
      (github.event_name == 'schedule' && github.sha) ||
51
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
52
    }}
53
  cancel-in-progress: true
54

55
env:
56
  clusterName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
57
  cilium_cli_ci_version:
58
  CILIUM_CLI_MODE: helm
59
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
60
  # renovate: datasource=github-releases depName=eksctl-io/eksctl
61
  eksctl_version: v0.171.0
62
  # renovate: datasource=github-releases depName=kubernetes/kubernetes
63
  kubectl_version: v1.29.2
64

65
jobs:
66
  commit-status-start:
67
    name: Commit Status Start
68
    runs-on: ubuntu-latest
69
    steps:
70
      - name: Set initial commit status
71
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
72
        with:
73
          sha: ${{ inputs.SHA || github.sha }}
74

75
  generate-matrix:
76
    name: Generate Matrix
77
    runs-on: ubuntu-latest
78
    outputs:
79
      matrix: ${{ steps.set-matrix.outputs.matrix }}
80
    steps:
81
      - name: Checkout context ref (trusted)
82
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
83
        with:
84
          ref: ${{ inputs.context-ref || github.sha }}
85
          persist-credentials: false
86

87
      - name: Convert YAML to JSON
88
        run: |
89
          work_dir=".github/actions/aws"
90
          destination_directory="/tmp/generated/aws"
91
          mkdir -p "${destination_directory}"
92

93
          yq -o=json "${work_dir}/k8s-versions.yaml" | jq . > "${destination_directory}/aws.json"
94

95
      - name: Generate Matrix
96
        id: set-matrix
97
        run: |
98
          cd /tmp/generated/aws
99

100
          # Use complete matrix in case of scheduled run
101
          # main -> event_name = schedule
102
          # other stable branches -> PR-number starting with v (e.g. v1.14)
103
          if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.PR-number }}" == v* ]];then
104
            cp aws.json /tmp/matrix.json
105
          else
106
            jq '{ "include": [ .include[] | select(.default) ] }' aws.json > /tmp/matrix.json
107
          fi
108

109
          echo "Generated matrix:"
110
          cat /tmp/matrix.json
111
          echo "matrix=$(jq -c . < /tmp/matrix.json)" >> $GITHUB_OUTPUT
112

113
  installation-and-connectivity:
114
    name: Installation and Connectivity Test
115
    needs: generate-matrix
116
    runs-on: ubuntu-latest
117
    timeout-minutes: 45
118
    env:
119
      job_name: "Installation and Connectivity Test"
120
    strategy:
121
      fail-fast: false
122
      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
123

124
    steps:
125
      - name: Checkout context ref (trusted)
126
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
127
        with:
128
          ref: ${{ inputs.context-ref || github.sha }}
129
          persist-credentials: false
130

131
      - name: Set Environment Variables
132
        uses: ./.github/actions/set-env-variables
133

134
      - name: Get Cilium's default values
135
        id: default_vars
136
        uses: ./.github/actions/helm-default
137
        with:
138
          image-tag: ${{ inputs.SHA }}
139
          chart-dir: ./untrusted/install/kubernetes/cilium
140

141
      - name: Set up job variables
142
        id: vars
143
        run: |
144
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
145
            OWNER="${{ inputs.PR-number }}"
146
          else
147
            OWNER="${{ github.ref_name }}"
148
            OWNER="${OWNER/./-}"
149
          fi
150

151
          # Set ipam.mode=cluster-pool to overwrite the ipam value set by the
152
          # cilium-cli which is setting it to 'eni' because it auto-detects
153
          # the cluster as being EKS.
154
          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
155
            --cluster-name=${{ env.clusterName }} \
156
            --helm-set=hubble.relay.enabled=true \
157
            --helm-set=enableIPv4Masquerade=false \
158
            --helm-set=cni.chainingMode=aws-cni \
159
            --helm-set=eni.enabled=false \
160
            --helm-set=ipam.mode=cluster-pool \
161
            --helm-set=routingMode=native \
162
            --helm-set=bandwidthManager.enabled=false \
163
            --wait=false"
164

165
          # L7 policies are not supported in chaining mode.
166
          CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled --hubble=false --collect-sysdump-on-failure \
167
            --test '!fqdn,!l7' --external-target amazon.com --external-ip 1.0.0.1 --external-other-ip 1.1.1.1"
168
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
169
          echo connectivity_test_defaults=${CONNECTIVITY_TEST_DEFAULTS} >> $GITHUB_OUTPUT
170
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT
171
          echo owner=${OWNER} >> $GITHUB_OUTPUT
172

173
      - name: Install Cilium CLI
174
        uses: cilium/cilium-cli@7306e3cdc6caee738157f08e3e1ba26179f104e5 # v0.15.23
175
        with:
176
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
177
          release-version: ${{ env.CILIUM_CLI_VERSION }}
178
          ci-version: ${{ env.cilium_cli_ci_version }}
179

180
      - name: Install kubectl
181
        run: |
182
          curl -sLO "https://dl.k8s.io/release/${{ env.kubectl_version }}/bin/linux/amd64/kubectl"
183
          curl -sLO "https://dl.k8s.io/${{ env.kubectl_version }}/bin/linux/amd64/kubectl.sha256"
184
          echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
185
          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
186
          kubectl version --client
187

188
      - name: Install eksctl CLI
189
        run: |
190
          curl -LO "https://github.com/eksctl-io/eksctl/releases/download/${{ env.eksctl_version }}/eksctl_$(uname -s)_amd64.tar.gz"
191
          sudo tar xzvfC eksctl_$(uname -s)_amd64.tar.gz /usr/bin
192
          rm eksctl_$(uname -s)_amd64.tar.gz
193

194
      - name: Set up AWS CLI credentials
195
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
196
        with:
197
          role-to-assume: ${{ secrets.AWS_PR_ASSUME_ROLE }}
198
          aws-region: ${{ matrix.region }}
199

200
      - name: Create EKS cluster
201
        uses: ./.github/actions/setup-eks-cluster
202
        with:
203
          cluster_name: ${{ env.clusterName }}
204
          region: ${{ matrix.region }}
205
          owner: "${{ steps.vars.outputs.owner }}"
206
          version: ${{ matrix.version }}
207
          spot: ${{ github.event_name != 'schedule' }}
208

209
      - name: Wait for images to be available
210
        timeout-minutes: 30
211
        shell: bash
212
        run: |
213
          for image in cilium-ci operator-generic-ci hubble-relay-ci ; do
214
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
215
          done
216

217
      # Warning: since this is a privileged workflow, subsequent workflow job
218
      # steps must take care not to execute untrusted code.
219
      - name: Checkout pull request branch (NOT TRUSTED)
220
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
221
        with:
222
          ref: ${{ steps.vars.outputs.sha }}
223
          persist-credentials: false
224
          path: untrusted
225
          sparse-checkout: |
226
            install/kubernetes/cilium
227

228
      - name: Install Cilium
229
        id: install-cilium
230
        run: |
231
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}
232

233
      - name: Wait for Cilium to be ready
234
        run: |
235
          cilium status --wait --wait-duration=10m
236
          kubectl get pods -n kube-system
237

238
      - name: Check that AWS iptables chains have not been removed
239
        run: |
240
          for pod in $(kubectl get po -n kube-system -l app.kubernetes.io/name=cilium-agent -o name); do
241
            echo "Checking ${pod}"
242
            if ! kubectl exec -n kube-system  ${pod} -c cilium-agent -- iptables-save | grep --silent ':AWS'; then
243
              echo "Expected AWS iptables chains are not present"
244
              exit 1
245
            fi
246
          done
247

248
      - name: Port forward Relay
249
        run: |
250
          cilium hubble port-forward&
251
          sleep 10s
252
          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
253

254
      - name: Make JUnit report directory
255
        run: |
256
          mkdir -p cilium-junits
257

258
      - name: Run connectivity test (${{ join(matrix.*, ', ') }})
259
        run: |
260
          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
261
          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}).xml" \
262
          --junit-property github_job_step="Run connectivity test (${{ join(matrix.*, ', ') }})"
263

264
      - name: Post-test information gathering
265
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
266
        run: |
267
          echo "=== Retrieve cluster state ==="
268
          kubectl get pods --all-namespaces -o wide
269
          cilium status
270
          cilium sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
271
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
272

273
      - name: Upload artifacts
274
        if: ${{ !success() }}
275
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
276
        with:
277
          name: cilium-sysdumps-${{ matrix.version }}
278
          path: cilium-sysdump-*.zip
279

280
      - name: Upload JUnits [junit]
281
        if: ${{ always() }}
282
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
283
        with:
284
          name: cilium-junits-${{ matrix.version }}
285
          path: cilium-junits/*.xml
286

287
      - name: Publish Test Results As GitHub Summary
288
        if: ${{ always() }}
289
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
290
        with:
291
          junit-directory: "cilium-junits"
292

293
  merge-upload:
294
    if: ${{ always() }}
295
    name: Merge and Upload Artifacts
296
    runs-on: ubuntu-latest
297
    needs: installation-and-connectivity
298
    steps:
299
      - name: Merge Sysdumps
300
        if: ${{ needs.installation-and-connectivity.result == 'failure' }}
301
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
302
        with:
303
          name: cilium-sysdumps
304
          pattern: cilium-sysdumps-*
305
          retention-days: 5
306
          delete-merged: true
307
        continue-on-error: true
308
      - name: Merge JUnits
309
        uses: actions/upload-artifact/merge@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
310
        with:
311
          name: cilium-junits
312
          pattern: cilium-junits-*
313
          retention-days: 5
314
          delete-merged: true
315

316
  commit-status-final:
317
    if: ${{ always() }}
318
    name: Commit Status Final
319
    needs: installation-and-connectivity
320
    runs-on: ubuntu-latest
321
    steps:
322
      - name: Set final commit status
323
        uses: myrotvorets/set-commit-status-action@38f3f27c7d52fb381273e95542f07f0fba301307 # v2.0.0  
324
        with:
325
          sha: ${{ inputs.SHA || github.sha }}
326
          status: ${{ needs.installation-and-connectivity.result }}
327

328
  cleanup:
329
    name: Cleanup EKS Clusters
330
    if: ${{ always() }}
331
    continue-on-error: true
332
    needs: [generate-matrix, installation-and-connectivity]
333
    runs-on: ubuntu-latest
334
    timeout-minutes: 45
335
    strategy:
336
      fail-fast: false
337
      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
338

339
    steps:
340
      - name: Install eksctl CLI
341
        run: |
342
          curl -LO "https://github.com/eksctl-io/eksctl/releases/download/${{ env.eksctl_version }}/eksctl_$(uname -s)_amd64.tar.gz"
343
          sudo tar xzvfC eksctl_$(uname -s)_amd64.tar.gz /usr/bin
344
          rm eksctl_$(uname -s)_amd64.tar.gz
345

346
      - name: Set up AWS CLI credentials
347
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
348
        with:
349
          role-to-assume: ${{ secrets.AWS_PR_ASSUME_ROLE }}
350
          aws-region: ${{ matrix.region }}
351

352
      - name: Clean up EKS
353
        run: |
354
          eksctl delete cluster --name ${{ env.clusterName }} --region ${{ matrix.region }}
355