cilium

1
name: Conformance K8s Upstream Network
2

3
# Any change in triggers needs to be reflected in the concurrency group.
4
on:
5
  pull_request:
6
    paths-ignore:
7
      - 'Documentation/**'
8
      - 'test/**'
9
  push:
10
    branches:
11
      - main
12
      - ft/main/**
13
    paths-ignore:
14
      - 'Documentation/**'
15
      - 'test/**'
16

17
permissions: read-all
18

19
concurrency:
20
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }}
21
  cancel-in-progress: true
22

23
env:
24
  # renovate: datasource=github-releases depName=kubernetes-sigs/kind
25
  kind_version: v0.22.0
26
  cluster_name: cilium-testing
27
  cilium_cli_ci_version:
28
  CILIUM_CLI_MODE: helm
29
  # renovate: datasource=docker depName=kindest/node
30
  k8s_version: v1.29.1
31

32
jobs:
33
  kubernetes-e2e-net-conformance:
34
    name: Installation and Conformance Test
35
    runs-on: ubuntu-latest
36
    timeout-minutes: 45
37
    strategy:
38
      fail-fast: false
39
      matrix:
40
        # TODO add "dual" and "ipv6", "ipv6" fails to install cilium
41
        ipFamily: ["ipv4"]
42
    env:
43
      IP_FAMILY: ${{ matrix.ipFamily }}
44

45
    steps:
46
      - name: Checkout main branch to access local actions
47
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
48
        with:
49
          ref: ${{ github.event.repository.default_branch }}
50
          persist-credentials: false
51
      - name: Set Environment Variables
52
        uses: ./.github/actions/set-env-variables
53

54
      - name: Enable ipv4 and ipv6 forwarding
55
        run: |
56
          sudo sysctl -w net.ipv6.conf.all.forwarding=1
57
          sudo sysctl -w net.ipv4.ip_forward=1
58

59
      - name: Set up environment (download Kubernetes dependencies)
60
        run: |
61
          TMP_DIR=$(mktemp -d)
62
          # Test binaries
63
          curl -L https://dl.k8s.io/${{ env.k8s_version }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
64
          tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
65
            --directory ${TMP_DIR} \
66
            --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
67
          # kubectl
68
          curl -L https://dl.k8s.io/${{ env.k8s_version }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
69
          # kind
70
          curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/${{ env.kind_version }}/kind-linux-amd64
71
          # Install
72
          sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
73
          sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
74
          sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
75
          sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
76
          sudo chmod +x /usr/local/bin/*
77
          sudo rm -rf ${TMP_DIR}
78

79
      - name: Create multi node cluster
80
        run: |
81
          cat <<EOF | /usr/local/bin/kind create cluster \
82
            --name ${{ env.cluster_name}}           \
83
            --image kindest/node:${{ env.k8s_version }}  \
84
            -v7 --wait 1m --retain --config=-
85
          kind: Cluster
86
          apiVersion: kind.x-k8s.io/v1alpha4
87
          networking:
88
            ipFamily: ${IP_FAMILY}
89
            kubeProxyMode: "none"
90
            disableDefaultCNI: true
91
          nodes:
92
          - role: control-plane
93
          - role: worker
94
          - role: worker
95
          EOF
96

97
      - name: Workaround CoreDNS for IPv6 airgapped
98
        if: ${{ matrix.ipFamily == 'ipv6' }}
99
        run: |
100
          # Patch CoreDNS to work in Github CI
101
          # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
102
          # to work in an offline environment:
103
          # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
104
          # 2. Github CI adds following domains to resolv.conf search field:
105
          # .net.
106
          # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
107
          # otherwise pods stops trying to resolve the domain.
108
          # Get the current config
109
          original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
110
          echo "Original CoreDNS config:"
111
          echo "${original_coredns}"
112
          # Patch it
113
          fixed_coredns=$(
114
            printf '%s' "${original_coredns}" | sed \
115
              -e 's/^.*kubernetes cluster\.local/& net/' \
116
              -e '/^.*upstream$/d' \
117
              -e '/^.*fallthrough.*$/d' \
118
              -e '/^.*forward . \/etc\/resolv.conf$/d' \
119
              -e '/^.*loop$/d' \
120
          )
121
          echo "Patched CoreDNS config:"
122
          echo "${fixed_coredns}"
123
          printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -
124

125
      - name: Get Cilium's default values
126
        id: default_vars
127
        uses: ./.github/actions/helm-default
128
        with:
129
          image-tag: ${{ github.event.pull_request.head.sha }}
130

131
      - name: Set up job variables
132
        id: vars
133
        run: |
134
          # Note: On Kind, we install Cilium with HostPort (portmap CNI chaining) enabled,
135
          # to ensure coverage of that feature in cilium connectivity test
136
          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
137
            --helm-set=cni.chainingMode=portmap \
138
            --helm-set=kubeProxyReplacement=true \
139
            --helm-set=sessionAffinity=true \
140
            --helm-set=identityChangeGracePeriod="0s""
141
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
142
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT
143

144
      - name: Install Cilium CLI
145
        uses: cilium/cilium-cli@7306e3cdc6caee738157f08e3e1ba26179f104e5 # v0.15.23
146
        with:
147
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
148
          release-version: ${{ env.CILIUM_CLI_VERSION }}
149
          ci-version: ${{ env.cilium_cli_ci_version }}
150

151
      - name: Checkout code
152
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
153
        with:
154
          ref: ${{ steps.vars.outputs.sha }}
155
          persist-credentials: false
156

157
      - name: Wait for images to be available
158
        timeout-minutes: 30
159
        shell: bash
160
        run: |
161
          for image in cilium-ci operator-generic-ci hubble-relay-ci ; do
162
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
163
          done
164

165
      - name: Install Cilium
166
        id: install-cilium
167
        run: |
168
          cilium install --wait ${{ steps.vars.outputs.cilium_install_defaults }}
169

170
      - name: Run Kubernetes sig-network conformance test
171
        run: |
172
          # output_dir
173
          mkdir -p _artifacts
174

175
          # get kubeconfig to pass to the e2e binary
176
          kind get kubeconfig --name ${{ env.cluster_name }} > _artifacts/kubeconfig.conf
177

178
          # Kubernetes e2e tests use ginkgo and tags to select the tests that should run based on two regex, focus and skip:
179
          # Focus tests:
180
          # \[Conformance\]|\[sig-network\]: Conformance tests are defined by the project to guarantee a consistent behaviour and some mandatory features on all clusters
181
          #                                  sig-network tests are defined by sig-networkto guarantee a consistent behaviour on all the the k8s network implementations
182
          # Skipped tests:
183
          # Disruptive|Serial : require to run in serial and perform disruptive operations on clusters (reboots, ...)
184
          # Federation|PerformanceDNS : unrelated sig-network tests
185
          # Feature : skip features that are not GA, however, some of them should be enabled, per example [Feature:ProxyTerminatingEndpoints]
186
          # DualStack : only with dualstack clusters
187
          # KubeProxy|kube-proxy : kube-proxy specifics
188
          # LoadBalancer|GCE|ExternalIP : require a cloud provider, some of them are GCE specifics
189
          # Aggregator : Flaky, https://github.com/cilium/cilium/issues/24622.
190
          # same.port.number.but.different.protocols|HostPort|should.serve.endpoints.on.same.port.and.different.protocols : #9207
191
          # rejected : Kubernetes expect Services without endpoints associated to REJECT the connection to notify the client, Cilium silently drops the packet
192
          # externalTrafficPolicy : needs investigation
193

194
          # Run tests
195
          export KUBERNETES_CONFORMANCE_TEST='y'
196
          export E2E_REPORT_DIR=${PWD}/_artifacts
197
          /usr/local/bin/ginkgo --nodes=5                \
198
            --focus="(HostPort.*\[Conformance\].*|Services.*\[Conformance\].*|Net.*ol.*)"     \
199
            --skip="(Legacy|HostPort.validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|should.allow.egress.access.to.server.in.CIDR.block|should.enforce.except.clause.while.egress.access.to.server.in.CIDR.block|should.ensure.an.IP.overlapping.both.IPBlock.CIDR.and.IPBlock.Except.is.allowed|Feature:SCTPConnectivity|should.serve.endpoints.on.same.port.and.different.protocols)" \
200
            /usr/local/bin/e2e.test                       \
201
            --                                            \
202
            --kubeconfig=${PWD}/_artifacts/kubeconfig.conf     \
203
            --provider=local                              \
204
            --dump-logs-on-failure=true                   \
205
            --report-dir=${E2E_REPORT_DIR}                \
206
            --disable-log-dump=true
207

208
      - name: Post-test information gathering
209
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
210
        run: |
211
          kubectl get pods --all-namespaces -o wide
212
          cilium status
213
          cilium sysdump --output-filename cilium-sysdump-final
214
          /usr/local/bin/kind export logs --name  ${{ env.cluster_name }} --loglevel=debug ./_artifacts/logs
215
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
216

217
      - name: Upload artifacts
218
        if: ${{ !success() }}
219
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
220
        with:
221
          name: cilium-sysdumps
222
          path: cilium-sysdump-*.zip
223
          retention-days: 5
224

225
      - name: Upload cluster logs
226
        if: ${{ !success() }}
227
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
228
        with:
229
          name: kind-logs
230
          path: ./_artifacts/logs
231
          retention-days: 5
232

233
      - name: Upload Kubernetes e2e Junit Reports
234
        if: ${{ success() }}
235
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
236
        with:
237
          name: kubernetes-e2e-junit
238
          path: './_artifacts/*.xml'
239

240
      - name: Publish Test Results As GitHub Summary
241
        if: ${{ always() }}
242
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
243
        with:
244
          junit-directory: "_artifacts"
245