test/e2e_node/podresources_test.go - master - githubmirror/kubernetes

test/e2e_node/podresources_test.go
2 617 строк92 KB
Kevin Torres
E2E tests
20 июл 2026, 20:26
20 июл 2026, 20:26c1e7998
/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2enode

import (
	"context"
	"errors"
	"fmt"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/onsi/ginkgo/v2"
	"github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
	"github.com/onsi/gomega/types"

	"google.golang.org/grpc"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
	kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1"
	"k8s.io/kubernetes/pkg/features"
	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
	apisgrpc "k8s.io/kubernetes/pkg/kubelet/apis/grpc"
	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
	"k8s.io/kubernetes/pkg/kubelet/cm/memorymanager"
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
	"k8s.io/kubernetes/pkg/kubelet/util"
	testutils "k8s.io/kubernetes/test/utils"
	admissionapi "k8s.io/pod-security-admission/api"
	"k8s.io/utils/cpuset"

	"k8s.io/kubernetes/test/e2e/feature"
	"k8s.io/kubernetes/test/e2e/framework"
	e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
)

const (
	defaultTopologyUnawareResourceName = "example.com/resource"
)

type containerDesc struct {
	cntName string
	// TODO: add resourceName/resourceAmount like in podDesc
	// TODO: add command like in podDesc to let users configure the command to use
	cpuRequest int // cpuRequest is in millicores
	memRequest int // memRequest is in megabytes
	cpuLimit   int // cpuLimit is in millicores
	memLimit   int // memLimit is in megabytes
}

func (desc containerDesc) CPURequestQty() resource.Quantity {
	qty := resource.NewMilliQuantity(int64(desc.cpuRequest), resource.DecimalSI)
	return *qty
}

func (desc containerDesc) MemRequestQty() resource.Quantity {
	qty := resource.NewQuantity(int64(desc.memRequest)*1024*1024, resource.DecimalSI)
	return *qty
}

func (desc containerDesc) CPULimitQty() resource.Quantity {
	qty := resource.NewMilliQuantity(int64(desc.cpuLimit), resource.DecimalSI)
	return *qty
}

func (desc containerDesc) MemLimitQty() resource.Quantity {
	qty := resource.NewQuantity(int64(desc.memLimit)*1024*1024, resource.DecimalSI)
	return *qty
}

func (desc containerDesc) CPURequestExclusive() int {
	if (desc.cpuRequest % 1000) != 0 {
		// exclusive cpus are request only if the quantity is integral;
		// hence, explicitly rule out non-integral requests
		return 0
	}
	return desc.cpuRequest / 1000
}

type podDesc struct {
	podName        string
	cntName        string
	resourceName   string
	resourceAmount int
	cpuRequest     int // cpuRequest is in millicores
	memRequest     int // memRequest is in megabytes
	cpuLimit       int // cpuLimit is in millicores
	memLimit       int // memLimit is in megabytes
	initContainers []initContainerDesc
	mainCntCommand []string
	restartPolicy  *v1.RestartPolicy
	// for historical reasons, the first app container data is embedded within the podDesc,
	// while every other else is treated as expected within `extraContainers`
	// TODO: unify all the container specs in `appContainers []containerDesc`
	extraContainers []containerDesc
	useFullSpec     bool // TODO get rid of this; require reviewing all the existing testcases
}

func (desc podDesc) CPURequestQty() resource.Quantity {
	qty := resource.NewMilliQuantity(int64(desc.cpuRequest), resource.DecimalSI)
	return *qty
}

func (desc podDesc) MemRequestQty() resource.Quantity {
	qty := resource.NewQuantity(int64(desc.memRequest)*1024*1024, resource.DecimalSI)
	return *qty
}

func (desc podDesc) CPULimitQty() resource.Quantity {
	qty := resource.NewMilliQuantity(int64(desc.cpuLimit), resource.DecimalSI)
	return *qty
}

func (desc podDesc) MemLimitQty() resource.Quantity {
	qty := resource.NewQuantity(int64(desc.memLimit)*1024*1024, resource.DecimalSI)
	return *qty
}

func (desc podDesc) CPURequestExclusive() int {
	if (desc.cpuRequest % 1000) != 0 {
		// exclusive cpus are request only if the quantity is integral;
		// hence, explicitly rule out non-integral requests
		return 0
	}
	return desc.cpuRequest / 1000
}

func (desc podDesc) RequiresCPU() bool {
	return desc.cpuRequest > 0
}

func (desc podDesc) RequiresDevices() bool {
	return desc.resourceName != "" && desc.resourceAmount > 0
}

type initContainerDesc struct {
	cntName        string
	resourceName   string
	resourceAmount int
	cpuRequest     int // cpuRequest is in millicores
	restartPolicy  *v1.ContainerRestartPolicy
}

func (desc initContainerDesc) CPURequestQty() resource.Quantity {
	qty := resource.NewMilliQuantity(int64(desc.cpuRequest), resource.DecimalSI)
	return *qty
}

func (desc initContainerDesc) CPURequestExclusive() int {
	if (desc.cpuRequest % 1000) != 0 {
		// exclusive cpus are request only if the quantity is integral;
		// hence, explicitly rule out non-integral requests
		return 0
	}
	return desc.cpuRequest / 1000
}

func (desc initContainerDesc) RequiresCPU() bool {
	return desc.cpuRequest > 0
}

func (desc initContainerDesc) RequiresDevices() bool {
	return desc.resourceName != "" && desc.resourceAmount > 0
}

type podDescOption func(*podDesc)

func WithFullSpec(pd *podDesc) {
	pd.useFullSpec = true
}

func makePodResourcesTestPod(desc podDesc, opts ...podDescOption) *v1.Pod {
	for _, opt := range opts {
		opt(&desc)
	}
	cnt := v1.Container{
		Name:  desc.cntName,
		Image: busyboxImage,
		Resources: v1.ResourceRequirements{
			Requests: v1.ResourceList{},
			Limits:   v1.ResourceList{},
		},
		Command: []string{"sh", "-c", "sleep 1d"},
	}
	if desc.useFullSpec {
		if desc.cpuLimit > 0 {
			cnt.Resources.Limits[v1.ResourceCPU] = desc.CPULimitQty()
		}
		if desc.cpuRequest > 0 {
			cnt.Resources.Requests[v1.ResourceCPU] = desc.CPURequestQty()
		}
		if desc.memLimit > 0 {
			cnt.Resources.Limits[v1.ResourceMemory] = desc.MemLimitQty()
		}
		if desc.memRequest > 0 {
			cnt.Resources.Requests[v1.ResourceMemory] = desc.MemRequestQty()
		}
	} else if desc.RequiresCPU() {
		cpuRequestQty := desc.CPURequestQty()
		cnt.Resources.Requests[v1.ResourceCPU] = cpuRequestQty
		cnt.Resources.Limits[v1.ResourceCPU] = cpuRequestQty
		// we don't really care, we only need to be in guaranteed QoS
		cnt.Resources.Requests[v1.ResourceMemory] = resource.MustParse("100Mi")
		cnt.Resources.Limits[v1.ResourceMemory] = resource.MustParse("100Mi")
	}
	if desc.RequiresDevices() {
		devQty := resource.NewQuantity(int64(desc.resourceAmount), resource.DecimalSI)
		cnt.Resources.Requests[v1.ResourceName(desc.resourceName)] = *devQty
		cnt.Resources.Limits[v1.ResourceName(desc.resourceName)] = *devQty
	}
	if len(desc.mainCntCommand) > 0 {
		cnt.Command = desc.mainCntCommand
	}
	restartPolicy := v1.RestartPolicyNever
	if desc.restartPolicy != nil {
		restartPolicy = *desc.restartPolicy
	}

	var appCnts []v1.Container
	appCnts = append(appCnts, cnt)

	var initCnts []v1.Container
	for _, cntDesc := range desc.initContainers {
		initCnt := v1.Container{
			Name:  cntDesc.cntName,
			Image: busyboxImage,
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{},
				Limits:   v1.ResourceList{},
			},
			Command:       []string{"sh", "-c", "sleep 5s"},
			RestartPolicy: cntDesc.restartPolicy,
		}
		if cntDesc.restartPolicy != nil && *cntDesc.restartPolicy == v1.ContainerRestartPolicyAlways {
			initCnt.Command = []string{"sh", "-c", "sleep 1d"}
		}
		if cntDesc.RequiresCPU() {
			cpuRequestQty := cntDesc.CPURequestQty()
			initCnt.Resources.Requests[v1.ResourceCPU] = cpuRequestQty
			initCnt.Resources.Limits[v1.ResourceCPU] = cpuRequestQty
			// we don't really care, we only need to be in guaranteed QoS
			initCnt.Resources.Requests[v1.ResourceMemory] = resource.MustParse("100Mi")
			initCnt.Resources.Limits[v1.ResourceMemory] = resource.MustParse("100Mi")
		}
		if cntDesc.RequiresDevices() {
			initCnt.Resources.Requests[v1.ResourceName(cntDesc.resourceName)] = resource.MustParse(fmt.Sprintf("%d", cntDesc.resourceAmount))
			initCnt.Resources.Limits[v1.ResourceName(cntDesc.resourceName)] = resource.MustParse(fmt.Sprintf("%d", cntDesc.resourceAmount))
		}
		initCnts = append(initCnts, initCnt)
	}

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: desc.podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy:  restartPolicy,
			InitContainers: initCnts,
			Containers:     appCnts,
		},
	}
}

func logPodResources(podIdx int, pr *kubeletpodresourcesv1.PodResources) {
	ns := pr.GetNamespace()
	cnts := pr.GetContainers()
	if len(cnts) == 0 {
		framework.Logf("#%02d/%02d/%02d - %s/%s/%s   No containers", podIdx, 0, 0, ns, pr.GetName(), "_")
		return
	}

	for cntIdx, cnt := range cnts {
		if len(cnt.Devices) == 0 {
			framework.Logf("#%02d/%02d/%02d - %s/%s/%s   cpus -> %v   resources -> none", podIdx, cntIdx, 0, ns, pr.GetName(), cnt.Name, cnt.CpuIds)
			continue
		}

		for devIdx, dev := range cnt.Devices {
			framework.Logf("#%02d/%02d/%02d - %s/%s/%s   cpus -> %v   %s -> %s", podIdx, cntIdx, devIdx, ns, pr.GetName(), cnt.Name, cnt.CpuIds, dev.ResourceName, strings.Join(dev.DeviceIds, ", "))
		}
	}
}

type podResMap map[string]map[string]*kubeletpodresourcesv1.ContainerResources

func convertToMap(podsResources []*kubeletpodresourcesv1.PodResources) podResMap {
	res := make(map[string]map[string]*kubeletpodresourcesv1.ContainerResources)
	for idx, podResource := range podsResources {
		// to make troubleshooting easier
		logPodResources(idx, podResource)

		cnts := make(map[string]*kubeletpodresourcesv1.ContainerResources)
		for _, cnt := range podResource.GetContainers() {
			cnts[cnt.GetName()] = cnt
		}
		res[podResource.GetName()] = cnts
	}
	return res
}

func getPodResourcesValues(ctx context.Context, cli kubeletpodresourcesv1.PodResourcesListerClient) (podResMap, error) {
	resp, err := cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{})
	if err != nil {
		return nil, err
	}
	return convertToMap(resp.GetPodResources()), nil
}

type testPodData struct {
	PodMap map[string]*v1.Pod
}

func newTestPodData() *testPodData {
	return &testPodData{
		PodMap: make(map[string]*v1.Pod),
	}
}

func (tpd *testPodData) createPodsForTest(ctx context.Context, f *framework.Framework, podReqs []podDesc) {
	ginkgo.GinkgoHelper()
	for _, podReq := range podReqs {
		pod := makePodResourcesTestPod(podReq)
		pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)

		framework.Logf("created pod %s", podReq.podName)
		tpd.PodMap[podReq.podName] = pod
	}
}

/* deletePodsForTest clean up all the pods run for a testcase. Must ensure proper cleanup */
func (tpd *testPodData) deletePodsForTest(ctx context.Context, f *framework.Framework) {
	ginkgo.GinkgoHelper()
	deletePodsAsync(ctx, f, tpd.PodMap)
}

/* deletePod removes pod during a test. Should do a best-effort clean up */
func (tpd *testPodData) deletePod(ctx context.Context, f *framework.Framework, podName string) {
	ginkgo.GinkgoHelper()
	_, ok := tpd.PodMap[podName]
	if !ok {
		return
	}
	deletePodSyncByName(ctx, f, podName)
	delete(tpd.PodMap, podName)
}

func findContainerDeviceByName(devs []*kubeletpodresourcesv1.ContainerDevices, resourceName string) *kubeletpodresourcesv1.ContainerDevices {
	for _, dev := range devs {
		if dev.ResourceName == resourceName {
			return dev
		}
	}
	return nil
}

func matchPodDescWithResourcesNamesOnly(expected []podDesc, found podResMap) error {
	framework.Logf("got %d pods expected %d", len(found), len(expected))
	if len(found) != len(expected) {
		return fmt.Errorf("found %d items expected %d", len(found), len(expected))
	}
	for _, podReq := range expected {
		framework.Logf("matching: %#v", podReq)

		if _, ok := found[podReq.podName]; !ok {
			return fmt.Errorf("no pod resources for pod %q", podReq.podName)
		}

	}
	return nil
}

func matchPodDescAndCountWithResources(expected []podDesc, found podResMap) error {
	framework.Logf("got %d pods expected %d", len(found), len(expected))
	if len(found) != len(expected) {
		return fmt.Errorf("found %d items expected %d", len(found), len(expected))
	}
	return matchPodDescWithResources(expected, found)
}

func matchPodDescWithResources(expected []podDesc, found podResMap) error {
	for _, podReq := range expected {
		framework.Logf("matching: %#v", podReq)

		podInfo, ok := found[podReq.podName]
		if !ok {
			return fmt.Errorf("no pod resources for pod %q", podReq.podName)
		}
		cntInfo, ok := podInfo[podReq.cntName]
		if !ok {
			return fmt.Errorf("no container resources for pod %q container %q", podReq.podName, podReq.cntName)
		}
		if podReq.RequiresCPU() {
			if exclusiveCpus := podReq.CPURequestExclusive(); exclusiveCpus != len(cntInfo.CpuIds) {
				if exclusiveCpus == 0 {
					return fmt.Errorf("pod %q container %q requested %d expected to be allocated CPUs from shared pool %v", podReq.podName, podReq.cntName, podReq.cpuRequest, cntInfo.CpuIds)
				}
				return fmt.Errorf("pod %q container %q expected %d cpus got %v", podReq.podName, podReq.cntName, exclusiveCpus, cntInfo.CpuIds)
			}
		}
		if podReq.RequiresDevices() {
			dev := findContainerDeviceByName(cntInfo.GetDevices(), podReq.resourceName)
			if dev == nil {
				return fmt.Errorf("pod %q container %q expected data for resource %q not found", podReq.podName, podReq.cntName, podReq.resourceName)
			}
			if len(dev.DeviceIds) != podReq.resourceAmount {
				return fmt.Errorf("pod %q container %q resource %q expected %d items got %v", podReq.podName, podReq.cntName, podReq.resourceName, podReq.resourceAmount, dev.DeviceIds)
			}
		} else {
			devs := cntInfo.GetDevices()
			if len(devs) > 0 {
				return fmt.Errorf("pod %q container %q expected no resources, got %v", podReq.podName, podReq.cntName, devs)
			}
		}
		if cnts, ok := found[defaultTopologyUnawareResourceName]; ok {
			for _, cnt := range cnts {
				for _, cd := range cnt.GetDevices() {
					if cd.ResourceName != defaultTopologyUnawareResourceName {
						continue
					}
					if cd.Topology != nil {
						//we expect nil topology
						return fmt.Errorf("Nil topology is expected")
					}
				}

			}
		}

		// check init containers
		for _, initCntDesc := range podReq.initContainers {
			if initCntDesc.restartPolicy == nil || *initCntDesc.restartPolicy != v1.ContainerRestartPolicyAlways {
				// If the init container is not restartable, we don't expect it
				// to be reported.
				_, ok := podInfo[initCntDesc.cntName]
				if ok {
					return fmt.Errorf("pod %q regular init container %q should not be reported", podReq.podName, initCntDesc.cntName)
				}
				continue
			}

			cntInfo, ok := podInfo[initCntDesc.cntName]
			if !ok {
				return fmt.Errorf("no container resources for pod %q container %q", podReq.podName, initCntDesc.cntName)
			}
			if initCntDesc.RequiresCPU() {
				if exclusiveCpus := initCntDesc.CPURequestExclusive(); exclusiveCpus != len(cntInfo.CpuIds) {
					if exclusiveCpus == 0 {
						return fmt.Errorf("pod %q container %q requested %d expected to be allocated CPUs from shared pool %v", podReq.podName, initCntDesc.cntName, initCntDesc.cpuRequest, cntInfo.CpuIds)
					}
					return fmt.Errorf("pod %q container %q expected %d cpus got %v", podReq.podName, initCntDesc.cntName, exclusiveCpus, cntInfo.CpuIds)
				}
			}
			if initCntDesc.RequiresDevices() {
				dev := findContainerDeviceByName(cntInfo.GetDevices(), initCntDesc.resourceName)
				if dev == nil {
					return fmt.Errorf("pod %q container %q expected data for resource %q not found", podReq.podName, initCntDesc.cntName, initCntDesc.resourceName)
				}
				if len(dev.DeviceIds) != initCntDesc.resourceAmount {
					return fmt.Errorf("pod %q container %q resource %q expected %d items got %v", podReq.podName, initCntDesc.cntName, initCntDesc.resourceName, initCntDesc.resourceAmount, dev.DeviceIds)
				}
			} else {
				devs := cntInfo.GetDevices()
				if len(devs) > 0 {
					return fmt.Errorf("pod %q container %q expected no resources, got %v", podReq.podName, initCntDesc.cntName, devs)
				}
			}
			if cnts, ok := found[defaultTopologyUnawareResourceName]; ok {
				for _, cnt := range cnts {
					for _, cd := range cnt.GetDevices() {
						if cd.ResourceName != defaultTopologyUnawareResourceName {
							continue
						}
						if cd.Topology != nil {
							// we expect nil topology
							return fmt.Errorf("Nil topology is expected")
						}
					}
				}
			}
		}
	}
	return nil
}

func expectPodResources(ctx context.Context, cli kubeletpodresourcesv1.PodResourcesListerClient, expected []podDesc) {
	ginkgo.GinkgoHelper()
	gomega.Eventually(ctx, func(ctx context.Context) error {
		found, err := getPodResourcesValues(ctx, cli)
		if err != nil {
			return err
		}
		return matchPodDescWithResources(expected, found)
	}, time.Minute, 10*time.Second).Should(gomega.Succeed())
}

func filterOutDesc(descs []podDesc, name string) []podDesc {
	var ret []podDesc
	for _, desc := range descs {
		if desc.podName == name {
			continue
		}
		ret = append(ret, desc)
	}
	return ret
}

func podresourcesListTests(ctx context.Context, f *framework.Framework, cli kubeletpodresourcesv1.PodResourcesListerClient, sd *sriovData, sidecarContainersEnabled bool) {
	ginkgo.GinkgoHelper()

	var tpd *testPodData

	var found podResMap
	var expected []podDesc
	var extra podDesc

	expectedBasePods := 0 /* nothing but pods we create */
	if sd != nil {
		expectedBasePods = 1 // sriovdp
	}

	var err error
	ginkgo.By("checking the output when no pods are present")
	found, err = getPodResourcesValues(ctx, cli)
	framework.ExpectNoError(err, "getPodResourcesValues() failed err: %v", err)
	gomega.ExpectWithOffset(1, found).To(gomega.HaveLen(expectedBasePods), "base pod expectation mismatch")

	tpd = newTestPodData()
	ginkgo.By("checking the output when only pods which don't require resources are present")
	expected = []podDesc{
		{
			podName: "pod-00",
			cntName: "cnt-00",
		},
		{
			podName: "pod-01",
			cntName: "cnt-00",
		},
	}

	tpd.createPodsForTest(ctx, f, expected)
	expectPodResources(ctx, cli, expected)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when only a subset of pods require resources")
	if sd != nil {
		expected = []podDesc{
			{
				podName: "pod-00",
				cntName: "cnt-00",
			},
			{
				podName:        "pod-01",
				cntName:        "cnt-00",
				resourceName:   sd.resourceName,
				resourceAmount: 1,
				cpuRequest:     1000,
			},
			{
				podName:    "pod-02",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName:        "pod-03",
				cntName:        "cnt-00",
				resourceName:   sd.resourceName,
				resourceAmount: 1,
				cpuRequest:     1000,
			},
		}
	} else {
		expected = []podDesc{
			{
				podName: "pod-00",
				cntName: "cnt-00",
			},
			{
				podName:    "pod-01",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName:    "pod-02",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName:    "pod-03",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
		}

	}
	tpd.createPodsForTest(ctx, f, expected)
	expectPodResources(ctx, cli, expected)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when creating pods which require resources between calls")
	if sd != nil {
		expected = []podDesc{
			{
				podName: "pod-00",
				cntName: "cnt-00",
			},
			{
				podName:        "pod-01",
				cntName:        "cnt-00",
				resourceName:   sd.resourceName,
				resourceAmount: 1,
				cpuRequest:     1000,
			},
			{
				podName:    "pod-02",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
		}
	} else {
		expected = []podDesc{
			{
				podName: "pod-00",
				cntName: "cnt-00",
			},
			{
				podName:    "pod-01",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName:    "pod-02",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
		}
	}

	tpd.createPodsForTest(ctx, f, expected)
	expectPodResources(ctx, cli, expected)

	if sd != nil {
		extra = podDesc{
			podName:        "pod-03",
			cntName:        "cnt-00",
			resourceName:   sd.resourceName,
			resourceAmount: 1,
			cpuRequest:     1000,
		}
	} else {
		extra = podDesc{
			podName:    "pod-03",
			cntName:    "cnt-00",
			cpuRequest: 1000,
		}

	}

	tpd.createPodsForTest(ctx, f, []podDesc{
		extra,
	})

	expected = append(expected, extra)
	expectPodResources(ctx, cli, expected)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when deleting pods which require resources between calls")

	if sd != nil {
		expected = []podDesc{
			{
				podName:    "pod-00",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName:        "pod-01",
				cntName:        "cnt-00",
				resourceName:   sd.resourceName,
				resourceAmount: 1,
				cpuRequest:     2000,
			},
			{
				podName: "pod-02",
				cntName: "cnt-00",
			},
			{
				podName:        "pod-03",
				cntName:        "cnt-00",
				resourceName:   sd.resourceName,
				resourceAmount: 1,
				cpuRequest:     1000,
			},
		}
	} else {
		expected = []podDesc{
			{
				podName:    "pod-00",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName:    "pod-01",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
			{
				podName: "pod-02",
				cntName: "cnt-00",
			},
			{
				podName:    "pod-03",
				cntName:    "cnt-00",
				cpuRequest: 1000,
			},
		}
	}
	tpd.createPodsForTest(ctx, f, expected)
	expectPodResources(ctx, cli, expected)

	tpd.deletePod(ctx, f, "pod-01")
	expectedPostDelete := filterOutDesc(expected, "pod-01")
	expectPodResources(ctx, cli, expectedPostDelete)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when pods request non integral CPUs")
	if sd != nil {
		expected = []podDesc{
			{
				podName:    "pod-00",
				cntName:    "cnt-00",
				cpuRequest: 1500,
			},
			{
				podName:        "pod-01",
				cntName:        "cnt-00",
				resourceName:   sd.resourceName,
				resourceAmount: 1,
				cpuRequest:     1500,
			},
		}
	} else {
		expected = []podDesc{
			{
				podName:    "pod-00",
				cntName:    "cnt-00",
				cpuRequest: 1500,
			},
		}

	}
	tpd.createPodsForTest(ctx, f, expected)
	expectPodResources(ctx, cli, expected)
	tpd.deletePodsForTest(ctx, f)

	if sidecarContainersEnabled {
		containerRestartPolicyAlways := v1.ContainerRestartPolicyAlways

		tpd = newTestPodData()
		ginkgo.By("checking the output when pods have init containers")
		if sd != nil {
			expected = []podDesc{
				{
					podName:    "pod-00",
					cntName:    "regular-00",
					cpuRequest: 1000,
					initContainers: []initContainerDesc{
						{
							cntName:        "init-00",
							resourceName:   sd.resourceName,
							resourceAmount: 1,
							cpuRequest:     1000,
						},
					},
				},
				{
					podName:    "pod-01",
					cntName:    "regular-00",
					cpuRequest: 1000,
					initContainers: []initContainerDesc{
						{
							cntName:        "restartable-init-00",
							resourceName:   sd.resourceName,
							resourceAmount: 1,
							cpuRequest:     1000,
							restartPolicy:  &containerRestartPolicyAlways,
						},
					},
				},
			}
		} else {
			expected = []podDesc{
				{
					podName:    "pod-00",
					cntName:    "regular-00",
					cpuRequest: 1000,
					initContainers: []initContainerDesc{
						{
							cntName:    "init-00",
							cpuRequest: 1000,
						},
					},
				},
				{
					podName:    "pod-01",
					cntName:    "regular-00",
					cpuRequest: 1000,
					initContainers: []initContainerDesc{
						{
							cntName:       "restartable-init-00",
							cpuRequest:    1000,
							restartPolicy: &containerRestartPolicyAlways,
						},
					},
				},
			}
		}

		tpd.createPodsForTest(ctx, f, expected)
		expectPodResources(ctx, cli, expected)
		tpd.deletePodsForTest(ctx, f)
	}
}

func podresourcesGetAllocatableResourcesTests(ctx context.Context, cli kubeletpodresourcesv1.PodResourcesListerClient, sd *sriovData, onlineCPUs, reservedSystemCPUs cpuset.CPUSet) {
	ginkgo.GinkgoHelper()

	ginkgo.By("checking the devices known to the kubelet")
	resp, err := cli.GetAllocatableResources(ctx, &kubeletpodresourcesv1.AllocatableResourcesRequest{})
	framework.ExpectNoError(err, "cannot get allocatable CPUs from podresources")
	allocatableCPUs, devs := demuxCPUsAndDevicesFromGetAllocatableResources(resp)

	if onlineCPUs.Size() == 0 {
		ginkgo.By("expecting no CPUs reported")
		gomega.Expect(onlineCPUs.Size()).To(gomega.Equal(reservedSystemCPUs.Size()), "with no online CPUs, no CPUs should be reserved")
	} else {
		ginkgo.By(fmt.Sprintf("expecting online CPUs reported - online=%v (%d) reserved=%v (%d)", onlineCPUs, onlineCPUs.Size(), reservedSystemCPUs, reservedSystemCPUs.Size()))
		if reservedSystemCPUs.Size() > onlineCPUs.Size() {
			ginkgo.Fail("more reserved CPUs than online")
		}
		expectedCPUs := onlineCPUs.Difference(reservedSystemCPUs)

		ginkgo.By(fmt.Sprintf("expecting CPUs '%v'='%v'", allocatableCPUs, expectedCPUs))
		gomega.Expect(allocatableCPUs.Equals(expectedCPUs)).To(gomega.BeTrueBecause("mismatch expecting CPUs"))
	}

	if sd == nil { // no devices in the environment, so expect no devices
		ginkgo.By("expecting no devices reported")
		gomega.Expect(devs).To(gomega.BeEmpty(), fmt.Sprintf("got unexpected devices %#v", devs))
		return
	}

	ginkgo.By(fmt.Sprintf("expecting some %q devices reported", sd.resourceName))
	gomega.Expect(devs).ToNot(gomega.BeEmpty())
	for _, dev := range devs {
		gomega.Expect(dev.ResourceName).To(gomega.Equal(sd.resourceName))
		gomega.Expect(dev.DeviceIds).ToNot(gomega.BeEmpty())
	}
}

func demuxCPUsAndDevicesFromGetAllocatableResources(resp *kubeletpodresourcesv1.AllocatableResourcesResponse) (cpuset.CPUSet, []*kubeletpodresourcesv1.ContainerDevices) {
	devs := resp.GetDevices()
	var cpus []int
	for _, cpuid := range resp.GetCpuIds() {
		cpus = append(cpus, int(cpuid))
	}
	return cpuset.New(cpus...), devs
}

func podresourcesGetWithRetry(ctx context.Context, cli kubeletpodresourcesv1.PodResourcesListerClient, podNamespace, podName string) *kubeletpodresourcesv1.GetPodResourcesResponse {
	ginkgo.GinkgoHelper()
	var err error
	var resp *kubeletpodresourcesv1.GetPodResourcesResponse
	gomega.Eventually(func() error {
		resp, err = cli.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{PodName: podName, PodNamespace: podNamespace})
		framework.Logf("Get result: %v, err: %v", resp, err)
		return err
	}).WithContext(ctx).WithPolling(2*time.Second).WithTimeout(30*time.Second).Should(gomega.Succeed(), "Expected Get to succeed for %s/%s", podNamespace, podName)
	return resp
}

func podresourcesGetTests(ctx context.Context, f *framework.Framework, cli kubeletpodresourcesv1.PodResourcesListerClient, sidecarContainersEnabled bool) {
	ginkgo.GinkgoHelper()

	ginkgo.By("checking the output when no pods are present")
	expected := []podDesc{}

	var err error
	var resp *kubeletpodresourcesv1.GetPodResourcesResponse
	gomega.Eventually(func() error {
		resp, err = cli.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{PodName: "test", PodNamespace: f.Namespace.Name})
		framework.Logf("Get result: %v, err: %v", resp, err)
		return err
	}).WithContext(ctx).WithPolling(2*time.Second).WithTimeout(30*time.Second).ShouldNot(gomega.Succeed(), "Expected Get to fail for %s/%s", f.Namespace.Name, "test")

	podResourceList := []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
	res := convertToMap(podResourceList)
	err = matchPodDescWithResources(expected, res)
	framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)

	tpd := newTestPodData()
	ginkgo.By("checking the output when we try to get an inexistent pod")
	expected = []podDesc{}
	descs := []podDesc{
		{
			podName: "pod-00",
			cntName: "cnt-00",
		},
		{
			podName: "pod-01",
			cntName: "cnt-01",
		},
		{
			podName:    "pod-02",
			cntName:    "cnt-02",
			cpuRequest: 1000,
		},
		{
			podName: "pod-03",
			cntName: "cnt-03",
		},
	}
	tpd.createPodsForTest(ctx, f, descs)

	gomega.Eventually(func() error {
		resp, err = cli.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{PodName: "pod-AA", PodNamespace: f.Namespace.Name})
		framework.Logf("Get result: %v, err: %v", res, err)
		return err
	}).WithContext(ctx).WithPolling(2*time.Second).WithTimeout(30*time.Second).ShouldNot(gomega.Succeed(), "Expected Get to fail for %s/%s", f.Namespace.Name, "pod-AA")

	podResourceList = []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
	res = convertToMap(podResourceList)
	err = matchPodDescWithResources(expected, res)
	framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when only pods which don't require resources are present")

	expected = []podDesc{
		{
			podName: "pod-00",
			cntName: "cnt-00",
		},
	}
	tpd.createPodsForTest(ctx, f, expected)
	expectListAndGetConsistent(ctx, cli, "pod-00", f.Namespace.Name)

	resp = podresourcesGetWithRetry(ctx, cli, f.Namespace.Name, "pod-00")
	podResourceList = []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
	res = convertToMap(podResourceList)
	err = matchPodDescWithResources(expected, res)
	framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when only pod require CPU")

	expected = []podDesc{
		{
			podName:    "pod-01",
			cntName:    "cnt-00",
			cpuRequest: 1000,
		},
	}
	tpd.createPodsForTest(ctx, f, expected)
	expectListAndGetConsistent(ctx, cli, "pod-01", f.Namespace.Name)

	resp = podresourcesGetWithRetry(ctx, cli, f.Namespace.Name, "pod-01")
	podResourceList = []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
	res = convertToMap(podResourceList)
	err = matchPodDescWithResources(expected, res)
	framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)
	tpd.deletePodsForTest(ctx, f)

	tpd = newTestPodData()
	ginkgo.By("checking the output when a pod has multiple containers and only one of them requires exclusive CPUs")

	expected = []podDesc{
		{
			podName:    "pod-01",
			cntName:    "cnt-00",
			cpuRequest: 1000,
			cpuLimit:   1000,
			memRequest: 256,
			memLimit:   256,
			extraContainers: []containerDesc{
				{
					cntName:    "cnt-01",
					cpuRequest: 600,
				},
				{
					cntName:    "cnt-02",
					cpuRequest: 400,
				},
			},
			useFullSpec: true,
		},
	}
	tpd.createPodsForTest(ctx, f, expected)
	expectListAndGetConsistent(ctx, cli, "pod-01", f.Namespace.Name)

	resp = podresourcesGetWithRetry(ctx, cli, f.Namespace.Name, "pod-01")
	podResourceList = []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
	res = convertToMap(podResourceList)
	err = matchPodDescWithResources(expected, res)
	framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)
	tpd.deletePodsForTest(ctx, f)

	restartNever := v1.RestartPolicyNever
	tpd = newTestPodData()
	ginkgo.By("checking Get() returns an error for a terminated pod")

	completedDesc := podDesc{
		podName:        "pod-01",
		cntName:        "cnt-00",
		cpuRequest:     1000,
		restartPolicy:  &restartNever,
		mainCntCommand: []string{"sh", "-c", "/bin/true"},
	}

	completedPod := e2epod.NewPodClient(f).Create(ctx, makePodResourcesTestPod(completedDesc))
	framework.Logf("created pod %s", completedDesc.podName)
	tpd.PodMap[completedDesc.podName] = completedPod
	// Wait for the pod to complete to avoid races.
	err = e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, completedPod.Name, "Pod Succeeded", 2*time.Minute, testutils.PodSucceeded)
	framework.ExpectNoError(err, "pod did not succeed as expected")

	gomega.Eventually(func() error {
		resp, err = cli.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{PodName: completedDesc.podName, PodNamespace: f.Namespace.Name})
		framework.Logf("Get result: %v, err: %v", res, err)
		return err
	}).WithContext(ctx).WithPolling(2*time.Second).WithTimeout(30*time.Second).ShouldNot(gomega.Succeed(), "Expected Get to fail for %s/%s", f.Namespace.Name, completedDesc.podName)

	// Returned PodResources for a terminated pod must be empty.
	pr := resp.GetPodResources()
	if pr != nil {
		gomega.Expect(pr.GetContainers()).To(gomega.BeEmpty(),
			"expected no container resources in response for terminated pod; got: %#v", pr.GetContainers())
	}
	tpd.deletePodsForTest(ctx, f)

	if sidecarContainersEnabled {
		containerRestartPolicyAlways := v1.ContainerRestartPolicyAlways

		tpd = newTestPodData()
		ginkgo.By("checking the output when only pod with init containers require CPU")
		expected = []podDesc{
			{
				podName:    "pod-01",
				cntName:    "cnt-00",
				cpuRequest: 1000,
				initContainers: []initContainerDesc{
					{
						cntName:    "init-00",
						cpuRequest: 1000,
					},
					{
						cntName:       "restartable-init-01",
						cpuRequest:    1000,
						restartPolicy: &containerRestartPolicyAlways,
					},
				},
			},
		}
		tpd.createPodsForTest(ctx, f, expected)

		resp = podresourcesGetWithRetry(ctx, cli, f.Namespace.Name, "pod-01")
		podResourceList = []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
		res = convertToMap(podResourceList)
		err = matchPodDescWithResources(expected, res)
		framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)
		tpd.deletePodsForTest(ctx, f)
	}
}

// Serial because the test updates kubelet configuration.
var _ = SIGDescribe("POD Resources API", framework.WithSerial(), feature.PodResourcesAPI, func() {
	f := framework.NewDefaultFramework("podresources-test")
	addBeforeEachForCleaningUpPods(f)
	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged

	var reservedSystemCPUs cpuset.CPUSet

	ginkgo.BeforeEach(func() {
		reservedSystemCPUs = cpuset.New(1)
	})

	ginkgo.Context("with SRIOV devices in the system", func() {
		ginkgo.BeforeEach(func() {
			requireSRIOVDevices()
		})

		ginkgo.Context("with CPU manager Static policy", func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				// this is a very rough check. We just want to rule out system that does NOT have enough resources
				_, cpuAlloc, _ := getLocalNodeCPUDetails(ctx, f)

				if cpuAlloc < minCoreCount {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
				}
			})

			// empty context to apply kubelet config changes
			ginkgo.Context("with reserved system CPUs", func() {
				tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
					// Set the CPU Manager policy to static.
					initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)

					// Set the CPU Manager reconcile period to 1 second.
					initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}

					cpus := reservedSystemCPUs.String()
					framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
					initialConfig.ReservedSystemCPUs = cpus
				})

				framework.It("should return the expected responses", framework.WithNodeConformance(), func(ctx context.Context) {
					onlineCPUs, err := getOnlineCPUs()
					framework.ExpectNoError(err, "getOnlineCPUs() failed err: %v", err)

					configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
					sd := setupSRIOVConfigOrFail(ctx, f, configMap)
					ginkgo.DeferCleanup(teardownSRIOVConfigOrFail, f, sd)

					waitForSRIOVResources(ctx, f, sd)

					endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
					framework.ExpectNoError(err, "LocalEndpoint() failed err: %v", err)

					cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
					framework.ExpectNoError(err, "GetV1Client() failed err: %v", err)
					defer framework.ExpectNoError(conn.Close())

					waitForSRIOVResources(ctx, f, sd)

					ginkgo.By("checking List()")
					podresourcesListTests(ctx, f, cli, sd, true)
					ginkgo.By("checking GetAllocatableResources()")
					podresourcesGetAllocatableResourcesTests(ctx, cli, sd, onlineCPUs, reservedSystemCPUs)
				})
			})
		})

		ginkgo.Context("with CPU manager None policy", func() {
			ginkgo.It("should return the expected responses", func(ctx context.Context) {
				// current default is "none" policy - no need to restart the kubelet

				requireSRIOVDevices()

				configMap := getSRIOVDevicePluginConfigMap(framework.TestContext.SriovdpConfigMapFile)
				sd := setupSRIOVConfigOrFail(ctx, f, configMap)
				ginkgo.DeferCleanup(teardownSRIOVConfigOrFail, f, sd)

				waitForSRIOVResources(ctx, f, sd)

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err, "LocalEndpoint() failed err: %v", err)

				cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err, "GetV1Client() failed err: %v", err)
				defer func() {
					framework.ExpectNoError(conn.Close())
				}()

				waitForSRIOVResources(ctx, f, sd)

				// intentionally passing empty cpuset instead of onlineCPUs because with none policy
				// we should get no allocatable cpus - no exclusively allocatable CPUs, depends on policy static
				podresourcesGetAllocatableResourcesTests(ctx, cli, sd, cpuset.New(), cpuset.New())
			})
		})
	})

	framework.Context("without SRIOV devices in the system", framework.WithFlaky(), func() {
		ginkgo.BeforeEach(func() {
			requireLackOfSRIOVDevices()
		})

		ginkgo.Context("with CPU manager Static policy", func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				// this is a very rough check. We just want to rule out system that does NOT have enough resources
				_, cpuAlloc, _ := getLocalNodeCPUDetails(ctx, f)

				if cpuAlloc < minCoreCount {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
				}
			})

			ginkgo.Context("with reserved system CPUs", func() {
				tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
					// Set the CPU Manager policy to static.
					initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)

					// Set the CPU Manager reconcile period to 1 second.
					initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}

					cpus := reservedSystemCPUs.String()
					framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
					initialConfig.ReservedSystemCPUs = cpus
				})

				ginkgo.It("should succeed when calling Get for a valid pod", func(ctx context.Context) {
					ginkgo.By("checking Get succeeds for a valid pod")
					pd := podDesc{
						podName:    "fg-enabled-pod",
						cntName:    "fg-enabled-cnt",
						cpuRequest: 1000,
					}
					pod := makePodResourcesTestPod(pd)
					pod = e2epod.NewPodClient(f).Create(ctx, pod)
					defer e2epod.NewPodClient(f).DeleteSync(ctx, pod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
					err := e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Ready", 2*time.Minute, testutils.PodRunningReady)
					framework.ExpectNoError(err)

					// Kubelet restarts when applying CPUManager static policy;
					// podresources socket may not be immediately ready.
					// Retry with a fresh connection.
					waitForPodResourcesV1Serving(ctx)

					var (
						res     *kubeletpodresourcesv1.GetPodResourcesResponse
						lastErr error
					)

					// Once list is available, Get() should too.
					// Retry time kept shorter.
					gomega.Eventually(func() error {
						lastErr = withPodResourcesV1Client(ctx, func(cli kubeletpodresourcesv1.PodResourcesListerClient) error {
							reqCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
							defer cancel()

							r, err := cli.Get(reqCtx, &kubeletpodresourcesv1.GetPodResourcesRequest{
								PodName:      pod.Name,
								PodNamespace: pod.Namespace,
							})
							if err != nil {
								return err
							}
							res = r
							return nil
						})
						return lastErr
					}).WithTimeout(15*time.Second).WithPolling(1*time.Second).Should(
						gomega.Succeed(),
						"Expected Get to succeed (last err: %v)", lastErr,
					)

					framework.Logf("Get result: %v", res)
					gomega.Expect(res).ToNot(gomega.BeNil(), "expected not nil Get response")
					gomega.Expect(res.PodResources.Name).To(gomega.Equal(pod.Name))
					gomega.Expect(res.PodResources.Containers).To(gomega.HaveLen(1), "expected one container")
					container := res.PodResources.Containers[0]
					gomega.Expect(container.Name).To(gomega.Equal(pd.cntName), "expected container name match")
					gomega.Expect(container.CpuIds).ToNot(gomega.BeEmpty(), "expected CPU IDs to be reported")
					gomega.Expect(container.CpuIds).To(gomega.HaveLen(pd.CPURequestExclusive()), "expected one exclusive CPU")
					gomega.Expect(container.Devices).To(gomega.BeEmpty(), "expected no devices")
				})

				ginkgo.It("should return the expected responses", func(ctx context.Context) {
					onlineCPUs, err := getOnlineCPUs()
					framework.ExpectNoError(err, "getOnlineCPUs() failed err: %v", err)

					waitForPodResourcesV1Serving(ctx)

					endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
					framework.ExpectNoError(err, "LocalEndpoint() failed err: %v", err)

					cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
					framework.ExpectNoError(err, "GetV1Client() failed err: %v", err)
					defer func() {
						framework.ExpectNoError(conn.Close())
					}()

					podresourcesListTests(ctx, f, cli, nil, false)
					podresourcesGetAllocatableResourcesTests(ctx, cli, nil, onlineCPUs, reservedSystemCPUs)
					podresourcesGetTests(ctx, f, cli, false)
				})

				framework.It("should return the expected responses", framework.WithNodeConformance(), func(ctx context.Context) {
					onlineCPUs, err := getOnlineCPUs()
					framework.ExpectNoError(err, "getOnlineCPUs() failed err: %v", err)

					waitForPodResourcesV1Serving(ctx)

					endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
					framework.ExpectNoError(err, "LocalEndpoint() failed err: %v", err)

					cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
					framework.ExpectNoError(err, "GetV1Client() failed err: %v", err)
					defer func() {
						framework.ExpectNoError(conn.Close())
					}()

					podresourcesListTests(ctx, f, cli, nil, true)
					podresourcesGetAllocatableResourcesTests(ctx, cli, nil, onlineCPUs, reservedSystemCPUs)
					podresourcesGetTests(ctx, f, cli, true)
				})
			})
		})

		ginkgo.Context("with CPU manager None policy", func() {
			ginkgo.It("should return the expected responses", func(ctx context.Context) {

				waitForPodResourcesV1Serving(ctx)

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err, "LocalEndpoint() failed err: %v", err)

				cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err, "GetV1Client() failed err: %v", err)
				defer func() {
					framework.ExpectNoError(conn.Close())
				}()

				// intentionally passing empty cpuset instead of onlineCPUs because with none policy
				// we should get no allocatable cpus - no exclusively allocatable CPUs, depends on policy static
				podresourcesGetAllocatableResourcesTests(ctx, cli, nil, cpuset.New(), cpuset.New())
			})
		})

	})

	ginkgo.When("checking core resource managers assignments", func() {
		var podresConn *grpc.ClientConn
		var podMap map[string]*v1.Pod
		var cpuAlloc int64

		ginkgo.BeforeEach(func(ctx context.Context) {
			podMap = make(map[string]*v1.Pod)
		})

		ginkgo.AfterEach(func(ctx context.Context) {
			if podresConn != nil {
				framework.ExpectNoError(podresConn.Close())
			}
			deletePodsAsync(ctx, f, podMap)
		})

		ginkgo.BeforeEach(func(ctx context.Context) {
			// this is a very rough check. We just want to rule out system that does NOT have enough resources
			_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
			if cpuAlloc < minCoreCount {
				e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
			}
		})

		ginkgo.When("listing with restricted list output enabled", func() {

			tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
				initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
				initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Minute} // set it long enough it is practically disabled
				cpus := reservedSystemCPUs.String()
				framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
				initialConfig.ReservedSystemCPUs = cpus
			})

			ginkgo.It("should report only the running pods regardless of the QoS", func(ctx context.Context) {
				// TODO: compute automatically
				if cpuAlloc < 8 {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
				}

				restartNever := v1.RestartPolicyNever
				restartAlways := v1.RestartPolicyAlways
				descs := []podDesc{
					{
						cntName:        "e2e-test-cnt-gu-term-1",
						podName:        "e2e-test-pod-gu-term-1",
						cpuLimit:       1000,
						memLimit:       256,
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:        "e2e-test-cnt-gu-term-2",
						podName:        "e2e-test-pod-gu-term-2",
						cpuLimit:       200,
						memLimit:       256,
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:       "e2e-test-cnt-gu-run-1",
						podName:       "e2e-test-pod-gu-run-1",
						cpuLimit:      200,
						memLimit:      256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-gu-run-2",
						podName:       "e2e-test-pod-gu-run-2",
						cpuLimit:      1000,
						memLimit:      256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-bu-term-1",
						podName:        "e2e-test-pod-bu-term-1",
						cpuRequest:     400,
						memRequest:     256,
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:       "e2e-test-cnt-bu-run-1",
						podName:       "e2e-test-pod-bu-run-1",
						cpuRequest:    400,
						memRequest:    256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-be-term-1",
						podName:        "e2e-test-pod-be-term-1",
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:       "e2e-test-cnt-be-run-1",
						podName:       "e2e-test-pod-be-run-1",
						restartPolicy: &restartAlways,
					},
				}
				expected := []podDesc{
					{
						cntName:       "e2e-test-cnt-gu-run-1",
						podName:       "e2e-test-pod-gu-run-1",
						cpuLimit:      200,
						memLimit:      256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-gu-run-2",
						podName:       "e2e-test-pod-gu-run-2",
						cpuLimit:      1000,
						memLimit:      256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-bu-run-1",
						podName:       "e2e-test-pod-bu-run-1",
						cpuRequest:    400,
						memRequest:    256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-be-run-1",
						podName:       "e2e-test-pod-be-run-1",
						restartPolicy: &restartAlways,
					},
				}

				for _, desc := range descs {
					pod := makePodResourcesTestPod(desc, WithFullSpec)
					if desc.restartPolicy == &restartAlways {
						pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
						podMap[string(pod.UID)] = pod
					} else {
						pod = e2epod.NewPodClient(f).Create(ctx, pod)
						pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
						framework.ExpectNoError(err)
						podMap[string(pod.UID)] = pod
						err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
						framework.ExpectNoError(err)
					}
				}

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)

				var cli kubeletpodresourcesv1.PodResourcesListerClient
				cli, podresConn, err = podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)

				gomega.Consistently(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					return matchPodDescAndCountWithResources(expected, found)
				}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
			})

			ginkgo.It("should report only the running pods among many", func(ctx context.Context) {
				// TODO: compute automatically
				if cpuAlloc < 8 {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
				}
				restartNever := v1.RestartPolicyNever
				restartAlways := v1.RestartPolicyAlways
				descs := []podDesc{
					{
						cntName:       "e2e-test-cnt-payload-gu-1",
						podName:       "e2e-test-pod-payload-gu-1",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-mixed-cpu-mem",
						podName:        "e2e-test-pod-mixed-cpu-mem",
						cpuRequest:     1000,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
						restartPolicy:  &restartNever,
					},
					{
						cntName:       "e2e-test-cnt-payload-gu-2",
						podName:       "e2e-test-pod-payload-gu-2",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-payload-gu-3",
						podName:       "e2e-test-pod-payload-gu-3",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-mixed-mem",
						podName:        "e2e-test-pod-mixed-mem",
						cpuRequest:     1000,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
						restartPolicy:  &restartNever,
					},
				}
				expected := []podDesc{
					{
						cntName:       "e2e-test-cnt-payload-gu-1",
						podName:       "e2e-test-pod-payload-gu-1",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-payload-gu-2",
						podName:       "e2e-test-pod-payload-gu-2",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-payload-gu-3",
						podName:       "e2e-test-pod-payload-gu-3",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
				}

				for _, desc := range descs {
					pod := makePodResourcesTestPod(desc)
					if desc.restartPolicy == &restartAlways {
						pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
						podMap[string(pod.UID)] = pod
					} else {
						pod = e2epod.NewPodClient(f).Create(ctx, pod)
						pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
						framework.ExpectNoError(err)
						podMap[string(pod.UID)] = pod
						err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
						framework.ExpectNoError(err)
					}
				}

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)

				var cli kubeletpodresourcesv1.PodResourcesListerClient
				cli, podresConn, err = podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)

				gomega.Consistently(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					return matchPodDescAndCountWithResources(expected, found)
				}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
			})

			ginkgo.DescribeTable("all pods terminated and requiring", func(ctx context.Context, cpuReqMilli, podCount int) {
				cpuReqTot := int64((cpuReqMilli * podCount) / 1000)
				ginkgo.By(fmt.Sprintf("needed: %d cores", cpuReqTot))
				if cpuAlloc < cpuReqTot {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, cpuReqTot)
				}

				restartNever := v1.RestartPolicyNever

				for idx := range podCount {
					pd := podDesc{
						cntName:        fmt.Sprintf("e2e-test-cnt-%d", idx),
						podName:        fmt.Sprintf("e2e-test-pod-%d", idx),
						cpuRequest:     cpuReqMilli,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
						restartPolicy:  &restartNever,
					}
					pod := makePodResourcesTestPod(pd)
					pod = e2epod.NewPodClient(f).Create(ctx, pod)
					pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
					framework.ExpectNoError(err)
					podMap[string(pod.UID)] = pod
					err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
					framework.ExpectNoError(err)
				}
				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)

				var cli kubeletpodresourcesv1.PodResourcesListerClient
				cli, podresConn, err = podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)

				gomega.Eventually(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					if len(found) > 0 {
						return fmt.Errorf("returned unexpected pods: %v", found)
					}
					return nil
				}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())

				gomega.Consistently(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					if len(found) > 0 {
						return fmt.Errorf("returned unexpected pods: %v", found)
					}
					return nil
				}).WithContext(ctx).WithTimeout(10 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
			},
				ginkgo.Entry("cpu and mem single", context.TODO(), 1000, 1),
				ginkgo.Entry("cpu and mem multi", context.TODO(), 1000, 3),
				ginkgo.Entry("mem only single", context.TODO(), 200, 1),
				ginkgo.Entry("mem only multi", context.TODO(), 200, 3),
			)
		})

		ginkgo.When("listing with restricted list output disabled for backward compatible defaults", func() {

			tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
				initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
				initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Minute} // set it long enough it is practically disabled
				cpus := reservedSystemCPUs.String()
				framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
				initialConfig.ReservedSystemCPUs = cpus
				if initialConfig.FeatureGates == nil {
					initialConfig.FeatureGates = make(map[string]bool)
				}
				initialConfig.FeatureGates["KubeletPodResourcesListUseActivePods"] = false
			})

			ginkgo.It("should report all the known pods regardless of the QoS", func(ctx context.Context) {
				// TODO: compute automatically
				if cpuAlloc < 8 {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
				}

				restartNever := v1.RestartPolicyNever
				restartAlways := v1.RestartPolicyAlways
				descs := []podDesc{
					{
						cntName:        "e2e-test-cnt-gu-term-1",
						podName:        "e2e-test-pod-gu-term-1",
						cpuLimit:       1000,
						memLimit:       256,
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:        "e2e-test-cnt-gu-term-2",
						podName:        "e2e-test-pod-gu-term-2",
						cpuLimit:       200,
						memLimit:       256,
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:       "e2e-test-cnt-gu-run-1",
						podName:       "e2e-test-pod-gu-run-1",
						cpuLimit:      200,
						memLimit:      256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-gu-run-2",
						podName:       "e2e-test-pod-gu-run-2",
						cpuLimit:      1000,
						memLimit:      256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-bu-term-1",
						podName:        "e2e-test-pod-bu-term-1",
						cpuRequest:     400,
						memRequest:     256,
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:       "e2e-test-cnt-bu-run-1",
						podName:       "e2e-test-pod-bu-run-1",
						cpuRequest:    400,
						memRequest:    256,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-be-term-1",
						podName:        "e2e-test-pod-be-term-1",
						restartPolicy:  &restartNever,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
					},
					{
						cntName:       "e2e-test-cnt-be-run-1",
						podName:       "e2e-test-pod-be-run-1",
						restartPolicy: &restartAlways,
					},
				}
				expected := descs

				for _, desc := range descs {
					pod := makePodResourcesTestPod(desc, WithFullSpec)
					if desc.restartPolicy == &restartAlways {
						pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
						podMap[string(pod.UID)] = pod
					} else {
						pod = e2epod.NewPodClient(f).Create(ctx, pod)
						pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
						framework.ExpectNoError(err)
						podMap[string(pod.UID)] = pod
						err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
						framework.ExpectNoError(err)
					}
				}

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)

				var cli kubeletpodresourcesv1.PodResourcesListerClient
				cli, podresConn, err = podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)

				gomega.Consistently(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					return matchPodDescAndCountWithResources(expected, found)
				}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
			})

			ginkgo.It("should report all the known pods", func(ctx context.Context) {
				// TODO: compute automatically
				if cpuAlloc < 8 {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, 8)
				}
				restartNever := v1.RestartPolicyNever
				restartAlways := v1.RestartPolicyAlways
				descs := []podDesc{
					{
						cntName:       "e2e-test-cnt-payload-gu-1",
						podName:       "e2e-test-pod-payload-gu-1",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-mixed-cpu-mem",
						podName:        "e2e-test-pod-mixed-cpu-mem",
						cpuRequest:     1000,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
						restartPolicy:  &restartNever,
					},
					{
						cntName:       "e2e-test-cnt-payload-gu-2",
						podName:       "e2e-test-pod-payload-gu-2",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:       "e2e-test-cnt-payload-gu-3",
						podName:       "e2e-test-pod-payload-gu-3",
						cpuRequest:    1000,
						restartPolicy: &restartAlways,
					},
					{
						cntName:        "e2e-test-cnt-mixed-mem",
						podName:        "e2e-test-pod-mixed-mem",
						cpuRequest:     1000,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
						restartPolicy:  &restartNever,
					},
				}
				expected := descs

				for _, desc := range descs {
					pod := makePodResourcesTestPod(desc)
					if desc.restartPolicy == &restartAlways {
						pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
						podMap[string(pod.UID)] = pod
					} else {
						pod = e2epod.NewPodClient(f).Create(ctx, pod)
						pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
						framework.ExpectNoError(err)
						podMap[string(pod.UID)] = pod
						err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
						framework.ExpectNoError(err)
					}
				}

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)

				var cli kubeletpodresourcesv1.PodResourcesListerClient
				cli, podresConn, err = podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)

				gomega.Consistently(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					// all but the last pod will have inconsistent resource reporting.
					// See: https://github.com/kubernetes/kubernetes/issues/132020#issuecomment-2921478368
					// this is why we introduced the FeatureGate  flag in the first place
					return matchPodDescWithResourcesNamesOnly(expected, found)
				}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
			})

			ginkgo.DescribeTable("all pods terminated and requiring", func(ctx context.Context, cpuReqMilli, podCount int) {
				cpuReqTot := int64((cpuReqMilli * podCount) / 1000)
				ginkgo.By(fmt.Sprintf("needed: %d cores", cpuReqTot))
				if cpuAlloc < cpuReqTot {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU %d allocatable < %d", cpuAlloc, cpuReqTot)
				}

				restartNever := v1.RestartPolicyNever

				var expected []podDesc
				for idx := range podCount {
					pd := podDesc{
						cntName:        fmt.Sprintf("e2e-test-cnt-%d", idx),
						podName:        fmt.Sprintf("e2e-test-pod-%d", idx),
						cpuRequest:     cpuReqMilli,
						mainCntCommand: []string{"sh", "-c", "/bin/true"},
						restartPolicy:  &restartNever,
					}
					expected = append(expected, pd)
					pod := makePodResourcesTestPod(pd)
					pod = e2epod.NewPodClient(f).Create(ctx, pod)
					pod, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
					framework.ExpectNoError(err)
					podMap[string(pod.UID)] = pod
					err = e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "Pod Succeeded", time.Minute*2, testutils.PodSucceeded)
					framework.ExpectNoError(err)
				}
				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)

				var cli kubeletpodresourcesv1.PodResourcesListerClient
				cli, podresConn, err = podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)

				// After all pods terminate, PodResources should eventually stop reporting them.
				// gomega.Consistently was used here, but CI showed transient failures like:
				// "Failed after 0.001s.: received 1 pods, expected 0" immediately after termination due to lags
				// Use Eventually to allow the system to converge before asserting emptiness.
				gomega.Eventually(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					// all but the last pod will have inconsistent resource reporting.
					// See: https://github.com/kubernetes/kubernetes/issues/132020#issuecomment-2921478368
					// this is why we introduced the FeatureGate in the first place
					return matchPodDescWithResourcesNamesOnly(expected, found)
				}).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())
				// for more robust check, also do a Consistently check
				gomega.Consistently(func(ctx context.Context) error {
					found, err := getPodResourcesValues(ctx, cli)
					if err != nil {
						return err
					}
					return matchPodDescWithResourcesNamesOnly(expected, found)
				}).WithContext(ctx).WithTimeout(10 * time.Second).WithPolling(2 * time.Second).Should(gomega.Succeed())

			},
				ginkgo.Entry("cpu and mem single", context.TODO(), 1000, 1),
				ginkgo.Entry("cpu and mem multi", context.TODO(), 1000, 3),
				ginkgo.Entry("mem only single", context.TODO(), 200, 1),
				ginkgo.Entry("mem only multi", context.TODO(), 200, 3),
			)
		})
	})

	ginkgo.Context("with a topology-unaware device plugin, which reports resources w/o hardware topology", func() {
		ginkgo.Context("with CPU manager Static policy", func() {
			ginkgo.BeforeEach(func(ctx context.Context) {
				// this is a very rough check. We just want to rule out system that does NOT have enough resources
				_, cpuAlloc, _ := getLocalNodeCPUDetails(ctx, f)

				if cpuAlloc < minCoreCount {
					e2eskipper.Skipf("Skipping CPU Manager tests since the CPU allocatable < %d", minCoreCount)
				}
			})

			// empty context to apply kubelet config changes
			ginkgo.Context("", func() {
				tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
					// Set the CPU Manager policy to static.
					initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)

					// Set the CPU Manager reconcile period to 1 second.
					initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}

					cpus := reservedSystemCPUs.String()
					framework.Logf("configurePodResourcesInKubelet: using reservedSystemCPUs=%q", cpus)
					initialConfig.ReservedSystemCPUs = cpus
				})

				ginkgo.It("should return proper podresources the same as before the restart of kubelet", func(ctx context.Context) {
					dpPod := setupSampleDevicePluginOrFail(ctx, f)
					ginkgo.DeferCleanup(teardownSampleDevicePluginOrFail, f, dpPod)

					waitForTopologyUnawareResources(ctx, f)

					endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
					framework.ExpectNoError(err, "LocalEndpoint() failed err: %v", err)

					cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
					framework.ExpectNoError(err, "GetV1Client() failed err: %v", err)
					defer func() {
						framework.ExpectNoError(conn.Close())
					}()

					ginkgo.By("checking List and resources topology unaware resource should be without topology")

					allocatableResponse, _ := cli.GetAllocatableResources(ctx, &kubeletpodresourcesv1.AllocatableResourcesRequest{})
					for _, dev := range allocatableResponse.GetDevices() {
						if dev.ResourceName != defaultTopologyUnawareResourceName {
							continue
						}
						gomega.Expect(dev.Topology).To(gomega.BeNil(), "Topology is expected to be empty for topology unaware resources")
					}

					desc := podDesc{
						podName:        "pod-01",
						cntName:        "cnt-01",
						resourceName:   defaultTopologyUnawareResourceName,
						resourceAmount: 1,
						cpuRequest:     1000,
					}

					tpd := newTestPodData()
					tpd.createPodsForTest(ctx, f, []podDesc{
						desc,
					})

					expectPodResources(ctx, cli, []podDesc{desc})

					ginkgo.By("Restarting Kubelet")
					restartKubelet(ctx, true)

					// we need to wait for the node to be reported ready before we can safely query
					// the podresources endpoint again. Otherwise we will have false negatives.
					ginkgo.By("Wait for node to be ready")
					waitForTopologyUnawareResources(ctx, f)

					expectPodResources(ctx, cli, []podDesc{desc})
					tpd.deletePodsForTest(ctx, f)
				})
			})
		})
	})

	f.Context("when querying /metrics", f.WithNodeConformance(), func() {
		ginkgo.BeforeEach(func(ctx context.Context) {
			// ensure APIs have been called at least once
			endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
			framework.ExpectNoError(err, "LocalEndpoint() failed err %v", err)

			cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
			framework.ExpectNoError(err, "GetV1Client() failed err %v", err)
			defer func() {
				framework.ExpectNoError(conn.Close())
			}()

			_, err = cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{})
			framework.ExpectNoError(err, "List() failed err %v", err)

			_, err = cli.GetAllocatableResources(ctx, &kubeletpodresourcesv1.AllocatableResourcesRequest{})
			framework.ExpectNoError(err, "GetAllocatableResources() failed err %v", err)

			desc := podDesc{
				podName: "pod-01",
				cntName: "cnt-01",
			}
			tpd := newTestPodData()
			tpd.createPodsForTest(ctx, f, []podDesc{
				desc,
			})
			expectPodResources(ctx, cli, []podDesc{desc})

			expected := []podDesc{}

			resp := podresourcesGetWithRetry(ctx, cli, f.Namespace.Name, "pod-01")
			podResourceList := []*kubeletpodresourcesv1.PodResources{resp.GetPodResources()}
			res := convertToMap(podResourceList)
			err = matchPodDescWithResources(expected, res)
			framework.ExpectNoError(err, "matchPodDescWithResources() failed err %v", err)
			tpd.deletePodsForTest(ctx, f)
		})

		ginkgo.It("should report the values for the podresources metrics", func(ctx context.Context) {
			// we updated the kubelet config in BeforeEach, so we can assume we start fresh.
			// being [Serial], we can also assume noone else but us is running pods.
			ginkgo.By("Checking the value of the podresources metrics")

			matchResourceMetrics := gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
				"kubelet_pod_resources_endpoint_requests_total": gstruct.MatchAllElements(nodeID, gstruct.Elements{
					"": timelessSampleAtLeast(1),
				}),
				"kubelet_pod_resources_endpoint_requests_list": gstruct.MatchAllElements(nodeID, gstruct.Elements{
					"": timelessSampleAtLeast(1),
				}),
				"kubelet_pod_resources_endpoint_requests_get_allocatable": gstruct.MatchAllElements(nodeID, gstruct.Elements{
					"": timelessSampleAtLeast(1),
				}),
				"kubelet_pod_resources_endpoint_requests_get": gstruct.MatchAllElements(nodeID, gstruct.Elements{
					"": timelessSampleAtLeast(1),
				}),
				// not checking errors: the calls don't have non-catastrophic (e.g. out of memory) error conditions yet.
			})

			ginkgo.By("Giving the Kubelet time to start up and produce metrics")
			gomega.Eventually(ctx, getPodResourcesMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
			ginkgo.By("Ensuring the metrics match the expectations a few more times")
			gomega.Consistently(ctx, getPodResourcesMetrics, 1*time.Minute, 15*time.Second).Should(matchResourceMetrics)
		})
	})

	framework.Context("with the builtin rate limit values", framework.WithFlaky(), func() {
		ginkgo.It("should hit throttling when calling podresources List in a tight loop", func(ctx context.Context) {
			// ensure APIs have been called at least once
			endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
			framework.ExpectNoError(err, "LocalEndpoint() failed err %v", err)

			ginkgo.By("Connecting to the kubelet endpoint")
			cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
			framework.ExpectNoError(err, "GetV1Client() failed err %v", err)
			defer func() {
				framework.ExpectNoError(conn.Close())
			}()

			tries := podresources.DefaultQPS * 2 // This should also be greater than DefaultBurstTokens
			errs := []error{}

			ginkgo.By(fmt.Sprintf("Issuing %d List() calls in a tight loop", tries))
			startTime := time.Now()
			for range tries {
				_, err = cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{})
				errs = append(errs, err)
			}
			elapsed := time.Since(startTime)

			ginkgo.By(fmt.Sprintf("Checking return codes for %d List() calls in %v", tries, elapsed))

			framework.ExpectNoError(errs[0], "the first List() call unexpectedly failed with %v", errs[0])
			// we would expect (burst) successes and then (tries-burst) errors on a clean test environment running with
			// enough CPU power. CI is usually harsher. So we relax constraints, expecting at least _a_ failure, while
			// we are likely to get much more. But we can't predict yet how more we should expect, so we prefer to relax
			// constraints than to risk flakes at this stage.
			errLimitExceededCount := 0
			for _, err := range errs[1:] {
				if errors.Is(err, apisgrpc.ErrorLimitExceeded) {
					errLimitExceededCount++
				}
			}
			gomega.Expect(errLimitExceededCount).ToNot(gomega.BeZero(), "never hit the rate limit trying %d calls in %v", tries, elapsed)

			framework.Logf("got %d/%d rate limit errors, at least one needed, the more the better", errLimitExceededCount, tries)

			// this is not needed for this test. We're done. But we need to play nice with *other* tests which may run just after,
			// and which need to query the API. If they run "too fast", they can still be throttled because the throttling period
			// is not exhausted yet, yielding false negatives, leading to flakes.
			// We can't reset the period for the rate limit, we just wait "long enough" to make sure we absorb the burst
			// and other queries are not rejected because happening to soon
			ginkgo.By("Cooling down to reset the podresources API rate limit")
			time.Sleep(5 * time.Second)
		})
	})
})

func requireLackOfSRIOVDevices() {
	if sriovdevCount, err := countSRIOVDevices(); err != nil || sriovdevCount > 0 {
		e2eskipper.Skipf("this test is meant to run on a system with no configured VF from SRIOV device")
	}
}

func getOnlineCPUs() (cpuset.CPUSet, error) {
	onlineCPUList, err := os.ReadFile("/sys/devices/system/cpu/online")
	if err != nil {
		return cpuset.New(), err
	}
	return cpuset.Parse(strings.TrimSpace(string(onlineCPUList)))
}

func setupSampleDevicePluginOrFail(ctx context.Context, f *framework.Framework) *v1.Pod {
	e2enode.WaitForNodeToBeReady(ctx, f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)

	dp := getSampleDevicePluginPod(kubeletdevicepluginv1beta1.DevicePluginPath, "dp")
	dp.Spec.NodeName = framework.TestContext.NodeName

	ginkgo.By("Create the sample device plugin pod")

	dpPod := e2epod.NewPodClient(f).CreateSync(ctx, dp)

	err := e2epod.WaitForPodCondition(ctx, f.ClientSet, dpPod.Namespace, dpPod.Name, "Ready", 120*time.Second, testutils.PodRunningReady)
	if err != nil {
		framework.Logf("Sample Device Pod %v took too long to enter running/ready: %v", dp.Name, err)
	}
	framework.ExpectNoError(err, "WaitForPodCondition() failed err: %v", err)

	return dpPod
}

func teardownSampleDevicePluginOrFail(ctx context.Context, f *framework.Framework, pod *v1.Pod) {
	gp := int64(0)
	deleteOptions := metav1.DeleteOptions{
		GracePeriodSeconds: &gp,
	}
	ginkgo.By(fmt.Sprintf("Delete sample device plugin pod %s/%s", pod.Namespace, pod.Name))
	err := f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, deleteOptions)

	framework.ExpectNoError(err, "Failed to delete Pod %v in Namespace %v", pod.Name, pod.Namespace)
	waitForAllContainerRemoval(ctx, pod.Name, pod.Namespace)
}

func waitForTopologyUnawareResources(ctx context.Context, f *framework.Framework) {
	ginkgo.By(fmt.Sprintf("Waiting for %q resources to become available on the local node", defaultTopologyUnawareResourceName))

	gomega.Eventually(ctx, func(ctx context.Context) bool {
		node := getLocalNode(ctx, f)
		resourceAmount := e2enode.CountSampleDeviceAllocatable(node)
		return resourceAmount > 0
	}, 2*time.Minute, framework.Poll).Should(gomega.BeTrueBecause("expected %q resources to be available, got no resources", defaultTopologyUnawareResourceName))
}

func getPodResourcesMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) {
	// we are running out of good names, so we need to be unnecessarily specific to avoid clashes
	ginkgo.By("getting Pod Resources metrics from the metrics API")
	return e2emetrics.GrabKubeletMetricsWithoutProxy(ctx, nodeNameOrIP()+":10255", "/metrics")
}

func timelessSampleAtLeast(lower interface{}) types.GomegaMatcher {
	return gstruct.PointTo(gstruct.MatchAllFields(gstruct.Fields{
		// We already check Metric when matching the Id
		"Metric":    gstruct.Ignore(),
		"Value":     gomega.BeNumerically(">=", lower),
		"Timestamp": gstruct.Ignore(),
		"Histogram": gstruct.Ignore(),
	}))
}

func getPodResourcesFromList(ctx context.Context, cli kubeletpodresourcesv1.PodResourcesListerClient, podName, podNamespace string) (*kubeletpodresourcesv1.PodResources, error) {
	resp, err := cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{})
	if err != nil {
		return nil, err
	}

	for _, pr := range resp.GetPodResources() {
		if pr.GetName() == podName && pr.GetNamespace() == podNamespace {
			return pr, nil
		}
	}
	return nil, fmt.Errorf("pod %s/%s not found in List() response", podNamespace, podName)
}

func sanitizeMemory(mem []*kubeletpodresourcesv1.ContainerMemory) {
	// We sort memory blocks to ensure deterministic ordering of memory types
	// (since they can be returned in non-deterministic orders from map traversals)
	// and clear Topology info because NUMA node layouts are hardware-dependent and
	// not part of the API List vs Get consistency checks.
	//nolint:modernize // keep sort.Slice for compatibility with supported Go versions
	sort.Slice(mem, func(i, j int) bool {
		return mem[i].GetMemoryType() < mem[j].GetMemoryType()
	})
	for _, m := range mem {
		m.Topology = nil
	}
}

func preparePodResourcesListVsGet(pr *kubeletpodresourcesv1.PodResources) *kubeletpodresourcesv1.PodResources {
	if pr == nil {
		return nil
	}

	out := proto.Clone(pr).(*kubeletpodresourcesv1.PodResources)

	// sort containers by name.
	//nolint:modernize // keep sort.Slice for compatibility with supported Go versions
	sort.Slice(out.Containers, func(i, j int) bool {
		return out.Containers[i].GetName() < out.Containers[j].GetName()
	})

	// sort pod-level CPU IDs.
	//nolint:modernize // keep sort.Slice for compatibility with supported Go versions
	sort.Slice(out.CpuIds, func(i, j int) bool {
		return out.CpuIds[i] < out.CpuIds[j]
	})

	sanitizeMemory(out.Memory)

	for _, c := range out.Containers {
		// sort CPU IDs.
		//nolint:modernize // keep sort.Slice for compatibility with supported Go versions
		sort.Slice(c.CpuIds, func(i, j int) bool {
			return c.CpuIds[i] < c.CpuIds[j]
		})

		// sort devices by resource name, then sort device IDs.
		//nolint:modernize // keep sort.Slice for compatibility with supported Go versions
		sort.Slice(c.Devices, func(i, j int) bool {
			return c.Devices[i].GetResourceName() < c.Devices[j].GetResourceName()
		})
		for _, d := range c.Devices {
			//nolint:modernize // keep sort.Strings for compatibility with supported Go versions
			sort.Strings(d.DeviceIds)
			// Topology isn't part of the List/Get consistency check.
			d.Topology = nil
		}

		sanitizeMemory(c.Memory)
		c.DynamicResources = nil
	}

	return out
}

func comparePodResourcesListVsGet(listPR, getPR *kubeletpodresourcesv1.PodResources) error {
	if listPR == nil || getPR == nil {
		return fmt.Errorf("nil PodResources: list=%v get=%v", listPR, getPR)
	}
	if listPR.GetName() != getPR.GetName() || listPR.GetNamespace() != getPR.GetNamespace() {
		return fmt.Errorf("pod identity mismatch: list=%s/%s get=%s/%s",
			listPR.GetNamespace(), listPR.GetName(), getPR.GetNamespace(), getPR.GetName())
	}

	lPR := preparePodResourcesListVsGet(listPR)
	gPR := preparePodResourcesListVsGet(getPR)

	if proto.Equal(lPR, gPR) {
		return nil
	}

	jsonOpts := protojson.MarshalOptions{UseProtoNames: true, EmitUnpopulated: true}
	ljson, lerr := jsonOpts.Marshal(lPR)
	gjson, gerr := jsonOpts.Marshal(gPR)
	if lerr == nil && gerr == nil {
		if diff := cmp.Diff(string(ljson), string(gjson)); diff != "" {
			return fmt.Errorf("List() vs Get() PodResources mismatch (-list +get):\n%s", diff)
		}
	}

	return fmt.Errorf("List() vs Get() PodResources mismatch")
}

func expectListAndGetConsistent(ctx context.Context, cli kubeletpodresourcesv1.PodResourcesListerClient, podName, podNamespace string) {
	gomega.Eventually(func(ctx context.Context) error {
		// retry in case of kubelet/podresources restarts.
		return withPodResourcesV1Client(ctx, func(c kubeletpodresourcesv1.PodResourcesListerClient) error {
			listPR, err := getPodResourcesFromList(ctx, c, podName, podNamespace)
			if err != nil {
				return err
			}
			getResp, err := c.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{
				PodName:      podName,
				PodNamespace: podNamespace,
			})
			if err != nil {
				return err
			}
			return comparePodResourcesListVsGet(listPR, getResp.GetPodResources())
		})
	}).
		WithContext(ctx).
		WithPolling(5 * time.Second).
		WithTimeout(1 * time.Minute).
		Should(gomega.Succeed())
}

// withPodResourcesV1Client dials the kubelet podresources unix socket, executes fn, and closes the conn.
// intended to be used from Eventually() to avoid holding a stale connection across kubelet restarts.
func withPodResourcesV1Client(
	ctx context.Context,
	fn func(cli kubeletpodresourcesv1.PodResourcesListerClient) error,
) error {
	endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
	if err != nil {
		return err
	}

	cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
	if err != nil {
		return err
	}

	defer func() { _ = conn.Close() }()

	return fn(cli)
}

// waitForPodResourcesV1Serving blocks until the PodResources v1 endpoint is serving requests.
// gRPC server may temporarily refuse connections even after pods become Ready,
// due to kubelet restarts.
// CI log: failed [FAILED] Expected Get to succeed with the feature gate enabled: rpc error: code = Canceled desc = grpc: the client connection is closing.
func waitForPodResourcesV1Serving(ctx context.Context) {
	var lastErr error
	gomega.Eventually(func() error {
		lastErr = withPodResourcesV1Client(ctx, func(cli kubeletpodresourcesv1.PodResourcesListerClient) error {
			reqCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			_, err := cli.List(reqCtx, &kubeletpodresourcesv1.ListPodResourcesRequest{})
			return err
		})
		return lastErr
	}).WithTimeout(1*time.Minute).WithPolling(1*time.Second).Should(
		gomega.Succeed(),
		"PodResources endpoint did not become ready (last err: %v)", lastErr,
	)
}

func makeMixPodWithPodLevelResources(podName, podCPURequest, containerCPURequest, memRequest string) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "exclusive-container",
					Image: busyboxImage,
					Resources: v1.ResourceRequirements{
						Requests: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse(containerCPURequest),
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
						Limits: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse(containerCPURequest),
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
					},
					Command: []string{"sh", "-c", "sleep 1d"},
				},
				{
					Name:  "shared-container",
					Image: busyboxImage,
					Resources: v1.ResourceRequirements{
						Requests: v1.ResourceList{
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
						Limits: v1.ResourceList{
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
					},
					Command: []string{"sh", "-c", "sleep 1d"},
				},
			},
			Resources: &v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse(podCPURequest),
					v1.ResourceMemory: resource.MustParse(memRequest),
				},
				Limits: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse(podCPURequest),
					v1.ResourceMemory: resource.MustParse(memRequest),
				},
			},
		},
	}
}

func configureStaticResourceManagers(initialConfig *kubeletconfig.KubeletConfiguration, reservedCPUs string) {
	initialConfig.CPUManagerPolicy = string(cpumanager.PolicyStatic)
	initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second}
	initialConfig.ReservedSystemCPUs = reservedCPUs

	initialConfig.MemoryManagerPolicy = string(memorymanager.PolicyTypeStatic)
	initialConfig.ReservedMemory = []kubeletconfig.MemoryReservation{
		{
			NumaNode: 0,
			Limits: v1.ResourceList{
				v1.ResourceMemory: resource.MustParse("1100Mi"),
			},
		},
	}
	if initialConfig.SystemReserved == nil {
		initialConfig.SystemReserved = map[string]string{}
	}
	initialConfig.SystemReserved[string(v1.ResourceMemory)] = "500Mi"
	if initialConfig.KubeReserved == nil {
		initialConfig.KubeReserved = map[string]string{}
	}
	initialConfig.KubeReserved[string(v1.ResourceMemory)] = "500Mi"
	if initialConfig.EvictionHard == nil {
		initialConfig.EvictionHard = map[string]string{}
	}
	initialConfig.EvictionHard["memory.available"] = "100Mi"
	if initialConfig.FeatureGates == nil {
		initialConfig.FeatureGates = make(map[string]bool)
	}
	initialConfig.FeatureGates[string(features.PodLevelResources)] = true
	initialConfig.FeatureGates[string(features.PodLevelResourceManagers)] = true
}

var _ = SIGDescribe("Pod Resources API Pod Level Resources", framework.WithSerial(), feature.PodResourcesAPI, feature.PodLevelResources, feature.PodLevelResourceManagers, framework.WithFeatureGate(features.PodLevelResources), framework.WithFeatureGate(features.PodLevelResourceManagers), func() {
	f := framework.NewDefaultFramework("podresources-pod-level-resources-test")
	addBeforeEachForCleaningUpPods(f)
	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged

	var reservedSystemCPUs cpuset.CPUSet

	ginkgo.BeforeEach(func(ctx context.Context) {
		reservedSystemCPUs = cpuset.New(1)
		// Ensure system has enough CPUs (at least 2 cores)
		_, cpuAlloc, _ := getLocalNodeCPUDetails(ctx, f)
		if cpuAlloc < 2 {
			e2eskipper.Skipf("Skipping tests since the CPU allocatable < 2 cores")
		}
	})

	ginkgo.Context("with CPU manager Static policy", func() {
		ginkgo.Context("when the topology manager scope is 'pod'", func() {
			tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
				configureStaticResourceManagers(initialConfig, reservedSystemCPUs.String())
				initialConfig.TopologyManagerPolicy = string(topologymanager.PolicyRestricted)
				initialConfig.TopologyManagerScope = string(topologymanager.PodTopologyScope)
			})

			ginkgo.It("should return pod-level CPU and Memory, container-level CPU and Memory, and empty container-level CPU and Memory for shared container in a guaranteed pod", func(ctx context.Context) {
				waitForPodResourcesV1Serving(ctx)

				// Create a pod with pod-level resources requesting 2 exclusive CPUs and 200Mi memory,
				// containing an exclusive container requesting 1 CPU and 100Mi memory and a shared
				// container requesting 100Mi memory
				pod := makeMixPodWithPodLevelResources("pod-scope-pod", "2", "1", "200Mi")

				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				defer e2epod.NewPodClient(f).DeleteSync(ctx, pod.Name, metav1.DeleteOptions{}, 2*time.Minute)

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)
				cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)
				defer func() { framework.ExpectNoError(conn.Close()) }()

				// Fetch and match
				gomega.Eventually(ctx, func(ctx context.Context) error {
					resp, err := cli.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{
						PodName:      pod.Name,
						PodNamespace: pod.Namespace,
					})
					if err != nil {
						return err
					}

					podRes := resp.GetPodResources()
					if len(podRes.GetCpuIds()) != 2 {
						return fmt.Errorf("expected 2 pod-level CPUs, got %v", podRes.GetCpuIds())
					}
					if len(podRes.GetMemory()) != 1 {
						return fmt.Errorf("expected 1 pod-level memory block, got %v", podRes.GetMemory())
					}
					if podRes.GetMemory()[0].GetSize() != 200*1024*1024 {
						return fmt.Errorf("expected 200Mi pod-level memory block size, got %d", podRes.GetMemory()[0].GetSize())
					}

					var foundExclusive, foundShared bool
					for _, cRes := range podRes.GetContainers() {
						if cRes.GetName() == "exclusive-container" {
							foundExclusive = true
							if len(cRes.GetCpuIds()) != 1 {
								return fmt.Errorf("expected 1 container-level CPU for exclusive-container, got %v", cRes.GetCpuIds())
							}
							if len(cRes.GetMemory()) != 1 {
								return fmt.Errorf("expected 1 container-level memory block for exclusive-container, got %v", cRes.GetMemory())
							}
							if cRes.GetMemory()[0].GetSize() != 100*1024*1024 {
								return fmt.Errorf("expected 100Mi container-level memory block size for exclusive-container, got %d", cRes.GetMemory()[0].GetSize())
							}
						}
						if cRes.GetName() == "shared-container" {
							foundShared = true
							if len(cRes.GetCpuIds()) > 0 {
								return fmt.Errorf("expected empty container-level CPUs for shared-container, got %v", cRes.GetCpuIds())
							}
							if len(cRes.GetMemory()) > 0 {
								return fmt.Errorf("expected empty container-level memory for shared-container, got %v", cRes.GetMemory())
							}
						}
					}
					if !foundExclusive || !foundShared {
						return fmt.Errorf("exclusive-container or shared-container resources not found")
					}
					return nil
				}).WithTimeout(1 * time.Minute).WithPolling(5 * time.Second).Should(gomega.Succeed())

				expectListAndGetConsistent(ctx, cli, pod.Name, pod.Namespace)
			})
		})

		ginkgo.Context("when the topology manager scope is 'container'", func() {
			tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {
				configureStaticResourceManagers(initialConfig, reservedSystemCPUs.String())
				initialConfig.TopologyManagerPolicy = string(topologymanager.PolicyRestricted)
				initialConfig.TopologyManagerScope = string(topologymanager.ContainerTopologyScope)
			})

			ginkgo.It("should return empty pod-level CPU and Memory, container-level CPU and Memory, and empty container-level CPU and Memory for shared container in a guaranteed pod", func(ctx context.Context) {
				waitForPodResourcesV1Serving(ctx)

				// Create a pod with pod-level resources requesting 2 exclusive CPUs and 200Mi memory,
				// containing an exclusive container requesting 1 CPU and 100Mi memory and a shared
				// container requesting 100Mi memory
				pod := makeMixPodWithPodLevelResources("container-scope-pod", "2", "1", "200Mi")

				ginkgo.By("creating the test pod")
				pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
				defer e2epod.NewPodClient(f).DeleteSync(ctx, pod.Name, metav1.DeleteOptions{}, 2*time.Minute)

				endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
				framework.ExpectNoError(err)
				cli, conn, err := podresources.GetV1Client(ctx, endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
				framework.ExpectNoError(err)
				defer func() { framework.ExpectNoError(conn.Close()) }()

				// Fetch and match
				gomega.Eventually(ctx, func(ctx context.Context) error {
					resp, err := cli.Get(ctx, &kubeletpodresourcesv1.GetPodResourcesRequest{
						PodName:      pod.Name,
						PodNamespace: pod.Namespace,
					})
					if err != nil {
						return err
					}

					podRes := resp.GetPodResources()
					if len(podRes.GetCpuIds()) > 0 {
						return fmt.Errorf("expected empty pod-level CPU, got %v", podRes.GetCpuIds())
					}
					if len(podRes.GetMemory()) > 0 {
						return fmt.Errorf("expected empty pod-level memory, got %v", podRes.GetMemory())
					}

					var foundExclusive, foundShared bool
					for _, cRes := range podRes.GetContainers() {
						if cRes.GetName() == "exclusive-container" {
							foundExclusive = true
							if len(cRes.GetCpuIds()) != 1 {
								return fmt.Errorf("expected 1 container-level CPU for exclusive-container, got %v", cRes.GetCpuIds())
							}
							if len(cRes.GetMemory()) != 1 {
								return fmt.Errorf("expected 1 container-level memory block for exclusive-container, got %v", cRes.GetMemory())
							}
							if cRes.GetMemory()[0].GetSize() != 100*1024*1024 {
								return fmt.Errorf("expected 100Mi container-level memory block size for exclusive-container, got %d", cRes.GetMemory()[0].GetSize())
							}
						}
						if cRes.GetName() == "shared-container" {
							foundShared = true
							if len(cRes.GetCpuIds()) > 0 {
								return fmt.Errorf("expected empty container-level CPUs for shared-container, got %v", cRes.GetCpuIds())
							}
							if len(cRes.GetMemory()) > 0 {
								return fmt.Errorf("expected empty container-level memory for shared-container, got %v", cRes.GetMemory())
							}
						}
					}
					if !foundExclusive || !foundShared {
						return fmt.Errorf("exclusive-container or shared-container resources not found")
					}
					return nil
				}).WithTimeout(1 * time.Minute).WithPolling(5 * time.Second).Should(gomega.Succeed())

				expectListAndGetConsistent(ctx, cli, pod.Name, pod.Namespace)
			})
		})
	})
})