test/e2e/node/pod_resize.go - master - githubmirror/kubernetes

test/e2e/node/pod_resize.go
1 562 строки70 KB
Natasha Sarkar
e2e: tests covering memory-backed volume resize scenarios
21 июл 2026, 02:47
21 июл 2026, 02:4707a3ac5
/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"context"
	"errors"
	"fmt"
	"strconv"
	"time"

	v1 "k8s.io/api/core/v1"
	schedulingv1 "k8s.io/api/scheduling/v1"
	apiequality "k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	clientset "k8s.io/client-go/kubernetes"
	helpers "k8s.io/component-helpers/resource"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/test/e2e/common/node/framework/cgroups"
	"k8s.io/kubernetes/test/e2e/common/node/framework/podresize"
	"k8s.io/kubernetes/test/e2e/framework"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"

	"github.com/onsi/ginkgo/v2"
	"github.com/onsi/gomega"
	gomegatypes "github.com/onsi/gomega/types"
)

func doPodResizeResourceQuotaTests(f *framework.Framework) {
	originalContainers := []podresize.ResizableContainerInfo{
		{
			Name:      "c1",
			Resources: &cgroups.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"},
		},
	}

	ginkgo.BeforeEach(func(ctx context.Context) {
		resourceQuota := v1.ResourceQuota{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "resize-resource-quota",
				Namespace: f.Namespace.Name,
			},
			Spec: v1.ResourceQuotaSpec{
				Hard: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("800m"),
					v1.ResourceMemory: resource.MustParse("800Mi"),
				},
			},
		}

		ginkgo.By("Creating a ResourceQuota")
		_, rqErr := f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, &resourceQuota, metav1.CreateOptions{})
		framework.ExpectNoError(rqErr, "failed to create resource quota")
		// pod creation using this quota will fail until the quota status is populated, so we need to wait to
		// prevent races with the resourcequota controller
		ginkgo.By("Waiting for ResourceQuota status to populate")
		quotaStatusErr := waitForResourceQuota(ctx, f.ClientSet, f.Namespace.Name, resourceQuota.Name)
		framework.ExpectNoError(quotaStatusErr, "resource quota status failed to populate")
	})

	ginkgo.DescribeTable("pod-resize-resource-quota-test",
		func(ctx context.Context, desiredContainers []podresize.ResizableContainerInfo, expectedContainers []podresize.ResizableContainerInfo, wantError string) {
			tStamp := strconv.Itoa(time.Now().Nanosecond())
			testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, originalContainers, nil)
			testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1)
			testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, originalContainers, nil)
			testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2)

			ginkgo.By("creating pods")
			podClient := e2epod.NewPodClient(f)
			newPods := podClient.CreateBatch(ctx, []*v1.Pod{testPod1, testPod2})

			ginkgo.By("verifying initial pod resources, and policy are as expected")
			podresize.VerifyPodResources(newPods[0], originalContainers, nil)

			ginkgo.By("patching pod for resize with resource-quota")
			patchString := podresize.MakeResizePatch(originalContainers, desiredContainers, nil, nil)

			if wantError == "" {
				patchedPod, pErr := f.ClientSet.CoreV1().Pods(newPods[0].Namespace).Patch(ctx,
					newPods[0].Name, types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize")
				framework.ExpectNoError(pErr, "failed to patch pod for resize")

				expected := podresize.UpdateExpectedContainerRestarts(ctx, patchedPod, expectedContainers)
				ginkgo.By("verifying pod resources are as expected post patch, pre-actuation")
				podresize.VerifyPodResources(patchedPod, expected, nil)

				ginkgo.By("waiting for resize to be actuated")
				resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, newPods[0], expected)
				podresize.ExpectPodResized(ctx, f, resizedPod, expected)

				ginkgo.By("verifying pod resources after resize")
				podresize.VerifyPodResources(resizedPod, expected, nil)

			} else {
				var patchedPod *v1.Pod
				framework.ExpectNoError(framework.Gomega().
					// Use Eventually because we need to wait for the resource-quota controller to sync.
					Eventually(ctx, func(ctx context.Context) error {
						var pErr error
						patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPods[0].Namespace).Patch(ctx,
							newPods[0].Name, types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize")
						return pErr
					}).
					WithTimeout(f.Timeouts.PodStart).
					Should(gomega.MatchError(gomega.ContainSubstring(wantError))))

				expected := podresize.UpdateExpectedContainerRestarts(ctx, patchedPod, expectedContainers)
				ginkgo.By("verifying pod patched for resize with error remains unchanged")
				patchedPod, pErrEx2 := podClient.Get(ctx, newPods[0].Name, metav1.GetOptions{})
				framework.ExpectNoError(pErrEx2, "failed to get pod post failed resize")
				podresize.VerifyPodResources(patchedPod, expected, nil)
				framework.ExpectNoError(podresize.VerifyPodStatusResources(patchedPod, expected))
			}
		},

		ginkgo.Entry("exceed maximum CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "600m", CPULim: "600m", MemReq: "400Mi", MemLim: "400Mi"},
				},
			},
			originalContainers,
			"exceeded quota: resize-resource-quota, requested: cpu=300m, used: cpu=600m, limited: cpu=800m",
		),

		ginkgo.Entry("exceed maximum Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "250m", CPULim: "250m", MemReq: "750Mi", MemLim: "750Mi"},
				},
			},
			originalContainers,
			"exceeded quota: resize-resource-quota, requested: memory=450Mi, used: memory=600Mi, limited: memory=800Mi",
		),

		ginkgo.Entry("exceed maximum CPU and Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "600m", CPULim: "600m", MemReq: "750Mi", MemLim: "750Mi"},
				},
			},
			originalContainers,
			"exceeded quota: resize-resource-quota, requested: cpu=300m,memory=450Mi, used: cpu=600m,memory=600Mi, limited: cpu=800m,memory=800Mi",
		),

		ginkgo.Entry("valid increase of CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "300Mi", MemLim: "300Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "300Mi", MemLim: "300Mi"},
				},
			},
			"",
		),

		ginkgo.Entry("valid increase of Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			"",
		),

		ginkgo.Entry("valid increase for both CPU and Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"},
				},
			},
			"",
		),
	)
}

func doPodResizeLimitRangerTests(f *framework.Framework) {
	originalContainers := []podresize.ResizableContainerInfo{
		{
			Name:      "c1",
			Resources: &cgroups.ContainerResources{CPUReq: "300m", CPULim: "300m", MemReq: "300Mi", MemLim: "300Mi"},
		},
	}

	ginkgo.BeforeEach(func(ctx context.Context) {
		ginkgo.By("Creating a LimitRanger")
		lr := &v1.LimitRange{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "resize-limit-ranger",
				Namespace: f.Namespace.Name,
			},
			Spec: v1.LimitRangeSpec{
				Limits: []v1.LimitRangeItem{
					{
						Type: v1.LimitTypeContainer,
						Max: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse("500m"),
							v1.ResourceMemory: resource.MustParse("500Mi"),
						},
						Min: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse("50m"),
							v1.ResourceMemory: resource.MustParse("50Mi"),
						},
						Default: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse("100m"),
							v1.ResourceMemory: resource.MustParse("100Mi"),
						},
						DefaultRequest: v1.ResourceList{
							v1.ResourceCPU:    resource.MustParse("50m"),
							v1.ResourceMemory: resource.MustParse("50Mi"),
						},
					},
				},
			},
		}
		_, lrErr := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Create(context.Background(), lr, metav1.CreateOptions{})
		framework.ExpectNoError(lrErr, "failed to create limit ranger")

		ginkgo.By("Fetching the LimitRange to ensure it has proper values")
		gotLr, lrErr := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Get(ctx, lr.Name, metav1.GetOptions{})
		framework.ExpectNoError(lrErr)

		if !apiequality.Semantic.DeepEqual(lr.Spec.Limits, gotLr.Spec.Limits) {
			framework.Failf("retrieved LimitRange does not match created one: got %v; expected %v", gotLr.Spec.Limits, lr.Spec.Limits)
		}
	})

	ginkgo.DescribeTable("pod-resize-limit-ranger-test",
		func(ctx context.Context, desiredContainers []podresize.ResizableContainerInfo, expectedContainers []podresize.ResizableContainerInfo, wantErrors []string) {
			tStamp := strconv.Itoa(time.Now().Nanosecond())
			testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, originalContainers, nil)
			testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1)
			testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, originalContainers, nil)
			testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2)

			ginkgo.By("creating pods")
			podClient := e2epod.NewPodClient(f)
			newPods := podClient.CreateBatch(ctx, []*v1.Pod{testPod1, testPod2})

			ginkgo.By("verifying initial pod resources, and policy are as expected")
			podresize.VerifyPodResources(newPods[0], originalContainers, nil)

			ginkgo.By("patching pod for resize with limit-ranger")
			patchString := podresize.MakeResizePatch(originalContainers, desiredContainers, nil, nil)

			if len(wantErrors) == 0 {
				patchedPod, pErr := f.ClientSet.CoreV1().Pods(newPods[0].Namespace).Patch(ctx,
					newPods[0].Name, types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize")
				framework.ExpectNoError(pErr, "failed to patch pod for resize")

				expected := podresize.UpdateExpectedContainerRestarts(ctx, patchedPod, expectedContainers)
				ginkgo.By("verifying pod resources are as expected post patch, pre-actuation")
				podresize.VerifyPodResources(patchedPod, expected, nil)

				ginkgo.By("waiting for resize to be actuated")
				resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, newPods[0], expected)
				podresize.ExpectPodResized(ctx, f, resizedPod, expected)

				ginkgo.By("verifying pod resources after resize")
				podresize.VerifyPodResources(resizedPod, expected, nil)

			} else {
				var patchedPod *v1.Pod
				var errMatchers []gomegatypes.GomegaMatcher
				for _, wantErr := range wantErrors {
					errMatchers = append(errMatchers, gomega.ContainSubstring(wantErr))
				}

				framework.ExpectNoError(framework.Gomega().
					// Use Eventually because we need to wait for the limit-ranger controller to sync.
					Eventually(ctx, func(ctx context.Context) error {
						var pErr error
						patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPods[0].Namespace).Patch(ctx,
							newPods[0].Name, types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{}, "resize")
						return pErr
					}).
					WithTimeout(f.Timeouts.PodStart).
					Should(gomega.MatchError(gomega.And(errMatchers...))))

				expected := podresize.UpdateExpectedContainerRestarts(ctx, patchedPod, expectedContainers)
				ginkgo.By("verifying pod patched for resize with error remains unchanged")
				patchedPod, pErrEx2 := podClient.Get(ctx, newPods[0].Name, metav1.GetOptions{})
				framework.ExpectNoError(pErrEx2, "failed to get pod post failed resize")
				podresize.VerifyPodResources(patchedPod, expected, nil)
				framework.ExpectNoError(podresize.VerifyPodStatusResources(patchedPod, expected))
			}
		},

		ginkgo.Entry("exceed maximum CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "600m", CPULim: "600m", MemReq: "250Mi", MemLim: "250Mi"},
				},
			},
			originalContainers,
			[]string{"forbidden: maximum cpu usage per Container is 500m, but limit is 600m"},
		),

		ginkgo.Entry("exceed maximum Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "250m", CPULim: "250m", MemReq: "750Mi", MemLim: "750Mi"},
				},
			},
			originalContainers,
			[]string{"forbidden: maximum memory usage per Container is 500Mi, but limit is 750Mi"},
		),

		ginkgo.Entry("exceed maximum Memory and CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "600m", CPULim: "600m", MemReq: "600Mi", MemLim: "600Mi"},
				},
			},
			originalContainers,
			[]string{"maximum cpu usage per Container is 500m, but limit is 600m", "maximum memory usage per Container is 500Mi, but limit is 600Mi"},
		),

		ginkgo.Entry("request below min CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "10m", CPULim: "10m", MemReq: "400Mi", MemLim: "400Mi"},
				},
			},
			originalContainers,
			[]string{"forbidden: minimum cpu usage per Container is 50m, but request is 10m"},
		),

		ginkgo.Entry("request below min Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "250m", CPULim: "250m", MemReq: "10Mi", MemLim: "10Mi"},
				},
			},
			originalContainers,
			[]string{"forbidden: minimum memory usage per Container is 50Mi, but request is 10Mi"},
		),

		ginkgo.Entry("request below min CPU and min Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "10m", CPULim: "10m", MemReq: "10Mi", MemLim: "10Mi"},
				},
			},
			originalContainers,
			[]string{"minimum cpu usage per Container is 50m, but request is 10m", "minimum memory usage per Container is 50Mi, but request is 10Mi"},
		),

		ginkgo.Entry("valid increase of CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "300Mi", MemLim: "300Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "300Mi", MemLim: "300Mi"},
				},
			},
			nil,
		),

		ginkgo.Entry("valid increase of Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			nil,
		),

		ginkgo.Entry("valid increase of CPU and Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "400Mi", MemLim: "400Mi"},
				},
			},
			nil,
		),

		ginkgo.Entry("valid decrease of Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "400m", CPULim: "400m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			nil,
		),

		ginkgo.Entry("valid decrease of CPU",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "350m", CPULim: "350m", MemReq: "350Mi", MemLim: "350Mi"},
				},
			},
			nil,
		),

		ginkgo.Entry("valid decrease of CPU and Memory",
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "250m", CPULim: "250m", MemReq: "250Mi", MemLim: "250Mi"},
				},
			},
			[]podresize.ResizableContainerInfo{
				{
					Name:      "c1",
					Resources: &cgroups.ContainerResources{CPUReq: "250m", CPULim: "250m", MemReq: "250Mi", MemLim: "250Mi"},
				},
			},
			nil,
		),
	)
}

func doPodResizeSchedulerTests(f *framework.Framework) {
	ginkgo.It("pod-resize-scheduler-tests", func(ctx context.Context) {
		podClient := e2epod.NewPodClient(f)
		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
		framework.ExpectNoError(err, "failed to get running nodes")
		gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty())
		framework.Logf("Found %d schedulable nodes", len(nodes.Items))

		ginkgo.By("Find node CPU resources available for allocation!")
		node := nodes.Items[0]
		nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")
		framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.",
			node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue())

		//
		// Scheduler focused pod resize E2E test case #1:
		//     1. Create pod1 and pod2 on node such that pod1 has enough CPU to be scheduled, but pod2 does not.
		//     2. Resize pod2 down so that it fits on the node and can be scheduled.
		//     3. Verify that pod2 gets scheduled and comes up and running.
		//
		testPod1CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/2, resource.DecimalSI)
		testPod2CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue(), resource.DecimalSI)
		testPod2CPUQuantityResized := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/2, resource.DecimalSI)
		framework.Logf("TEST1: testPod1 initial CPU request is '%dm'", testPod1CPUQuantity.MilliValue())
		framework.Logf("TEST1: testPod2 initial CPU request is '%dm'", testPod2CPUQuantity.MilliValue())
		framework.Logf("TEST1: testPod2 resized CPU request is '%dm'", testPod2CPUQuantityResized.MilliValue())

		c1 := []podresize.ResizableContainerInfo{
			{
				Name:      "c1",
				Resources: &cgroups.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()},
			},
		}
		c2 := []podresize.ResizableContainerInfo{
			{
				Name:      "c2",
				Resources: &cgroups.ContainerResources{CPUReq: testPod2CPUQuantity.String(), CPULim: testPod2CPUQuantity.String()},
			},
		}
		patchTestpod2ToFitNode := fmt.Sprintf(`{
				"spec": {
					"containers": [
						{
							"name":      "c2",
							"resources": {"requests": {"cpu": "%dm"}, "limits": {"cpu": "%dm"}}
						}
					]
				}
			}`, testPod2CPUQuantityResized.MilliValue(), testPod2CPUQuantityResized.MilliValue())

		tStamp := strconv.Itoa(time.Now().Nanosecond())
		testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, c1, nil)
		testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1)
		testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, c2, nil)
		testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2)
		e2epod.SetNodeAffinity(&testPod1.Spec, node.Name)
		e2epod.SetNodeAffinity(&testPod2.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("TEST1: Create pod '%s' that fits the node '%s'", testPod1.Name, node.Name))
		testPod1 = podClient.CreateSync(ctx, testPod1)
		gomega.Expect(testPod1.Status.Phase).To(gomega.Equal(v1.PodRunning))
		gomega.Expect(testPod1.Generation).To(gomega.BeEquivalentTo(1))

		ginkgo.By(fmt.Sprintf("TEST1: Create pod '%s' that won't fit node '%s' with pod '%s' on it", testPod2.Name, node.Name, testPod1.Name))
		testPod2 = podClient.Create(ctx, testPod2)
		err = e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, testPod2.Name, testPod2.Namespace)
		framework.ExpectNoError(err)
		gomega.Expect(testPod2.Status.Phase).To(gomega.Equal(v1.PodPending))
		gomega.Expect(testPod2.Generation).To(gomega.BeEquivalentTo(1))

		ginkgo.By(fmt.Sprintf("TEST1: Resize pod '%s' to fit in node '%s'", testPod2.Name, node.Name))
		testPod2, pErr := f.ClientSet.CoreV1().Pods(testPod2.Namespace).Patch(ctx,
			testPod2.Name, types.StrategicMergePatchType, []byte(patchTestpod2ToFitNode), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(pErr, "failed to patch pod for resize")
		gomega.Expect(testPod2.Generation).To(gomega.BeEquivalentTo(2))

		ginkgo.By(fmt.Sprintf("TEST1: Verify that pod '%s' is running after resize", testPod2.Name))
		framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, testPod2))

		// Scheduler focused pod resize E2E test case #2
		//     1. With pod1 + pod2 running on node above, create pod3 that requests more CPU than available, verify pending.
		//     2. Resize pod1 down so that pod3 gets room to be scheduled.
		//     3. Verify that pod3 is scheduled and running.
		//
		nodeAllocatableCPU2, nodeAvailableCPU2, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")
		framework.Logf("TEST2: Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.",
			node.Name, nodeAllocatableCPU2.MilliValue(), nodeAvailableCPU2.MilliValue())
		testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU2.MilliValue()+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI)
		testPod1CPUQuantityResized := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/3, resource.DecimalSI)
		framework.Logf("TEST2: testPod1 MilliCPUs after resize '%dm'", testPod1CPUQuantityResized.MilliValue())

		c3 := []podresize.ResizableContainerInfo{
			{
				Name:      "c3",
				Resources: &cgroups.ContainerResources{CPUReq: testPod3CPUQuantity.String(), CPULim: testPod3CPUQuantity.String()},
			},
		}
		patchTestpod1ToMakeSpaceForPod3 := fmt.Sprintf(`{
				"spec": {
					"containers": [
						{
							"name":      "c1",
							"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
						}
					]
				}
			}`, testPod1CPUQuantityResized.MilliValue(), testPod1CPUQuantityResized.MilliValue())

		tStamp = strconv.Itoa(time.Now().Nanosecond())
		testPod3 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, c3, nil)
		testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3)
		e2epod.SetNodeAffinity(&testPod3.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("TEST2: Create testPod3 '%s' that cannot fit node '%s' due to insufficient CPU.", testPod3.Name, node.Name))
		testPod3 = podClient.Create(ctx, testPod3)
		p3Err := e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, testPod3.Name, testPod3.Namespace)
		framework.ExpectNoError(p3Err, "failed to create pod3 or pod3 did not become pending!")
		gomega.Expect(testPod3.Status.Phase).To(gomega.Equal(v1.PodPending))
		gomega.Expect(testPod3.Generation).To(gomega.BeEquivalentTo(1))

		ginkgo.By(fmt.Sprintf("TEST2: Resize pod '%s' to make enough space for pod '%s'", testPod1.Name, testPod3.Name))
		testPod1, p1Err := f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(ctx,
			testPod1.Name, types.StrategicMergePatchType, []byte(patchTestpod1ToMakeSpaceForPod3), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(p1Err, "failed to patch pod for resize")
		gomega.Expect(testPod1.Generation).To(gomega.BeEquivalentTo(2))

		ginkgo.By(fmt.Sprintf("TEST2: Verify pod '%s' is running after successfully resizing pod '%s'", testPod3.Name, testPod1.Name))
		framework.Logf("TEST2: Pod '%s' CPU requests '%dm'", testPod1.Name, testPod1.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())
		framework.Logf("TEST2: Pod '%s' CPU requests '%dm'", testPod2.Name, testPod2.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())
		framework.Logf("TEST2: Pod '%s' CPU requests '%dm'", testPod3.Name, testPod3.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())
		framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, testPod3))

		// Scheduler focused pod resize E2E test case #3
		//     1. With pod1 + pod2 + pod3 running on node above, attempt to scale up pod1 to requests more CPU than available, verify deferred.
		//     2. Delete pod2 + pod3 to make room for pod3.
		//     3. Verify that pod1 resize has completed.
		//     4. Attempt to scale up pod1 to request more cpu than the node has, verify infeasible.
		patchTestpod1ExceedNodeAvailable := fmt.Sprintf(`{
			"spec": {
				"containers": [
					{
						"name":      "c1",
						"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
					}
				]
			}
		}`, testPod1CPUQuantity.MilliValue(), testPod1CPUQuantity.MilliValue())

		testPod1CPUExceedingAllocatable := resource.NewMilliQuantity(nodeAllocatableCPU.MilliValue()*2, resource.DecimalSI)
		patchTestpod1ExceedNodeAllocatable := fmt.Sprintf(`{
			"spec": {
				"containers": [
					{
						"name":      "c1",
						"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
					}
				]
			}
		}`, testPod1CPUExceedingAllocatable.MilliValue(), testPod1CPUExceedingAllocatable.MilliValue())

		ginkgo.By(fmt.Sprintf("TEST3: Resize pod '%s' exceed node available capacity", testPod1.Name))
		testPod1, p1Err = f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(ctx,
			testPod1.Name, types.StrategicMergePatchType, []byte(patchTestpod1ExceedNodeAvailable), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(p1Err, "failed to patch pod for resize")
		gomega.Expect(testPod1.Generation).To(gomega.BeEquivalentTo(3))
		waitForPodDeferred(ctx, f, testPod1)

		ginkgo.By("deleting pods 2 and 3")
		e2epod.DeletePodsWithWait(ctx, f.ClientSet, []*v1.Pod{testPod2, testPod3})

		ginkgo.By(fmt.Sprintf("TEST3: Verify pod '%s' is resized successfully after pod deletion '%s' and '%s", testPod1.Name, testPod2.Name, testPod3.Name))
		expected := []podresize.ResizableContainerInfo{
			{
				Name:         "c1",
				Resources:    &cgroups.ContainerResources{CPUReq: testPod1CPUQuantity.String(), CPULim: testPod1CPUQuantity.String()},
				RestartCount: testPod1.Status.ContainerStatuses[0].RestartCount,
			},
		}
		resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod1, expected)
		podresize.ExpectPodResized(ctx, f, resizedPod, expected)

		ginkgo.By(fmt.Sprintf("TEST3: Resize pod '%s' to exceed the node capacity", testPod1.Name))
		_, p1Err = f.ClientSet.CoreV1().Pods(testPod1.Namespace).Patch(ctx,
			testPod1.Name, types.StrategicMergePatchType, []byte(patchTestpod1ExceedNodeAllocatable), metav1.PatchOptions{}, "resize")

		gomega.Expect(p1Err).To(gomega.HaveOccurred(), "expected patching pod to fail due to exceeding node allocatable")

		var statusErr *apierrors.StatusError
		gomega.Expect(errors.As(p1Err, &statusErr)).To(gomega.BeTrueBecause("Expected error to be a StatusError, but got: %v", p1Err))

		gomega.Expect(statusErr.ErrStatus.Reason).To(gomega.Equal(metav1.StatusReasonForbidden))
		gomega.Expect(statusErr.Error()).To(gomega.ContainSubstring("node didn't have enough allocatable resources"))
		gomega.Expect(statusErr.ErrStatus.Details.Causes[0].Type).To(gomega.Equal(metav1.CauseType("NodeCapacity")))

		ginkgo.By("deleting pod 1")
		delErr1 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod1)
		framework.ExpectNoError(delErr1, "failed to delete pod %s", testPod1.Name)
	})
}

func doPodResizeRetryDeferredTests(f *framework.Framework) {
	ginkgo.It("pod-resize-retry-deferred-test-1", func(ctx context.Context) {
		// Deferred resize E2E test case #1 (2-Pod Scenario):
		//     1. Assume Node CPU is divided into 10 "parts".
		//     2. Create Pod A (5 parts) and Pod B (3 parts).
		//     3. Verify remaining free CPU is < 3 parts (cannot fit Pod B's requested increase).
		//     4. Resize Pod B to 6 parts. Verify the resize is deferred.
		//     5. Resize Pod A down to 2 parts (frees up exactly 3 parts).
		//     6. Verify Pod B's deferred resize has now completed successfully.

		podClient := e2epod.NewPodClient(f)
		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
		framework.ExpectNoError(err, "failed to get running nodes")
		gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty())
		framework.Logf("Found %d schedulable nodes", len(nodes.Items))

		ginkgo.By("Find node CPU resources available for allocation!")
		node := nodes.Items[0]
		nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")
		framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.",
			node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue())

		// Conceptually divide the available CPU into 10 parts.
		onePartCPU := nodeAvailableCPU.MilliValue() / 10
		framework.Logf("Node '%s' currently available MilliCPUs = %dm (1 part = %dm)", node.Name, nodeAvailableCPU.MilliValue(), onePartCPU)

		// Pod A: 5 parts, Pod B: 3 parts
		podACPU := resource.NewMilliQuantity(onePartCPU*5, resource.DecimalSI)
		podBCPU := resource.NewMilliQuantity(onePartCPU*3, resource.DecimalSI)

		framework.Logf("Pod A initial CPU request is '%dm'", podACPU.MilliValue())
		framework.Logf("Pod B initial CPU request is '%dm'", podBCPU.MilliValue())

		cA := []podresize.ResizableContainerInfo{
			{Name: "ca", Resources: &cgroups.ContainerResources{CPUReq: podACPU.String(), CPULim: podACPU.String()}},
		}
		cB := []podresize.ResizableContainerInfo{
			{Name: "cb", Resources: &cgroups.ContainerResources{CPUReq: podBCPU.String(), CPULim: podBCPU.String()}},
		}

		tStamp := strconv.Itoa(time.Now().Nanosecond())

		podA := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod-a", tStamp, cA, nil)
		podA = e2epod.MustMixinRestrictedPodSecurity(podA)
		e2epod.SetNodeAffinity(&podA.Spec, node.Name)

		podB := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod-b", tStamp, cB, nil)
		podB = e2epod.MustMixinRestrictedPodSecurity(podB)
		e2epod.SetNodeAffinity(&podB.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create Pod A '%s' and Pod B '%s'", podA.Name, podB.Name))

		podA = podClient.CreateSync(ctx, podA)
		gomega.Expect(podA.Status.Phase).To(gomega.Equal(v1.PodRunning))
		gomega.Expect(podA.Generation).To(gomega.BeEquivalentTo(1))

		podB = podClient.CreateSync(ctx, podB)
		gomega.Expect(podB.Status.Phase).To(gomega.Equal(v1.PodRunning))
		gomega.Expect(podB.Generation).To(gomega.BeEquivalentTo(1))

		ginkgo.By("Verify remaining CPU is less than the 3 parts required for Pod B's resize")
		_, nodeAvailableCPU2, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")

		threePartsCPU := onePartCPU * 3
		framework.Logf("Remaining CPU = %dm. Pod B needs %dm more to resize.", nodeAvailableCPU2.MilliValue(), threePartsCPU)
		gomega.Expect(nodeAvailableCPU2.MilliValue()).To(gomega.BeNumerically("<", threePartsCPU),
			"Available CPU should be strictly less than 3 parts to ensure Pod B's resize is deferred")

		// Calculations for Resizing
		// Pod B: Resize to 6 parts (needs 3 more parts)
		podBResizedCPU := resource.NewMilliQuantity(onePartCPU*6, resource.DecimalSI)
		// Pod A: Resize to 2 parts (drops 3 parts, freeing them up)
		podAResizedCPU := resource.NewMilliQuantity(onePartCPU*2, resource.DecimalSI)

		patchPodBToDeferred := fmt.Sprintf(`{
        	"spec": {
            	"containers": [
                	{
                    	"name":      "cb",
                    	"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
                	}
            	]
        	}
    	}`, podBResizedCPU.MilliValue(), podBResizedCPU.MilliValue())

		patchPodAToMakeSpace := fmt.Sprintf(`{
        	"spec": {
           		"containers": [
                	{
                    	"name":      "ca",
                    	"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
                	}
            	]
        	}
    	}`, podAResizedCPU.MilliValue(), podAResizedCPU.MilliValue())

		ginkgo.By(fmt.Sprintf("Resize Pod B '%s' to 6 parts (will be deferred due to insufficient CPU)", podB.Name))
		podB, err = f.ClientSet.CoreV1().Pods(podB.Namespace).Patch(ctx,
			podB.Name, types.StrategicMergePatchType, []byte(patchPodBToDeferred), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch Pod B for resize")
		waitForPodDeferred(ctx, f, podB)

		ginkgo.By(fmt.Sprintf("Resize Pod A '%s' down to 2 parts, freeing up exactly 3 parts for Pod B", podA.Name))
		podA, err = f.ClientSet.CoreV1().Pods(podA.Namespace).Patch(ctx,
			podA.Name, types.StrategicMergePatchType, []byte(patchPodAToMakeSpace), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch Pod A to free up space")
		gomega.Expect(podA.Generation).To(gomega.BeEquivalentTo(2))

		ginkgo.By("Verify Pod B successfully actuates the deferred resize after space is freed")
		expected := []podresize.ResizableContainerInfo{
			{
				Name:      "cb",
				Resources: &cgroups.ContainerResources{CPUReq: podBResizedCPU.String(), CPULim: podBResizedCPU.String()},
			},
		}
		resizedPodB := podresize.WaitForPodResizeActuation(ctx, f, podClient, podB, expected)
		podresize.ExpectPodResized(ctx, f, resizedPodB, expected)

		ginkgo.By("Cleaning up test pods")
		e2epod.DeletePodsWithWait(ctx, f.ClientSet, []*v1.Pod{podA, podB})
	})

	ginkgo.It("pod-resize-retry-deferred-test-2", func(ctx context.Context) {
		// Deferred resize E2E test case #2:
		//	   1. Create 5 pods on the node, where the first one has 2/3 of the node allocatable CPU,
		//        and the remaining ones each have 1/16 of the node allocatable CPU.
		//     2. Resize all remaining pods to request 2/3 of the node allocatable CPU, verify deferred.
		//     3. Delete the first pod, verify the pod with the highest priority has its resize accepted.
		//     4. Repeat step 3 until all but the last pod has been deleted.

		podClient := e2epod.NewPodClient(f)
		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
		framework.ExpectNoError(err, "failed to get running nodes")
		gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty())
		framework.Logf("Found %d schedulable nodes", len(nodes.Items))

		ginkgo.By("Find node CPU and memory resources available for allocation!")
		node := nodes.Items[0]

		nodeAllocatableCPU, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")
		framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.",
			node.Name, nodeAllocatableCPU.MilliValue(), nodeAvailableCPU.MilliValue())

		majorityCPUQuantity := resource.NewMilliQuantity(2*nodeAvailableCPU.MilliValue()/3, resource.DecimalSI)
		littleCPUQuantity := resource.NewMilliQuantity(nodeAvailableCPU.MilliValue()/16, resource.DecimalSI)
		containerWithMajorityCPU := []podresize.ResizableContainerInfo{
			{
				Name:      "c",
				Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()},
			},
		}
		containerWithLittleCPU := []podresize.ResizableContainerInfo{
			{
				Name:      "c",
				Resources: &cgroups.ContainerResources{CPUReq: littleCPUQuantity.String(), CPULim: littleCPUQuantity.String()},
			},
		}
		containerWithLittleCPUGuaranteedQoS := []podresize.ResizableContainerInfo{
			{
				Name: "c",
				Resources: &cgroups.ContainerResources{
					CPUReq: littleCPUQuantity.String(),
					CPULim: littleCPUQuantity.String(),
					MemReq: "100Mi",
					MemLim: "100Mi",
				},
			},
		}

		tStamp := strconv.Itoa(time.Now().Nanosecond())
		testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, containerWithMajorityCPU, nil)
		testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1)
		e2epod.SetNodeAffinity(&testPod1.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create pod '%s' with 2/3 of the node cpu", testPod1.Name))
		testPod1 = podClient.CreateSync(ctx, testPod1)
		gomega.Expect(testPod1.Status.Phase).To(gomega.Equal(v1.PodRunning))

		// Create pod2 with 1/16 of the node allocatable CPU, with high priority based on priority class.
		testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, containerWithLittleCPU, nil)
		testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2)
		pc, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{
			ObjectMeta: metav1.ObjectMeta{
				Name: fmt.Sprintf("testpod2-priority-class-%s", testPod1.Namespace),
			},
			Value: 1000,
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err)
		defer func() {
			framework.ExpectNoError(f.ClientSet.SchedulingV1().PriorityClasses().Delete(ctx, pc.Name, metav1.DeleteOptions{}))
		}()

		testPod2.Spec.PriorityClassName = pc.Name
		e2epod.SetNodeAffinity(&testPod2.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu and high priority class", testPod2.Name))
		testPod2 = podClient.CreateSync(ctx, testPod2)
		gomega.Expect(testPod2.Status.Phase).To(gomega.Equal(v1.PodRunning))

		// Create pod3 with 1/16 of the node allocatable CPU, that is a "guaranteed" pod (all others should be "burstable").
		testPod3 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, containerWithLittleCPUGuaranteedQoS, nil)
		testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3)
		e2epod.SetNodeAffinity(&testPod3.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu and guaranteed qos", testPod3.Name))
		testPod3 = podClient.CreateSync(ctx, testPod3)
		gomega.Expect(testPod3.Status.Phase).To(gomega.Equal(v1.PodRunning))

		// Create pod4 with 1/16 of the node allocatable CPU.
		testPod4 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod4", tStamp, containerWithLittleCPU, nil)
		testPod4 = e2epod.MustMixinRestrictedPodSecurity(testPod4)
		e2epod.SetNodeAffinity(&testPod4.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu", testPod4.Name))
		testPod4 = podClient.CreateSync(ctx, testPod4)
		gomega.Expect(testPod4.Status.Phase).To(gomega.Equal(v1.PodRunning))

		// Create pod5 with 1/16 of the node allocatable CPU.
		testPod5 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod5", tStamp, containerWithLittleCPU, nil)
		testPod5 = e2epod.MustMixinRestrictedPodSecurity(testPod5)
		e2epod.SetNodeAffinity(&testPod5.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create pod '%s' with 1/16 of the node cpu", testPod5.Name))
		testPod5 = podClient.CreateSync(ctx, testPod5)
		gomega.Expect(testPod5.Status.Phase).To(gomega.Equal(v1.PodRunning))

		patchTestPod := fmt.Sprintf(`{
			"spec": {
				"containers": [
					{
						"name":      "c",
						"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
					}
				]
			}
		}`, majorityCPUQuantity.MilliValue(), majorityCPUQuantity.MilliValue())

		// Resize requests are done in an arbitrary order, to verify that the priority based on priority class
		// or qos class takes precedent over the order of the requests.

		// Attempt pod4 resize request to 2/3 of the node allocatable CPU, verify deferred.
		ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod4.Name))
		testPod4, err = f.ClientSet.CoreV1().Pods(testPod4.Namespace).Patch(ctx,
			testPod4.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch pod for resize")
		waitForPodDeferred(ctx, f, testPod4)

		// Attempt pod3 resize request to 2/3 of the node allocatable CPU, verify deferred.
		ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod3.Name))
		testPod3, err = f.ClientSet.CoreV1().Pods(testPod3.Namespace).Patch(ctx,
			testPod3.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch pod for resize")
		waitForPodDeferred(ctx, f, testPod3)

		// Attempt pod2 resize request to 2/3 of the node allocatable CPU, verify deferred.
		ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod2.Name))
		testPod2, err = f.ClientSet.CoreV1().Pods(testPod2.Namespace).Patch(ctx,
			testPod2.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch pod for resize")
		waitForPodDeferred(ctx, f, testPod2)

		// Attempt pod5 resize request to 2/3 of the node allocatable CPU, verify deferred.
		ginkgo.By(fmt.Sprintf("Resize pod '%s'", testPod5.Name))
		testPod5, err = f.ClientSet.CoreV1().Pods(testPod5.Namespace).Patch(ctx,
			testPod5.Name, types.StrategicMergePatchType, []byte(patchTestPod), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch pod for resize")
		waitForPodDeferred(ctx, f, testPod5)

		// Delete pod1. Verify pod2's resize has completed, while the others are still deferred.
		ginkgo.By("deleting pod1")
		delErr1 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod1)
		framework.ExpectNoError(delErr1, "failed to delete pod %s", testPod1.Name)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod '%s' deleted", testPod2.Name, testPod1.Name))
		expected := []podresize.ResizableContainerInfo{
			{
				Name:      "c",
				Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()},
			},
		}
		resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expected)
		podresize.ExpectPodResized(ctx, f, resizedPod, expected)
		waitForPodDeferred(ctx, f, testPod3)
		waitForPodDeferred(ctx, f, testPod4)
		waitForPodDeferred(ctx, f, testPod5)

		// Delete pod2. Verify pod3's resize has completed, while the others are still deferred.
		ginkgo.By("deleting pod2")
		delErr2 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod2)
		framework.ExpectNoError(delErr2, "failed to delete pod %s", testPod2.Name)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod '%s' deleted", testPod3.Name, testPod2.Name))
		expected = []podresize.ResizableContainerInfo{
			{
				Name: "c",
				Resources: &cgroups.ContainerResources{
					CPUReq: majorityCPUQuantity.String(),
					CPULim: majorityCPUQuantity.String(),
					MemReq: "100Mi",
					MemLim: "100Mi",
				},
			},
		}
		resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expected)
		podresize.ExpectPodResized(ctx, f, resizedPod, expected)
		waitForPodDeferred(ctx, f, testPod4)
		waitForPodDeferred(ctx, f, testPod5)

		// Delete pod3. Verify pod4's resize has completed, while the others are still deferred.
		ginkgo.By("deleting pod3")
		delErr3 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod3)
		framework.ExpectNoError(delErr3, "failed to delete pod %s", testPod3.Name)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod '%s' deleted", testPod4.Name, testPod3.Name))
		expected = []podresize.ResizableContainerInfo{
			{
				Name:      "c",
				Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()},
			},
		}
		resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expected)
		podresize.ExpectPodResized(ctx, f, resizedPod, expected)
		waitForPodDeferred(ctx, f, testPod5)

		// Delete pod4. Verify pod5's resize has completed.
		ginkgo.By("deleting pod4")
		delErr4 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod4)
		framework.ExpectNoError(delErr4, "failed to delete pod %s", testPod4.Name)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod '%s' deleted", testPod5.Name, testPod4.Name))
		expected = []podresize.ResizableContainerInfo{
			{
				Name:      "c",
				Resources: &cgroups.ContainerResources{CPUReq: majorityCPUQuantity.String(), CPULim: majorityCPUQuantity.String()},
			},
		}
		resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod5, expected)
		podresize.ExpectPodResized(ctx, f, resizedPod, expected)

		ginkgo.By("deleting pod5")
		delErr5 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod5)
		framework.ExpectNoError(delErr5, "failed to delete pod %s", testPod5.Name)
	})

	ginkgo.It("pod-resize-retry-deferred-test-3", func(ctx context.Context) {
		// Deferred resize E2E test case #3:
		// 	   1. Create pod1, pod2, pod3, and pod4 on node, each starting with 1/4 of the node's allocatable CPU and and 1/5 the memory.
		// 	   2. Resize pod2 to request more CPU than available (1/3), along with a decrease in memory (1/24), verify the resize is deferred.
		//	   3. Resize pod3 to request more memory than available (2/3), along with a decrease in CPU (1/24), verify the resize is deferred.
		// 	   4. Resize pod4 to request more CPU than available (5/8), verify the resize is deferred.
		//	   5. The deferred resizes above are chosen carefully such that:
		//	      - deletion of pod1 should make room for pod2's resize (but not pod3 or pod4).
		//	      - pod2's resize should make room for pod3's resize (but not pod4).
		//	      - pod3's resize should make room for pod4's resize.
		//     6. Delete pod1, verify the chain of deferred resizes is actuated.

		podClient := e2epod.NewPodClient(f)
		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
		framework.ExpectNoError(err, "failed to get running nodes")
		gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty())
		framework.Logf("Found %d schedulable nodes", len(nodes.Items))

		ginkgo.By("Find node CPU and memory resources available for allocation!")
		node := nodes.Items[0]

		nodeAllocatableCPU, initNodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")
		framework.Logf("Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.",
			node.Name, nodeAllocatableCPU.MilliValue(), initNodeAvailableCPU.MilliValue())

		nodeAllocatableMem, initNodeAvailableMem, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceMemory)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")
		framework.Logf("Node '%s': NodeAllocatable Memory = %d. Memory currently available to allocate = %d.",
			node.Name, nodeAllocatableMem.Value(), initNodeAvailableMem.Value())

		initialCPUQuantity := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/4, resource.DecimalSI)
		initialMemoryQuantity := resource.NewQuantity(initNodeAvailableMem.Value()/5, resource.DecimalSI)
		framework.Logf("initial CPU request is '%dm'", initialCPUQuantity.MilliValue())
		framework.Logf("initial Memory request is '%d'", initialMemoryQuantity.Value())

		c := []podresize.ResizableContainerInfo{
			{
				Name: "c",
				Resources: &cgroups.ContainerResources{
					CPUReq: initialCPUQuantity.String(),
					MemReq: initialMemoryQuantity.String(),
				},
			},
		}

		tStamp := strconv.Itoa(time.Now().Nanosecond())
		testPod1 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod1", tStamp, c, nil)
		testPod1 = e2epod.MustMixinRestrictedPodSecurity(testPod1)
		e2epod.SetNodeAffinity(&testPod1.Spec, node.Name)

		testPod2 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod2", tStamp, c, nil)
		testPod2 = e2epod.MustMixinRestrictedPodSecurity(testPod2)
		e2epod.SetNodeAffinity(&testPod2.Spec, node.Name)

		testPod3 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod3", tStamp, c, nil)
		testPod3 = e2epod.MustMixinRestrictedPodSecurity(testPod3)
		e2epod.SetNodeAffinity(&testPod3.Spec, node.Name)

		testPod4 := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod4", tStamp, c, nil)
		testPod4 = e2epod.MustMixinRestrictedPodSecurity(testPod4)
		e2epod.SetNodeAffinity(&testPod4.Spec, node.Name)

		testPods := []*v1.Pod{testPod1, testPod2, testPod3, testPod4}
		ginkgo.By(fmt.Sprintf("Create pods that fits the node '%s'", node.Name))
		podClient.CreateBatch(ctx, testPods)

		testPod2CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/3, resource.DecimalSI)
		testPod2MemoryQuantityResized := resource.NewQuantity(initNodeAvailableMem.Value()/24, resource.DecimalSI)

		testPod3CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/24, resource.DecimalSI)
		testPod3MemoryQuantityResized := resource.NewQuantity(2*initNodeAvailableMem.Value()/3, resource.DecimalSI)

		testPod4CPUQuantityResized := resource.NewMilliQuantity(initNodeAvailableCPU.MilliValue()/2+initNodeAvailableCPU.MilliValue()/8, resource.DecimalSI)
		testPod4MemoryQuantityResized := initialMemoryQuantity

		expectedTestPod2Resized := []podresize.ResizableContainerInfo{
			{
				Name: "c",
				Resources: &cgroups.ContainerResources{
					CPUReq: testPod2CPUQuantityResized.String(),
					MemReq: testPod2MemoryQuantityResized.String(),
				},
			},
		}
		expectedTestPod3Resized := []podresize.ResizableContainerInfo{
			{
				Name: "c",
				Resources: &cgroups.ContainerResources{
					CPUReq: testPod3CPUQuantityResized.String(),
					MemReq: testPod3MemoryQuantityResized.String(),
				},
			},
		}
		expectedTestPod4Resized := []podresize.ResizableContainerInfo{
			{
				Name: "c",
				Resources: &cgroups.ContainerResources{
					CPUReq: testPod4CPUQuantityResized.String(),
					MemReq: testPod4MemoryQuantityResized.String(),
				},
			},
		}

		patchTestPod2ToDeferred := podresize.MakeResizePatch(c, expectedTestPod2Resized, nil, nil)
		patchTestPod3ToDeferred := podresize.MakeResizePatch(c, expectedTestPod3Resized, nil, nil)
		patchTestPod4ToDeferred := podresize.MakeResizePatch(c, expectedTestPod4Resized, nil, nil)

		patches := []string{string(patchTestPod2ToDeferred), string(patchTestPod3ToDeferred), string(patchTestPod4ToDeferred)}

		for i := range patches {
			testPod := testPods[i+1]
			patch := patches[i]
			ginkgo.By(fmt.Sprintf("Resize pod '%s' that cannot fit node due to insufficient CPU or memory", testPod.Name))
			testPod, err = f.ClientSet.CoreV1().Pods(testPod.Namespace).Patch(ctx,
				testPod.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "resize")
			framework.ExpectNoError(err, "failed to patch pod for resize")
			waitForPodDeferred(ctx, f, testPod)
		}

		ginkgo.By("deleting pod 1")
		delErr1 := e2epod.DeletePodWithWait(ctx, f.ClientSet, testPod1)
		framework.ExpectNoError(delErr1, "failed to delete pod %s", testPod1.Name)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod deletion '%s'", testPod2.Name, testPod1.Name))
		resizedPod := podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod2, expectedTestPod2Resized)
		podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod2Resized)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod3.Name, testPod2.Name))
		resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod3, expectedTestPod3Resized)
		podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod3Resized)

		ginkgo.By(fmt.Sprintf("Verify pod '%s' is resized successfully after pod resize '%s'", testPod4.Name, testPod3.Name))
		resizedPod = podresize.WaitForPodResizeActuation(ctx, f, podClient, testPod4, expectedTestPod4Resized)
		podresize.ExpectPodResized(ctx, f, resizedPod, expectedTestPod4Resized)

		ginkgo.By("deleting pods")
		e2epod.DeletePodsWithWait(ctx, f.ClientSet, testPods)
	})
}

func doPodResizeDeferredPreemptionTests(f *framework.Framework) {
	ginkgo.It("preempt a lower-priority pod to satisfy a deferred resize", func(ctx context.Context) {
		podClient := e2epod.NewPodClient(f)
		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
		framework.ExpectNoError(err, "failed to get running nodes")
		gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty())

		node := nodes.Items[0]
		_, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get CPU resources available for allocation")

		availableMillis := nodeAvailableCPU.MilliValue()
		framework.Logf("Node '%s' available CPU = %dm", node.Name, availableMillis)

		// Victim pod: 50% of available CPU, low priority
		victimCPU := resource.NewMilliQuantity(int64(float64(availableMillis)*0.5), resource.DecimalSI)
		// Mid-priority pod: 20% of available CPU, mid priority
		midPriCPU := resource.NewMilliQuantity(int64(float64(availableMillis)*0.2), resource.DecimalSI)
		// Preemptor pod: 20% of available CPU, high priority
		preemptorCPU := resource.NewMilliQuantity(int64(float64(availableMillis)*0.2), resource.DecimalSI)

		victimContainers := []podresize.ResizableContainerInfo{
			{Name: "ca", Resources: &cgroups.ContainerResources{CPUReq: victimCPU.String(), CPULim: victimCPU.String()}},
		}
		midPriContainers := []podresize.ResizableContainerInfo{
			{Name: "cc", Resources: &cgroups.ContainerResources{CPUReq: midPriCPU.String(), CPULim: midPriCPU.String()}},
		}
		preemptorContainers := []podresize.ResizableContainerInfo{
			{Name: "cb", Resources: &cgroups.ContainerResources{CPUReq: preemptorCPU.String(), CPULim: preemptorCPU.String()}},
		}

		tStamp := strconv.Itoa(time.Now().Nanosecond())
		victim := podresize.MakePodWithResizableContainers(f.Namespace.Name, "victim-pod", tStamp, victimContainers, nil)
		victim = e2epod.MustMixinRestrictedPodSecurity(victim)
		e2epod.SetNodeAffinity(&victim.Spec, node.Name)

		midPriPod := podresize.MakePodWithResizableContainers(f.Namespace.Name, "midpri-pod", tStamp, midPriContainers, nil)
		midPriPod = e2epod.MustMixinRestrictedPodSecurity(midPriPod)
		e2epod.SetNodeAffinity(&midPriPod.Spec, node.Name)

		preemptor := podresize.MakePodWithResizableContainers(f.Namespace.Name, "preemptor-pod", tStamp, preemptorContainers, nil)
		preemptor = e2epod.MustMixinRestrictedPodSecurity(preemptor)
		e2epod.SetNodeAffinity(&preemptor.Spec, node.Name)

		// Create Priority Classes
		pcPreemptor, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{
			ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("preemptor-priority-%s", f.Namespace.Name)},
			Value:      1000,
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err)
		defer func() {
			_ = f.ClientSet.SchedulingV1().PriorityClasses().Delete(ctx, pcPreemptor.Name, metav1.DeleteOptions{})
		}()

		pcMid, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{
			ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("midpri-priority-%s", f.Namespace.Name)},
			Value:      500,
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err)
		defer func() {
			_ = f.ClientSet.SchedulingV1().PriorityClasses().Delete(ctx, pcMid.Name, metav1.DeleteOptions{})
		}()

		pcVictim, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{
			ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("victim-priority-%s", f.Namespace.Name)},
			Value:      100,
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err)
		defer func() {
			_ = f.ClientSet.SchedulingV1().PriorityClasses().Delete(ctx, pcVictim.Name, metav1.DeleteOptions{})
		}()

		victim.Spec.PriorityClassName = pcVictim.Name
		midPriPod.Spec.PriorityClassName = pcMid.Name
		preemptor.Spec.PriorityClassName = pcPreemptor.Name

		ginkgo.By(fmt.Sprintf("Patch node '%s' to disable resize preemption initially", node.Name))
		patchDisablePreemption := []byte(`{"spec": {"podPreemptionPolicy": {"disableResizePreemption": ["test-disable-preemption"]}}}`)
		_, err = f.ClientSet.CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, patchDisablePreemption, metav1.PatchOptions{})
		framework.ExpectNoError(err, "failed to patch node to disable resize preemption")
		defer func() {
			patchReset := []byte(`{"spec": {"podPreemptionPolicy": null}}`)
			_, _ = f.ClientSet.CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patchReset, metav1.PatchOptions{})
		}()

		ginkgo.By(fmt.Sprintf("Create victim pod '%s' (50%% CPU), mid-pri pod '%s' (20%% CPU) and preemptor pod '%s' (20%% CPU) in batch", victim.Name, midPriPod.Name, preemptor.Name))
		createdPods := podClient.CreateBatch(ctx, []*v1.Pod{victim, midPriPod, preemptor})
		victim = createdPods[0]
		midPriPod = createdPods[1]
		preemptor = createdPods[2]

		// Resize midPriPod to 45% CPU and preemptor to 50% CPU
		// midPriPod target (45%) + preemptor target (50%) = 95% <= 100% node capacity (midPriPod is NOT evicted)
		midPriTargetCPU := resource.NewMilliQuantity(int64(float64(availableMillis)*0.45), resource.DecimalSI)
		preemptorTargetCPU := resource.NewMilliQuantity(int64(float64(availableMillis)*0.50), resource.DecimalSI)

		patchMidPriToDeferred := fmt.Sprintf(`{
			"spec": {
				"containers": [
					{
						"name":      "cc",
						"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
					}
				]
			}
		}`, midPriTargetCPU.MilliValue(), midPriTargetCPU.MilliValue())
		patchPreemptorToDeferred := fmt.Sprintf(`{
			"spec": {
				"containers": [
					{
						"name":      "cb",
						"resources": {"requests": {"cpu": "%dm"},"limits": {"cpu": "%dm"}}
					}
				]
			}
		}`, preemptorTargetCPU.MilliValue(), preemptorTargetCPU.MilliValue())

		ginkgo.By(fmt.Sprintf("Resize mid-priority pod '%s' to 45%% CPU (exceeds free node capacity, should defer)", midPriPod.Name))
		midPriPod, err = f.ClientSet.CoreV1().Pods(midPriPod.Namespace).Patch(ctx,
			midPriPod.Name, types.StrategicMergePatchType, []byte(patchMidPriToDeferred), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch mid-pri pod for resize")
		waitForPodDeferred(ctx, f, midPriPod)

		ginkgo.By(fmt.Sprintf("Resize high-priority preemptor pod '%s' to 50%% CPU (exceeds free node capacity, should defer)", preemptor.Name))
		preemptor, err = f.ClientSet.CoreV1().Pods(preemptor.Namespace).Patch(ctx,
			preemptor.Name, types.StrategicMergePatchType, []byte(patchPreemptorToDeferred), metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(err, "failed to patch preemptor pod for resize")
		waitForPodDeferred(ctx, f, preemptor)

		ginkgo.By(fmt.Sprintf("Assert that victim pod '%s' is NOT preempted while policy is disabled", victim.Name))
		gomega.Consistently(ctx, func(g gomega.Gomega) {
			p, err := f.ClientSet.CoreV1().Pods(victim.Namespace).Get(ctx, victim.Name, metav1.GetOptions{})
			g.Expect(err).To(gomega.Succeed())
			g.Expect(p.Status.Phase).To(gomega.Equal(v1.PodRunning))
		}, 10*time.Second, 2*time.Second).Should(gomega.Succeed(), "victim pod should remain running while preemption is disabled")

		ginkgo.By(fmt.Sprintf("Remove disable policy from node '%s' to enable resize preemption", node.Name))
		patchReset := []byte(`{"spec": {"podPreemptionPolicy": null}}`)
		_, err = f.ClientSet.CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patchReset, metav1.PatchOptions{})
		framework.ExpectNoError(err, "failed to patch node to enable resize preemption")

		ginkgo.By(fmt.Sprintf("Verify that victim pod '%s' gets preempted/evicted by high-priority preemptor after enabling resize preemption", victim.Name))
		err = e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, victim.Name, victim.Namespace, 5*time.Minute)
		framework.ExpectNoError(err, "victim pod was not deleted/preempted within timeout after policy removal")

		ginkgo.By("Verify that preemptor pod's resize is successfully actuated after preemption")
		expectedPreemptorResized := []podresize.ResizableContainerInfo{
			{
				Name:      "cb",
				Resources: &cgroups.ContainerResources{CPUReq: preemptorTargetCPU.String(), CPULim: preemptorTargetCPU.String()},
			},
		}
		resizedPreemptor := podresize.WaitForPodResizeActuation(ctx, f, podClient, preemptor, expectedPreemptorResized)
		podresize.ExpectPodResized(ctx, f, resizedPreemptor, expectedPreemptorResized)

		ginkgo.By(fmt.Sprintf("Verify that mid-priority pod '%s' is preserved and successfully actuated after victim eviction", midPriPod.Name))
		expectedMidPriResized := []podresize.ResizableContainerInfo{
			{
				Name:      "cc",
				Resources: &cgroups.ContainerResources{CPUReq: midPriTargetCPU.String(), CPULim: midPriTargetCPU.String()},
			},
		}
		resizedMidPri := podresize.WaitForPodResizeActuation(ctx, f, podClient, midPriPod, expectedMidPriResized)
		podresize.ExpectPodResized(ctx, f, resizedMidPri, expectedMidPriResized)
	})
}

var _ = SIGDescribe(framework.WithSerial(), "Pod InPlace Resize Container (scheduler-focused)", framework.WithFeatureGate(features.InPlacePodVerticalScaling), func() {
	f := framework.NewDefaultFramework("pod-resize-scheduler-tests")
	ginkgo.BeforeEach(func(ctx context.Context) {
		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
		framework.ExpectNoError(err)
		if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) {
			e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping")
		}
	})
	doPodResizeSchedulerTests(f)
})

var _ = SIGDescribe(framework.WithSerial(), "Pod InPlace Resize Container (deferred-resizes)", framework.WithFeatureGate(features.InPlacePodVerticalScaling), func() {
	f := framework.NewDefaultFramework("pod-resize-deferred-resize-tests")
	ginkgo.BeforeEach(func(ctx context.Context) {
		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
		framework.ExpectNoError(err)
		if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) {
			e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping")
		}
	})
	doPodResizeRetryDeferredTests(f)
})

var _ = SIGDescribe("Pod InPlace Resize Container (resource-quota)", framework.WithFeatureGate(features.InPlacePodVerticalScaling), func() {
	f := framework.NewDefaultFramework("pod-resize-resource-quota-tests")

	ginkgo.BeforeEach(func(ctx context.Context) {
		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
		framework.ExpectNoError(err)
		if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) {
			e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping")
		}
	})
	doPodResizeResourceQuotaTests(f)
})

var _ = SIGDescribe("Pod InPlace Resize Container (limit-ranger)", framework.WithFeatureGate(features.InPlacePodVerticalScaling), func() {
	f := framework.NewDefaultFramework("pod-resize-limit-ranger-tests")

	ginkgo.BeforeEach(func(ctx context.Context) {
		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
		framework.ExpectNoError(err)
		if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) {
			e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping")
		}
	})
	doPodResizeLimitRangerTests(f)
})

var _ = SIGDescribe(framework.WithSerial(), framework.WithSlow(), "Pod InPlace Resize Container (deferred-resize-preemption)",
	framework.WithFeatureGate(features.InPlacePodVerticalScaling),
	framework.WithFeatureGate(features.InPlacePodVerticalScalingSchedulerPreemption),
	func() {
		f := framework.NewDefaultFramework("pod-resize-deferred-preemption-tests")
		ginkgo.BeforeEach(func(ctx context.Context) {
			node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
			framework.ExpectNoError(err)
			if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) {
				e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping")
			}
		})
		doPodResizeDeferredPreemptionTests(f)
	},
)

func waitForResourceQuota(ctx context.Context, c clientset.Interface, ns, quotaName string) error {
	return framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (v1.ResourceList, error) {
		quota, err := c.CoreV1().ResourceQuotas(ns).Get(ctx, quotaName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		return quota.Status.Used, nil
	})).WithTimeout(framework.PollShortTimeout).ShouldNot(gomega.BeEmpty())
}

func waitForPodDeferred(ctx context.Context, f *framework.Framework, testPod *v1.Pod) {
	framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, testPod.Namespace, testPod.Name, "display pod resize status as deferred", f.Timeouts.PodStart, func(pod *v1.Pod) (bool, error) {
		return helpers.IsPodResizeDeferred(pod), nil
	}))
}

func doPodResizeMemoryVolumeDeferredTests(f *framework.Framework) {
	ginkgo.It("should isolate volume updates when container resource resize is deferred", func(ctx context.Context) {
		podClient := e2epod.NewPodClient(f)
		nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet)
		framework.ExpectNoError(err, "failed to get running nodes")
		gomega.Expect(nodes.Items).ShouldNot(gomega.BeEmpty())
		node := nodes.Items[0]

		_, nodeAvailableCPU, err := e2enode.GetNodeAllocatableAndAvailableQuantities(ctx, f.ClientSet, &node, v1.ResourceCPU)
		framework.ExpectNoError(err, "failed to get available CPU")

		// Conceptually divide the available CPU into 10 parts.
		onePartCPU := nodeAvailableCPU.MilliValue() / 10
		framework.Logf("Node '%s' currently available MilliCPUs = %dm (1 part = %dm)", node.Name, nodeAvailableCPU.MilliValue(), onePartCPU)

		// Pod A (filler pod): 5 parts
		podACPU := resource.NewMilliQuantity(onePartCPU*5, resource.DecimalSI)
		// Pod B (test pod): 3 parts
		podBCPU := resource.NewMilliQuantity(onePartCPU*3, resource.DecimalSI)

		cA := []podresize.ResizableContainerInfo{
			{Name: "ca", Resources: &cgroups.ContainerResources{CPUReq: podACPU.String(), CPULim: podACPU.String()}},
		}
		original := []podresize.ResizableContainerInfo{{
			Name:      "c1",
			Resources: &cgroups.ContainerResources{CPUReq: podBCPU.String(), CPULim: podBCPU.String()},
		}}

		tStamp := strconv.Itoa(time.Now().Nanosecond())
		podA := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod-a", tStamp, cA, nil)
		podA = e2epod.MustMixinRestrictedPodSecurity(podA)
		e2epod.SetNodeAffinity(&podA.Spec, node.Name)

		qty64Mi := resource.MustParse("64Mi")
		qty128Mi := resource.MustParse("128Mi")
		testPod := podresize.MakePodWithResizableContainers(f.Namespace.Name, "testpod-b", tStamp, original, nil)
		testPod.Spec.Volumes = []v1.Volume{
			{
				Name: "mem-vol",
				VolumeSource: v1.VolumeSource{
					EmptyDir: &v1.EmptyDirVolumeSource{
						Medium:    v1.StorageMediumMemory,
						SizeLimit: &qty64Mi,
					},
				},
			},
		}
		testPod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
			{
				Name:      "mem-vol",
				MountPath: "/cache",
			},
		}
		testPod = e2epod.MustMixinRestrictedPodSecurity(testPod)
		e2epod.SetNodeAffinity(&testPod.Spec, node.Name)

		ginkgo.By(fmt.Sprintf("Create Pod A '%s' and Pod B '%s'", podA.Name, testPod.Name))
		podA = podClient.CreateSync(ctx, podA)
		gomega.Expect(podA.Status.Phase).To(gomega.Equal(v1.PodRunning))

		podB := podClient.CreateSync(ctx, testPod)
		gomega.Expect(podB.Status.Phase).To(gomega.Equal(v1.PodRunning))

		ginkgo.By("verifying initial mounted emptyDir volume size via df inside container")
		stdout, _, err := e2epod.ExecCommandInContainerWithFullOutput(f, podB.Name, "c1", "df", "-m", "/cache")
		framework.ExpectNoError(err, "failed to run df")
		gomega.Expect(stdout).To(gomega.ContainSubstring(strconv.FormatInt(qty64Mi.Value()/(1024*1024), 10)))

		initialRestarts := podB.Status.ContainerStatuses[0].RestartCount

		// Pod B: Resize request to 6 parts (needs 3 more parts, which exceeds remaining available CPU)
		podBResizedCPU := resource.NewMilliQuantity(onePartCPU*6, resource.DecimalSI)

		ginkgo.By(fmt.Sprintf("patching Pod B CPU request to %dm (exceeds remaining available, will be deferred) and sizeLimit to %s", podBResizedCPU.MilliValue(), qty128Mi.String()))
		patchBytes := fmt.Appendf(nil, `{
			"spec": {
				"containers": [
					{
						"name": "c1",
						"resources": {
							"requests": {"cpu": "%dm"},
							"limits": {"cpu": "%dm"}
						}
					}
				],
				"volumes": [
					{
						"name": "mem-vol",
						"emptyDir": {
							"sizeLimit": "%s"
						}
					}
				]
			}
		}`, podBResizedCPU.MilliValue(), podBResizedCPU.MilliValue(), qty128Mi.String())

		patchedPod, pErr := f.ClientSet.CoreV1().Pods(podB.Namespace).Patch(ctx, podB.Name,
			types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "resize")
		framework.ExpectNoError(pErr, "failed to patch pod for resize and sizeLimit")

		ginkgo.By("waiting for pod resize to be marked as deferred")
		waitForPodDeferred(ctx, f, patchedPod)

		ginkgo.By("verifying that the volume mount inside the container remains at 64Mi (atomic isolation)")
		stdout, _, err = e2epod.ExecCommandInContainerWithFullOutput(f, patchedPod.Name, "c1", "df", "-m", "/cache")
		framework.ExpectNoError(err, "failed to run df inside container")
		gomega.Expect(stdout).To(gomega.ContainSubstring(strconv.FormatInt(qty64Mi.Value()/(1024*1024), 10)))
		gomega.Expect(stdout).ToNot(gomega.ContainSubstring(strconv.FormatInt(qty128Mi.Value()/(1024*1024), 10)))

		gotPod, getErr := podClient.Get(ctx, patchedPod.Name, metav1.GetOptions{})
		framework.ExpectNoError(getErr)
		gomega.Expect(gotPod.Status.ContainerStatuses[0].RestartCount).To(gomega.Equal(initialRestarts))

		ginkgo.By("deleting pods")
		e2epod.DeletePodsWithWait(ctx, f.ClientSet, []*v1.Pod{podA, patchedPod})
	})
}

var _ = SIGDescribe(framework.WithSerial(), "Pod InPlace Resize Memory-Backed Volume (deferred-resizes)", framework.WithFeatureGate(features.InPlacePodVerticalScalingMemoryBackedVolumes), func() {
	f := framework.NewDefaultFramework("pod-resize-memory-vol-deferred-tests")
	ginkgo.BeforeEach(func(ctx context.Context) {
		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
		framework.ExpectNoError(err)
		if framework.NodeOSDistroIs("windows") || e2enode.IsARM64(node) {
			e2eskipper.Skipf("runtime does not support InPlacePodVerticalScaling -- skipping")
		}
	})
	doPodResizeMemoryVolumeDeferredTests(f)
})