Skip to content

Commit f94d424

Browse files
committed
Migrate OCP-67564: node drain should block when PodDisruptionBudget minAvailable equals 100%
Migrates test from openshift-tests-private to origin. Test validates that node drain is properly blocked when a PodDisruptionBudget has minAvailable=100% with an empty selector. The test: 1. Creates a deployment with 1 replica 2. Creates a PodDisruptionBudget with minAvailable=100% and empty selector 3. Attempts to drain a node 4. Verifies the drain operation is blocked 5. Deletes the PDB and verifies drain succeeds This is a disruptive test marked as informing. Updates: - Add test to test/extended/node/node_e2e/node.go - Add helper functions to test/extended/node/node_utils.go: - GetSingleWorkerNode: retrieves a worker node name - WaitClusterOperatorAvailable: waits for cluster operators to be available - Document test in test/extended/node/README.md Relates: https://issues.redhat.com/browse/OCPBUGS-15035
1 parent 38c4fba commit f94d424

3 files changed

Lines changed: 195 additions & 0 deletions

File tree

test/extended/node/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ This directory contains OpenShift end-to-end tests for node-related features.
1919
- **image_volume.go** - Tests mounting container images as volumes in pods, including subPath and error handling
2020
- **node_swap.go** - Tests default kubelet swap settings (failSwapOn and swapBehavior) and rejection of user overrides
2121
- **zstd_chunked.go** - Tests building and running images with zstd:chunked compression format
22+
- **node_e2e/node.go** - PodDisruptionBudget drain blocking (OCP-67564) - Tests that node drain is blocked when PDB has minAvailable=100% with empty selector [Disruptive] [Lifecycle:informing]
2223

2324
## Directory Structure
2425

test/extended/node/node_e2e/node.go

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,15 @@ import (
99

1010
g "github.com/onsi/ginkgo/v2"
1111
o "github.com/onsi/gomega"
12+
ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
1213

1314
configv1 "github.com/openshift/api/config/v1"
1415
"github.com/openshift/origin/test/extended/imagepolicy"
16+
appsv1 "k8s.io/api/apps/v1"
17+
corev1 "k8s.io/api/core/v1"
18+
policyv1 "k8s.io/api/policy/v1"
1519
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
20+
"k8s.io/apimachinery/pkg/util/intstr"
1621
utilrand "k8s.io/apimachinery/pkg/util/rand"
1722
"k8s.io/apimachinery/pkg/util/wait"
1823
e2e "k8s.io/kubernetes/test/e2e/framework"
@@ -164,6 +169,153 @@ var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager",
164169
e2e.Logf("/dev/fuse mount output: %s", output)
165170
o.Expect(output).To(o.ContainSubstring("fuse"), "dev fuse is not mounted inside pod")
166171
})
172+
173+
//author: bgudi@redhat.com
174+
//migrated from openshift-tests-private
175+
//automates: https://issues.redhat.com/browse/OCPBUGS-15035
176+
g.It("[OTP] node's drain should block when PodDisruptionBudget minAvailable equals 100 percentage and selector is empty [Disruptive] [OCP-67564]", ote.Informing(), func() {
177+
ctx := context.Background()
178+
179+
// Skip on SNO/External topologies where there might not be dedicated worker nodes
180+
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{})
181+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get cluster infrastructure")
182+
if infra.Status.ControlPlaneTopology == "SingleReplica" || infra.Status.ControlPlaneTopology == "External" {
183+
g.Skip("Skipping on SNO/External topology - requires dedicated worker nodes")
184+
}
185+
186+
oc.SetupProject()
187+
namespace := oc.Namespace()
188+
189+
g.By("Create a deployment with 6 replicas")
190+
replicas := int32(6)
191+
deployment := &appsv1.Deployment{
192+
ObjectMeta: metav1.ObjectMeta{
193+
Name: "hello-openshift",
194+
Namespace: namespace,
195+
Labels: map[string]string{
196+
"app": "myapp",
197+
},
198+
},
199+
Spec: appsv1.DeploymentSpec{
200+
Replicas: &replicas,
201+
Selector: &metav1.LabelSelector{
202+
MatchLabels: map[string]string{
203+
"app": "myapp",
204+
},
205+
},
206+
Template: corev1.PodTemplateSpec{
207+
ObjectMeta: metav1.ObjectMeta{
208+
Name: "myapp",
209+
Labels: map[string]string{
210+
"app": "myapp",
211+
},
212+
},
213+
Spec: corev1.PodSpec{
214+
SecurityContext: &corev1.PodSecurityContext{
215+
RunAsNonRoot: &[]bool{true}[0],
216+
SeccompProfile: &corev1.SeccompProfile{
217+
Type: corev1.SeccompProfileTypeRuntimeDefault,
218+
},
219+
},
220+
Containers: []corev1.Container{
221+
{
222+
Name: "myapp",
223+
Image: "quay.io/openshifttest/hello-openshift@sha256:4200f438cf2e9446f6bcff9d67ceea1f69ed07a2f83363b7fb52529f7ddd8a83",
224+
SecurityContext: &corev1.SecurityContext{
225+
AllowPrivilegeEscalation: &[]bool{false}[0],
226+
Capabilities: &corev1.Capabilities{
227+
Drop: []corev1.Capability{"ALL"},
228+
},
229+
},
230+
},
231+
},
232+
},
233+
},
234+
},
235+
}
236+
_, err = oc.KubeClient().AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{})
237+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create deployment")
238+
g.DeferCleanup(oc.KubeClient().AppsV1().Deployments(namespace).Delete, ctx, "hello-openshift", metav1.DeleteOptions{})
239+
240+
g.By("Wait for deployment to be ready")
241+
err = wait.PollUntilContextTimeout(ctx, 3*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
242+
deploy, pollErr := oc.KubeClient().AppsV1().Deployments(namespace).Get(ctx, "hello-openshift", metav1.GetOptions{})
243+
if pollErr != nil {
244+
e2e.Logf("Error getting deployment: %v", pollErr)
245+
return false, nil
246+
}
247+
if deploy.Status.ReadyReplicas == replicas {
248+
e2e.Logf("Deployment is ready with %d replicas", deploy.Status.ReadyReplicas)
249+
return true, nil
250+
}
251+
e2e.Logf("Waiting for deployment, ready replicas: %d/%d", deploy.Status.ReadyReplicas, replicas)
252+
return false, nil
253+
})
254+
o.Expect(err).NotTo(o.HaveOccurred(), "deployment did not become ready")
255+
256+
g.By("Create PodDisruptionBudget with 100% minAvailable")
257+
pdb := &policyv1.PodDisruptionBudget{
258+
ObjectMeta: metav1.ObjectMeta{
259+
Name: "my-pdb",
260+
Namespace: namespace,
261+
},
262+
Spec: policyv1.PodDisruptionBudgetSpec{
263+
MinAvailable: &intstr.IntOrString{
264+
Type: intstr.String,
265+
StrVal: "100%",
266+
},
267+
Selector: &metav1.LabelSelector{},
268+
},
269+
}
270+
_, err = oc.KubeClient().PolicyV1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{})
271+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create PodDisruptionBudget")
272+
g.DeferCleanup(oc.KubeClient().PolicyV1().PodDisruptionBudgets(namespace).Delete, ctx, "my-pdb", metav1.DeleteOptions{})
273+
274+
g.By("Get a single worker node")
275+
workerNode, err := nodeutils.GetSingleWorkerNode(ctx, oc)
276+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get worker node")
277+
e2e.Logf("Selected worker node: %s", workerNode)
278+
279+
g.By("Obtain the pods running on the selected worker node")
280+
podsInWorker, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", namespace, "-o=jsonpath={.items[?(@.spec.nodeName=='"+workerNode+"')].metadata.name}").Output()
281+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pods on worker node")
282+
o.Expect(len(strings.Fields(podsInWorker))).Should(o.BeNumerically(">", 0), "no pods found on worker node")
283+
284+
g.By("Make sure that PDB's DisruptionAllowed condition is False")
285+
var pdbStatus string
286+
err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 30*time.Second, true, func(pollCtx context.Context) (bool, error) {
287+
var pollErr error
288+
pdbStatus, pollErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("poddisruptionbudget", "my-pdb", "-n", namespace, "-o=jsonpath={.status.conditions[?(@.type==\"DisruptionAllowed\")].status}").Output()
289+
if pollErr != nil {
290+
e2e.Logf("Error getting PDB status: %v", pollErr)
291+
return false, nil
292+
}
293+
if pdbStatus != "" {
294+
return true, nil
295+
}
296+
e2e.Logf("Waiting for PDB DisruptionAllowed condition to appear")
297+
return false, nil
298+
})
299+
o.Expect(err).NotTo(o.HaveOccurred(), "PDB DisruptionAllowed condition not found")
300+
o.Expect(pdbStatus).Should(o.Equal("False"), "PDB DisruptionAllowed should be False")
301+
302+
g.By("Drain the selected worker node")
303+
g.DeferCleanup(func() {
304+
err := nodeutils.WaitClusterOperatorAvailable(ctx, oc)
305+
o.Expect(err).NotTo(o.HaveOccurred(), "cluster operators failed to return to available state after node drain")
306+
})
307+
g.DeferCleanup(oc.AsAdmin().WithoutNamespace().Run("adm").Args("uncordon", workerNode).Execute)
308+
309+
out, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("drain", workerNode, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=30s").Output()
310+
o.Expect(err).To(o.HaveOccurred(), "drain operation should have been blocked but it wasn't")
311+
o.Expect(strings.Contains(out, "Cannot evict pod as it would violate the pod's disruption budget")).Should(o.BeTrue(), "drain output missing PDB violation error message")
312+
o.Expect(strings.Contains(out, "There are pending nodes to be drained")).Should(o.BeTrue(), "drain output missing pending nodes error message")
313+
314+
g.By("Verify that the pods were not drained from the node")
315+
podsAfterDrain, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", namespace, "-o=jsonpath={.items[?(@.spec.nodeName=='"+workerNode+"')].metadata.name}").Output()
316+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pods after drain attempt")
317+
o.Expect(podsInWorker).Should(o.BeIdenticalTo(podsAfterDrain), "pods should not have been evicted from the node")
318+
})
167319
})
168320

169321
// author: asahay@redhat.com

test/extended/node/node_utils.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,3 +764,45 @@ func GetFirstReadyWorkerNode(oc *exutil.CLI) string {
764764
o.Expect(false).To(o.BeTrue(), "no Ready worker node found among %v", workers)
765765
return "" // unreachable; satisfies compiler
766766
}
767+
768+
// GetSingleWorkerNode returns the name of a single worker node
769+
func GetSingleWorkerNode(ctx context.Context, oc *exutil.CLI) (string, error) {
770+
nodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker")
771+
if err != nil {
772+
return "", err
773+
}
774+
if len(nodes) == 0 {
775+
return "", fmt.Errorf("no worker nodes found")
776+
}
777+
framework.Logf("Worker Node Name is %v", nodes[0].Name)
778+
return nodes[0].Name, nil
779+
}
780+
781+
// WaitClusterOperatorAvailable waits for all cluster operators to be available
782+
func WaitClusterOperatorAvailable(ctx context.Context, oc *exutil.CLI) error {
783+
timeout := 30 * time.Minute
784+
785+
waitErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
786+
availableCOStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusteroperator", "-o=jsonpath={.items[*].status.conditions[?(@.type==\"Available\")].status}").Output()
787+
if err != nil {
788+
framework.Logf("Error getting cluster operators: %v", err)
789+
return false, nil
790+
}
791+
if availableCOStatus == "" {
792+
framework.Logf("No cluster operator status found")
793+
return false, nil
794+
}
795+
statuses := strings.Fields(availableCOStatus)
796+
for _, status := range statuses {
797+
if status != "True" {
798+
framework.Logf("Some Cluster Operator is still Unavailable")
799+
return false, nil
800+
}
801+
}
802+
return true, nil
803+
})
804+
if waitErr != nil {
805+
return fmt.Errorf("some cluster operator is still unavailable after timeout: %w", waitErr)
806+
}
807+
return nil
808+
}

0 commit comments

Comments
 (0)