From f491802b446f4d504505b7507a18c7139ea0bdc9 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 7 Jun 2022 09:37:13 -0700 Subject: [PATCH] Update flaky tests for v1.24 (#5625) * Update flaky tests for v1.24 * Consolidate flaky-test regex into file Signed-off-by: Derek Nola --- scripts/flaky-tests | 5 +++++ scripts/test-setup-sonobuoy | 6 +----- scripts/test-setup-sonobuoy-etcd | 6 +----- scripts/test-setup-sonobuoy-mysql | 6 +----- scripts/test-setup-sonobuoy-postgres | 6 +----- 5 files changed, 9 insertions(+), 20 deletions(-) create mode 100644 scripts/flaky-tests diff --git a/scripts/flaky-tests b/scripts/flaky-tests new file mode 100644 index 0000000000..1b965e0ed6 --- /dev/null +++ b/scripts/flaky-tests @@ -0,0 +1,5 @@ +[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance] +[Fail] [sig-node] KubeletManagedEtcHosts [It] should test kubelet managed /etc/hosts file [LinuxOnly] [NodeConformance] [Conformance] +[Fail] [sig-api-machinery] AdmissionWebhook [Privileged:ClusterAdmin] [It] should be able to deny pod and configmap creation [Conformance] +[Fail] [sig-apps] Deployment [It] should run the lifecycle of a Deployment [Conformance] +[Fail] [sig-api-machinery] AdmissionWebhook [Privileged:ClusterAdmin] [It] listing validating webhooks should work [Conformance] \ No newline at end of file diff --git a/scripts/test-setup-sonobuoy b/scripts/test-setup-sonobuoy index 484262d160..57a07e9cfd 100755 --- a/scripts/test-setup-sonobuoy +++ b/scripts/test-setup-sonobuoy @@ -19,11 +19,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log) # Ignore sonobuoy failures if only these flaky tests have failed - local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]' - local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]' - local flakyTest3='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]' - local flakyTest4='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]' - flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" ) + flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/scripts/test-setup-sonobuoy-etcd b/scripts/test-setup-sonobuoy-etcd index a27258f0e9..83e27a00f4 100644 --- a/scripts/test-setup-sonobuoy-etcd +++ b/scripts/test-setup-sonobuoy-etcd @@ -20,11 +20,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log) # Ignore sonobuoy failures if only these flaky tests have failed - local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]' - local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]' - local flakyTest3='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]' - local flakyTest4='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]' - flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" ) + flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/scripts/test-setup-sonobuoy-mysql b/scripts/test-setup-sonobuoy-mysql index f4ad5df8d5..8862af5c5d 100644 --- a/scripts/test-setup-sonobuoy-mysql +++ b/scripts/test-setup-sonobuoy-mysql @@ -61,11 +61,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log) # Ignore sonobuoy failures if only these flaky tests have failed - local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]' - local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]' - local flakyTest3='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]' - local flakyTest4='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]' - flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" ) + flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/scripts/test-setup-sonobuoy-postgres b/scripts/test-setup-sonobuoy-postgres index a922d224ea..062459f717 100644 --- a/scripts/test-setup-sonobuoy-postgres +++ b/scripts/test-setup-sonobuoy-postgres @@ -61,11 +61,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log) # Ignore sonobuoy failures if only these flaky tests have failed - local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]' - local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]' - local flakyTest3='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]' - local flakyTest4='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]' - flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" ) + flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] }