Containerd restart testlet (#6696)

* Add containerd testlet to startup integration
* Fix all log dumps
* Stop server gracefully

Signed-off-by: Derek Nola <derek.nola@suse.com>

tests/integration/certrotation/certrotation_int_test.go

@@ -77,10 +77,15 @@ var _ = Describe("certificate rotation", Ordered, func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(server)
+			testutil.K3sSaveLog(server, false)
 		}
 		Expect(testutil.K3sKillServer(server)).To(Succeed())
 		Expect(testutil.K3sCleanup(-1, "")).To(Succeed())

tests/integration/custometcdargs/custometcdargs_int_test.go

@@ -52,10 +52,15 @@ var _ = Describe("custom etcd args", Ordered, func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(customEtcdArgsServer)
+			testutil.K3sSaveLog(customEtcdArgsServer, false)
 		}
 		Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())

tests/integration/dualstack/dualstack_int_test.go

@@ -53,10 +53,15 @@ var _ = Describe("dual stack", Ordered, func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() && os.Getenv("CI") != "true" {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(dualStackServer)
+			testutil.K3sSaveLog(dualStackServer, false)
 		}
 		Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())

tests/integration/etcdrestore/etcd_restore_int_test.go

@@ -106,10 +106,15 @@ var _ = Describe("etcd snapshot restore", Ordered, func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(server1)
+			testutil.K3sSaveLog(server1, false)
 		}
 		Expect(testutil.K3sKillServer(server1)).To(Succeed())
 		Expect(testutil.K3sKillServer(server2)).To(Succeed())

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -113,10 +113,15 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(server)
+			testutil.K3sSaveLog(server, false)
 		}
 		Expect(testutil.K3sKillServer(server)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())

tests/integration/integration.go

@@ -229,7 +229,23 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 	return &K3sServer{cmd, f}, err
 }
 
-// K3sKillServer terminates the running K3s server and its children
+// K3sStopServer gracefully stops the running K3s server and does not kill its children.
+// Equivalent to stopping the K3s service
+func K3sStopServer(server *K3sServer) error {
+	if server.log != nil {
+		server.log.Close()
+	}
+	if err := server.cmd.Process.Kill(); err != nil {
+		return errors.Wrap(err, "failed to kill k3s process")
+	}
+	if _, err := server.cmd.Process.Wait(); err != nil {
+		return errors.Wrap(err, "failed to wait for k3s process exit")
+	}
+	return nil
+}
+
+// K3sKillServer terminates the running K3s server and its children.
+// Equivalent to k3s-killall.sh
 func K3sKillServer(server *K3sServer) error {
 	if server.log != nil {
 		server.log.Close()
@@ -292,8 +308,11 @@ func K3sCleanup(k3sTestLock int, dataDir string) error {
 	return nil
 }
 
-func K3sDumpLog(server *K3sServer) error {
+func K3sSaveLog(server *K3sServer, dump bool) error {
 	server.log.Close()
+	if !dump {
+		return nil
+	}
 	log, err := os.Open(server.log.Name())
 	if err != nil {
 		return err

tests/integration/localstorage/localstorage_int_test.go

@@ -82,10 +82,15 @@ var _ = Describe("local storage", func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(localStorageServer)
+			testutil.K3sSaveLog(localStorageServer, false)
 		}
 		Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -141,10 +141,15 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
 	})
 })
 
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(secretsEncryptionServer)
+			testutil.K3sSaveLog(secretsEncryptionServer, false)
 		}
 		Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, secretsEncryptionDataDir)).To(Succeed())

tests/integration/startup/startup_int_test.go

@@ -208,12 +208,60 @@ var _ = Describe("startup tests", Ordered, func() {
 			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		})
 	})
+	// Check for regression of containerd restarting pods
+	// https://github.com/containerd/containerd/issues/7843
+	When("a server with a dummy pod", func() {
+		It("is created with no arguments", func() {
+			var err error
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
+		})
+		It("creates a new pod", func() {
+			Expect(testutil.K3sCmd("kubectl apply -f ./testdata/dummy.yaml")).
+				To(ContainSubstring("pod/dummy created"))
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl get event -n kube-system --field-selector involvedObject.name=dummy")
+			}, "60s", "5s").Should(ContainSubstring("Started container dummy"))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sStopServer(startupServer)).To(Succeed())
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
+		})
+		It("has the dummy pod not restarted", func() {
+			Consistently(func(g Gomega) {
+				res, err := testutil.K3sCmd("kubectl get event -n kube-system --field-selector involvedObject.name=dummy")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(res).NotTo(ContainSubstring("Pod sandbox changed, it will be killed and re-created"))
+				g.Expect(res).NotTo(ContainSubstring("Stopping container dummy"))
+			}, "30s", "5s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
 })
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		if CurrentSpecReport().Failed() {
+		if failed {
-			testutil.K3sDumpLog(startupServer)
+			testutil.K3sSaveLog(startupServer, false)
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
 		}
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
 	}

tests/integration/startup/testdata/dummy.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dummy
+  namespace: kube-system
+spec:
+  containers:
+  - name: dummy
+    image: ranchertest/mytestcontainer
+    imagePullPolicy: IfNotPresent