From 674a05478f78f9ad2b204917a309eb238072d6db Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 12 Jan 2023 13:43:31 -0800 Subject: [PATCH] Containerd restart testlet (#6696) * Add containerd testlet to startup integration * Fix all log dumps * Stop server gracefully Signed-off-by: Derek Nola --- .../certrotation/certrotation_int_test.go | 9 +++- .../custometcdargs/custometcdargs_int_test.go | 9 +++- .../dualstack/dualstack_int_test.go | 9 +++- .../etcdrestore/etcd_restore_int_test.go | 9 +++- .../etcdsnapshot/etcdsnapshot_int_test.go | 9 +++- tests/integration/integration.go | 23 +++++++- .../localstorage/localstorage_int_test.go | 9 +++- .../secretsencryption_int_test.go | 9 +++- tests/integration/startup/startup_int_test.go | 52 ++++++++++++++++++- tests/integration/startup/testdata/dummy.yaml | 11 ++++ 10 files changed, 131 insertions(+), 18 deletions(-) create mode 100644 tests/integration/startup/testdata/dummy.yaml diff --git a/tests/integration/certrotation/certrotation_int_test.go b/tests/integration/certrotation/certrotation_int_test.go index 740a15c77a..ffd7fad53a 100644 --- a/tests/integration/certrotation/certrotation_int_test.go +++ b/tests/integration/certrotation/certrotation_int_test.go @@ -77,10 +77,15 @@ var _ = Describe("certificate rotation", Ordered, func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(server) + if failed { + testutil.K3sSaveLog(server, false) } Expect(testutil.K3sKillServer(server)).To(Succeed()) Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) diff --git a/tests/integration/custometcdargs/custometcdargs_int_test.go b/tests/integration/custometcdargs/custometcdargs_int_test.go index 9fe94be59f..20ea9803d7 100644 --- a/tests/integration/custometcdargs/custometcdargs_int_test.go +++ b/tests/integration/custometcdargs/custometcdargs_int_test.go @@ -52,10 +52,15 @@ var _ = Describe("custom etcd args", Ordered, func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(customEtcdArgsServer) + if failed { + testutil.K3sSaveLog(customEtcdArgsServer, false) } Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed()) Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index 63391f58d3..81bbc208e9 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -53,10 +53,15 @@ var _ = Describe("dual stack", Ordered, func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() && os.Getenv("CI") != "true" { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(dualStackServer) + if failed { + testutil.K3sSaveLog(dualStackServer, false) } Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed()) Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index 5c982c3a0b..22bb0f2b6e 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -106,10 +106,15 @@ var _ = Describe("etcd snapshot restore", Ordered, func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(server1) + if failed { + testutil.K3sSaveLog(server1, false) } Expect(testutil.K3sKillServer(server1)).To(Succeed()) Expect(testutil.K3sKillServer(server2)).To(Succeed()) diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index d7d7a8dfdf..1df2a208be 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -113,10 +113,15 @@ var _ = Describe("etcd snapshots", Ordered, func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(server) + if failed { + testutil.K3sSaveLog(server, false) } Expect(testutil.K3sKillServer(server)).To(Succeed()) Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) diff --git a/tests/integration/integration.go b/tests/integration/integration.go index e09f06b085..be60e7d8cf 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -229,7 +229,23 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) { return &K3sServer{cmd, f}, err } -// K3sKillServer terminates the running K3s server and its children +// K3sStopServer gracefully stops the running K3s server and does not kill its children. +// Equivalent to stopping the K3s service +func K3sStopServer(server *K3sServer) error { + if server.log != nil { + server.log.Close() + } + if err := server.cmd.Process.Kill(); err != nil { + return errors.Wrap(err, "failed to kill k3s process") + } + if _, err := server.cmd.Process.Wait(); err != nil { + return errors.Wrap(err, "failed to wait for k3s process exit") + } + return nil +} + +// K3sKillServer terminates the running K3s server and its children. +// Equivalent to k3s-killall.sh func K3sKillServer(server *K3sServer) error { if server.log != nil { server.log.Close() @@ -292,8 +308,11 @@ func K3sCleanup(k3sTestLock int, dataDir string) error { return nil } -func K3sDumpLog(server *K3sServer) error { +func K3sSaveLog(server *K3sServer, dump bool) error { server.log.Close() + if !dump { + return nil + } log, err := os.Open(server.log.Name()) if err != nil { return err diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index c9cdd5594e..be1ebd6e40 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -82,10 +82,15 @@ var _ = Describe("local storage", func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(localStorageServer) + if failed { + testutil.K3sSaveLog(localStorageServer, false) } Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed()) Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go index 9532cf207c..8bab882cdb 100644 --- a/tests/integration/secretsencryption/secretsencryption_int_test.go +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -141,10 +141,15 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { }) }) +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(secretsEncryptionServer) + if failed { + testutil.K3sSaveLog(secretsEncryptionServer, false) } Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) Expect(testutil.K3sCleanup(testLock, secretsEncryptionDataDir)).To(Succeed()) diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index 2a4d58bff4..e161ba0280 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -208,12 +208,60 @@ var _ = Describe("startup tests", Ordered, func() { Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) }) }) + // Check for regression of containerd restarting pods + // https://github.com/containerd/containerd/issues/7843 + When("a server with a dummy pod", func() { + It("is created with no arguments", func() { + var err error + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "120s", "5s").Should(Succeed()) + }) + It("creates a new pod", func() { + Expect(testutil.K3sCmd("kubectl apply -f ./testdata/dummy.yaml")). + To(ContainSubstring("pod/dummy created")) + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl get event -n kube-system --field-selector involvedObject.name=dummy") + }, "60s", "5s").Should(ContainSubstring("Started container dummy")) + }) + It("restarts the server", func() { + var err error + Expect(testutil.K3sStopServer(startupServer)).To(Succeed()) + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) + }) + It("has the dummy pod not restarted", func() { + Consistently(func(g Gomega) { + res, err := testutil.K3sCmd("kubectl get event -n kube-system --field-selector involvedObject.name=dummy") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(res).NotTo(ContainSubstring("Pod sandbox changed, it will be killed and re-created")) + g.Expect(res).NotTo(ContainSubstring("Stopping container dummy")) + }, "30s", "5s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() }) var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - if CurrentSpecReport().Failed() { - testutil.K3sDumpLog(startupServer) + if failed { + testutil.K3sSaveLog(startupServer, false) + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) } Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) } diff --git a/tests/integration/startup/testdata/dummy.yaml b/tests/integration/startup/testdata/dummy.yaml new file mode 100644 index 0000000000..7ef64b0d29 --- /dev/null +++ b/tests/integration/startup/testdata/dummy.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: dummy + namespace: kube-system +spec: + containers: + - name: dummy + image: ranchertest/mytestcontainer + imagePullPolicy: IfNotPresent