diff --git a/pkg/etcd/int_tests/restore/etcd_restore_int_test.go b/pkg/etcd/int_tests/restore/etcd_restore_int_test.go new file mode 100644 index 0000000000..84ebb7016a --- /dev/null +++ b/pkg/etcd/int_tests/restore/etcd_restore_int_test.go @@ -0,0 +1,121 @@ +package restore_test + +import ( + "strings" + "testing" + + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + testutil "github.com/rancher/k3s/tests/util" +) + +var server1, server2 *testutil.K3sServer +var tmpdDataDir = "/tmp/restoredatadir" +var clientCACertHash string +var restoreServerArgs = []string{"--cluster-init", "-t", "test", "-d", tmpdDataDir} +var _ = BeforeSuite(func() { + if !testutil.IsExistingServer() { + var err error + server1, err = testutil.K3sStartServer(restoreServerArgs...) + Expect(err).ToNot(HaveOccurred()) + } +}) + +var _ = Describe("etcd snapshot restore", func() { + BeforeEach(func() { + if testutil.IsExistingServer() && !testutil.ServerArgsPresent(restoreServerArgs) { + Skip("Test needs k3s server with: " + strings.Join(restoreServerArgs, " ")) + } + }) + When("a snapshot is restored on existing node", func() { + It("etcd starts up with no problems", func() { + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "pods", "-A") + }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + }) + It("create a workload", func() { + result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml") + Expect(result).To(ContainSubstring("deployment.apps/nginx-deployment created")) + Expect(err).NotTo(HaveOccurred()) + }) + It("saves an etcd snapshot", func() { + Expect(testutil.K3sCmd("etcd-snapshot", "save", "-d", tmpdDataDir, "--name", "snapshot-to-restore")). + To(ContainSubstring("saved")) + }) + It("list snapshots", func() { + Expect(testutil.K3sCmd("etcd-snapshot", "ls", "-d", tmpdDataDir)). + To(MatchRegexp(`://` + tmpdDataDir + `/server/db/snapshots/snapshot-to-restore`)) + }) + // create another workload + It("create a workload 2", func() { + result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl2.yaml") + Expect(result).To(ContainSubstring("deployment.apps/nginx-deployment-post-snapshot created")) + Expect(err).NotTo(HaveOccurred()) + }) + It("get Client CA cert hash", func() { + // get md5sum of the CA certs + var err error + clientCACertHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '") + Expect(err).ToNot(HaveOccurred()) + }) + It("stop k3s", func() { + Expect(testutil.K3sKillServer(server1, true)).To(Succeed()) + }) + It("restore the snapshot", func() { + // get snapshot file + filePath, err := testutil.RunCommand(`sudo find ` + tmpdDataDir + `/server -name "*snapshot-to-restore*"`) + Expect(err).ToNot(HaveOccurred()) + filePath = strings.TrimSuffix(filePath, "\n") + Eventually(func() (string, error) { + return testutil.K3sCmd("server", "-d", tmpdDataDir, "--cluster-reset", "--token", "test", "--cluster-reset-restore-path", filePath) + }, "360s", "5s").Should(ContainSubstring(`Etcd is running, restart without --cluster-reset flag now`)) + }) + It("start k3s server", func() { + var err error + server2, err = testutil.K3sStartServer(restoreServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("starts up with no problems", func() { + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "pods", "-A") + }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + }) + It("Make sure Workload 1 exists", func() { + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment") + }, "360s", "5s").Should(ContainSubstring("3/3")) + }) + It("Make sure Workload 2 does not exists", func() { + res, err := testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment-post-snapshot") + Expect(err).To(HaveOccurred()) + Expect(res).To(ContainSubstring("not found")) + }) + It("check if CA cert hash matches", func() { + // get md5sum of the CA certs + var err error + clientCACertHash2, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '") + Expect(err).ToNot(HaveOccurred()) + Expect(clientCACertHash2).To(Equal(clientCACertHash)) + }) + It("stop k3s", func() { + Expect(testutil.K3sKillServer(server2, false)).To(Succeed()) + }) + }) +}) + +var _ = AfterSuite(func() { + if !testutil.IsExistingServer() { + Expect(testutil.K3sKillServer(server1, false)).To(Succeed()) + Expect(testutil.K3sCleanup(server1, true, tmpdDataDir)).To(Succeed()) + Expect(testutil.K3sKillServer(server2, false)).To(Succeed()) + Expect(testutil.K3sCleanup(server2, true, tmpdDataDir)).To(Succeed()) + } +}) + +func Test_RestoreSnapshotIntegrationEtcd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, "Etcd Suite", []Reporter{ + reporters.NewJUnitReporter("/tmp/results/junit-etcd-restore.xml"), + }) +} diff --git a/pkg/etcd/int_tests/restore/testdata/temp_depl.yaml b/pkg/etcd/int_tests/restore/testdata/temp_depl.yaml new file mode 100644 index 0000000000..3649247c1b --- /dev/null +++ b/pkg/etcd/int_tests/restore/testdata/temp_depl.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 \ No newline at end of file diff --git a/pkg/etcd/int_tests/restore/testdata/temp_depl2.yaml b/pkg/etcd/int_tests/restore/testdata/temp_depl2.yaml new file mode 100644 index 0000000000..8cea5e6f2d --- /dev/null +++ b/pkg/etcd/int_tests/restore/testdata/temp_depl2.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-post-snapshot + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 \ No newline at end of file diff --git a/pkg/etcd/etcd_int_test.go b/pkg/etcd/int_tests/snapshot/etcd_int_test.go similarity index 98% rename from pkg/etcd/etcd_int_test.go rename to pkg/etcd/int_tests/snapshot/etcd_int_test.go index 4cbf608561..1821ce0f25 100644 --- a/pkg/etcd/etcd_int_test.go +++ b/pkg/etcd/int_tests/snapshot/etcd_int_test.go @@ -1,4 +1,4 @@ -package etcd_test +package snapshot_test import ( "regexp" @@ -113,7 +113,7 @@ var _ = Describe("etcd snapshots", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { Expect(testutil.K3sKillServer(server, false)).To(Succeed()) - Expect(testutil.K3sCleanup(server, true)).To(Succeed()) + Expect(testutil.K3sCleanup(server, true, "")).To(Succeed()) } }) diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index 73f682a508..b72094cd68 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -53,7 +53,7 @@ var _ = Describe("dual stack", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() && os.Getenv("CI") != "true" { Expect(testutil.K3sKillServer(dualStackServer, false)).To(Succeed()) - Expect(testutil.K3sCleanup(dualStackServer, true)).To(Succeed()) + Expect(testutil.K3sCleanup(dualStackServer, true, "")).To(Succeed()) } }) diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index 83398ed809..bf48c6751e 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -82,7 +82,7 @@ var _ = Describe("local storage", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { Expect(testutil.K3sKillServer(localStorageServer, false)).To(Succeed()) - Expect(testutil.K3sCleanup(localStorageServer, true)).To(Succeed()) + Expect(testutil.K3sCleanup(localStorageServer, true, "")).To(Succeed()) } }) diff --git a/tests/util/integration.go b/tests/util/integration.go index 339e503922..1d7860b0d9 100644 --- a/tests/util/integration.go +++ b/tests/util/integration.go @@ -2,8 +2,10 @@ package util import ( "bufio" + "bytes" "encoding/json" "errors" + "fmt" "os" "os/exec" "os/user" @@ -18,7 +20,7 @@ import ( // Compile-time variable var existingServer = "False" -const lockFile = "/var/lock/k3s-test.lock" +const lockFile = "/tmp/k3s-test.lock" type K3sServer struct { cmd *exec.Cmd @@ -146,9 +148,7 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) { for _, arg := range inputArgs { cmdArgs = append(cmdArgs, strings.Fields(arg)...) } - k3sBin := findK3sExecutable() - k3sCmd := append([]string{"server"}, cmdArgs...) cmd := exec.Command(k3sBin, k3sCmd...) // Give the server a new group id so we can kill it and its children later @@ -176,7 +176,7 @@ func K3sKillServer(server *K3sServer, releaseLock bool) error { } // K3sCleanup attempts to cleanup networking and files leftover from an integration test -func K3sCleanup(server *K3sServer, releaseLock bool) error { +func K3sCleanup(server *K3sServer, releaseLock bool, dataDir string) error { if cni0Link, err := netlink.LinkByName("cni0"); err == nil { links, _ := netlink.LinkList() for _, link := range links { @@ -193,7 +193,10 @@ func K3sCleanup(server *K3sServer, releaseLock bool) error { if flannelV6, err := netlink.LinkByName("flannel-v6.1"); err == nil { netlink.LinkDel(flannelV6) } - if err := os.RemoveAll("/var/lib/rancher/k3s"); err != nil { + if dataDir == "" { + dataDir = "/var/lib/rancher/k3s" + } + if err := os.RemoveAll(dataDir); err != nil { return err } if releaseLock { @@ -201,3 +204,15 @@ func K3sCleanup(server *K3sServer, releaseLock bool) error { } return nil } + +// RunCommand Runs command on the cluster accessing the cluster through kubeconfig file +func RunCommand(cmd string) (string, error) { + c := exec.Command("bash", "-c", cmd) + var out bytes.Buffer + c.Stdout = &out + err := c.Run() + if err != nil { + return "", fmt.Errorf("%s", err) + } + return out.String(), nil +}