Test Coverage Reports for E2E tests (#7526)

* Move coverage writer into agent and server
* Add coverage report to E2E PR tests
* Add codecov upload to drone

Signed-off-by: Derek Nola <derek.nola@suse.com>

.drone.yml

@@ -709,7 +709,7 @@ steps:
   image: rancher/dapper:v0.5.0
   commands:
     - DOCKER_BUILDKIT=1 docker build --target test-e2e -t test-e2e -f Dockerfile.test .
-    - SKIP_VALIDATE=true SKIP_AIRGAP=true dapper ci
+    - SKIP_VALIDATE=true SKIP_AIRGAP=true GOCOVER=1 dapper ci
     - cp dist/artifacts/* /tmp/artifacts/
   volumes:
   - name: cache
@@ -725,6 +725,7 @@ steps:
     memory: 10Gi
   environment:
     E2E_REGISTRY: 'true'
+    E2E_GOCOVER: 'true'
   commands:
   - mkdir -p dist/artifacts
   - cp /tmp/artifacts/* dist/artifacts/
@@ -743,20 +744,24 @@ steps:
   - cd tests/e2e/validatecluster
   - vagrant destroy -f
   - go test -v -timeout=45m ./validatecluster_test.go -ci -local
+  - cp ./coverage.out /tmp/artifacts/validate-coverage.out
   - cd ../secretsencryption
   - vagrant destroy -f
   - go test -v -timeout=30m ./secretsencryption_test.go -ci -local
+  - cp ./coverage.out /tmp/artifacts/se-coverage.out
   - cd ../startup
   - vagrant destroy -f
   - go test -v -timeout=30m ./startup_test.go -ci -local
+  - cp ./coverage.out /tmp/artifacts/startup-coverage.out
   - |
     if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then
       cd ../upgradecluster
       vagrant destroy -f
       E2E_RELEASE_CHANNEL="latest" go test -v -timeout=45m ./upgradecluster_test.go  -ci -local
+      cp ./coverage.out /tmp/artifacts/upgrade-coverage.out
     fi
   - docker stop registry && docker rm registry
-
+  
   volumes:
   - name: libvirt
     path: /var/run/libvirt/
@@ -765,6 +770,26 @@ steps:
   - name: cache
     path: /tmp/artifacts
 
+- name: upload to codecov
+  image: robertstettner/drone-codecov
+  settings:
+    token: 
+      from_secret: codecov_token
+    files:
+      - /tmp/artifacts/validate-coverage.out
+      - /tmp/artifacts/se-coverage.out
+      - /tmp/artifacts/startup-coverage.out
+      - /tmp/artifacts/upgrade-coverage.out
+    flags: 
+      - e2etests
+  when:
+    event:
+    - pull_request
+
+  volumes:
+  - name: cache
+    path: /tmp/artifacts
+
 volumes:
 - name: docker
   host:

Dockerfile.test

@@ -44,10 +44,10 @@ RUN vagrant box add generic/ubuntu2004 --provider libvirt --force
 RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
     chmod +x ./kubectl; \
     mv ./kubectl /usr/local/bin/kubectl
-ENV GO_VERSION 1.19.2
-RUN curl -O -L "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \
+RUN GO_VERSION=$(curl https://go.dev/VERSION?m=text); \
+    curl -O -L "https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz"; \
     rm -rf /usr/local/go; \
-    tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz;
+    tar -C /usr/local -xzf ${GO_VERSION}.linux-amd64.tar.gz;
 
 ENV PATH="${PATH}:/usr/local/go/bin"
 

pkg/cli/agent/agent.go

@@ -76,5 +76,7 @@ func Run(ctx *cli.Context) error {
 
 	contextCtx := signals.SetupSignalContext()
 
+	go cmds.WriteCoverage(contextCtx)
+
 	return agent.Run(contextCtx, cfg)
 }

pkg/cli/cmds/cover_default.go

@@ -0,0 +1,7 @@
+//go:build !linux || !cover
+
+package cmds
+
+import "context"
+
+func WriteCoverage(ctx context.Context) {}

pkg/cli/cmds/cover_linux.go

@@ -1,6 +1,6 @@
-//go:build cover
+//go:build linux && cover
 
-package server
+package cmds
 
 import (
 	"context"
@@ -13,7 +13,7 @@ import (
 
 // writeCoverage checks if GOCOVERDIR is set on startup and writes coverage files to that directory
 // every 20 seconds. This is done to ensure that the coverage files are written even if the process is killed.
-func writeCoverage(ctx context.Context) {
+func WriteCoverage(ctx context.Context) {
 	if k, ok := os.LookupEnv("GOCOVERDIR"); ok {
 		for {
 			select {

pkg/cli/server/server.go

@@ -440,6 +440,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		return err
 	}
 
+	go cmds.WriteCoverage(ctx)
+
 	go func() {
 		if !serverConfig.ControlConfig.DisableAPIServer {
 			<-serverConfig.ControlConfig.Runtime.APIServerReady

pkg/server/no_cover.go

@@ -1,7 +0,0 @@
-//go:build !cover
-
-package server
-
-import "context"
-
-func writeCoverage(ctx context.Context) {}

pkg/server/server.go

@@ -80,7 +80,6 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
 			return errors.Wrap(err, "startup hook")
 		}
 	}
-	go writeCoverage(ctx)
 	go startOnAPIServerReady(ctx, config)
 
 	if err := printTokens(&config.ControlConfig); err != nil {

tests/e2e/secretsencryption/Vagrantfile

@@ -5,6 +5,7 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+GOCOVER = (ENV['E2E_GOCOVER'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
 NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
@@ -25,6 +26,10 @@ def provision(vm, role, role_num, node_num)
   
   vm.provision "shell", inline: "ping -c 2 k3s.io"
   
+  if !GOCOVER.empty?
+    addCoverageDir(vm, role)
+  end
+
   if role.include?("server") && role_num == 0
     vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
       k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1 --secrets-encryption]

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -306,6 +306,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if os.Getenv("E2E_GOCOVER") != "" {
+		Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed())
+	}
 	if failed && !*ci {
 		fmt.Println("FAILED!")
 	} else {

tests/e2e/startup/Vagrantfile

@@ -5,6 +5,7 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+GOCOVER = (ENV['E2E_GOCOVER'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
 NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
@@ -26,7 +27,7 @@ def provision(vm, role, role_num, node_num)
   node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
   
   vm.provision "shell", inline: "ping -c 2 k3s.io"
-  
+
   if role.include?("server")
     vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
       k3s.args = "server "
@@ -52,6 +53,10 @@ def provision(vm, role, role_num, node_num)
     end
   end
 
+  if !GOCOVER.empty?
+    addCoverageDir(vm, role)
+  end
+
   if vm.box.to_s.include?("microos")
     vm.provision 'k3s-reload', type: 'reload', run: 'once'
   end

tests/e2e/startup/startup_test.go

@@ -274,6 +274,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if os.Getenv("E2E_GOCOVER") != "" {
+		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
+	}
 	if failed && !*ci {
 		fmt.Println("FAILED!")
 	} else {

tests/e2e/testutils.go

@@ -489,6 +489,29 @@ func UpgradeCluster(nodeNames []string, local bool) error {
 	return nil
 }
 
+func GetCoverageReport(nodeNames []string) error {
+	covDirs := []string{}
+	for _, nodeName := range nodeNames {
+		covDir := nodeName + "-cov"
+		covDirs = append(covDirs, covDir)
+		os.MkdirAll(covDir, 0755)
+		cmd := "vagrant scp " + nodeName + ":/tmp/k3scov/* " + covDir
+		if _, err := RunCommand(cmd); err != nil {
+			return err
+		}
+	}
+	cmd := "go tool covdata textfmt -i " + strings.Join(covDirs, ",") + " -o coverage.out"
+	if out, err := RunCommand(cmd); err != nil {
+		return fmt.Errorf("failed to generate coverage report: %s, %v", out, err)
+	}
+	for _, covDir := range covDirs {
+		if err := os.RemoveAll(covDir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // getPodIPs returns the IPs of all pods
 func GetPodIPs(kubeConfigFile string) ([]ObjIP, error) {
 	cmd := `kubectl get pods -A -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIPs[*].ip}{"\n"}{end}' --kubeconfig=` + kubeConfigFile

tests/e2e/upgradecluster/Vagrantfile

@@ -7,6 +7,7 @@ RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
 REGISTRY = (ENV['E2E_REGISTRY'] || "")
+GOCOVER = (ENV['E2E_GOCOVER'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
 NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
@@ -46,6 +47,10 @@ def provision(vm, role, role_num, node_num)
     vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ]
   end
 
+  if !GOCOVER.empty?
+    addCoverageDir(vm, role)
+  end
+
   if role.include?("server") && role_num == 0
     vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
       k3s.args = "server "

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -385,6 +385,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if os.Getenv("E2E_GOCOVER") != "" {
+		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
+	}
 	if failed && !*ci {
 		fmt.Println("FAILED!")
 	} else {

tests/e2e/vagrantdefaults.rb

@@ -34,6 +34,16 @@ def getInstallType(vm, release_version, branch)
   end
 end
 
+def addCoverageDir(vm, role)
+  service = role.include?("agent") ? "k3s-agent" : "k3s" 
+    script = <<~SHELL
+      mkdir -p /tmp/k3scov
+      echo -e 'GOCOVERDIR=/tmp/k3scov' >> /etc/default/#{service}
+      systemctl daemon-reload
+    SHELL
+    vm.provision "go coverage", type: "shell", inline: script 
+end
+
 def getHardenedArg(vm, hardened, scripts_location)
   if hardened.empty? 
     return ""

tests/e2e/validatecluster/Vagrantfile

@@ -9,6 +9,7 @@ EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
 HARDENED = (ENV['E2E_HARDENED'] || "")
 REGISTRY = (ENV['E2E_REGISTRY'] || "")
 RANCHER = (ENV['E2E_RANCHER'] || "")
+GOCOVER = (ENV['E2E_GOCOVER'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
 NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
@@ -39,6 +40,9 @@ def provision(vm, role, role_num, node_num)
     vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ]
   end
   
+  if !GOCOVER.empty?
+    addCoverageDir(vm, role)
+  end
 
   if role.include?("server") && role_num == 0
     vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
@@ -133,7 +137,7 @@ def getDBType(role, role_num, vm)
 end
 
 Vagrant.configure("2") do |config|
-  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
+  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload", "vagrant-scp"]
   # Default provider is libvirt, virtualbox is only provided as a backup
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS

tests/e2e/validatecluster/validatecluster_test.go

@@ -404,6 +404,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if os.Getenv("E2E_GOCOVER") != "" {
+		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
+	}
 	if failed && !*ci {
 		fmt.Println("FAILED!")
 	} else {

tests/integration/longhorn/longhorn_int_test.go

@@ -139,7 +139,7 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !testutil.IsExistingServer() {
+	if !testutil.IsExistingServer() && server != nil {
 		if failed {
 			testutil.K3sSaveLog(server, false)
 		}