diff --git a/e2e/Dockerfile b/conformance/Dockerfile similarity index 100% rename from e2e/Dockerfile rename to conformance/Dockerfile diff --git a/e2e/run-test.sh b/conformance/run-test.sh similarity index 100% rename from e2e/run-test.sh rename to conformance/run-test.sh diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml deleted file mode 100644 index 74ea3500f7..0000000000 --- a/e2e/docker-compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: '3' -services: - server: - image: rancher/k3s:v0.1.0-rc8 - command: server - environment: - - K3S_CLUSTER_SECRET=somethingtotallyrandom - volumes: - - config:/etc/rancher/k3s - tmpfs: - - /run - - /var/run - privileged: true - - node: - image: rancher/k3s:v0.1.0-rc8 - tmpfs: - - /run - - /var/run - privileged: true - environment: - - K3S_URL=https://server:6443 - - K3S_CLUSTER_SECRET=somethingtotallyrandom - - sonobuoy: - build: . - volumes: - - config:/etc/rancher/k3s - -volumes: - config: {} diff --git a/tests/TESTING.md b/tests/TESTING.md index 8449970b8b..56ccce2dc4 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -1,7 +1,12 @@ # Testing Standards in K3s -Go testing in K3s comes in 3 forms: Unit, Integration, and End-to-End (E2E). This -document will explain *when* each test should be written and *how* each test should be +Testing in K3s comes in 4 forms: +- [Unit](#unit-tests) +- [Integration](#integration-tests) +- [Smoke](#smoke-tests) +- [End-to-End (E2E)](#end-to-end-e2e-tests) + +This document will explain *when* each test should be written and *how* each test should be generated, formatted, and run. Note: all shell commands given are relative to the root k3s repo directory. @@ -60,12 +65,12 @@ To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions. Integration tests can be placed in two areas: 1. Next to the go package they intend to test. -2. In `tests/integration/` for package agnostic testing. +2. In `tests/integration/` for package agnostic testing. Package specific integration tests should use the `_test` package. Package agnostic integration tests should use the `integration` package. -All integration test files should be named: `_int_test.go` -All integration test functions should be named: `Test_Integration`. +All integration test files should be named: `_int_test.go`. +All integration test functions should be named: `Test_Integration`. See the [etcd snapshot test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_int_test.go) as a package specific example. See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage/localstorage_int_test.go) as a package agnostic example. @@ -172,21 +177,33 @@ ___ ## End-to-End (E2E) Tests -End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on separate testing utilities and are self-contained within the `test/e2e` directory. E2E tests cover complete K3s single and multi-cluster configuration and administration: bringup, update, teardown etc. -E2E tests are run nightly as part of K3s quality assurance (QA). +E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA). + +### Framework +End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on [Vagrant](https://www.vagrantup.com/) to provide the underlying cluster configuration. + +Currently tested operating systems are: +- [Ubuntu 20.04](https://app.vagrantup.com/generic/boxes/ubuntu2004) +- [Leap 15.3](https://app.vagrantup.com/opensuse/boxes/Leap-15.3.x86_64) (stand-in for SLE-Server) +- [MicroOS](https://app.vagrantup.com/dweomer/boxes/microos.amd64) (stand-in for SLE-Micro) ### Format -All E2E tests should be placed under the `e2e` package. +All E2E tests should be placed under `tests/e2e/`. All E2E test functions should be named: `Test_E2E`. -See the [upgrade cluster test](https://github.com/k3s-io/k3s/blob/master/tests/e2e/upgradecluster_test.go) as an example. +A E2E test consists of two parts: +1. `Vagrantfile`: a vagrant file which describes and configures the VMs upon which the cluster and test will run +2. `.go`: A go test file which calls `vagrant up` and controls the actual testing of the cluster + +See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example. ### Running -Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. +Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider. +Once setup is complete, E2E tests can be run with: ```bash -go test ./tests/e2e... -run E2E +go test ./tests/e2e/... -run E2E ``` ## Contributing New Or Updated Tests diff --git a/tests/e2e/scripts/build.sh b/tests/e2e/scripts/build.sh deleted file mode 100755 index 7024fd3904..0000000000 --- a/tests/e2e/scripts/build.sh +++ /dev/null @@ -1 +0,0 @@ -/usr/local/bin/docker build -f ../Dockerfile.build -t k3s_create_cluster:v1.0 . diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh new file mode 100644 index 0000000000..9f8780114d --- /dev/null +++ b/tests/e2e/scripts/latest_commit.sh @@ -0,0 +1,7 @@ +# Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build +curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=5&sha=$1" | jq -r '.[] | .sha' &> $2 +curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum +while [ $? -ne 0 ]; do + sed -i 1d $2 + curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 $2).sha256sum +done \ No newline at end of file diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile new file mode 100644 index 0000000000..857afc906b --- /dev/null +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -0,0 +1,109 @@ +ENV['VAGRANT_NO_PARALLEL'] = 'no' +NODE_ROLES = (ENV['NODE_ROLES'] || + ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) +NODE_BOXES = (ENV['NODE_BOXES'] || + ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) +RELEASE_CHANNEL = (ENV['RELEASE_CHANNEL'] || "latest") +RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "") +NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i +# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks +NETWORK_PREFIX = "10.10.10" +install_type = "" + +def provision(vm, roles, role_num, node_num) + vm.box = NODE_BOXES[node_num] + vm.hostname = "#{roles[0]}-#{role_num}" + # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 + vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" + + osConfigure(vm) + + + if !RELEASE_VERSION.empty? + install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" + elsif RELEASE_CHANNEL == "commit" + vm.provision "shell", path: "../scripts/latest_commit.sh", args: ["master", "/tmp/k3s_commits"] + install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)" + else + vm.provision "latest version", type: "shell", + inline: "curl -w '%{url_effective}' -L -s -S https://update.k3s.io/v1-release/channels/#{RELEASE_CHANNEL} -o /dev/null | sed -e 's|.*/||' &> /tmp/k3s_version" + install_type = "INSTALL_K3S_VERSION=$(cat\ /tmp/k3s_version)" + end + + vm.provision "shell", inline: "ping -c 2 k3s.io" + + if roles.include?("server") && role_num == 0 + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.installer_url = 'https://get.k3s.io' + k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + elsif roles.include?("server") && role_num != 0 + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.installer_url = 'https://get.k3s.io' + k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + end + if roles.include?("agent") + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.installer_url = 'https://get.k3s.io' + k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + end + if vm.box.include?("microos") + vm.provision 'k3s-reload', type: 'reload', run: 'once' + end +end + +def osConfigure(vm) + + if vm.box.include?("ubuntu2004") + vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0", run: 'once' + vm.provision "shell", inline: "apt install -y jq", run: 'once' + end + if vm.box.include?("Leap") + vm.provision "shell", inline: "zypper install -y jq", run: 'once' + end + if vm.box.include?("microos") + vm.provision "shell", inline: "transactional-update pkg install -y jq", run: 'once' + vm.provision 'reload', run: 'once' + end + +end + +Vagrant.configure("2") do |config| + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] + # Default provider is libvirt, virtualbox is only provided as a backup + config.vm.provider "libvirt" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + end + config.vm.provider "virtualbox" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + end + + if NODE_ROLES.kind_of?(String) + NODE_ROLES = NODE_ROLES.split(" ", -1) + end + if NODE_BOXES.kind_of?(String) + NODE_BOXES = NODE_BOXES.split(" ", -1) + end + + # Must iterate on the index, vagrant does not understand iterating + # over the node roles themselves + NODE_ROLES.length.times do |i| + name = NODE_ROLES[i] + config.vm.define name do |node| + roles = name.split("-", -1) + role_num = roles.pop.to_i + provision(node.vm, roles, role_num, i) + end + end +end diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 557c9c11af..5cd8072983 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -4,10 +4,12 @@ NODE_ROLES = (ENV['NODE_ROLES'] || NODE_BOXES = (ENV['NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "") NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks NETWORK_PREFIX = "10.10.10" +install_type = "" def provision(vm, roles, role_num, node_num) vm.box = NODE_BOXES[node_num] @@ -17,32 +19,27 @@ def provision(vm, roles, role_num, node_num) osConfigure(vm) - # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build - $check_commit = <<-'SCRIPT' - curl -s -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/k3s-io/k3s/commits?per_page=5&sha=#{GITHUB_BRANCH}' | jq -r '.[] | .sha' &> /tmp/k3s_commits - curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 /tmp/k3s_commits).sha256sum - while [ $? -ne 0 ]; do - sed -i 1d /tmp/k3s_commits - curl -s --fail https://storage.googleapis.com/k3s-ci-builds/k3s-$(head -n 1 /tmp/k3s_commits).sha256sum - done - SCRIPT - vm.provision "shell", - inline: $check_commit - + if !RELEASE_VERSION.empty? + install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}" + else + # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build + vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/k3s_commits"] + install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)" + end vm.provision "shell", inline: "ping -c 2 k3s.io" if roles.include?("server") && role_num == 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end elsif roles.include?("server") && role_num != 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end @@ -50,7 +47,7 @@ def provision(vm, roles, role_num, node_num) vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'https://get.k3s.io' k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end @@ -69,7 +66,8 @@ def osConfigure(vm) vm.provision "shell", inline: "zypper install -y jq" end if vm.box.include?("microos") - vm.provision "shell", inline: "transactional-update pkg install -y jq", reboot: true + vm.provision "shell", inline: "transactional-update pkg install -y jq" + vm.provision 'reload', run: 'once' end end