From c77efe64e1521d48c91987b92f406d616fe964eb Mon Sep 17 00:00:00 2001 From: Jacob Blain Christen Date: Mon, 15 Nov 2021 11:50:38 -0700 Subject: [PATCH] tests/vagrant: refactor vagrant smoke tests (#4484) - updated `tests/TESTING.md` - cgroup, snapshotter, and install tests all under tests/vagrant - cgroup and snapshotter workflows trigger for all code changes on all branches (excluding markdown docs, install script, and other vagrant tests) - install workflow triggers for relevant script and fixture changes, only on master or pull-requests that target it - integration and unit test workflows should not trigger for install script changes nor anything under tests/vagrant Signed-off-by: Jacob Blain Christen --- .github/workflows/cgroup.yaml | 91 +++++++++++ .github/workflows/cgroup2.yaml | 80 ---------- .github/workflows/install.yaml | 150 +++++++----------- .github/workflows/integration.yaml | 10 +- .github/workflows/snapshotter.yaml | 83 +++++----- .github/workflows/unitcoverage.yaml | 10 +- tests/TESTING.md | 76 ++++++++- tests/cgroup2/.gitignore | 5 - tests/cgroup2/Vagrantfile | 61 ------- tests/vagrant/.gitignore | 1 + .../cgroup/unified/fedora-34/Vagrantfile | 120 ++++++++++++++ .../install/centos-7/Vagrantfile | 47 +++--- .../install/centos-8/Vagrantfile | 47 +++--- .../install/fedora-coreos/Vagrantfile | 49 +++--- .../install/opensuse-leap/Vagrantfile | 47 +++--- .../install/opensuse-microos/Vagrantfile | 45 +++--- .../install/ubuntu-focal/Vagrantfile | 49 +++--- .../btrfs/opensuse-leap/Vagrantfile | 55 ++++--- 18 files changed, 597 insertions(+), 429 deletions(-) create mode 100644 .github/workflows/cgroup.yaml delete mode 100644 .github/workflows/cgroup2.yaml delete mode 100644 tests/cgroup2/.gitignore delete mode 100644 tests/cgroup2/Vagrantfile create mode 100644 tests/vagrant/.gitignore create mode 100644 tests/vagrant/cgroup/unified/fedora-34/Vagrantfile rename tests/{ => vagrant}/install/centos-7/Vagrantfile (70%) rename tests/{ => vagrant}/install/centos-8/Vagrantfile (70%) rename tests/{ => vagrant}/install/fedora-coreos/Vagrantfile (65%) rename tests/{ => vagrant}/install/opensuse-leap/Vagrantfile (67%) rename tests/{ => vagrant}/install/opensuse-microos/Vagrantfile (72%) rename tests/{ => vagrant}/install/ubuntu-focal/Vagrantfile (65%) rename tests/{ => vagrant}/snapshotter/btrfs/opensuse-leap/Vagrantfile (65%) diff --git a/.github/workflows/cgroup.yaml b/.github/workflows/cgroup.yaml new file mode 100644 index 0000000000..84f2c96192 --- /dev/null +++ b/.github/workflows/cgroup.yaml @@ -0,0 +1,91 @@ +name: Control Group +on: + push: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/vagrant/cgroup/**" + - ".github/**" + - "!.github/workflows/cgroup.yaml" + pull_request: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/vagrant/cgroup/**" + - ".github/**" + - "!.github/workflows/cgroup.yaml" + workflow_dispatch: {} +jobs: + prep: + name: "Prepare" + runs-on: ubuntu-20.04 + timeout-minutes: 40 + steps: + - name: "Checkout" + uses: actions/checkout@v2 + with: { fetch-depth: 1 } + - name: "Build" + run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make + - name: "Upload" + uses: actions/upload-artifact@v2 + with: { name: k3s, path: dist/artifacts/k3s } + test: + name: "Smoke Test" + needs: prep + # nested virtualization is only available on macOS hosts + runs-on: macos-10.15 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + vm: [fedora-34] + mode: [unified] + max-parallel: 1 + defaults: + run: + working-directory: tests/vagrant/cgroup/${{ matrix.mode }}/${{ matrix.vm }} + steps: + - name: "Checkout" + uses: actions/checkout@v2 + with: { fetch-depth: 1 } + - name: "Download Binary" + uses: actions/download-artifact@v2 + with: { name: k3s, path: dist/artifacts/ } + - name: "Vagrant Cache" + uses: actions/cache@v2 + with: + path: | + ~/.vagrant.d/boxes + ~/.vagrant.d/gems + key: cgroup-${{ hashFiles(format('tests/vagrant/cgroup/{0}/{1}/Vagrantfile', matrix.mode, matrix.vm)) }} + id: vagrant-cache + continue-on-error: true + - name: "Vagrant Plugin(s)" + run: vagrant plugin install vagrant-k3s + - name: "Vagrant Up ⏩ Install K3s" + run: vagrant up + - name: "K3s Start" # start k3s rootfull + run: vagrant ssh -- sudo systemctl start k3s-server + - name: "K3s Ready" # wait for k3s to be ready + run: vagrant provision --provision-with=k3s-ready + - name: "K3s Status" # kubectl get node,all -A -o wide + run: vagrant provision --provision-with=k3s-status + - name: "Sonobuoy (--mode=quick)" + env: {TEST_RESULTS_PATH: rootfull} + run: vagrant provision --provision-with=k3s-sonobuoy + - name: "K3s Stop" # stop k3s rootfull + run: vagrant ssh -- sudo systemctl stop k3s-server + - name: "Vagrant Reload" + run: vagrant reload + - name: "[Rootless] Starting K3s" + run: vagrant ssh -- systemctl --user start k3s-rootless + - name: "[Rootless] K3s Ready" + env: {TEST_KUBECONFIG: /home/vagrant/.kube/k3s.yaml} + run: vagrant provision --provision-with=k3s-ready + - name: "[Rootless] Sonobuoy (--mode=quick)" + env: {TEST_KUBECONFIG: /home/vagrant/.kube/k3s.yaml, TEST_RESULTS_PATH: rootless} + run: vagrant provision --provision-with=k3s-sonobuoy diff --git a/.github/workflows/cgroup2.yaml b/.github/workflows/cgroup2.yaml deleted file mode 100644 index 7b63a90f57..0000000000 --- a/.github/workflows/cgroup2.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: cgroup2 -on: - push: - paths-ignore: - - "**.md" - - "install.sh" - - "tests/install" - pull_request: - paths-ignore: - - "**.md" - - "install.sh" - - "tests/install" - workflow_dispatch: {} -jobs: - build: - name: "Build" - runs-on: ubuntu-20.04 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "Make" - run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make - - name: "Upload k3s binary" - uses: actions/upload-artifact@v2 - with: - name: k3s - path: dist/artifacts/k3s - test: - name: "Test" - needs: build - # nested virtualization is only available on macOS hosts - runs-on: macos-10.15 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "Download k3s binary" - uses: actions/download-artifact@v2 - with: - name: k3s - path: ./tests/cgroup2 - - name: "Boot Fedora VM" - run: | - cp -r k3s.service k3s-rootless.service ./tests/util ./tests/cgroup2 - cd ./tests/cgroup2 - vagrant up - vagrant ssh-config >> ~/.ssh/config - - name: "Starting k3s" - run: | - ssh default -- sudo systemctl start k3s - # Sonobuoy requires CoreDNS to be ready - - name: "Waiting for CoreDNS to be ready" - run: | - ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml /vagrant/util/wait-for-coredns.sh - # Vagrant is slow, so we set --mode=quick here - - name: "Run Sonobuoy (--mode=quick)" - run: | - ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --mode=quick --wait - - name: "Stopping k3s" - run: | - ssh default -- sudo systemctl stop k3s - # FIXME: rootful k3s processes are still running even after `systemctl stop k3s`, so we reboot the VM here. - # This reboot is also useful for ensuring `systemctl daemon-reload`: https://github.com/rootless-containers/rootlesscontaine.rs/issues/32 - cd ./tests/cgroup2 - vagrant halt - vagrant up - - name: "[Rootless] Starting k3s-rootless" - run: | - ssh default -- systemctl --user start k3s-rootless - - name: "[Rootless] Waiting for CoreDNS to be ready" - run: | - ssh default -- KUBECONFIG=/home/vagrant/.kube/k3s.yaml /vagrant/util/wait-for-coredns.sh - - name: "[Rootless] Run Sonobuoy (--mode=quick)" - run: | - ssh default -- KUBECONFIG=/home/vagrant/.kube/k3s.yaml sonobuoy run --mode=quick --wait diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml index fdb8d5d869..20477943cf 100644 --- a/.github/workflows/install.yaml +++ b/.github/workflows/install.yaml @@ -1,111 +1,71 @@ -name: Installer +name: Install Script on: push: + branches: [main, master] paths: + - "channel.yaml" - "install.sh" - - "tests/install/**" + - "tests/vagrant/install/**" pull_request: + branches: [main, master] paths: - "install.sh" - - "tests/install/**" + - "tests/vagrant/install/**" workflow_dispatch: {} jobs: - install-centos-7: - name: "CentOS 7" - # nested virtualization is only available on macOS hosts + test: + name: "Smoke Test" runs-on: macos-10.15 timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + channel: [stable] + vm: [centos-7, centos-8, fedora-coreos, opensuse-leap, opensuse-microos, ubuntu-focal] + include: + - {channel: latest, vm: centos-8} + - {channel: latest, vm: ubuntu-focal} + - {channel: latest, vm: opensuse-leap} + - {channel: testing, vm: centos-8} + - {channel: testing, vm: opensuse-leap} + - {channel: testing, vm: ubuntu-focal} + max-parallel: 2 + defaults: + run: + working-directory: tests/vagrant/install/${{ matrix.vm }} + env: + INSTALL_K3S_CHANNEL: ${{ matrix.channel }} steps: - name: "Checkout" uses: actions/checkout@v2 + with: {fetch-depth: 1} + - name: "Vagrant Cache" + uses: actions/cache@v2 with: - fetch-depth: 1 - - name: "VagrantPlugin::K3S" - working-directory: tests/install/centos-7 - run: vagrant plugin install vagrant-k3s - - name: "Vagrant::⬆" - working-directory: tests/install/centos-7 - run: vagrant up - install-centos-8: - name: "CentOS 8" - # nested virtualization is only available on macOS hosts - runs-on: macos-10.15 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "VagrantPlugin::K3S" - working-directory: tests/install/centos-8 - run: vagrant plugin install vagrant-k3s - - name: "Vagrant::⬆" - working-directory: tests/install/centos-8 - run: vagrant up - install-fedora-coreos: - name: "Fedora CoreOS" - # nested virtualization is only available on macOS hosts - runs-on: macos-10.15 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "VagrantPlugin::K3S" - working-directory: tests/install/fedora-coreos - run: vagrant plugin install vagrant-k3s - - name: "Vagrant::⬆" - working-directory: tests/install/fedora-coreos - run: vagrant up - install-opensuse-leap: - name: "openSUSE Leap" - # nested virtualization is only available on macOS hosts - runs-on: macos-10.15 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "VagrantPlugin::K3S" - working-directory: tests/install/opensuse-leap - run: vagrant plugin install vagrant-k3s - - name: "Vagrant::⬆" - working-directory: tests/install/opensuse-leap - run: vagrant up - install-opensuse-microos: - name: "openSUSE MicroOS" - # nested virtualization is only available on macOS hosts - runs-on: macos-10.15 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "VagrantPlugin::Reload" - working-directory: tests/install/opensuse-microos - run: vagrant plugin install vagrant-reload - - name: "VagrantPlugin::K3S" - working-directory: tests/install/opensuse-microos - run: vagrant plugin install vagrant-k3s - - name: "Vagrant::⬆" - working-directory: tests/install/opensuse-microos - run: vagrant up - install-ubuntu-focal: - name: "Ubuntu Focal" - # nested virtualization is only available on macOS hosts - runs-on: macos-10.15 - timeout-minutes: 40 - steps: - - name: "Checkout" - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: "VagrantPlugin::K3S" - working-directory: tests/install/ubuntu-focal - run: vagrant plugin install vagrant-k3s - - name: "Vagrant::⬆" - working-directory: tests/install/ubuntu-focal + path: | + ~/.vagrant.d/boxes + ~/.vagrant.d/gems + key: install-${{ hashFiles(format('tests/vagrant/install/{0}/Vagrantfile', matrix.vm)) }} + id: vagrant-cache + continue-on-error: true + - name: "Vagrant Plugin(s)" + run: vagrant plugin install vagrant-k3s vagrant-reload + - name: "Vagrant Up ⏩ Install K3s" run: vagrant up + - name: "⏳ Node" + run: vagrant provision --provision-with=k3s-wait-for-node + - name: "⏳ CoreDNS" + run: vagrant provision --provision-with=k3s-wait-for-coredns + - name: "⏳ Local Storage" + run: vagrant provision --provision-with=k3s-wait-for-local-storage + continue-on-error: true + - name: "⏳ Metrics Server" + run: vagrant provision --provision-with=k3s-wait-for-metrics-server + continue-on-error: true + - name: "⏳ Traefik" + run: vagrant provision --provision-with=k3s-wait-for-traefik + continue-on-error: true + - name: "k3s-status" + run: vagrant provision --provision-with=k3s-status + - name: "k3s-procps" + run: vagrant provision --provision-with=k3s-procps diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 2ed245b494..6ea8098708 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -3,13 +3,19 @@ on: push: paths-ignore: - "**.md" + - "channel.yaml" - "install.sh" - - "tests/install" + - "tests/vagrant/**" + - ".github/**" + - "!.github/workflows/integration.yaml" pull_request: paths-ignore: - "**.md" + - "channel.yaml" - "install.sh" - - "tests/install" + - "tests/vagrant/**" + - ".github/**" + - "!.github/workflows/integration.yaml" workflow_dispatch: {} jobs: build: diff --git a/.github/workflows/snapshotter.yaml b/.github/workflows/snapshotter.yaml index 16cc52c837..c875817f37 100644 --- a/.github/workflows/snapshotter.yaml +++ b/.github/workflows/snapshotter.yaml @@ -1,71 +1,80 @@ -name: Snapshotter Testing +name: Snapshotter on: push: paths-ignore: + - "**.md" + - "channel.yaml" - "install.sh" - - "tests/cgroup2/**" - - "tests/install/**" - - "tests/integration/**" - - "tests/unitcoverage/**" + - "tests/**" + - "!tests/vagrant/snapshotter/**" + - ".github/**" + - "!.github/workflows/snapshotter.yaml" pull_request: paths-ignore: + - "**.md" + - "channel.yaml" - "install.sh" - - "tests/cgroup2/**" - - "tests/install/**" - - "tests/integration/**" - - "tests/unitcoverage/**" + - "tests/**" + - "!tests/vagrant/snapshotter/**" + - ".github/**" + - "!.github/workflows/snapshotter.yaml" workflow_dispatch: {} jobs: - build: - name: "Build" + prep: + name: "Prepare" runs-on: ubuntu-20.04 timeout-minutes: 40 steps: - name: "Checkout" uses: actions/checkout@v2 - with: - fetch-depth: 1 + with: { fetch-depth: 1 } - name: "Build" run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make - name: "Upload Binary" uses: actions/upload-artifact@v2 - with: - name: k3s - path: dist/artifacts/k3s - - name: "Upload Images" - uses: actions/upload-artifact@v2 - with: - name: k3s-airgap-images-amd64.tar - path: dist/artifacts/k3s-airgap-images-amd64.tar + with: { name: k3s, path: dist/artifacts/k3s } test: - name: "Test" + name: "Smoke Test" + needs: prep # nested virtualization is only available on macOS hosts runs-on: macos-10.15 - needs: build timeout-minutes: 40 strategy: + fail-fast: false matrix: vm: [opensuse-leap] + snapshotter: [btrfs] + max-parallel: 1 + defaults: + run: + working-directory: tests/vagrant/snapshotter/${{ matrix.snapshotter }}/${{ matrix.vm }} + env: + VAGRANT_EXPERIMENTAL: disks steps: - name: "Checkout" uses: actions/checkout@v2 - with: - fetch-depth: 1 + with: { fetch-depth: 1 } - name: "Download Binary" uses: actions/download-artifact@v2 + with: { name: k3s, path: dist/artifacts/ } + - name: "Vagrant Cache" + uses: actions/cache@v2 with: - name: k3s - path: dist/artifacts/ - - name: "Download Images" - uses: actions/download-artifact@v2 - with: - name: k3s-airgap-images-amd64.tar - path: dist/artifacts/ + path: | + ~/.vagrant.d/boxes + ~/.vagrant.d/gems + key: snapshotter-${{ hashFiles(format('tests/vagrant/snapshotter/{0}/{1}/Vagrantfile', matrix.snapshotter, matrix.vm)) }} + id: vagrant-cache + continue-on-error: true - name: "Vagrant Plugin(s)" - working-directory: tests/snapshotter/btrfs/${{ matrix.vm }} run: vagrant plugin install vagrant-k3s - - name: "Vagrant VM" - working-directory: tests/snapshotter/btrfs/${{ matrix.vm }} - env: - VAGRANT_EXPERIMENTAL: disks + - name: "Vagrant Up ⏩ Install K3s" run: vagrant up + - name: "⏳ Node" + run: vagrant provision --provision-with=k3s-wait-for-node + - name: "⏳ CoreDNS" + run: vagrant provision --provision-with=k3s-wait-for-coredns + - name: "k3s-status" # kubectl get node,all -A -o wide + run: vagrant provision --provision-with=k3s-status + - name: "k3s-snapshots" # if no snapshots then we fail + run: vagrant provision --provision-with=k3s-snapshots diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml index af06d9647d..a33104f4e6 100644 --- a/.github/workflows/unitcoverage.yaml +++ b/.github/workflows/unitcoverage.yaml @@ -3,13 +3,19 @@ on: push: paths-ignore: - "**.md" + - "channel.yaml" - "install.sh" - - "tests/install" + - "tests/vagrant/**" + - ".github/**" + - "!.github/workflows/unitcoverage.yaml" pull_request: paths-ignore: - "**.md" + - "channel.yaml" - "install.sh" - - "tests/install" + - "tests/vagrant/**" + - ".github/**" + - "!.github/workflows/unitcoverage.yaml" workflow_dispatch: {} jobs: test: diff --git a/tests/TESTING.md b/tests/TESTING.md index 983d58c746..84c6cce81a 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -4,7 +4,7 @@ Go testing in K3s comes in 3 forms: Unit, Integration, and End-to-End (E2E). Thi document will explain *when* each test should be written and *how* each test should be generated, formatted, and run. -Note: all shell commands given are relateive to the root k3s repo directory. +Note: all shell commands given are relative to the root k3s repo directory. ___ ## Unit Tests @@ -95,6 +95,80 @@ sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy results + The number of vCPU for the guest to use. +- `TEST_VM_MEMORY` (default :arrow_right: 2048)
+ The number of megabytes of memory for the guest to use. +- `TEST_VM_BOOT_TIMEOUT` (default :arrow_right: 600)
+ The time in seconds that Vagrant will wait for the machine to boot and be accessible. + +### Running + +The **Install Script** tests can be run by changing to the fixture directory and invoking `vagrant up`, e.g.: +```shell +cd tests/vagrant/install/centos-8 +vagrant up +# the following provisioners are optional. the do not run by default but are invoked +# explicitly by github actions workflow to avoid certain timeout issues on slow runners +vagrant provision --provision-with=k3s-wait-for-node +vagrant provision --provision-with=k3s-wait-for-coredns +vagrant provision --provision-with=k3s-wait-for-local-storage +vagrant provision --provision-with=k3s-wait-for-metrics-server +vagrant provision --provision-with=k3s-wait-for-traefik +vagrant provision --provision-with=k3s-status +vagrant provision --provision-with=k3s-procps +``` + +The **Control Groups** and **Snapshotter** tests require that k3s binary is built at `dist/artifacts/k3s`. +They are invoked similarly, i.e. `vagrant up`, but with different sets of named shell provisioners. +Take a look at the individual Vagrantfiles and/or the Github Actions workflows that harness them to get +an idea of how they can be invoked. + +___ + ## End-to-End (E2E) Tests End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on separate testing utilities and are self-contained within the `test/e2e` directory. E2E tests cover complete K3s single and multi-cluster configuration and administration: bringup, update, teardown etc. diff --git a/tests/cgroup2/.gitignore b/tests/cgroup2/.gitignore deleted file mode 100644 index 7d75037630..0000000000 --- a/tests/cgroup2/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -k3s -k3s.service -k3s-rootless.service -util/ -.vagrant/ diff --git a/tests/cgroup2/Vagrantfile b/tests/cgroup2/Vagrantfile deleted file mode 100644 index 3d0fef70eb..0000000000 --- a/tests/cgroup2/Vagrantfile +++ /dev/null @@ -1,61 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrant box for testing k3s with cgroup v2. -# Invoked via k3s/.github/workflows/cgroup2.yaml . -# -# The following files need to be present in this directory: -# - k3s -# - k3s.service -# - k3s-rootless.service -# - util/ -Vagrant.configure("2") do |config| - config.vm.box = "fedora/34-cloud-base" - memory = 2048 - cpus = 2 - config.vm.provider :virtualbox do |v| - v.memory = memory - v.cpus = cpus - end - config.vm.provider :libvirt do |v| - v.memory = memory - v.cpus = cpus - end - config.vm.provision "install-k3s", type: "shell", run: "once" do |sh| - sh.inline = <<~SHELL - set -eux -o pipefail - - # Install k3s binary - install -m 755 /vagrant/k3s /usr/local/bin - ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl - - # Install k3s systemd service (not launched here) - cp -f /vagrant/k3s.service /etc/systemd/system/k3s.service - touch /etc/systemd/system/k3s.service.env - systemctl daemon-reload - - # Install sonobuoy binary - curl -fsSL https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.20.0/sonobuoy_0.20.0_linux_amd64.tar.gz | tar xzvC /usr/local/bin sonobuoy - - # [Rootless] Configure sysctl - echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/rootless.conf - sysctl --system - - # [Rootless] Enable cgroup v2 delegation - mkdir -p /etc/systemd/system/user@.service.d - cat <<-EOF > /etc/systemd/system/user@.service.d/delegate.conf -[Service] -Delegate=yes -EOF - systemctl daemon-reload - - # [Rootless] Enable systemd lingering - loginctl enable-linger vagrant - - # [Rootless] Install k3s-rootless systemd service (not launched here) - mkdir -p /home/vagrant/.config/systemd/user - cp -f /vagrant/k3s-rootless.service /home/vagrant/.config/systemd/user/k3s-rootless.service - chown -R vagrant:vagrant /home/vagrant/.config - SHELL - end -end diff --git a/tests/vagrant/.gitignore b/tests/vagrant/.gitignore new file mode 100644 index 0000000000..a977916f65 --- /dev/null +++ b/tests/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant/ diff --git a/tests/vagrant/cgroup/unified/fedora-34/Vagrantfile b/tests/vagrant/cgroup/unified/fedora-34/Vagrantfile new file mode 100644 index 0000000000..3e1d614d79 --- /dev/null +++ b/tests/vagrant/cgroup/unified/fedora-34/Vagrantfile @@ -0,0 +1,120 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# +# Vagrant box for testing k3s with cgroup v2. + +ENV['TEST_UNITFILE_ROOTFULL'] ||= '../../../../../k3s.service' +ENV['TEST_UNITFILE_ROOTLESS'] ||= '../../../../../k3s-rootless.service' + +Vagrant.configure("2") do |config| + config.vagrant.plugins = { + 'vagrant-k3s' => {:version => '~> 0.1.3'}, + } + config.vm.box = "fedora/34-cloud-base" + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '../../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false, + rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*' + + config.vm.define 'cgroup-unified', primary: true do |test| + test.vm.hostname = 'smoke' + test.vm.provision :file, run: 'always', source: ENV['TEST_UNITFILE_ROOTFULL'], destination: 'k3s-rootfull.service' + test.vm.provision :file, run: 'always', source: ENV['TEST_UNITFILE_ROOTLESS'], destination: 'k3s-rootless.service' + test.vm.provision 'k3s-prepare', type: "shell", run: "once", privileged: true do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + + # Install k3s binary + install -vm 755 /vagrant/k3s /usr/local/bin + + # Install k3s SELinux policy + dnf install -y https://github.com/k3s-io/k3s-selinux/releases/download/v0.5.testing.2/k3s-selinux-0.5-2.el8.noarch.rpm + + # Install k3s systemd service (not launched here) + install -vm 644 -T /home/vagrant/k3s-rootfull.service /etc/systemd/system/k3s-server.service + touch /etc/systemd/system/k3s-server.service.env + systemctl daemon-reload + + # Install sonobuoy binary + curl -fsSL https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.20.0/sonobuoy_0.20.0_linux_amd64.tar.gz | tar xzvC /usr/local/bin sonobuoy + + # [Rootless] Configure sysctl + echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/rootless.conf + sysctl --system + # [Rootless] Enable cgroup v2 delegation + mkdir -p /etc/systemd/system/user@.service.d + cat <<-EOF > /etc/systemd/system/user@.service.d/delegate.conf +[Service] +Delegate=yes +EOF + systemctl daemon-reload + + # [Rootless] Enable systemd lingering + loginctl enable-linger vagrant + + # [Rootless] Install k3s-rootless systemd service (not launched here) + mkdir -p /home/vagrant/.config/systemd/user + cp -f /home/vagrant/k3s-rootless.service /home/vagrant/.config/systemd/user/k3s-rootless.service + chown -R vagrant:vagrant /home/vagrant/.config + SHELL + end + test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.args = %w[server] + k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=vagrant INSTALL_K3S_SKIP_ENABLE=true] + k3s.config = { + 'disable' => %w[local-storage metrics-server servicelb traefik], + 'disable-helm-controller' => true, + 'disable-network-policy' => true, + 'write-kubeconfig-mode' => '0644', + } + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + test.vm.provision "k3s-ready", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { + :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin", + :KUBECONFIG => ENV['TEST_KUBECONFIG'] || '/etc/rancher/k3s/k3s.yaml', + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + echo 'Waiting for node to be ready ...' + time timeout 500 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + time timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + SHELL + end + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { + :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin", + :KUBECONFIG => ENV['TEST_KUBECONFIG'] || '/etc/rancher/k3s/k3s.yaml', + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + kubectl get node,all -A -o wide + SHELL + end + test.vm.provision "k3s-sonobuoy", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { + :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin", + :KUBECONFIG => ENV['TEST_KUBECONFIG'] || '/etc/rancher/k3s/k3s.yaml', + :RESULTS_PATH => ENV['TEST_RESULTS_PATH'] || '.', + } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + sonobuoy run --mode=quick --wait + sonobuoy retrieve ${RESULTS_PATH} + sonobuoy results $(ls -rt ${RESULTS_PATH}/*.tar.gz | tail -1) | grep Status | grep passed + SHELL + end + end + + config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + + %w[libvirt virtualbox].each do |p| + config.vm.provider p do |v| + v.cpus = ENV['TEST_VM_CPUS'] || 2 + v.memory = ENV['TEST_VM_MEMORY'] || 2048 + end + end +end diff --git a/tests/install/centos-7/Vagrantfile b/tests/vagrant/install/centos-7/Vagrantfile similarity index 70% rename from tests/install/centos-7/Vagrantfile rename to tests/vagrant/install/centos-7/Vagrantfile index 810429a391..12cde07a20 100644 --- a/tests/install/centos-7/Vagrantfile +++ b/tests/vagrant/install/centos-7/Vagrantfile @@ -2,71 +2,74 @@ # vi: set ft=ruby : # -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' +ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, } + config.vm.box = "dweomer/centos-7.9-amd64" + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define 'install-centos-7', primary: true do |test| - test.vm.box = 'centos/7' - test.vm.hostname = 'install' - test.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + }) k3s.config = { :selinux => true, :token => 'vagrant', } k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-wait-for-node", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -74,17 +77,19 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-status-selinux", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eux -o pipefail - ps auxZ | grep container_ | grep -v grep + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep SHELL end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + + %w[libvirt virtualbox vmware_desktop].each do |p| config.vm.provider p do |v| v.cpus = ENV['TEST_VM_CPUS'] || 2 v.memory = ENV['TEST_VM_MEMORY'] || 2048 @@ -94,6 +99,4 @@ Vagrant.configure("2") do |config| v.gui = false v.check_guest_additions = false end - - config.vm.synced_folder '.', '/vagrant', disabled: true -end \ No newline at end of file +end diff --git a/tests/install/centos-8/Vagrantfile b/tests/vagrant/install/centos-8/Vagrantfile similarity index 70% rename from tests/install/centos-8/Vagrantfile rename to tests/vagrant/install/centos-8/Vagrantfile index 879026ff63..3f1d792170 100644 --- a/tests/install/centos-8/Vagrantfile +++ b/tests/vagrant/install/centos-8/Vagrantfile @@ -2,71 +2,74 @@ # vi: set ft=ruby : # -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' +ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, } + config.vm.box = "dweomer/centos-8.4-amd64" + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define 'install-centos-8', primary: true do |test| - test.vm.box = 'centos/stream8' - test.vm.hostname = 'install' - test.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + }) k3s.config = { :selinux => true, :token => 'vagrant', } k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-wait-for-node", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -74,17 +77,19 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-status-selinux", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eux -o pipefail - ps auxZ | grep container_ | grep -v grep + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep SHELL end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + + %w[libvirt virtualbox vmware_desktop].each do |p| config.vm.provider p do |v| v.cpus = ENV['TEST_VM_CPUS'] || 2 v.memory = ENV['TEST_VM_MEMORY'] || 2048 @@ -94,6 +99,4 @@ Vagrant.configure("2") do |config| v.gui = false v.check_guest_additions = false end - - config.vm.synced_folder '.', '/vagrant', disabled: true -end \ No newline at end of file +end diff --git a/tests/install/fedora-coreos/Vagrantfile b/tests/vagrant/install/fedora-coreos/Vagrantfile similarity index 65% rename from tests/install/fedora-coreos/Vagrantfile rename to tests/vagrant/install/fedora-coreos/Vagrantfile index f6ff84961d..66fbe9dc6b 100644 --- a/tests/install/fedora-coreos/Vagrantfile +++ b/tests/vagrant/install/fedora-coreos/Vagrantfile @@ -2,71 +2,74 @@ # vi: set ft=ruby : # -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' +ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, } + config.vm.box = 'dhml/fedora-coreos-34.20210725.3.0-20210813' + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define 'install-fedora-coreos', primary: true do |test| - test.vm.box = 'dhml/fedora-coreos-34.20210725.3.0-20210813' - test.vm.hostname = 'install' - test.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///var/home/core/install.sh' k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + }) k3s.config = { :selinux => true, :token => 'vagrant', } k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-wait-for-node", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -74,9 +77,19 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end + test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep + SHELL + end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + + %w[libvirt virtualbox vmware_desktop].each do |p| config.vm.provider p do |v| v.cpus = ENV['TEST_VM_CPUS'] || 2 v.memory = ENV['TEST_VM_MEMORY'] || 2048 @@ -86,6 +99,4 @@ Vagrant.configure("2") do |config| v.gui = false v.check_guest_additions = false end - - config.vm.synced_folder '.', '/vagrant', disabled: true end diff --git a/tests/install/opensuse-leap/Vagrantfile b/tests/vagrant/install/opensuse-leap/Vagrantfile similarity index 67% rename from tests/install/opensuse-leap/Vagrantfile rename to tests/vagrant/install/opensuse-leap/Vagrantfile index 8d57462ee7..f0f8370bd4 100644 --- a/tests/install/opensuse-leap/Vagrantfile +++ b/tests/vagrant/install/opensuse-leap/Vagrantfile @@ -2,71 +2,76 @@ # vi: set ft=ruby : # -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' +ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, } + config.vm.box = 'opensuse/Leap-15.3.x86_64' + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define 'install-opensuse-leap', primary: true do |test| - test.vm.box = 'opensuse/Leap-15.3.x86_64' - test.vm.hostname = 'install' + test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' # /sbin/apparmor_parser is needed by the 1.21 kubelet if the value of /sys/module/apparmor/parameters/enabled is Y test.vm.provision 'k3s-prepare', type: 'shell', run: 'once', inline: 'zypper install -y apparmor-parser' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_SELINUX_RPM=true] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + :INSTALL_K3S_SKIP_SELINUX_RPM => 'true', + }) k3s.config = { :token => 'vagrant', } k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-wait-for-node", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -74,9 +79,17 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end + test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep + SHELL + end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + %w[libvirt virtualbox vmware_desktop].each do |p| config.vm.provider p do |v| v.cpus = ENV['TEST_VM_CPUS'] || 2 v.memory = ENV['TEST_VM_MEMORY'] || 2048 @@ -86,6 +99,4 @@ Vagrant.configure("2") do |config| v.gui = false v.check_guest_additions = false end - - config.vm.synced_folder '.', '/vagrant', disabled: true end diff --git a/tests/install/opensuse-microos/Vagrantfile b/tests/vagrant/install/opensuse-microos/Vagrantfile similarity index 72% rename from tests/install/opensuse-microos/Vagrantfile rename to tests/vagrant/install/opensuse-microos/Vagrantfile index 2b40b0eeed..30934ba045 100644 --- a/tests/install/opensuse-microos/Vagrantfile +++ b/tests/vagrant/install/opensuse-microos/Vagrantfile @@ -2,23 +2,26 @@ # vi: set ft=ruby : # -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' +ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, 'vagrant-reload' => {}, } + config.vm.box = 'dweomer/microos.amd64' + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define 'install-microos', primary: true do |test| - test.vm.box = 'dweomer/microos.amd64' - test.vm.hostname = 'install' - test.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' + test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + }) k3s.config = { :selinux => true, :token => 'vagrant', @@ -27,49 +30,49 @@ Vagrant.configure("2") do |config| end # install.sh will move the snapshot forward when installing k3s-selinux policy, so, we reboot to pick that up test.vm.provision 'k3s-reload', type: 'reload', run: 'once' - test.vm.provision "k3s-wait-for-node", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -77,17 +80,17 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-status-selinux", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eux -o pipefail - ps auxZ | grep container_ | grep -v grep + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep SHELL end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + %w[libvirt virtualbox vmware_desktop].each do |p| config.vm.provider p do |v| v.cpus = ENV['TEST_VM_CPUS'] || 2 v.memory = ENV['TEST_VM_MEMORY'] || 2048 @@ -97,6 +100,4 @@ Vagrant.configure("2") do |config| v.gui = false v.check_guest_additions = false end - - config.vm.synced_folder '.', '/vagrant', disabled: true -end \ No newline at end of file +end diff --git a/tests/install/ubuntu-focal/Vagrantfile b/tests/vagrant/install/ubuntu-focal/Vagrantfile similarity index 65% rename from tests/install/ubuntu-focal/Vagrantfile rename to tests/vagrant/install/ubuntu-focal/Vagrantfile index 54fdd4c538..eafd61d609 100644 --- a/tests/install/ubuntu-focal/Vagrantfile +++ b/tests/vagrant/install/ubuntu-focal/Vagrantfile @@ -2,71 +2,74 @@ # vi: set ft=ruby : # -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' +ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, } + config.vm.box = 'generic/ubuntu2004' + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define 'install-ubuntu-focal', primary: true do |test| - test.vm.box = 'generic/ubuntu2004' - test.vm.hostname = 'install' + test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' - # sh is a symlink to dash, so, we need to sidestep this: https://github.com/dweomer/vagrant-k3s/blob/v0.1.2/lib/vagrant-k3s/provisioner.rb#L71 - test.vm.provision 'k3s-prepare', type: 'shell', run: 'once', inline: 'ln -nsf bash /usr/bin/sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_SELINUX_RPM=true] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + :INSTALL_K3S_SKIP_SELINUX_RPM => 'true', + }) k3s.config = { :token => 'vagrant', } k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-wait-for-node", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail - time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -74,9 +77,17 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end + test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep + SHELL + end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + %w[libvirt virtualbox vmware_desktop].each do |p| config.vm.provider p do |v| v.cpus = ENV['TEST_VM_CPUS'] || 2 v.memory = ENV['TEST_VM_MEMORY'] || 2048 @@ -86,6 +97,4 @@ Vagrant.configure("2") do |config| v.gui = false v.check_guest_additions = false end - - config.vm.synced_folder '.', '/vagrant', disabled: true end \ No newline at end of file diff --git a/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile b/tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile similarity index 65% rename from tests/snapshotter/btrfs/opensuse-leap/Vagrantfile rename to tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile index 975224a19f..c2f24057d4 100644 --- a/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile +++ b/tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile @@ -1,17 +1,20 @@ # -*- mode: ruby -*- # vi: set ft=ruby : # - -ENV['TEST_VM_NAME'] ||= 'snapshotter-btrfs' -ENV['TEST_VM_HOSTNAME'] ||= 'test' +# Vagrant box for testing k3s with the btrfs snapshotter. Usage: +# VAGRANT_EXPERIMENTAL=disks vagrant up Vagrant.configure("2") do |config| config.vagrant.plugins = { 'vagrant-k3s' => {:version => '~> 0.1.3'}, } - config.vm.define ENV['TEST_VM_NAME'], primary: true do |test| - test.vm.box = "opensuse/Leap-15.3.x86_64" - test.vm.hostname = ENV['TEST_VM_HOSTNAME'] + config.vm.box = "opensuse/Leap-15.3.x86_64" + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '../../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false, + rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*' + + config.vm.define 'snapshotter-btrfs', primary: true do |test| + test.vm.hostname = 'smoke' test.vm.provision 'k3s-prepare', type: 'shell', run: 'once', privileged: true do |sh| sh.inline = <<~EOF #!/usr/bin/env bash @@ -30,10 +33,8 @@ Vagrant.configure("2") do |config| umount /mnt mount -t btrfs -o subvol=@k3s /dev/${BTRFS_DEV} /var/lib/rancher/k3s fi - if [ -e /vagrant/k3s ]; then - cp -vf /vagrant/k3s /usr/local/bin/ - chmod -v +x /usr/local/bin/k3s - fi + # Install k3s binary + install -m 755 /vagrant/k3s /usr/local/bin if [ -e /vagrant/*.tar ]; then mkdir -vp /var/lib/rancher/k3s/agent/images for tar in /vagrant/*.tar; do @@ -53,38 +54,49 @@ Vagrant.configure("2") do |config| } k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-ready", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eu -o pipefail echo 'Waiting for node to be ready ...' - time timeout 120 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' + kubectl get node,all -A -o wide + SHELL + end + test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' SHELL end - test.vm.provision "k3s-status", type: "shell", run: "once" do |sh| + test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL #!/usr/bin/env bash set -eux -o pipefail kubectl get node,all -A -o wide + SHELL + end + test.vm.provision "k3s-snapshots", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs SHELL end end - %w[hyperv libvirt virtualbox vmware_desktop].each do |p| + %w[libvirt virtualbox].each do |p| config.vm.provider p do |v| - v.cpus = ENV['CPUS'] || 2 - v.memory = ENV['MEMORY'] || 2048 + v.cpus = ENV['TEST_VM_CPUS'] || 2 + v.memory = ENV['TEST_VM_MEMORY'] || 2048 end end - config.vm.provider :hyperv do |v,o| - o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks" - end - config.vm.provider :libvirt do |v,o| v.storage :file, :size => '8G' end @@ -94,7 +106,4 @@ Vagrant.configure("2") do |config| v.check_guest_additions = false o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks" end - - config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: ['1', 'true'].include?(ENV['RSYNC_DISABLE']), - rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*' end