From fa37d03395881e1ed26cb1817d098061fa00e9bb Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 29 Feb 2024 15:41:56 -0800 Subject: [PATCH] Update install test OS matrix (#9480) * Remove old cgroupsv2 test * Consolidate install test clauses into functions * Unpin vagrant-k3s plugin version, run latest * Add ubuntu-2204 as install test, remove ubuntu-focal * Update nightly install matrix * Move to Leap 15.5 * Consolidate vagrant box caching key to improve cache hits on all VM testing Signed-off-by: Derek Nola --- .github/workflows/cgroup.yaml | 88 ------------ .github/workflows/install.yaml | 7 +- .github/workflows/nightly-install.yaml | 12 +- .github/workflows/snapshotter.yaml | 2 +- tests/cgroup/unified/fedora/Vagrantfile | 132 ------------------ tests/e2e/token/token_test.go | 2 +- tests/install/centos-7/Vagrantfile | 94 +++---------- tests/install/fedora/Vagrantfile | 91 +++--------- tests/install/install_util.rb | 92 ++++++++++++ tests/install/opensuse-leap/Vagrantfile | 83 +++-------- tests/install/opensuse-microos/Vagrantfile | 5 +- tests/install/rocky-8/Vagrantfile | 90 +++--------- tests/install/rocky-9/Vagrantfile | 92 +++--------- tests/install/ubuntu-2204/Vagrantfile | 59 ++++++++ tests/install/ubuntu-focal/Vagrantfile | 110 --------------- .../btrfs/opensuse-leap/Vagrantfile | 6 +- 16 files changed, 264 insertions(+), 701 deletions(-) delete mode 100644 .github/workflows/cgroup.yaml delete mode 100644 tests/cgroup/unified/fedora/Vagrantfile create mode 100644 tests/install/install_util.rb create mode 100644 tests/install/ubuntu-2204/Vagrantfile delete mode 100644 tests/install/ubuntu-focal/Vagrantfile diff --git a/.github/workflows/cgroup.yaml b/.github/workflows/cgroup.yaml deleted file mode 100644 index 9f0ef3ce3c..0000000000 --- a/.github/workflows/cgroup.yaml +++ /dev/null @@ -1,88 +0,0 @@ -name: Control Group -on: - push: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/**" - - "!tests/cgroup/**" - - ".github/**" - - "!.github/workflows/cgroup.yaml" - pull_request: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/**" - - "!tests/cgroup/**" - - ".github/**" - - "!.github/workflows/cgroup.yaml" - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - build: - uses: ./.github/workflows/build-k3s.yaml - test: - name: "Conformance Test" - needs: build - # nested virtualization is only available on macOS hosts - runs-on: macos-12 - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - vm: [fedora] - mode: [unified] - max-parallel: 1 - defaults: - run: - working-directory: tests/cgroup/${{ matrix.mode }}/${{ matrix.vm }} - steps: - - name: "Checkout" - uses: actions/checkout@v4 - with: { fetch-depth: 1 } - - name: "Download Binary" - uses: actions/download-artifact@v3 - with: { name: k3s, path: dist/artifacts/ } - - name: "Vagrant Cache" - uses: actions/cache@v3 - with: - path: | - ~/.vagrant.d/boxes - ~/.vagrant.d/gems - key: cgroup-${{ hashFiles(format('tests/cgroup/{0}/{1}/Vagrantfile', matrix.mode, matrix.vm)) }} - id: vagrant-cache - continue-on-error: true - - name: "Vagrant Plugin(s)" - run: vagrant plugin install vagrant-k3s vagrant-reload - - name: "Vagrant Up" - run: vagrant up - - name: "K3s Prepare" - run: vagrant provision --provision-with=k3s-prepare - - name: ⏬ "K3s Install" - run: vagrant provision --provision-with=k3s-install - - name: ⏩ "K3s Start" - run: vagrant provision --provision-with=k3s-start - - name: "K3s Ready" # wait for k3s to be ready - run: vagrant provision --provision-with=k3s-ready - - name: "K3s Status" # kubectl get node,all -A -o wide - run: vagrant provision --provision-with=k3s-status - - name: "Sonobuoy (--mode=quick)" - env: {TEST_RESULTS_PATH: rootfull} - run: vagrant provision --provision-with=k3s-sonobuoy - - name: "K3s Stop" # stop k3s rootfull - run: vagrant ssh -- sudo systemctl stop k3s-server - - name: "Vagrant Reload" - run: vagrant reload - - name: "[Rootless] Starting K3s" - run: vagrant ssh -- systemctl --user start k3s-rootless - - name: "[Rootless] K3s Ready" - env: {TEST_KUBECONFIG: /home/vagrant/.kube/k3s.yaml} - run: vagrant provision --provision-with=k3s-ready - # - name: "[Rootless] Sonobuoy (--mode=quick)" - # env: {TEST_KUBECONFIG: /home/vagrant/.kube/k3s.yaml, TEST_RESULTS_PATH: rootless} - # run: vagrant provision --provision-with=k3s-sonobuoy diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml index 148685c08d..71a86a5e66 100644 --- a/.github/workflows/install.yaml +++ b/.github/workflows/install.yaml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - vm: [centos-7, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-focal] + vm: [centos-7, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2204] max-parallel: 2 defaults: run: @@ -47,7 +47,7 @@ jobs: path: | ~/.vagrant.d/boxes ~/.vagrant.d/gems - key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }} + key: vagrant-box-${{ matrix.vm }} id: vagrant-cache continue-on-error: true - name: "Vagrant Plugin(s)" @@ -65,6 +65,9 @@ jobs: vagrant scp k3s /tmp/k3s vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s" vagrant provision --provision-with=k3s-upload + - name: Add binary to PATH + if: matrix.vm == 'centos-7' || matrix.vm == 'rocky-8' || matrix.vm == 'rocky-9' || matrix.vm == 'opensuse-leap' + run: vagrant provision --provision-with=add-bin-path - name: "⏩ Install K3s" run: | vagrant provision --provision-with=k3s-prepare diff --git a/.github/workflows/nightly-install.yaml b/.github/workflows/nightly-install.yaml index a907a51f97..417219c623 100644 --- a/.github/workflows/nightly-install.yaml +++ b/.github/workflows/nightly-install.yaml @@ -15,13 +15,9 @@ jobs: strategy: fail-fast: false matrix: - channel: [stable] - vm: [centos-7, rocky-8, fedora, opensuse-leap, ubuntu-focal] - include: - - {channel: latest, vm: rocky-8} - - {channel: latest, vm: ubuntu-focal} - - {channel: latest, vm: opensuse-leap} - max-parallel: 2 + channel: [stable, latest] + vm: [rocky-8, fedora, opensuse-leap, ubuntu-2204] + max-parallel: 4 defaults: run: working-directory: tests/install/${{ matrix.vm }} @@ -37,7 +33,7 @@ jobs: path: | ~/.vagrant.d/boxes ~/.vagrant.d/gems - key: install-${{ matrix.vm }}-${{ hashFiles('tests/install/${{ matrix.vm }}/Vagrantfile') }} + key: vagrant-box-${{ matrix.vm }} id: vagrant-cache continue-on-error: true - name: "Vagrant Plugin(s)" diff --git a/.github/workflows/snapshotter.yaml b/.github/workflows/snapshotter.yaml index c342f068dc..396c8bd825 100644 --- a/.github/workflows/snapshotter.yaml +++ b/.github/workflows/snapshotter.yaml @@ -56,7 +56,7 @@ jobs: path: | ~/.vagrant.d/boxes ~/.vagrant.d/gems - key: snapshotter-${{ hashFiles(format('tests/snapshotter/{0}/{1}/Vagrantfile', matrix.snapshotter, matrix.vm)) }} + key: vagrant-box-${{ matrix.vm }} id: vagrant-cache continue-on-error: true - name: "Vagrant Plugin(s)" diff --git a/tests/cgroup/unified/fedora/Vagrantfile b/tests/cgroup/unified/fedora/Vagrantfile deleted file mode 100644 index b4c9c9f85b..0000000000 --- a/tests/cgroup/unified/fedora/Vagrantfile +++ /dev/null @@ -1,132 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : -# -# Vagrant box for testing k3s with cgroup v2. - -ENV['TEST_UNITFILE_ROOTFULL'] ||= '../../../../k3s.service' -ENV['TEST_UNITFILE_ROOTLESS'] ||= '../../../../k3s-rootless.service' - -Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } - config.vm.box = "generic/fedora37" - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false, - rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*' - - config.vm.define 'cgroup-unified', primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision "disable-firewall", type: "shell", inline: "systemctl stop firewalld && systemctl disable firewalld" - test.vm.provision :file, run: 'always', source: ENV['TEST_UNITFILE_ROOTFULL'], destination: 'k3s-rootfull.service' - test.vm.provision :file, run: 'always', source: ENV['TEST_UNITFILE_ROOTLESS'], destination: 'k3s-rootless.service' - test.vm.provision 'k3s-prepare', type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once', privileged: true do |sh| - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - - # Install k3s binary - install -vm 755 /vagrant/k3s /usr/local/bin - - # Install k3s SELinux policy - dnf install -y https://github.com/k3s-io/k3s-selinux/releases/download/v1.2.stable.2/k3s-selinux-1.2-2.el8.noarch.rpm - - # Install k3s systemd service (not launched here) - install -vm 644 -T /home/vagrant/k3s-rootfull.service /etc/systemd/system/k3s-server.service - touch /etc/systemd/system/k3s-server.service.env - systemctl daemon-reload - - # Install sonobuoy binary - SONOBUOY_VER=0.56.16 - curl -fsSL https://github.com/vmware-tanzu/sonobuoy/releases/download/v"$SONOBUOY_VER"/sonobuoy_"$SONOBUOY_VER"_linux_amd64.tar.gz | tar xzvC /usr/local/bin sonobuoy - - # [Rootless] Configure sysctl - dnf install -y fuse-overlayfs fuse - echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/rootless.conf - sysctl --system - # [Rootless] Enable cgroup v2 delegation - mkdir -p /etc/systemd/system/user@.service.d - cat <<-EOF > /etc/systemd/system/user@.service.d/delegate.conf -[Service] -Delegate=yes -EOF - - systemctl daemon-reload - - # [Rootless] Enable systemd lingering - loginctl enable-linger vagrant - - # [Rootless] Install k3s-rootless systemd service (not launched here) - mkdir -p /home/vagrant/.config/systemd/user - cp -f /home/vagrant/k3s-rootless.service /home/vagrant/.config/systemd/user/k3s-rootless.service - chown -R vagrant:vagrant /home/vagrant/.config - SHELL - end - - test.vm.provision 'k3s-install', type: 'k3s', run: ENV['CI'] == 'true' ? 'never' : 'once' do |k3s| - k3s.args = %w[server] - k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=vagrant INSTALL_K3S_SKIP_ENABLE=true] - k3s.config = <<~YAML - disable: - - local-storage - - metrics-server - - servicelb - - traefik - disable-helm-controller: true - disable-network-policy: true - write-kubeconfig-mode: '0644' - YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "k3s-start", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.inline = "systemctl start k3s-server" - end - test.vm.provision "k3s-ready", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { - :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin", - :KUBECONFIG => ENV['TEST_KUBECONFIG'] || '/etc/rancher/k3s/k3s.yaml', - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 500 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - time timeout 500 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { - :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin", - :KUBECONFIG => ENV['TEST_KUBECONFIG'] || '/etc/rancher/k3s/k3s.yaml', - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-sonobuoy", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { - :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin", - :KUBECONFIG => ENV['TEST_KUBECONFIG'] || '/etc/rancher/k3s/k3s.yaml', - :RESULTS_PATH => ENV['TEST_RESULTS_PATH'] || '.', - } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - sonobuoy run --mode=quick --wait - sonobuoy retrieve ${RESULTS_PATH} - sonobuoy results $(ls -rt ${RESULTS_PATH}/*.tar.gz | tail -1) | grep Status | grep passed - SHELL - end - end - - config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' - - %w[libvirt virtualbox].each do |p| - config.vm.provider p do |v| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 2048 - end - end -end diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 09ab793838..dc7c746859 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -14,7 +14,7 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.4.x86_64 +// generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.5.x86_64 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") diff --git a/tests/install/centos-7/Vagrantfile b/tests/install/centos-7/Vagrantfile index 78186523d7..f7be36b3a1 100644 --- a/tests/install/centos-7/Vagrantfile +++ b/tests/install/centos-7/Vagrantfile @@ -5,15 +5,17 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } - config.vm.box = "dweomer/centos-7.9-amd64" + config.vagrant.plugins = ["vagrant-k3s"] + config.vm.box = "generic/centos7" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true + # Load in helper functions + load "../install_util.rb" + config.vm.define 'install-centos-7', primary: true do |test| test.vm.hostname = 'smoke' + test.vm.provision "add-bin-path", type: "shell", inline: "echo \"export PATH=/usr/local/bin:\$PATH\" >> ~/.bashrc" test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' @@ -25,76 +27,22 @@ Vagrant.configure("2") do |config| selinux: true token: 'vagrant' YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'k3s|kube|container' | grep -v grep - SHELL end + + waitForNodeReady(test.vm) + + waitForCoreDns(test.vm) + + waitForLocalStorage(test.vm) + + waitForMetricsServer(test.vm) + + waitForTraefik(test.vm) + + kubectlStatus(test.vm) + + checkK3sProcesses(test.vm) + end config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' diff --git a/tests/install/fedora/Vagrantfile b/tests/install/fedora/Vagrantfile index 4ae8cb0775..09bf843dcf 100644 --- a/tests/install/fedora/Vagrantfile +++ b/tests/install/fedora/Vagrantfile @@ -5,13 +5,14 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } + config.vagrant.plugins = ["vagrant-k3s"] config.vm.box = 'generic/fedora37' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true + # Load in helper functions + load "../install_util.rb" + config.vm.define 'install-fedora', primary: true do |test| test.vm.hostname = 'smoke' test.vm.provision "disable-firewall", type: "shell", inline: "systemctl stop firewalld" @@ -26,76 +27,22 @@ Vagrant.configure("2") do |config| selinux: true token: 'vagrant' YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'k3s|kube|container' | grep -v grep - SHELL end + + waitForNodeReady(test.vm) + + waitForCoreDns(test.vm) + + waitForLocalStorage(test.vm) + + waitForMetricsServer(test.vm) + + waitForTraefik(test.vm) + + kubectlStatus(test.vm) + + checkK3sProcesses(test.vm) + end config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' diff --git a/tests/install/install_util.rb b/tests/install/install_util.rb new file mode 100644 index 0000000000..c3012a609e --- /dev/null +++ b/tests/install/install_util.rb @@ -0,0 +1,92 @@ +def waitForNodeReady(vm) + vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + echo 'Waiting for node to be ready ...' + time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' + kubectl get node,all -A -o wide + SHELL + end + end + + def waitForCoreDns(vm) + vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + function describe-coredns { + RC=$? + if [[ $RC -ne 0 ]]; then + kubectl describe node + kubectl --namespace kube-system describe pod -l k8s-app=kube-dns + kubectl --namespace kube-system logs -l k8s-app=kube-dns + fi + exit $RC + } + trap describe-coredns EXIT + time timeout 120 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' + SHELL + end + end + + def waitForLocalStorage(vm) + vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time timeout 120 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' + SHELL + end + end + + # Metrics takes the longest to start, so we give it 3 minutes + def waitForMetricsServer(vm) + vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time timeout 180 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' + SHELL + end + end + + def waitForTraefik(vm) + vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eu -o pipefail + time timeout 120 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' + SHELL + end + end + + def kubectlStatus(vm) + vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + kubectl get node,all -A -o wide + SHELL + end + end + + def checkK3sProcesses(vm) + vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + ps auxZ | grep -E 'k3s|kube|container' | grep -v grep + SHELL + end + end + + def checkCGroupV2(vm) + vm.provision "cgroupv2", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| + sh.inline = <<~SHELL + #!/usr/bin/env bash + set -eux -o pipefail + k3s check-config | grep 'cgroups V2 mounted' + SHELL + end + end \ No newline at end of file diff --git a/tests/install/opensuse-leap/Vagrantfile b/tests/install/opensuse-leap/Vagrantfile index ede7aeb084..dd08103df9 100644 --- a/tests/install/opensuse-leap/Vagrantfile +++ b/tests/install/opensuse-leap/Vagrantfile @@ -5,15 +5,17 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } - config.vm.box = 'opensuse/Leap-15.4.x86_64' + config.vagrant.plugins = ["vagrant-k3s"] + config.vm.box = 'opensuse/Leap-15.5.x86_64' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true + # Load in helper functions + load "../install_util.rb" + config.vm.define 'install-opensuse-leap', primary: true do |test| test.vm.hostname = 'smoke' + test.vm.provision "add-bin-path", type: "shell", inline: "echo \"export PATH=/usr/local/bin:\$PATH\" >> ~/.bashrc" test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' # /sbin/apparmor_parser is needed by the 1.21 kubelet if the value of /sys/module/apparmor/parameters/enabled is Y test.vm.provision 'k3s-prepare', type: 'shell', run: 'once', inline: 'zypper install -y apparmor-parser' @@ -27,8 +29,8 @@ Vagrant.configure("2") do |config| k3s.config = <<~YAML token: 'vagrant' YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end + test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } sh.inline = <<~SHELL @@ -39,64 +41,19 @@ Vagrant.configure("2") do |config| kubectl get node,all -A -o wide SHELL end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'k3s|kube|container' | grep -v grep - SHELL - end + + waitForCoreDns(test.vm) + + waitForLocalStorage(test.vm) + + waitForMetricsServer(test.vm) + + waitForTraefik(test.vm) + + kubectlStatus(test.vm) + + checkK3sProcesses(test.vm) + end %w[libvirt virtualbox vmware_desktop].each do |p| diff --git a/tests/install/opensuse-microos/Vagrantfile b/tests/install/opensuse-microos/Vagrantfile index 08cb085612..a3f1c679c6 100644 --- a/tests/install/opensuse-microos/Vagrantfile +++ b/tests/install/opensuse-microos/Vagrantfile @@ -5,10 +5,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - 'vagrant-reload' => {}, - } + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] config.vm.box = 'dweomer/microos.amd64' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true diff --git a/tests/install/rocky-8/Vagrantfile b/tests/install/rocky-8/Vagrantfile index 052fe1a388..cc755ec895 100644 --- a/tests/install/rocky-8/Vagrantfile +++ b/tests/install/rocky-8/Vagrantfile @@ -5,16 +5,18 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } + config.vagrant.plugins = ["vagrant-k3s"] config.vm.box = "generic/rocky8" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true + # Load in helper functions + load "../install_util.rb" + config.vm.define 'install-rocky-8', primary: true do |test| test.vm.hostname = 'smoke' test.vm.provision "disable-firewall", type: "shell", inline: "systemctl stop firewalld" + test.vm.provision "add-bin-path", type: "shell", inline: "echo \"export PATH=/usr/local/bin:\$PATH\" >> ~/.bashrc" test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' @@ -28,74 +30,20 @@ Vagrant.configure("2") do |config| YAML k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end - test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'k3s|kube|container' | grep -v grep - SHELL - end + waitForNodeReady(test.vm) + + waitForCoreDns(test.vm) + + waitForLocalStorage(test.vm) + + waitForMetricsServer(test.vm) + + waitForTraefik(test.vm) + + kubectlStatus(test.vm) + + checkK3sProcesses(test.vm) + end config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' diff --git a/tests/install/rocky-9/Vagrantfile b/tests/install/rocky-9/Vagrantfile index b1373171a9..9ec4bba82f 100644 --- a/tests/install/rocky-9/Vagrantfile +++ b/tests/install/rocky-9/Vagrantfile @@ -6,16 +6,18 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' ENV['INSTALL_K3S_CHANNEL'] ||= 'testing' Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } + config.vagrant.plugins = ["vagrant-k3s"] config.vm.box = "generic/rocky9" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true + # Load in helper functions + load "../install_util.rb" + config.vm.define 'install-rocky-9', primary: true do |test| test.vm.hostname = 'smoke' test.vm.provision "disable-firewall", type: "shell", inline: "systemctl stop firewalld" + test.vm.provision "add-bin-path", type: "shell", inline: "echo \"export PATH=/usr/local/bin:\$PATH\" >> ~/.bashrc" test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.installer_url = 'file:///home/vagrant/install.sh' @@ -27,76 +29,22 @@ Vagrant.configure("2") do |config| selinux: true token: 'vagrant' YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'k3s|kube|container' | grep -v grep - SHELL end + waitForNodeReady(test.vm) + + waitForCoreDns(test.vm) + + waitForLocalStorage(test.vm) + + waitForMetricsServer(test.vm) + + waitForTraefik(test.vm) + + kubectlStatus(test.vm) + + checkK3sProcesses(test.vm) + + checkCGroupV2(test.vm) end config.vm.provision 'selinux-status', type: 'shell', run: 'once', inline: 'sestatus' diff --git a/tests/install/ubuntu-2204/Vagrantfile b/tests/install/ubuntu-2204/Vagrantfile new file mode 100644 index 0000000000..7b23ebeff6 --- /dev/null +++ b/tests/install/ubuntu-2204/Vagrantfile @@ -0,0 +1,59 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# + +ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' + +Vagrant.configure("2") do |config| + config.vagrant.plugins = ["vagrant-k3s"] + config.vm.box = 'generic/ubuntu2204' + config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds + config.vm.synced_folder '.', '/vagrant', disabled: true + + # Load in helper functions + load "../install_util.rb" + + config.vm.define 'install-ubuntu-2204', primary: true do |test| + test.vm.hostname = 'smoke' + test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' + test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.installer_url = 'file:///home/vagrant/install.sh' + k3s.args = %w[server] + k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ + :INSTALL_K3S_NAME => 'server', + :INSTALL_K3S_SKIP_SELINUX_RPM => 'true', + }) + k3s.config = <<~YAML + token: 'vagrant' + YAML + end + + waitForNodeReady(test.vm) + + waitForCoreDns(test.vm) + + waitForLocalStorage(test.vm) + + waitForMetricsServer(test.vm) + + waitForTraefik(test.vm) + + kubectlStatus(test.vm) + + checkK3sProcesses(test.vm) + + checkCGroupV2(test.vm) + + end + + %w[libvirt virtualbox vmware_desktop].each do |p| + config.vm.provider p do |v| + v.cpus = ENV['TEST_VM_CPUS'] || 2 + v.memory = ENV['TEST_VM_MEMORY'] || 2048 + end + end + config.vm.provider :virtualbox do |v,o| + v.gui = false + v.check_guest_additions = false + end +end diff --git a/tests/install/ubuntu-focal/Vagrantfile b/tests/install/ubuntu-focal/Vagrantfile deleted file mode 100644 index 4599949ba5..0000000000 --- a/tests/install/ubuntu-focal/Vagrantfile +++ /dev/null @@ -1,110 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : -# - -ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' - -Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } - config.vm.box = 'generic/ubuntu2004' - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.define 'install-ubuntu-focal', primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' - test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.installer_url = 'file:///home/vagrant/install.sh' - k3s.args = %w[server] - k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({ - :INSTALL_K3S_NAME => 'server', - :INSTALL_K3S_SKIP_SELINUX_RPM => 'true', - }) - k3s.config = <<~YAML - token: 'vagrant' - YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostname) 2>/dev/null); do sleep 5; done' - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/local-path-provisioner 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-metrics-server", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/metrics-server 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-wait-for-traefik", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/traefik 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-procps", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - ps auxZ | grep -E 'k3s|kube|container' | grep -v grep - SHELL - end - end - - %w[libvirt virtualbox vmware_desktop].each do |p| - config.vm.provider p do |v| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 2048 - end - end - config.vm.provider :virtualbox do |v,o| - v.gui = false - v.check_guest_additions = false - end -end diff --git a/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile b/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile index daf0a746cb..167a448de6 100644 --- a/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile +++ b/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile @@ -5,10 +5,8 @@ # VAGRANT_EXPERIMENTAL=disks vagrant up Vagrant.configure("2") do |config| - config.vagrant.plugins = { - 'vagrant-k3s' => {:version => '~> 0.1.3'}, - } - config.vm.box = "opensuse/Leap-15.4.x86_64" + config.vagrant.plugins = ["vagrant-k3s"] + config.vm.box = "opensuse/Leap-15.5.x86_64" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false, rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'