diff --git a/.github/workflows/cgroup2.yaml b/.github/workflows/cgroup2.yaml index 8a2d1decaa..eac687dbd2 100644 --- a/.github/workflows/cgroup2.yaml +++ b/.github/workflows/cgroup2.yaml @@ -35,26 +35,35 @@ jobs: path: ./tests/cgroup2 - name: "Boot Fedora VM" run: | - cp k3s.service ./tests/cgroup2 + cp -r k3s.service k3s-rootless.service ./tests/testutil ./tests/cgroup2 cd ./tests/cgroup2 vagrant up vagrant ssh-config >> ~/.ssh/config - # Sonobuoy requires CoreDNS to be ready - - name: "Waiting fore CoreDNS to be ready" + - name: "Starting k3s" run: | - counter=0 - # `kubectl wait` fails when the pods with the specified label are not created yet - until ssh default -- sudo k3s kubectl wait --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns; do - sleep 10 - ((counter++)) - if [[ $counter -eq 10 ]]; then - echo "CoreDNS not running?" - ssh default -- sudo k3s kubectl get pods -A - ssh default -- sudo k3s kubectl get nodes -o wide - exit 1 - fi - done + ssh default -- sudo systemctl start k3s + # Sonobuoy requires CoreDNS to be ready + - name: "Waiting for CoreDNS to be ready" + run: | + ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml /vagrant/testutil/wait-for-coredns.sh # Vagrant is slow, so we set --mode=quick here - name: "Run Sonobuoy (--mode=quick)" run: | - ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml /usr/local/bin/sonobuoy run --mode=quick --wait + ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --mode=quick --wait + - name: "Stopping k3s" + run: | + ssh default -- sudo systemctl stop k3s + # FIXME: rootful k3s processes are still running even after `systemctl stop k3s`, so we reboot the VM here. + # This reboot is also useful for ensuring `systemctl daemon-reload`: https://github.com/rootless-containers/rootlesscontaine.rs/issues/32 + cd ./tests/cgroup2 + vagrant halt + vagrant up + - name: "[Rootless] Starting k3s-rootless" + run: | + ssh default -- systemctl --user start k3s-rootless + - name: "[Rootless] Waiting for CoreDNS to be ready" + run: | + ssh default -- KUBECONFIG=/home/vagrant/.kube/k3s.yaml /vagrant/testutil/wait-for-coredns.sh + - name: "[Rootless] Run Sonobuoy (--mode=quick)" + run: | + ssh default -- KUBECONFIG=/home/vagrant/.kube/k3s.yaml sonobuoy run --mode=quick --wait diff --git a/tests/cgroup2/.gitignore b/tests/cgroup2/.gitignore index 747971f830..6108fd1e3c 100644 --- a/tests/cgroup2/.gitignore +++ b/tests/cgroup2/.gitignore @@ -1,3 +1,5 @@ k3s k3s.service +k3s-rootless.service +testutil/ .vagrant/ diff --git a/tests/cgroup2/Vagrantfile b/tests/cgroup2/Vagrantfile index 555f0a6576..d8ac4cc0fb 100644 --- a/tests/cgroup2/Vagrantfile +++ b/tests/cgroup2/Vagrantfile @@ -7,8 +7,10 @@ # The following files need to be present in this directory: # - k3s # - k3s.service +# - k3s-rootless.service +# - testutil/ Vagrant.configure("2") do |config| - config.vm.box = "fedora/33-cloud-base" + config.vm.box = "fedora/34-cloud-base" memory = 2048 cpus = 2 config.vm.provider :virtualbox do |v| @@ -22,13 +24,38 @@ Vagrant.configure("2") do |config| config.vm.provision "install-k3s", type: "shell", run: "once" do |sh| sh.inline = <<~SHELL set -eux -o pipefail + + # Install k3s binary install -m 755 /vagrant/k3s /usr/local/bin + ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl + + # Install k3s systemd service (not launched here) cp -f /vagrant/k3s.service /etc/systemd/system/k3s.service touch /etc/systemd/system/k3s.service.env systemctl daemon-reload - systemctl enable --now k3s.service || { systemctl status --full --no-pager k3s.service ; exit 1; } + # Install sonobuoy binary curl -fsSL https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.20.0/sonobuoy_0.20.0_linux_amd64.tar.gz | tar xzvC /usr/local/bin sonobuoy + + # [Rootless] Configure sysctl + echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/rootless.conf + sysctl --system + + # [Rootless] Enable cgroup v2 delegation + mkdir -p /etc/systemd/system/user@.service.d + cat <<-EOF > /etc/systemd/system/user@.service.d/delegate.conf +[Service] +Delegate=yes +EOF + systemctl daemon-reload + + # [Rootless] Enable systemd lingering + loginctl enable-linger vagrant + + # [Rootless] Install k3s-rootless systemd service (not launched here) + mkdir -p /home/vagrant/.config/systemd/user + cp -f /vagrant/k3s-rootless.service /home/vagrant/.config/systemd/user/k3s-rootless.service + chown -R vagrant:vagrant /home/vagrant/.config SHELL end end diff --git a/tests/testutil/wait-for-coredns.sh b/tests/testutil/wait-for-coredns.sh new file mode 100755 index 0000000000..10141b327d --- /dev/null +++ b/tests/testutil/wait-for-coredns.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Wait for CoreDNS pods to be ready. + +set -x +echo "Waiting for CoreDNS pods to be ready..." +counter=0 +# `kubectl wait` fails when the pods with the specified label are not created yet +until kubectl wait --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns; do + ((counter++)) + if [[ $counter -eq 20 ]]; then + echo "CoreDNS not running?" + kubectl get pods -A + kubectl get nodes -o wide + exit 1 + fi + sleep 10 +done