cgroup2 CI: add rootless

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

.github/workflows/cgroup2.yaml

@@ -35,26 +35,35 @@ jobs:
         path: ./tests/cgroup2
     - name: "Boot Fedora VM"
       run: |
-        cp k3s.service ./tests/cgroup2
+        cp -r k3s.service k3s-rootless.service ./tests/testutil ./tests/cgroup2
         cd ./tests/cgroup2
         vagrant up
         vagrant ssh-config >> ~/.ssh/config
-    # Sonobuoy requires CoreDNS to be ready
-    - name: "Waiting fore CoreDNS to be ready"
+    - name: "Starting k3s"
       run: |
-        counter=0
-        # `kubectl wait` fails when the pods with the specified label are not created yet
-        until ssh default -- sudo k3s kubectl wait --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns; do
-          sleep 10
-          ((counter++))
-          if [[ $counter -eq 10 ]]; then
-            echo "CoreDNS not running?"
-            ssh default -- sudo k3s kubectl get pods -A
-            ssh default -- sudo k3s kubectl get nodes -o wide
-            exit 1
-          fi
-        done
+        ssh default -- sudo systemctl start k3s
+    # Sonobuoy requires CoreDNS to be ready
+    - name: "Waiting for CoreDNS to be ready"
+      run: |
+        ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml /vagrant/testutil/wait-for-coredns.sh
     # Vagrant is slow, so we set --mode=quick here
     - name: "Run Sonobuoy (--mode=quick)"
       run: |
-        ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml /usr/local/bin/sonobuoy run --mode=quick --wait
+        ssh default -- sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --mode=quick --wait
+    - name: "Stopping k3s"
+      run: |
+        ssh default -- sudo systemctl stop k3s
+        # FIXME: rootful k3s processes are still running even after `systemctl stop k3s`, so we reboot the VM here.
+        # This reboot is also useful for ensuring `systemctl daemon-reload`: https://github.com/rootless-containers/rootlesscontaine.rs/issues/32
+        cd ./tests/cgroup2
+        vagrant halt
+        vagrant up
+    - name: "[Rootless] Starting k3s-rootless"
+      run: |
+        ssh default -- systemctl --user start k3s-rootless
+    - name: "[Rootless] Waiting for CoreDNS to be ready"
+      run: |
+        ssh default -- KUBECONFIG=/home/vagrant/.kube/k3s.yaml /vagrant/testutil/wait-for-coredns.sh
+    - name: "[Rootless] Run Sonobuoy (--mode=quick)"
+      run: |
+        ssh default -- KUBECONFIG=/home/vagrant/.kube/k3s.yaml sonobuoy run --mode=quick --wait

tests/cgroup2/.gitignore

@@ -1,3 +1,5 @@
 k3s
 k3s.service
+k3s-rootless.service
+testutil/
 .vagrant/

tests/cgroup2/Vagrantfile

@@ -7,8 +7,10 @@
 # The following files need to be present in this directory:
 # - k3s
 # - k3s.service
+# - k3s-rootless.service
+# - testutil/
 Vagrant.configure("2") do |config|
-  config.vm.box = "fedora/33-cloud-base"
+  config.vm.box = "fedora/34-cloud-base"
   memory = 2048
   cpus = 2
   config.vm.provider :virtualbox do |v|
@@ -22,13 +24,38 @@ Vagrant.configure("2") do |config|
   config.vm.provision "install-k3s", type: "shell", run: "once" do |sh|
     sh.inline = <<~SHELL
     set -eux -o pipefail
+
+    # Install k3s binary
     install -m 755  /vagrant/k3s /usr/local/bin
+    ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
+
+    # Install k3s systemd service (not launched here)
     cp -f /vagrant/k3s.service /etc/systemd/system/k3s.service
     touch /etc/systemd/system/k3s.service.env
     systemctl daemon-reload
-    systemctl enable --now k3s.service || { systemctl status --full --no-pager k3s.service ; exit 1; }
 
+    # Install sonobuoy binary
     curl -fsSL https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.20.0/sonobuoy_0.20.0_linux_amd64.tar.gz | tar xzvC /usr/local/bin sonobuoy
+
+    # [Rootless] Configure sysctl
+    echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/rootless.conf
+    sysctl --system
+
+    # [Rootless] Enable cgroup v2 delegation
+    mkdir -p /etc/systemd/system/user@.service.d
+    cat <<-EOF > /etc/systemd/system/user@.service.d/delegate.conf
+[Service]
+Delegate=yes
+EOF
+    systemctl daemon-reload
+
+    # [Rootless] Enable systemd lingering
+    loginctl enable-linger vagrant
+
+    # [Rootless] Install k3s-rootless systemd service (not launched here)
+    mkdir -p /home/vagrant/.config/systemd/user
+    cp -f /vagrant/k3s-rootless.service /home/vagrant/.config/systemd/user/k3s-rootless.service
+    chown -R vagrant:vagrant /home/vagrant/.config
     SHELL
   end
 end

tests/testutil/wait-for-coredns.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Wait for CoreDNS pods to be ready.
+
+set -x
+echo "Waiting for CoreDNS pods to be ready..."
+counter=0
+# `kubectl wait` fails when the pods with the specified label are not created yet
+until kubectl wait --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns; do
+  ((counter++))
+  if [[ $counter -eq 20 ]]; then
+    echo "CoreDNS not running?"
+    kubectl get pods -A
+    kubectl get nodes -o wide
+    exit 1
+  fi
+  sleep 10
+done