E2E Rancher and Hardened script improvements (#6778)

* Improve test-pad rancher script

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Improve hardened script and added kube-bench utility script

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Apply same audits for 1.22 and older

Signed-off-by: Derek Nola <derek.nola@suse.com>

Signed-off-by: Derek Nola <derek.nola@suse.com>

tests/e2e/scripts/harden.sh

@@ -6,4 +6,32 @@ kernel.panic=10
 kernel.panic_on_oops=1
 kernel.keys.root_maxbytes=25000000
 " >> /etc/sysctl.d/90-kubelet.conf
-sysctl -p /etc/sysctl.d/90-kubelet.conf
+sysctl -p /etc/sysctl.d/90-kubelet.conf
+
+mkdir -p /var/lib/rancher/k3s/server
+mkdir -m 700 /var/lib/rancher/k3s/server/logs
+echo "apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+- level: Metadata" >> /var/lib/rancher/k3s/server/audit.yaml
+
+if [ "$1" = "psa" ]; then
+    echo "apiVersion: apiserver.config.k8s.io/v1
+kind: AdmissionConfiguration
+plugins:
+- name: PodSecurity
+  configuration:
+    apiVersion: pod-security.admission.config.k8s.io/v1beta1
+    kind: PodSecurityConfiguration
+    defaults:
+      enforce: \"restricted\"
+      enforce-version: \"latest\"
+      audit: \"restricted\"
+      audit-version: \"latest\"
+      warn: \"restricted\"
+      warn-version: \"latest\"
+    exemptions:
+      usernames: []
+      runtimeClasses: []
+      namespaces: [kube-system, cis-operator-system]" >> /var/lib/rancher/k3s/server/psa.yaml
+fi

tests/e2e/scripts/rancher.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 node_ip=$1
+blank_node=$2
+
+if "$blank_node"; then
+    echo "Adding rancher ip to /etc/hosts"
+    echo "$node_ip test-pad.rancher" >> /etc/hosts
+    exit 0
+fi
 
 echo  "Give K3s time to startup"
 sleep 10
@@ -38,12 +45,11 @@ metadata:
   name: rancher
 spec:
   targetNamespace: cattle-system
-  version: 2.6.5
   chart: rancher
   repo: https://releases.rancher.com/server-charts/latest
   set:
     ingress.tls.source: "rancher"
-    hostname: "$node_ip.nip.io"
+    hostname: "test-pad.rancher"
     replicas: 1
 EOF
 
@@ -60,4 +66,4 @@ while ! kubectl get secret --namespace cattle-system bootstrap-secret -o go-temp
     echo "waiting for bootstrap-secret..."
     sleep 20
 done
-echo https://"$node_ip".nip.io/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')
+echo https://test-pad.rancher/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')

tests/e2e/vagrantdefaults.rb

@@ -34,6 +34,41 @@ def getInstallType(vm, release_version, branch)
   end
 end
 
+def getHardenedArg(vm, hardened, scripts_location)
+  if hardened.empty? 
+    return ""
+  end
+  hardened_arg = <<~HARD
+    protect-kernel-defaults: true
+    secrets-encryption: true
+    kube-controller-manager-arg:
+      - 'terminated-pod-gc-threshold=10'
+      - 'use-service-account-credentials=true'
+    kubelet-arg:
+      - 'streaming-connection-idle-timeout=5m'
+      - 'make-iptables-util-chains=true'
+      - 'event-qps=0'
+    kube-apiserver-arg:
+      - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log'
+      - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml'
+      - 'audit-log-maxage=30'
+      - 'audit-log-maxbackup=10'
+      - 'audit-log-maxsize=100'
+      - 'service-account-lookup=true'
+  HARD
+  if hardened == "psp"
+    vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
+    hardened_arg += "  - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'"
+  elsif hardened == "psa"
+    vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh", args: [ "psa" ]
+    hardened_arg += "  - 'admission-control-config-file=/var/lib/rancher/k3s/server/psa.yaml'"
+  else 
+    puts "Invalid E2E_HARDENED option"
+    exit 1
+  end
+  return hardened_arg
+end
+
 def dockerInstall(vm)
   vm.provider "libvirt" do |v|
     v.memory = NODE_MEMORY + 1024

tests/e2e/validatecluster/Vagrantfile

@@ -33,11 +33,8 @@ def provision(vm, role, role_num, node_num)
   vm.provision "shell", inline: "ping -c 2 k3s.io"
   
   db_type = getDBType(role, role_num, vm)
-
-  if !HARDENED.empty?
-    vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
-    hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\""
-  end
+  hardened_arg = getHardenedArg(vm, HARDENED, scripts_location)
+ 
   if !REGISTRY.empty?
     vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ]
   end
@@ -50,7 +47,6 @@ def provision(vm, role, role_num, node_num)
         token: vagrant
         node-external-ip: #{NETWORK_PREFIX}.100
         flannel-iface: eth1
-        tls-san: #{NETWORK_PREFIX}.100.nip.io
         #{db_type}
         #{hardened_arg}
       YAML
@@ -97,7 +93,8 @@ def provision(vm, role, role_num, node_num)
   end
   # This step does not run by default and is designed to be called by higher level tools
   if !RANCHER.empty?
-    vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: node_ip
+    blank_node = role.include?("agent") 
+    vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: [ "#{NETWORK_PREFIX}.100", blank_node.to_s ]
   end
 end