From 75f77ab9517f7a5b1a6c569fbe00590d6635c445 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 26 Jan 2023 18:17:33 -0800 Subject: [PATCH] E2E Rancher and Hardened script improvements (#6778) * Improve test-pad rancher script Signed-off-by: Derek Nola * Improve hardened script and added kube-bench utility script Signed-off-by: Derek Nola * Apply same audits for 1.22 and older Signed-off-by: Derek Nola Signed-off-by: Derek Nola --- tests/e2e/scripts/harden.sh | 30 ++++++++++++++++++++++- tests/e2e/scripts/rancher.sh | 12 ++++++--- tests/e2e/vagrantdefaults.rb | 35 +++++++++++++++++++++++++++ tests/e2e/validatecluster/Vagrantfile | 11 +++------ 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/tests/e2e/scripts/harden.sh b/tests/e2e/scripts/harden.sh index 3b65139ab2..5aec44f6cb 100644 --- a/tests/e2e/scripts/harden.sh +++ b/tests/e2e/scripts/harden.sh @@ -6,4 +6,32 @@ kernel.panic=10 kernel.panic_on_oops=1 kernel.keys.root_maxbytes=25000000 " >> /etc/sysctl.d/90-kubelet.conf -sysctl -p /etc/sysctl.d/90-kubelet.conf \ No newline at end of file +sysctl -p /etc/sysctl.d/90-kubelet.conf + +mkdir -p /var/lib/rancher/k3s/server +mkdir -m 700 /var/lib/rancher/k3s/server/logs +echo "apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata" >> /var/lib/rancher/k3s/server/audit.yaml + +if [ "$1" = "psa" ]; then + echo "apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1beta1 + kind: PodSecurityConfiguration + defaults: + enforce: \"restricted\" + enforce-version: \"latest\" + audit: \"restricted\" + audit-version: \"latest\" + warn: \"restricted\" + warn-version: \"latest\" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [kube-system, cis-operator-system]" >> /var/lib/rancher/k3s/server/psa.yaml +fi \ No newline at end of file diff --git a/tests/e2e/scripts/rancher.sh b/tests/e2e/scripts/rancher.sh index 4c7d8efa16..a9b9ff6d3d 100644 --- a/tests/e2e/scripts/rancher.sh +++ b/tests/e2e/scripts/rancher.sh @@ -1,5 +1,12 @@ #!/bin/bash node_ip=$1 +blank_node=$2 + +if "$blank_node"; then + echo "Adding rancher ip to /etc/hosts" + echo "$node_ip test-pad.rancher" >> /etc/hosts + exit 0 +fi echo "Give K3s time to startup" sleep 10 @@ -38,12 +45,11 @@ metadata: name: rancher spec: targetNamespace: cattle-system - version: 2.6.5 chart: rancher repo: https://releases.rancher.com/server-charts/latest set: ingress.tls.source: "rancher" - hostname: "$node_ip.nip.io" + hostname: "test-pad.rancher" replicas: 1 EOF @@ -60,4 +66,4 @@ while ! kubectl get secret --namespace cattle-system bootstrap-secret -o go-temp echo "waiting for bootstrap-secret..." sleep 20 done -echo https://"$node_ip".nip.io/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}') \ No newline at end of file +echo https://test-pad.rancher/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}') \ No newline at end of file diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index 62496c69dd..95e0ae6744 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -34,6 +34,41 @@ def getInstallType(vm, release_version, branch) end end +def getHardenedArg(vm, hardened, scripts_location) + if hardened.empty? + return "" + end + hardened_arg = <<~HARD + protect-kernel-defaults: true + secrets-encryption: true + kube-controller-manager-arg: + - 'terminated-pod-gc-threshold=10' + - 'use-service-account-credentials=true' + kubelet-arg: + - 'streaming-connection-idle-timeout=5m' + - 'make-iptables-util-chains=true' + - 'event-qps=0' + kube-apiserver-arg: + - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log' + - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml' + - 'audit-log-maxage=30' + - 'audit-log-maxbackup=10' + - 'audit-log-maxsize=100' + - 'service-account-lookup=true' + HARD + if hardened == "psp" + vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh" + hardened_arg += " - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'" + elsif hardened == "psa" + vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh", args: [ "psa" ] + hardened_arg += " - 'admission-control-config-file=/var/lib/rancher/k3s/server/psa.yaml'" + else + puts "Invalid E2E_HARDENED option" + exit 1 + end + return hardened_arg +end + def dockerInstall(vm) vm.provider "libvirt" do |v| v.memory = NODE_MEMORY + 1024 diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 2df1d63efe..cda96f9966 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -33,11 +33,8 @@ def provision(vm, role, role_num, node_num) vm.provision "shell", inline: "ping -c 2 k3s.io" db_type = getDBType(role, role_num, vm) - - if !HARDENED.empty? - vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh" - hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\"" - end + hardened_arg = getHardenedArg(vm, HARDENED, scripts_location) + if !REGISTRY.empty? vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ] end @@ -50,7 +47,6 @@ def provision(vm, role, role_num, node_num) token: vagrant node-external-ip: #{NETWORK_PREFIX}.100 flannel-iface: eth1 - tls-san: #{NETWORK_PREFIX}.100.nip.io #{db_type} #{hardened_arg} YAML @@ -97,7 +93,8 @@ def provision(vm, role, role_num, node_num) end # This step does not run by default and is designed to be called by higher level tools if !RANCHER.empty? - vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: node_ip + blank_node = role.include?("agent") + vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: [ "#{NETWORK_PREFIX}.100", blank_node.to_s ] end end