E2E Rancher and Hardened script improvements (#6778)

* Improve test-pad rancher script

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Improve hardened script and added kube-bench utility script

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Apply same audits for 1.22 and older

Signed-off-by: Derek Nola <derek.nola@suse.com>

Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola 2023-01-26 18:17:33 -08:00 committed by GitHub
parent f0655f153e
commit 75f77ab951
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 77 additions and 11 deletions

View File

@ -6,4 +6,32 @@ kernel.panic=10
kernel.panic_on_oops=1
kernel.keys.root_maxbytes=25000000
" >> /etc/sysctl.d/90-kubelet.conf
sysctl -p /etc/sysctl.d/90-kubelet.conf
sysctl -p /etc/sysctl.d/90-kubelet.conf
mkdir -p /var/lib/rancher/k3s/server
mkdir -m 700 /var/lib/rancher/k3s/server/logs
echo "apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Metadata" >> /var/lib/rancher/k3s/server/audit.yaml
if [ "$1" = "psa" ]; then
echo "apiVersion: apiserver.config.k8s.io/v1
kind: AdmissionConfiguration
plugins:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1beta1
kind: PodSecurityConfiguration
defaults:
enforce: \"restricted\"
enforce-version: \"latest\"
audit: \"restricted\"
audit-version: \"latest\"
warn: \"restricted\"
warn-version: \"latest\"
exemptions:
usernames: []
runtimeClasses: []
namespaces: [kube-system, cis-operator-system]" >> /var/lib/rancher/k3s/server/psa.yaml
fi

View File

@ -1,5 +1,12 @@
#!/bin/bash
node_ip=$1
blank_node=$2
if "$blank_node"; then
echo "Adding rancher ip to /etc/hosts"
echo "$node_ip test-pad.rancher" >> /etc/hosts
exit 0
fi
echo "Give K3s time to startup"
sleep 10
@ -38,12 +45,11 @@ metadata:
name: rancher
spec:
targetNamespace: cattle-system
version: 2.6.5
chart: rancher
repo: https://releases.rancher.com/server-charts/latest
set:
ingress.tls.source: "rancher"
hostname: "$node_ip.nip.io"
hostname: "test-pad.rancher"
replicas: 1
EOF
@ -60,4 +66,4 @@ while ! kubectl get secret --namespace cattle-system bootstrap-secret -o go-temp
echo "waiting for bootstrap-secret..."
sleep 20
done
echo https://"$node_ip".nip.io/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')
echo https://test-pad.rancher/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')

View File

@ -34,6 +34,41 @@ def getInstallType(vm, release_version, branch)
end
end
def getHardenedArg(vm, hardened, scripts_location)
if hardened.empty?
return ""
end
hardened_arg = <<~HARD
protect-kernel-defaults: true
secrets-encryption: true
kube-controller-manager-arg:
- 'terminated-pod-gc-threshold=10'
- 'use-service-account-credentials=true'
kubelet-arg:
- 'streaming-connection-idle-timeout=5m'
- 'make-iptables-util-chains=true'
- 'event-qps=0'
kube-apiserver-arg:
- 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log'
- 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml'
- 'audit-log-maxage=30'
- 'audit-log-maxbackup=10'
- 'audit-log-maxsize=100'
- 'service-account-lookup=true'
HARD
if hardened == "psp"
vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
hardened_arg += " - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'"
elsif hardened == "psa"
vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh", args: [ "psa" ]
hardened_arg += " - 'admission-control-config-file=/var/lib/rancher/k3s/server/psa.yaml'"
else
puts "Invalid E2E_HARDENED option"
exit 1
end
return hardened_arg
end
def dockerInstall(vm)
vm.provider "libvirt" do |v|
v.memory = NODE_MEMORY + 1024

View File

@ -33,11 +33,8 @@ def provision(vm, role, role_num, node_num)
vm.provision "shell", inline: "ping -c 2 k3s.io"
db_type = getDBType(role, role_num, vm)
if !HARDENED.empty?
vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\""
end
hardened_arg = getHardenedArg(vm, HARDENED, scripts_location)
if !REGISTRY.empty?
vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ]
end
@ -50,7 +47,6 @@ def provision(vm, role, role_num, node_num)
token: vagrant
node-external-ip: #{NETWORK_PREFIX}.100
flannel-iface: eth1
tls-san: #{NETWORK_PREFIX}.100.nip.io
#{db_type}
#{hardened_arg}
YAML
@ -97,7 +93,8 @@ def provision(vm, role, role_num, node_num)
end
# This step does not run by default and is designed to be called by higher level tools
if !RANCHER.empty?
vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: node_ip
blank_node = role.include?("agent")
vm.provision "Install Rancher", type: "shell", run: "never", path: scripts_location + "/rancher.sh", args: [ "#{NETWORK_PREFIX}.100", blank_node.to_s ]
end
end