Fix embedded mirror blocked by SAR RBAC and re-enable test

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

.drone.yml

@@ -617,33 +617,24 @@ steps:
   - mkdir -p dist/artifacts
   - cp /tmp/artifacts/* dist/artifacts/
   - docker stop registry && docker rm registry
-  # Cleanup VMs running, happens if a previous test panics
-  # Cleanup inactive domains, happens if previous test is canceled
-  - |
-    VMS=$(virsh list --name | grep '_server-\|_agent-' || true)
-    if [ -n "$VMS" ]; then
-      for vm in $VMS
-      do
-        virsh destroy $vm
-        virsh undefine $vm --remove-all-storage
-      done
-    fi 
-    VMS=$(virsh list --name --inactive | grep '_server-\|_agent-' || true)
-    if [ -n "$VMS" ]; then
-      for vm in $VMS
-      do
-        virsh undefine $vm
-      done
-    fi 
+  # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled
+  - tests/e2e/scripts/cleanup_vms.sh
   - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2
-  - cd tests/e2e/validatecluster
-  - vagrant destroy -f
-  - go test -v -timeout=45m ./validatecluster_test.go -ci -local
-  - cp ./coverage.out /tmp/artifacts/validate-coverage.out
-  - cd ../secretsencryption
-  - vagrant destroy -f
-  - go test -v -timeout=30m ./secretsencryption_test.go -ci -local
-  - cp ./coverage.out /tmp/artifacts/se-coverage.out
+  - |
+    cd tests/e2e/validatecluster
+    vagrant destroy -f
+    go test -v -timeout=45m ./validatecluster_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/validate-coverage.out
+  - |
+    cd ../secretsencryption
+    vagrant destroy -f
+    go test -v -timeout=30m ./secretsencryption_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/se-coverage.out
+  - |
+    cd ../splitserver
+    vagrant destroy -f
+    go test -v -timeout=30m ./splitserver_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/split-coverage.out
   - |
     if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then
       cd ../upgradecluster
@@ -669,6 +660,7 @@ steps:
     files:
       - /tmp/artifacts/validate-coverage.out
       - /tmp/artifacts/se-coverage.out
+      - /tmp/artifacts/split-coverage.out
       - /tmp/artifacts/upgrade-coverage.out
     flags: 
       - e2etests

.github/workflows/e2e.yaml

@@ -36,8 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # TODO fix embeddedmirror and add it to the matrix
-        etest: [startup, s3, btrfs, externalip, privateregistry, wasm]
+        etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
       max-parallel: 3
     steps:
       - name: "Checkout"
@@ -116,4 +115,4 @@ jobs:
         chmod +x ./dist/artifacts/k3s
         . ./tests/docker/test-helpers
         . ./tests/docker/test-run-${{ matrix.dtest }}
-        echo "Did test-run-${{ matrix.dtest }} pass $?"
+        echo "Did test-run-${{ matrix.dtest }} pass $?"

.golangci.json

@@ -10,7 +10,10 @@
 		]
 	},
 	"run": {
-		"skip-dirs": [
+		"deadline": "5m"
+	},
+	"issues": {
+		"exclude-dirs": [
 			"build",
 			"contrib",
 			"manifests",
@@ -18,12 +21,9 @@
 			"scripts",
 			"vendor"
 		],
-		"skip-files": [
+		"exclude-files": [
 			"/zz_generated_"
 		],
-		"deadline": "5m"
-	},
-	"issues": {
 		"exclude-rules": [
 			{
 				"linters": "typecheck",
@@ -43,4 +43,4 @@
 			}
 		]
 	}
-}
+}

Dockerfile.dapper

@@ -1,4 +1,4 @@
-ARG GOLANG=golang:1.21.9-alpine3.18
+ARG GOLANG=golang:1.22.2-alpine3.18
 FROM ${GOLANG}
 
 # Set proxy environment variables
@@ -22,7 +22,7 @@ RUN apk -U --no-cache add \
 RUN python3 -m pip install awscli
 
 # Install Trivy
-ENV TRIVY_VERSION="0.50.1"
+ENV TRIVY_VERSION="0.51.4"
 RUN case "$(go env GOARCH)" in \
     arm64) TRIVY_ARCH="ARM64" ;; \
     amd64) TRIVY_ARCH="64bit" ;; \

Dockerfile.local

@@ -1,4 +1,4 @@
-ARG GOLANG=golang:1.21.9-alpine3.18
+ARG GOLANG=golang:1.22.2-alpine3.18
 FROM ${GOLANG} as infra
 
 ARG http_proxy=$http_proxy

Dockerfile.manifest

@@ -1,4 +1,4 @@
-ARG GOLANG=golang:1.21.9-alpine3.18
+ARG GOLANG=golang:1.22.2-alpine3.18
 FROM ${GOLANG}
 
 COPY --from=plugins/manifest:1.2.3 /bin/* /bin/

Dockerfile.test

@@ -1,4 +1,4 @@
-ARG GOLANG=golang:1.21.9-alpine3.18
+ARG GOLANG=golang:1.22.2-alpine3.18
 FROM ${GOLANG} as test-base
 
 RUN apk -U --no-cache add bash jq

channel.yaml

@@ -1,7 +1,7 @@
 # Example channels config
 channels:
 - name: stable
-  latest: v1.29.3+k3s1
+  latest: v1.29.5+k3s1
 - name: latest
   latestRegexp: .*
   excludeRegexp: (^[^+]+-|v1\.25\.5\+k3s1|v1\.26\.0\+k3s1)
@@ -56,6 +56,9 @@ channels:
 - name: v1.29
   latestRegexp: v1\.29\..*
   excludeRegexp: ^[^+]+-
+- name: v1.30
+  latestRegexp: v1\.30\..*
+  excludeRegexp: ^[^+]+-
 github:
   owner: k3s-io
   repo: k3s

cmd/k3s/main.go

@@ -19,7 +19,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/untar"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/resolvehome"
+	"github.com/rancher/wrangler/v3/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 	"github.com/urfave/cli"

conformance/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.20
 ENV SONOBUOY_VERSION 0.57.1
 RUN apk add curl tar gzip
 RUN curl -sfL https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz | tar xvzf - -C /usr/bin

contrib/util/check-config.sh

@@ -388,7 +388,7 @@ flags="
   CGROUPS CGROUP_PIDS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG
   KEYS
   VETH BRIDGE BRIDGE_NETFILTER
-  IP_NF_FILTER IP_NF_TARGET_MASQUERADE
+  IP_NF_FILTER IP_NF_TARGET_MASQUERADE IP_NF_TARGET_REJECT
   NETFILTER_XT_MATCH_ADDRTYPE NETFILTER_XT_MATCH_CONNTRACK NETFILTER_XT_MATCH_IPVS NETFILTER_XT_MATCH_COMMENT NETFILTER_XT_MATCH_MULTIPORT
   IP_NF_NAT NF_NAT
   POSIX_MQUEUE

docs/adrs/gh-branch-strategy.md

@@ -0,0 +1,21 @@
+# Branching Strategy in Github
+
+Proposal Date: 2024-05-23
+
+## Status
+
+Accepted
+
+## Context
+
+K3s is released at the same cadence as upstream Kubernetes. This requires management of multiple versions at any given point in time. The current branching strategy uses `release-v[MAJOR].[MINOR]`, with the `master` branch corresponding to the highest version released based on [semver](https://semver.org/). Github's Tags are then used to cut releases, which are just point-in-time snapshots of the specified branch at a given point. As there is the potential for bugs and regressions to be on present on any given branch, this branching and release strategy requires a code freeze to QA the branch without new potentially breaking changes going in.
+
+## Decision
+All code changes go into the `master` branch. We maintain branches for all current release versions in the format `release-v[MAJOR].[MINOR]`. When changes made in master are necessary in a release, they should be backported directly into the release branches. If ever there are changes required only in the release branches and not in master, such as when bumping the kubernetes version from upstream, those can be made directly into the release branches themselves.
+
+## Consequences
+
+- Allows for constant development, with code freeze only relevant for the release branches.
+- This requires maintaining one additional branch than the current workflow, which also means one additional issue.
+- Testing would be more constant from the master branch.
+- Minor release captain will have to cut the new branch as soon as they bring in that new minor version.

docs/contrib/development.md

@@ -73,7 +73,7 @@ As described in the [Testing documentation](../../tests/TESTING.md), all the smo
 
 These topics already have been addressed on their respective documents:
 
-- [Git Workflow](./git-workflow.md)
+- [Git Workflow](./git_workflow.md)
 - [Building](../../BUILDING.md)
 - [Testing](../../tests/TESTING.md)
 

go.mod

@@ -1,12 +1,12 @@
 module github.com/k3s-io/k3s
 
-go 1.21
+go 1.22.2
 
 replace (
 	github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.11.0
-	github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.12-k3s1 // k3s/release-1.28
-	github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.1.0
-	github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.15-k3s1
+	github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.12-k3s1.30-3 // k3s/release-1.30
+	github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.1.2
+	github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.17-k3s1
 	github.com/docker/distribution => github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/docker => github.com/docker/docker v25.0.4+incompatible
 	github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.9.0
@@ -18,114 +18,114 @@ replace (
 	github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.11.0
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common => github.com/prometheus/common v0.45.0
-	github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d
-	github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.20-k3s1
+	github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314
 	github.com/ugorji/go => github.com/ugorji/go v1.2.11
-	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1
-	go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/pkg/v3 => github.com/k3s-io/etcd/pkg/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1
-	go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1
+	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1
+	go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1
+	go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.13-k3s1
+	go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.13-k3s1
+	go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.13-k3s1
+	go.etcd.io/etcd/pkg/v3 => github.com/k3s-io/etcd/pkg/v3 v3.5.13-k3s1
+	go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1
+	go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1
 	go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful => go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0
 	golang.org/x/crypto => golang.org/x/crypto v0.17.0
 	golang.org/x/net => golang.org/x/net v0.17.0
-	golang.org/x/sys => golang.org/x/sys v0.13.0
+	golang.org/x/sys => golang.org/x/sys v0.18.0
 	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230525234035-dd9d682886f9
 	google.golang.org/grpc => google.golang.org/grpc v1.58.3
 	gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.6.0
-	k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.29.4-k3s1
-	k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.29.4-k3s1
-	k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.29.4-k3s1
-	k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.29.4-k3s1
-	k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.29.4-k3s1
-	k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.29.4-k3s1
-	k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.29.4-k3s1
-	k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.29.4-k3s1
-	k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.29.4-k3s1
-	k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.29.4-k3s1
-	k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.29.4-k3s1
-	k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.29.4-k3s1
-	k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.29.4-k3s1
-	k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.29.4-k3s1
-	k8s.io/dynamic-resource-allocation => github.com/k3s-io/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v1.29.4-k3s1
-	k8s.io/endpointslice => github.com/k3s-io/kubernetes/staging/src/k8s.io/endpointslice v1.29.4-k3s1
+	k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.30.1-k3s1
+	k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.30.1-k3s1
+	k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.30.1-k3s1
+	k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.30.1-k3s1
+	k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.30.1-k3s1
+	k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.30.1-k3s1
+	k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.30.1-k3s1
+	k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.30.1-k3s1
+	k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.30.1-k3s1
+	k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.30.1-k3s1
+	k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.30.1-k3s1
+	k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.30.1-k3s1
+	k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.30.1-k3s1
+	k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.30.1-k3s1
+	k8s.io/dynamic-resource-allocation => github.com/k3s-io/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v1.30.1-k3s1
+	k8s.io/endpointslice => github.com/k3s-io/kubernetes/staging/src/k8s.io/endpointslice v1.30.1-k3s1
 	k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s2 // k3s-release-1.x
-	k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.100.1-k3s1 // k3s-main
-	k8s.io/kms => github.com/k3s-io/kubernetes/staging/src/k8s.io/kms v1.29.4-k3s1
-	k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.29.4-k3s1
-	k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.29.4-k3s1
-	k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.29.4-k3s1
-	k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.29.4-k3s1
-	k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.29.4-k3s1
-	k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.29.4-k3s1
-	k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.29.4-k3s1
-	k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.29.4-k3s1
-	k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.29.4-k3s1
-	k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.4-k3s1
-	k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.29.4-k3s1
-	k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.4-k3s1
-	k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.29.4-k3s1
-	k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.29.4-k3s1
-	k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.29.4-k3s1
+	k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.120.1-k3s1 // k3s-main
+	k8s.io/kms => github.com/k3s-io/kubernetes/staging/src/k8s.io/kms v1.30.1-k3s1
+	k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.30.1-k3s1
+	k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.30.1-k3s1
+	k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.30.1-k3s1
+	k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.30.1-k3s1
+	k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.30.1-k3s1
+	k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.30.1-k3s1
+	k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.30.1-k3s1
+	k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.30.1-k3s1
+	k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.30.1-k3s1
+	k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.30.1-k3s1
+	k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.30.1-k3s1
+	k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.30.1-k3s1
+	k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.30.1-k3s1
+	k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.30.1-k3s1
+	k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.30.1-k3s1
 	sourcegraph.com/sourcegraph/go-diff => github.com/sourcegraph/go-diff v0.6.0
 )
 
 require (
-	github.com/Microsoft/hcsshim v0.11.4
+	github.com/Microsoft/hcsshim v0.12.3
 	github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000
 	github.com/containerd/aufs v1.0.0
 	github.com/containerd/cgroups/v3 v3.0.2
-	github.com/containerd/containerd v1.7.14
+	github.com/containerd/containerd v1.7.16
 	github.com/containerd/fuse-overlayfs-snapshotter v1.0.8
 	github.com/containerd/stargz-snapshotter v0.15.1
 	github.com/containerd/zfs v1.1.0
 	github.com/coreos/go-iptables v0.7.0
 	github.com/coreos/go-systemd/v22 v22.5.0
-	github.com/docker/docker v25.0.4+incompatible
+	github.com/docker/docker v25.0.5+incompatible
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
-	github.com/flannel-io/flannel v0.24.2
+	github.com/flannel-io/flannel v0.25.2
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-logr/logr v1.4.1
 	github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/go-test/deep v1.0.7
 	github.com/golang/mock v1.6.0
-	github.com/google/cadvisor v0.48.1
+	github.com/google/cadvisor v0.49.0
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.1
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/joho/godotenv v1.5.1
 	github.com/json-iterator/go v1.1.12
-	github.com/k3s-io/helm-controller v0.15.9
-	github.com/k3s-io/kine v0.11.7
+	github.com/k3s-io/helm-controller v0.16.1
+	github.com/k3s-io/kine v0.11.9
 	github.com/klauspost/compress v1.17.7
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
 	github.com/lib/pq v1.10.2
 	github.com/libp2p/go-libp2p v0.33.2
 	github.com/mattn/go-sqlite3 v1.14.19
-	github.com/minio/minio-go/v7 v7.0.33
+	github.com/minio/minio-go/v7 v7.0.70
 	github.com/mwitkow/go-http-dialer v0.0.0-20161116154839-378f744fb2b8
 	github.com/natefinch/lumberjack v2.0.0+incompatible
-	github.com/onsi/ginkgo/v2 v2.15.0
-	github.com/onsi/gomega v1.31.1
+	github.com/onsi/ginkgo/v2 v2.16.0
+	github.com/onsi/gomega v1.32.0
 	github.com/opencontainers/runc v1.1.12
 	github.com/opencontainers/selinux v1.11.0
 	github.com/otiai10/copy v1.7.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/common v0.49.0
-	github.com/rancher/dynamiclistener v0.3.6
-	github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29
+	github.com/rancher/dynamiclistener v0.6.0-rc1
+	github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5
 	github.com/rancher/remotedialer v0.3.0
 	github.com/rancher/wharfie v0.6.4
-	github.com/rancher/wrangler v1.1.1
+	github.com/rancher/wrangler/v3 v3.0.0-rc2
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rootless-containers/rootlesskit v1.0.1
 	github.com/sirupsen/logrus v1.9.3
@@ -135,33 +135,33 @@ require (
 	github.com/urfave/cli v1.22.14
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	github.com/yl2chen/cidranger v1.0.2
-	go.etcd.io/etcd/api/v3 v3.5.10
-	go.etcd.io/etcd/client/pkg/v3 v3.5.10
-	go.etcd.io/etcd/client/v3 v3.5.10
+	go.etcd.io/etcd/api/v3 v3.5.13
+	go.etcd.io/etcd/client/pkg/v3 v3.5.13
+	go.etcd.io/etcd/client/v3 v3.5.13
 	go.etcd.io/etcd/etcdutl/v3 v3.5.9
-	go.etcd.io/etcd/server/v3 v3.5.10
+	go.etcd.io/etcd/server/v3 v3.5.13
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.21.0
-	golang.org/x/net v0.23.0
-	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.18.0
-	google.golang.org/grpc v1.62.0
+	golang.org/x/crypto v0.22.0
+	golang.org/x/net v0.24.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/sys v0.19.0
+	google.golang.org/grpc v1.63.2
 	gopkg.in/yaml.v2 v2.4.0
 	inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252
-	k8s.io/api v0.29.4
-	k8s.io/apimachinery v0.29.4
-	k8s.io/apiserver v0.29.4
+	k8s.io/api v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/apiserver v0.30.1
 	k8s.io/cli-runtime v0.22.2
 	k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
-	k8s.io/cloud-provider v0.29.4
+	k8s.io/cloud-provider v0.30.1
 	k8s.io/cluster-bootstrap v0.0.0
-	k8s.io/component-base v0.29.4
-	k8s.io/component-helpers v0.29.4
-	k8s.io/cri-api v0.29.4-alpha.0
+	k8s.io/component-base v0.30.1
+	k8s.io/component-helpers v0.30.1
+	k8s.io/cri-api v0.30.1
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kube-proxy v0.0.0
 	k8s.io/kubectl v0.25.0
-	k8s.io/kubernetes v1.29.4
+	k8s.io/kubernetes v1.30.1
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	sigs.k8s.io/yaml v1.4.0
 )
@@ -172,38 +172,26 @@ require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
-	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
-	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
-	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b // indirect
 	github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/Rican7/retry v0.1.0 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
-	github.com/avast/retry-go/v4 v4.3.2 // indirect
+	github.com/avast/retry-go/v4 v4.6.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c // indirect
-	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/canonical/go-dqlite v1.5.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect
-	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
 	github.com/container-storage-interface/spec v1.8.0 // indirect
 	github.com/containerd/btrfs/v2 v2.0.0 // indirect
@@ -213,21 +201,20 @@ require (
 	github.com/containerd/fifo v1.1.0 // indirect
 	github.com/containerd/go-cni v1.1.9 // indirect
 	github.com/containerd/go-runc v1.0.0 // indirect
-	github.com/containerd/imgcrypt v1.1.7 // indirect
+	github.com/containerd/imgcrypt v1.1.8 // indirect
 	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/nri v0.6.0 // indirect
+	github.com/containerd/nri v0.6.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
-	github.com/containerd/ttrpc v1.2.3 // indirect
+	github.com/containerd/ttrpc v1.2.4 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/containernetworking/cni v1.1.2 // indirect
-	github.com/containernetworking/plugins v1.3.0 // indirect
-	github.com/containers/ocicrypt v1.1.6 // indirect
+	github.com/containernetworking/plugins v1.4.1 // indirect
+	github.com/containers/ocicrypt v1.1.10 // indirect
 	github.com/coreos/go-oidc v2.2.1+incompatible // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
-	github.com/danwinship/knftables v0.0.13 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/daviddengcn/go-colortext v1.0.0 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
@@ -253,29 +240,23 @@ require (
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/gin-gonic/gin v1.9.1 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
 	github.com/go-openapi/jsonreference v0.20.4 // indirect
 	github.com/go-openapi/swag v0.22.9 // indirect
-	github.com/go-playground/locales v0.14.1 // indirect
-	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.14.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
-	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/google/cel-go v0.17.7 // indirect
+	github.com/google/cel-go v0.17.8 // indirect
 	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-containerregistry v0.14.0 // indirect
@@ -324,7 +305,6 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libopenstorage/openstorage v1.0.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
@@ -400,7 +380,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -413,37 +393,31 @@ require (
 	github.com/quic-go/webtransport-go v0.6.0 // indirect
 	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
-	github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
 	github.com/shengdoushi/base58 v1.0.0 // indirect
-	github.com/slok/go-http-metrics v0.10.0 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
-	github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
+	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect
-	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/urfave/cli/v2 v2.26.0 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
-	github.com/vmware/govmomi v0.30.6 // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
-	github.com/xenitab/pkg/gin v0.0.9 // indirect
 	github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.etcd.io/bbolt v1.3.9 // indirect
-	go.etcd.io/etcd/client/v2 v2.305.10 // indirect
-	go.etcd.io/etcd/pkg/v3 v3.5.10 // indirect
-	go.etcd.io/etcd/raft/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.13 // indirect
+	go.etcd.io/etcd/pkg/v3 v3.5.13 // indirect
+	go.etcd.io/etcd/raft/v3 v3.5.13 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0 // indirect
@@ -461,14 +435,13 @@ require (
 	go.uber.org/fx v1.20.1 // indirect
 	go.uber.org/mock v0.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.18.0 // indirect
+	golang.org/x/tools v0.20.0 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect
@@ -485,30 +458,32 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.29.4 // indirect
-	k8s.io/code-generator v0.29.4 // indirect
+	k8s.io/apiextensions-apiserver v0.30.1 // indirect
+	k8s.io/code-generator v0.30.1 // indirect
 	k8s.io/controller-manager v0.25.4 // indirect
 	k8s.io/csi-translation-lib v0.0.0 // indirect
 	k8s.io/dynamic-resource-allocation v0.0.0 // indirect
 	k8s.io/endpointslice v0.0.0 // indirect
-	k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
+	k8s.io/gengo v0.0.0-20240228010128-51d4e06bde70 // indirect
+	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
 	k8s.io/kms v0.0.0 // indirect
-	k8s.io/kube-aggregator v0.29.4 // indirect
+	k8s.io/kube-aggregator v0.30.1 // indirect
 	k8s.io/kube-controller-manager v0.0.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kube-scheduler v0.0.0 // indirect
 	k8s.io/kubelet v0.28.6 // indirect
 	k8s.io/legacy-cloud-providers v0.0.0 // indirect
 	k8s.io/metrics v0.0.0 // indirect
-	k8s.io/mount-utils v0.29.4 // indirect
+	k8s.io/mount-utils v0.30.1 // indirect
 	k8s.io/pod-security-admission v0.0.0 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
-	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/knftables v0.0.14 // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	tags.cncf.io/container-device-interface v0.6.2 // indirect
-	tags.cncf.io/container-device-interface/specs-go v0.6.0 // indirect
+	tags.cncf.io/container-device-interface v0.7.2 // indirect
+	tags.cncf.io/container-device-interface/specs-go v0.7.0 // indirect
 )

go.sum

@@ -231,31 +231,9 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59M
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
-github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw=
-github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs=
-github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk=
-github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=
-github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=
-github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
-github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
-github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=
-github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
-github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac=
-github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
-github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
@@ -269,11 +247,13 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
-github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM=
 github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -291,6 +271,7 @@ github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3
 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
 github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
@@ -310,8 +291,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/avast/retry-go/v4 v4.3.2 h1:x4sTEu3jSwr7zNjya8NTdIN+U88u/jtO/q3OupBoDtM=
-github.com/avast/retry-go/v4 v4.3.2/go.mod h1:rg6XFaiuFYII0Xu3RDbZQkxCofFwruZKW8oEF1jpWiU=
+github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
+github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
@@ -333,12 +314,8 @@ github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c/go.mod
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q=
-github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
-github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
-github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
 github.com/canonical/go-dqlite v1.5.1 h1:1YjtIrFsC1A3XlgsX38ARAiKhvkZS63PqsEd8z3T4yU=
 github.com/canonical/go-dqlite v1.5.1/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y=
-github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -353,9 +330,6 @@ github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNS
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0 h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
-github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
-github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
-github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -403,36 +377,40 @@ github.com/containerd/go-cni v1.1.9 h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9
 github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM=
 github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0=
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/imgcrypt v1.1.7 h1:WSf9o9EQ0KGHiUx2ESFZ+PKf4nxK9BcvV/nJDX8RkB4=
-github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k=
+github.com/containerd/imgcrypt v1.1.8 h1:ZS7TuywcRNLoHpU0g+v4/PsKynl6TYlw5xDVWWoIyFA=
+github.com/containerd/imgcrypt v1.1.8/go.mod h1:x6QvFIkMyO2qGIY2zXc88ivEzcbgvLdWjoZyGqDap5U=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/nri v0.6.0 h1:hdztxwL0gCS1CrCa9bvD1SoJiFN4jBuRQhplCvCPMj8=
-github.com/containerd/nri v0.6.0/go.mod h1:F7OZfO4QTPqw5r87aq+syZJwiVvRYLIlHZiZDBV1W3A=
+github.com/containerd/nri v0.6.1 h1:xSQ6elnQ4Ynidm9u49ARK9wRKHs80HCUI+bkXOxV4mA=
+github.com/containerd/nri v0.6.1/go.mod h1:7+sX3wNx+LR7RzhjnJiUkFDhn18P5Bg/0VnJ/uXpRJM=
 github.com/containerd/stargz-snapshotter v0.15.1 h1:fpsP4kf/Z4n2EYnU0WT8ZCE3eiKDwikDhL6VwxIlgeA=
 github.com/containerd/stargz-snapshotter v0.15.1/go.mod h1:74D+J1m1RMXytLmWxegXWhtOSRHPWZKpKc2NdK3S+us=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
 github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
+github.com/containerd/ttrpc v1.1.2/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
 github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak=
-github.com/containerd/ttrpc v1.2.3-0.20231030150553-baadfd8e7956/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM=
-github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0=
 github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM=
+github.com/containerd/ttrpc v1.2.4 h1:eQCQK4h9dxDmpOb9QOOMh2NHTfzroH1IkmHiKZi05Oo=
+github.com/containerd/ttrpc v1.2.4/go.mod h1:ojvb8SJBSch0XkqNO0L0YX/5NxR3UnVk2LzFKBK0upc=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
 github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4=
 github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0=
 github.com/containerd/zfs v1.1.0 h1:n7OZ7jZumLIqNJqXrEc/paBM840mORnmGdJDmAmJZHM=
 github.com/containerd/zfs v1.1.0/go.mod h1:oZF9wBnrnQjpWLaPKEinrx3TQ9a+W/RJO7Zb41d8YLE=
+github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
 github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
 github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
 github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
+github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
 github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4=
-github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM=
-github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0=
-github.com/containers/ocicrypt v1.1.6 h1:uoG52u2e91RE4UqmBICZY8dNshgfvkdl3BW6jnxiFaI=
-github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc=
+github.com/containernetworking/plugins v1.4.1 h1:+sJRRv8PKhLkXIl6tH1D7RMi+CbbHutDGU+ErLBORWA=
+github.com/containernetworking/plugins v1.4.1/go.mod h1:n6FFGKcaY4o2o5msgu/UImtoC+fpQXM3076VHfHbj60=
+github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/LFMtH0FIRt34=
+github.com/containers/ocicrypt v1.1.10 h1:r7UR6o8+lyhkEywetubUUgcKFjOWOaWz8cEBrCPX0ic=
+github.com/containers/ocicrypt v1.1.10/go.mod h1:YfzSSr06PTHQwSTUKqDSjish9BeW1E4HUmreluQcMd8=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -470,8 +448,6 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
 github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
-github.com/danwinship/knftables v0.0.13 h1:89Ieiia6MMfXWQF9dyaou1CwBU8h8sHa2Zo3OlY2o04=
-github.com/danwinship/knftables v0.0.13/go.mod h1:OzipaBQqkQAIbVnafTGyHgfFbjWTJecrA7/XNLNMO5E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -497,8 +473,6 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUn
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v23.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
@@ -554,8 +528,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flannel-io/flannel v0.24.2 h1:dXMRlGvmQ7iPtKuGJifmvpfrME5U3TVWDWZ2L/QqPqc=
-github.com/flannel-io/flannel v0.24.2/go.mod h1:GvvhQS/xd5QM6oc9yeVz8KBbp5hWJZgPtwTKgpuLUPI=
+github.com/flannel-io/flannel v0.25.2 h1:ATQ4PhZqd2MUpLm+NKbAaNxm2PJSLE+mS9WUI4RkKPs=
+github.com/flannel-io/flannel v0.25.2/go.mod h1:o5FAm9Rl28TydPKw1cQFYWPopfQKIjlYrcdFzBusaGI=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
@@ -578,14 +552,9 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT
 github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw=
 github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
-github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
-github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
 github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
@@ -601,6 +570,9 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
+github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
@@ -619,7 +591,6 @@ github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5 h1:aj5xnNwNY2GCk38Vga4FMm4GSX1bDzu8Z5JcQQdmOqg=
 github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
 github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
@@ -634,14 +605,6 @@ github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZC
 github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
 github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
 github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
-github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
-github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
-github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
-github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
-github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
-github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -663,13 +626,11 @@ github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
-github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
-github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
@@ -710,10 +671,10 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
-github.com/google/cadvisor v0.48.1 h1:eyYTxKBd+KxI1kh6rst4JSTLUhfHQM34qGpp+0AMlSg=
-github.com/google/cadvisor v0.48.1/go.mod h1:ZkYbiiVdyoqBmI2ahZI8GlmirT78OAOER0z4EQugkxQ=
-github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ=
-github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
+github.com/google/cadvisor v0.49.0 h1:1PYeiORXmcFYi609M4Qvq5IzcvcVaWgYxDt78uH8jYA=
+github.com/google/cadvisor v0.49.0/go.mod h1:s6Fqwb2KiWG6leCegVhw4KW40tf9f7m+SF1aXiE8Wsk=
+github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto=
+github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
@@ -919,6 +880,8 @@ github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPw
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
@@ -939,96 +902,96 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/k3s-io/containerd v1.7.15-k3s1 h1:X+GVNp3FiBy8rZzTMXShQJBmycPVi8vcwzsRBLdvqhM=
-github.com/k3s-io/containerd v1.7.15-k3s1/go.mod h1:SOFk39t+bfDZC8jPYg11uxrzG3Fh30ZOociJwXfvk8Y=
-github.com/k3s-io/cri-dockerd v0.3.12-k3s1 h1:jGTy2U1Nn8d9o23NwLV1NAigTBvePTA7XaZMifZ01Q4=
-github.com/k3s-io/cri-dockerd v0.3.12-k3s1/go.mod h1:S98trivsinxuNGQANgrZ9ComFqQkVv7vUvsXSNBRCFs=
+github.com/k3s-io/containerd v1.7.17-k3s1 h1:jXPVFdg+vEwsx7amOvjPIx180ltbKBBZM5tfBaQtlzA=
+github.com/k3s-io/containerd v1.7.17-k3s1/go.mod h1:T36IsoYQp97IT+64ws3GTq27V+M3518W11PDvOlBKPQ=
+github.com/k3s-io/cri-dockerd v0.3.12-k3s1.30-3 h1:lmvoMmpiprwTdQFW5p3f+Y1ZRnx2YDKENSsUZsUCszc=
+github.com/k3s-io/cri-dockerd v0.3.12-k3s1.30-3/go.mod h1:L7HNeF+iZZ/btgefGZI5v7oB1TQgpFyWvbhmFzfsWAY=
 github.com/k3s-io/cri-tools v1.29.0-k3s1 h1:16IXZ5lbPCmZM8FkgSMAPkhI4O2wVGExe3qEZbisFT0=
 github.com/k3s-io/cri-tools v1.29.0-k3s1/go.mod h1:fZeWlv+qq4gZ005I13j4JcvgFb6ZobVTtON3PqM5JVc=
-github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1 h1:y4ont0HdnS7gtWNTXM8gahpKjAHtctgON/sjVRthlZY=
-github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k=
-github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1 h1:LJFtNHaBJg2BqFE3lRxWZkUsKTYLbh0s0NCXPMjW3cg=
-github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4=
-github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1 h1:/IyNFC677PfYafrm4sWPShbmw1bkpvEio6YaxxFA9cU=
-github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1/go.mod h1:0NBdNx9wbxtEQLwAQtrDHwx58m02vXpDcgSYI2seohQ=
-github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1 h1:Knr/8l7Sx92zUyevYO0gIO5P6EEc6ztvRO5EzSnMy+A=
-github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA=
-github.com/k3s-io/etcd/etcdutl/v3 v3.5.9-k3s1 h1:IkCP2oKkQwyu+ad4FuToJu9SOuEVQZwCpjXj6SJqwvs=
-github.com/k3s-io/etcd/etcdutl/v3 v3.5.9-k3s1/go.mod h1:rQ6z0HAAxVgYwBTWJbs3ei8gMYiNQzF51lQ2kI+6LZU=
-github.com/k3s-io/etcd/pkg/v3 v3.5.9-k3s1 h1:au8ekw/8/wNokQ5dHB7MEdStKMCNBNm4tDsPWEMqW4Y=
-github.com/k3s-io/etcd/pkg/v3 v3.5.9-k3s1/go.mod h1:BZl0SAShQFk0IpLWR78T/+pyt8AruMHhTNNX73hkNVY=
-github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1 h1:nlix2+EM1UDofoHgp/X2VHzMvJW7oYbZbEinblZusNc=
-github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg=
-github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZe8rfaBPfbXo=
-github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g=
-github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64=
-github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8=
-github.com/k3s-io/kine v0.11.7 h1:+I4TrxozQv4cdmD8RULI35r4o5G+A7gOD3F75lfjDP0=
-github.com/k3s-io/kine v0.11.7/go.mod h1:4C/zNVwl3FU1EubA2ju1Hq36JIjp8gAZaM+Hfnuvqt4=
-github.com/k3s-io/klog/v2 v2.100.1-k3s1 h1:xb/Ta8dpQuIZueQEw2YTZUYrKoILdBmPiITVkNmYPa0=
-github.com/k3s-io/klog/v2 v2.100.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-github.com/k3s-io/kube-router/v2 v2.1.0 h1:BWVFMS78Duw/MRdZ8HpvVboO0yjqkIFKs51rHpI2EWM=
-github.com/k3s-io/kube-router/v2 v2.1.0/go.mod h1:JU/k2Qqcph0myF1cRDLidz1SZdoSrPczuYcGxYRSP0A=
-github.com/k3s-io/kubernetes v1.29.4-k3s1 h1:G2Hgqlc7E3bp9vt5Ssc8TKs09dt3EOBPg/TuCEObeXQ=
-github.com/k3s-io/kubernetes v1.29.4-k3s1/go.mod h1:28sDhcb87LX5z3GWAKYmLrhrifxi4W9bEWua4DRTIvk=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.29.4-k3s1 h1:C5uxsjBuO6CaO8C2U4PNgPdtsVuPCB6X1j7oxVe0+EE=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.29.4-k3s1/go.mod h1:5QkiNrCD00SOAgo8cbr43DKfW7NqRF9NpGKMO5TnQ8U=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.29.4-k3s1 h1:aJR+Jp8Iw/8s9bFRmsHytw+9ZMzezMgIJYN2wHVPgck=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.29.4-k3s1/go.mod h1:PsITwF8Ox/3gLX4kJsonfI15kansSVgDJ8CMw8QIiNI=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.29.4-k3s1 h1:WDJ1TExZaX6FU0Sed/5wZ5XXGUK4Te8m40gucUHuTHQ=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.29.4-k3s1/go.mod h1:166Zl0XJZbCvhEHvgP+h2m9Gm5xHxyQdgT9CyQRI9gw=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.29.4-k3s1 h1:zti8v005bjlyyjdH/RSDbgqo8HyzKLZ4OwWUUXAsh6E=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.29.4-k3s1/go.mod h1:8oHn4XNhYd8BeMxP6iXndQKKHaWPh8q8hIhKPSKd9KE=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.29.4-k3s1 h1:1lJKqTX/q8cor81afDZPB7wxZK8BuRxbFszQZFYfPF8=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.29.4-k3s1/go.mod h1:vPbAZgZ+2z6OdCp9taACrBBLlZGVfdOa7oC4+6iC0y4=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.29.4-k3s1 h1:9VZAlCus6XtrrHM2UCaowziRv4zqSW3wpqhNM1V+gos=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.29.4-k3s1/go.mod h1:vizyXfZHS/27DtTTjZ5mPXS5geAMGB4kj5iZtIk+GdY=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.29.4-k3s1 h1:ogI3d7EqsNboot92J6BWCgrp079okAfr93Panj8hOzM=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.29.4-k3s1/go.mod h1:N4Rw9sJ86OQXDc5dUFx2MnIOSttMkGPtLZyEoCzry8Q=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.29.4-k3s1 h1:I12nueoXNrcIPUHbBBExxxTvRNDc+p8IEVCtDzzk3aI=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.29.4-k3s1/go.mod h1:7nqgik19CFiTPjB0wYiE96lCsAPpytEAPT8Eqlhdrd8=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.29.4-k3s1 h1:AwPvAxx7zLiC9iLP0Q86YcdANiqwHC+nSORbGoFyzrM=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.29.4-k3s1/go.mod h1:aOKsghBDeKfSUqZ5vaOK6151HkaMkDsFBMkVR25teFQ=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.29.4-k3s1 h1:DVKwwIQ6LfjiFf5QMlEm9e05vtPCDWPmwxaI70GTFag=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.29.4-k3s1/go.mod h1:6oRMFYEkMBlkkBVznsi7HH/kV5F7nhyd6cqXCRCdzdo=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.29.4-k3s1 h1:dVT8rtt54jcX5qnRVFTYReHX3HFHUxHVA709x2KFphs=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.29.4-k3s1/go.mod h1:Qbpzsy+Ip0ceNppIQKfluwnEJ7UDjGmNeHqI4zn8H0E=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.29.4-k3s1 h1:d9iNmLjBoxdRjh/eRWxpLCX5KTEZcESXGKDS4Ctb7fE=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.29.4-k3s1/go.mod h1:trkLHxYEftqdW4UG3vwDLSMOV2yZeDGh0dD6jdc32aM=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.29.4-k3s1 h1:um4ehT61vmKWkET3Ss6Je76Qb6RYC6yUpq6HYisEojE=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.29.4-k3s1/go.mod h1:9Tgle7RkZOzgRM2VCSvvXZZjsQqOC6dxLtYy6BF+5PE=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.29.4-k3s1 h1:Xo97oOVwENuqjLpDbjnR19Bfywv8UCylE0Dzm34NrnI=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.29.4-k3s1/go.mod h1:r3eLoJhpPmT1wErodG1VONmI2pSfblZu1YWg7pWFZHE=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v1.29.4-k3s1 h1:rskq3sXlDMyQxCjQAgv8OXhmJKYOODlZVyxpbwWking=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v1.29.4-k3s1/go.mod h1:bffvbAY3XpW5Fu6GQjVfvhXpHQSqUfsqeNW8BwdX3LI=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/endpointslice v1.29.4-k3s1 h1:kItEsAacN2dnOpT5ZohHKSjMCo8Ol6vJVuITL9goA+c=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/endpointslice v1.29.4-k3s1/go.mod h1:BeunpqVyw83zVvf8H0iLOZWNXGoEDrmAHVcJ0NFzedk=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kms v1.29.4-k3s1 h1:sRHNaULSggLvcRx/KKKWKfPZsQcJrvqqLMNOdYpl4Ig=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kms v1.29.4-k3s1/go.mod h1:4pCpiW+pKMCqRLVoGPw11lrvEriSyd8o7DSvCw2u9vY=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.29.4-k3s1 h1:6j6hMxqvqcGuz0c9lJlWQ6b287ENKVuuvWynfwQMCaw=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.29.4-k3s1/go.mod h1:PMbMTXY5M1iXvr0tK1ycFlUJeiUOUPIWRnLDX/xDEac=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.29.4-k3s1 h1:1653eySngs0UkoC8EGJ9mbZqAGKVENLalhdMKMxffhU=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.29.4-k3s1/go.mod h1:o+O+TlsY0xG7iLwJKjdpRaxDuf07JkEGfU73CcaVOi4=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.29.4-k3s1 h1:+sl3ujiuprbK8NkH395VOtlH+30ow0vYTOiIzsPzOGc=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.29.4-k3s1/go.mod h1:2XJjMQo8aHpiGiJSr5C/ED69X7crURrDNYSJTt6j+bY=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.29.4-k3s1 h1:oyldL+LCaZsuQV+Ufni1sOC4nmDDYZcpmwO/KWLkUOI=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.29.4-k3s1/go.mod h1:2vDuSYmBuOfKFZ/R0632vKiomLKN5cLtyxuqA62Rlos=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.29.4-k3s1 h1:MaJyKwloRvG69Vd5T5BL0Vy8c+FZAwiaJN6Fpsnz8p8=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.29.4-k3s1/go.mod h1:5wY3XB46NQ9oTQdYb92teimR+9yai6ssD9K+Rd6lmug=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.29.4-k3s1 h1:o03worHS+/OnwS0Xk+RMh9l23UAlEbOrWJZqcSwqr24=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.29.4-k3s1/go.mod h1:BIgdnnHLOvfduD5ggrkXS1GlUPsRdw5YblFcmhouhgQ=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.29.4-k3s1 h1:pVrs/Mtzh4L/vHLDWHEEKlDcDXdkxPmEVVknDJm56a8=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.29.4-k3s1/go.mod h1:97CWm18L5b4oLOLskR48LLoDZ3vgQgjtpcilI3+PTPY=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.29.4-k3s1 h1:rW78PStko7c6j5iBsA7oCm7iyWblN2BHWrBrygmK5no=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.29.4-k3s1/go.mod h1:qxF7PHN59cVDWlhEvESV5Oj0CvWpzoSTUmHzXXLoxqw=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.4-k3s1 h1:DbzJ1dinrbvavL9WRmkrweT3xL2o60j/Qb8Nn7BRE64=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.4-k3s1/go.mod h1:4KmkE88Y4LDYrotr6iqMrolXDcWWY7UqmroXTO/sxFw=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.4-k3s1 h1:c9h9Bz1z3+aXpuCqUQm15dVa59bwAxrKXSGjS6N+NFk=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.4-k3s1/go.mod h1:3nvUgy9DAoVbLCBJcIBDCldv+vAc7hcHl6xJFRSpvb4=
+github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1 h1:aq6fxlEKdwCooLE3HOR6227U51DEvOw3DEbriJxD2QM=
+github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c=
+github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1 h1:t2I25UtBvohVAhlyXpYjd/Lznm+ybxNhvs3cnEGsF4Y=
+github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8=
+github.com/k3s-io/etcd/client/v2 v2.305.13-k3s1 h1:lvIdlAI6xRIHSUJC43sJx9lmxehq2quGb+8z5TJldGg=
+github.com/k3s-io/etcd/client/v2 v2.305.13-k3s1/go.mod h1:iQnL7fepbiomdXMb3om1rHq96htNNGv2sJkEcZGDRRg=
+github.com/k3s-io/etcd/client/v3 v3.5.13-k3s1 h1:/D6KAEGVzwivnjxZ5CzVIykVloLoKB/TBeKw2tKKVQ0=
+github.com/k3s-io/etcd/client/v3 v3.5.13-k3s1/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI=
+github.com/k3s-io/etcd/etcdutl/v3 v3.5.13-k3s1 h1:fIt+PVHCeINM5fl9OfMI+o9BJKf951pRiVcCytFW97c=
+github.com/k3s-io/etcd/etcdutl/v3 v3.5.13-k3s1/go.mod h1:2vhvTIQobP+Cb04qzlcbKGvX6J5oq/N1kquk1yCDIQY=
+github.com/k3s-io/etcd/pkg/v3 v3.5.13-k3s1 h1:uLU/SnBuhtSkdBk830x0pseHSsQQvh99C3deG6nc9d0=
+github.com/k3s-io/etcd/pkg/v3 v3.5.13-k3s1/go.mod h1:N+4PLrp7agI/Viy+dUYpX7iRtSPvKq+w8Y14d1vX+m0=
+github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1 h1:yexUwAPPdmYfIMWOj6sSyJ2nEe8QOrFzNuvYGRAsm5E=
+github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1/go.mod h1:uUFibGLn2Ksm2URMxN1fICGhk8Wu96EfDQyuLhAcAmw=
+github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1 h1:Pqcxkg7V60c26ZpHoekP9QoUdLuduxFn827A/5CIwm4=
+github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1/go.mod h1:K/8nbsGupHqmr5MkgaZpLlH1QdX1pcNQLAkODy44XcQ=
+github.com/k3s-io/helm-controller v0.16.1 h1:4sdJSYdAeTvMjjq3Pt1ZcyenRTJIAvKojTWRg/i8Ne4=
+github.com/k3s-io/helm-controller v0.16.1/go.mod h1:AcSxEhOIUgeVvBTnJOAwcezBZXtYew/RhKwO5xp3RlM=
+github.com/k3s-io/kine v0.11.9 h1:7HfWSwtOowb7GuV6nECnNlFKShgRgVBLdWXj0/4t0sE=
+github.com/k3s-io/kine v0.11.9/go.mod h1:N8rc1GDmEvvYRuTxhKTZfSc4fm/vyI6GbDxwBjccAjs=
+github.com/k3s-io/klog/v2 v2.120.1-k3s1 h1:7twAHPFpZA21KdMnMNnj68STQMPldAxF2Zsaol57dxw=
+github.com/k3s-io/klog/v2 v2.120.1-k3s1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+github.com/k3s-io/kube-router/v2 v2.1.2 h1:/eLfIsELLsqqRW1skIJ2qe7bWL6IZZ9Hg3IniIgObXo=
+github.com/k3s-io/kube-router/v2 v2.1.2/go.mod h1:a7QUTzCmDayYvqh6tXSKEB/ICSuGCs64qD4aCtaJqAU=
+github.com/k3s-io/kubernetes v1.30.1-k3s1 h1:UTQE4dXUvfOL6ESIxTKsqr6NTCIF+feNtlU5znXo3Lo=
+github.com/k3s-io/kubernetes v1.30.1-k3s1/go.mod h1:yPbIk3MhmhGigX62FLJm+CphNtjxqCvAIFQXup6RKS0=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.30.1-k3s1 h1:VzECjZ5j1WQJPXYMcRd3TqfRKoQRdq90NowWd4S3F7Q=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.30.1-k3s1/go.mod h1:MT0Wu+tcrQr/oMnfrjzdSZ7mzCiwx4+KDhIqa/+Br/I=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.30.1-k3s1 h1:ltHkjPoasBzcfbIxSk2gXfE8YJyoqyMBu11cUamd9QI=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.30.1-k3s1/go.mod h1:DPOzzIf6n1/l8+PABfoqj81RjqyBLI6+/Pjkys3zrQo=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.30.1-k3s1 h1:2lTp0BSdcYsHKVZhitAPrIIDVCk+HdduPJDDf1+OwDs=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.30.1-k3s1/go.mod h1:t8MQn0aJW4Wz3tmdr/QDNyBZquwqtqEUKqbKhQKjbp0=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.30.1-k3s1 h1:y1zNkNPSd9UrDvw/rDwRwxineBUmIMYR+CvFnxoN8O4=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.30.1-k3s1/go.mod h1:CUVmmmibyDvJRSdzgx+XZGcX4B6BBIyQWJ0XaZQDQ7U=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.30.1-k3s1 h1:DpqKBM7paNMMDohYDGi6H0KITugYMzH3vmYMGHcVDkQ=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.30.1-k3s1/go.mod h1:qzsasJdVj8wPJy0h4FrdC+Cqc6LGDP0PRlMUHxXx4mM=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.30.1-k3s1 h1:MFzenRmfuazTzlpfuyKSNHlmb/rEYQAavZNwasOeLWo=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.30.1-k3s1/go.mod h1:a+0Y4gJdcOQ9xQVK4WvIoKe99lj/AoaSorN3vpDRXec=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.30.1-k3s1 h1:7TRV9qRJg9F3EIwLEnKaOAMLe1lruN9DqmCzwCdl+ow=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.30.1-k3s1/go.mod h1:xi+XvU2vbnQtFkiU1pqn+dxcXwQADYeC1is21ciK2ss=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.30.1-k3s1 h1:GGKIlt35K7/mfztUXeW3JLcIeExfseFuaq0shlMUgtM=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.30.1-k3s1/go.mod h1:UQkCN/yAyK9GkwLx9OvIewMG++WMMmqM1ol0dZeih0Q=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.30.1-k3s1 h1:m3+CxnfQ4TyTZ/5pwI/6jZzvbNMKigTxWiHsITANIxw=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.30.1-k3s1/go.mod h1:dJC5AAQqmx2mhue0I0usnIajihQLIT37R3DDuUppdPE=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.30.1-k3s1 h1:ayo+TxAmTo9TwFHd8C3f/J4gQSZakl2fh4IhR7OyXjw=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.30.1-k3s1/go.mod h1:z0h1kSKwTKBb7mekPRApPIiQKjQ/97LEewIX0U2fTco=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.30.1-k3s1 h1:2cdk0c59SPVhPDqG81GvjRbb8jDpS/dVEHkoovzXUo8=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.30.1-k3s1/go.mod h1:JnDaovXMrRoLaNWAFTTYwljG5uo1YoITYNw/8P0m/Zo=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.30.1-k3s1 h1:atcflTFHFSyH0Mj/QA8JA1mcMbHYZ1SNmr807qyhsik=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.30.1-k3s1/go.mod h1:tGylrLWeY7AO4OR0evvI0CxLsmJZy83yPrtysLsGDXg=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.30.1-k3s1 h1:Kyyj87dU+y6gqzAT1FOdVmxoSz8z2TRqQsB0dGOHezY=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.30.1-k3s1/go.mod h1:/fkfIpAg9LQ3JKsBg3Zqxq1kpwX7uK8K66o573HlRZc=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.30.1-k3s1 h1:QvelUQ74Lr8zh819zu4FKnNWebA5LosKTdUL7Tzrgb8=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.30.1-k3s1/go.mod h1:ef38HCcCShGOPx8s6rnAlzS1hYCCGmiLGw3A2GGZ1pg=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v1.30.1-k3s1 h1:hsbj1ITihZ2/De5FWDksla+XyjxTbR2deTSpsKzr5+c=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v1.30.1-k3s1/go.mod h1:bLEkcQbwz8O7Q8Mb6O42blnLPr8T3OX+FoGuNSQZjqM=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/endpointslice v1.30.1-k3s1 h1:oAGPDwBQ78rqS4JNPkVsftql/vOuVfTzp0rr0u8Y5Ew=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/endpointslice v1.30.1-k3s1/go.mod h1:BIBwT7suEpxuEi3NZ7UVDtHYnRaurx9FZShAXM4wHLQ=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kms v1.30.1-k3s1 h1:86+TKZsgrioflWqUtN81Vy8b+oNzzYqkSGOAnrY20L8=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kms v1.30.1-k3s1/go.mod h1:tig/CdAZHSLnfo7HOBGtZEUcX2ym3ksoloM6gnm3/ws=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.30.1-k3s1 h1:GVYC2WlsIdMFerrw6shbJJ+km4LDfVUGRLrQYlNd7V4=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.30.1-k3s1/go.mod h1:w5RFyNnjfeptLtXFfHgzLwXaVNy+ESLZQJCbRdFzjN8=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.30.1-k3s1 h1:/9tgJiN8u5FDlLwPhPXsuRevXYFMyZNv0r+s/KdRZa4=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.30.1-k3s1/go.mod h1:tGb0LmTLtAqnOfXUlj+Ex+mNXxTlmGdkJFcI+JgnyAs=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.30.1-k3s1 h1:lB0rEKqLS6rRQXs1AeDUQgrDcqem004Nc+U1lNfPLtk=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.30.1-k3s1/go.mod h1:jlYTzNX+XdkW78Tbu7zNj9n/lnSEKQNl8McQPhMhmwM=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.30.1-k3s1 h1:NotK16kegryLC9V4uS4Ajf4ETwVZFA3pULDCOiKpzAs=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.30.1-k3s1/go.mod h1:2tg98tG689zt0TE5sHNPDd49OztHg0/wqaXIz7RLEoI=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.30.1-k3s1 h1:cj6LOgCVI0MeFgR+w/A3VDOWVPleO0fHgyVBHTWSRq4=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.30.1-k3s1/go.mod h1:WuomGAL3Q6+EQK2bfHud4HD3RruJvIQRA4uuoZ4Ew+w=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.30.1-k3s1 h1:oM2qXZ6IRQoVQA2YuBF0UTHCDb8AQ9vkc0uWU4n44pg=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.30.1-k3s1/go.mod h1:J3s04GUInRh4RD9N0tbJpkP1lciQ3WqFk69BvbNRtaQ=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.30.1-k3s1 h1:oAC3riwV4Na9j0HFvFG0/frmLA4KsdgEGsnMugsTdPk=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.30.1-k3s1/go.mod h1:FJ958oArq2Ca4R+aGj08ySB5IZU3CBIhhDihWp6tqiI=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.30.1-k3s1 h1:JjZvVOI126V0ihcqPzVzXzYH2PaZLmnhRWNqGxETjmo=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.30.1-k3s1/go.mod h1:zuttgbAy71QJ952slY86DBJu3fexyrTMXqfaTU1i3dE=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.30.1-k3s1 h1:3FGHz9/OicopJHLoPtJZb5YOrlwdFtbYxlwQJT1pIvI=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.30.1-k3s1/go.mod h1:4xH05OdueH2hpDdvzFGddYb+1GoCt/1GzcYN7ci1S14=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.30.1-k3s1 h1:NZiyhgSpmzqeKMW4QLhjjhEbsLSojcp8SkK/EjjMTo8=
+github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.30.1-k3s1/go.mod h1:TkJPz7+fWPDnOlud9WxO+KEu1KMBxQ6i9xsWryJd0l8=
 github.com/k3s-io/runc v1.1.12-k3s1 h1:p2x48K2BbRdF8crLEB4xoJ1pdjSprlvNNGpYBBULHL4=
 github.com/k3s-io/runc v1.1.12-k3s1/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8=
-github.com/k3s-io/spegel v0.0.20-k3s1 h1:alwhmC5jbaXrVEImbAdvmND8DtCi97/cRABRSkiEiUw=
-github.com/k3s-io/spegel v0.0.20-k3s1/go.mod h1:4neUkvTVGk6+Z+oiX40k15F21EsA/RnbcJHjXHlACCs=
+github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314 h1:TrZb/yM0OtBuifPXlKaOfcxpJqzakA8+KsoO4c69ZLM=
+github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314/go.mod h1:bMHfSjj1+Zf5VITCZe/wLjuni6rYAj/DjPU/kIVnhfA=
 github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
 github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
@@ -1037,6 +1000,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
+github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
@@ -1068,8 +1032,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
-github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
 github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
@@ -1176,8 +1138,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.33 h1:jLEHTp9jg2zWBa5w9W1i8WXq6o+oGRcjsdk9HbFgdlc=
-github.com/minio/minio-go/v7 v7.0.33/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g=
+github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
@@ -1209,6 +1171,7 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
+github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI=
 github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
 github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc=
@@ -1299,6 +1262,7 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
@@ -1318,11 +1282,14 @@ github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3Ro
 github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
 github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
+github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
 github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
+github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM=
+github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
@@ -1340,14 +1307,16 @@ github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRah
 github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
-github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
-github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
+github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk=
+github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg=
 github.com/open-policy-agent/opa v0.59.0/go.mod h1:rdJSkEc4oQ+0074/3Fsgno5bkPsYxTjU5aLNmMujIvI=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
 github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -1381,8 +1350,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo=
-github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=
@@ -1432,16 +1401,16 @@ github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utp
 github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY=
 github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc=
-github.com/rancher/dynamiclistener v0.3.6 h1:iAFWeiFNra6tYlt4k+jINrK3hOxZ8mjW2S/9nA6sxKs=
-github.com/rancher/dynamiclistener v0.3.6/go.mod h1:VqBaJNi+bZmre0+gi+2Jb6jbn7ovHzRueW+M7QhVKsk=
-github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29 h1:+kige/h8/LnzWgPjB5NUIHz/pWiW/lFpqcTUkN5uulY=
-github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29/go.mod h1:kgk9kJVMj9FIrrXU0iyM6u/9Je4bEjPImqswkTVaKsQ=
+github.com/rancher/dynamiclistener v0.6.0-rc1 h1:Emwf9o7PMLdQNv4lvFx7xJKxDuDa4Y69GvVEGU9U9Js=
+github.com/rancher/dynamiclistener v0.6.0-rc1/go.mod h1:BIPgJ8xFSUyuTyGvRMVt++S1qjD3+7Ptvq1TXl6hcTM=
+github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5 h1:6K4RhfmCy7uxaw9OzCljNLfFcgD/q7SeF+/2gCQ3Tvw=
+github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5/go.mod h1:7WkdfPEvWAdnHVioMUkhpZkshJzjDY62ocHVhcbw89M=
 github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcpvX1sM=
 github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q=
 github.com/rancher/wharfie v0.6.4 h1:JwYB+q661n8ut/ysgsjKe0P0z6bHCCFoC+29995ME90=
 github.com/rancher/wharfie v0.6.4/go.mod h1:kWv97z0sMAbnVNT/oe+JFZJVKn4xkas7ZdFf6UifWis=
-github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d h1:RQBqHXyAN5gWqUazV637kqmYcy8M8K5bdvXszNciLcY=
-github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d/go.mod h1:XzP4tZsV3hRksIZHaeglVtMFgz3YNh3q/TgKdP37Sl8=
+github.com/rancher/wrangler/v3 v3.0.0-rc2 h1:XGSPPp6GXELqlLvwJp5MsdqyCPu6SCA4UKJ7rQJzE40=
+github.com/rancher/wrangler/v3 v3.0.0-rc2/go.mod h1:f54hh7gFkwwbjsieT2b63FowzTU8FvrBonPe//0CIXo=
 github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk=
 github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@@ -1460,8 +1429,6 @@ github.com/rootless-containers/rootlesskit v1.0.1 h1:jepqW1txFSowKSMAEkVhWH3Oa1T
 github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc=
-github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1470,7 +1437,9 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
 github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ=
+github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY=
@@ -1509,8 +1478,6 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM=
-github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
@@ -1548,8 +1515,9 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
+github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=
+github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M=
 github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -1586,13 +1554,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
-github.com/tonglil/buflogr v1.0.1 h1:WXFZLKxLfqcVSmckwiMCF8jJwjIgmStJmg63YKRF1p0=
-github.com/tonglil/buflogr v1.0.1/go.mod h1:yYWwvSpn/3uAaqjf6mJg/XMiAciaR0QcRJH2gJGDxNE=
-github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
-github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -1611,6 +1573,7 @@ github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4J
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
 github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
@@ -1618,8 +1581,6 @@ github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
-github.com/vmware/govmomi v0.30.6 h1:O3tjSwQBy0XwI5uK1/yVIfQ1LP9bAECEDUfifnyGs9U=
-github.com/vmware/govmomi v0.30.6/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
@@ -1632,8 +1593,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/xenitab/pkg/gin v0.0.9 h1:BGdxnKoXAJBkthQTwQdaRdN7jTiNO+/C8hIexBrasfU=
-github.com/xenitab/pkg/gin v0.0.9/go.mod h1:8rzqJ8X5KJOo31PBOD4/Wtlt2ac8hCjN1mpOf1YAFs4=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk=
@@ -1677,28 +1636,27 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40=
 go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
 go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA=
 go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ=
-go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU=
-go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ=
 go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
 go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI=
 go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
 go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
@@ -1706,23 +1664,22 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM=
 go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
 go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
 go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
-go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI=
 go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
+go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
 go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
 go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
-go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk=
-go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM=
 go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
 go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
 go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
+go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU=
 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
 go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
-go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
@@ -1732,7 +1689,6 @@ go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
@@ -1740,7 +1696,6 @@ go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc=
 go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
 go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk=
 go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
@@ -1752,20 +1707,18 @@ go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
-golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
@@ -1832,9 +1785,11 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1894,16 +1849,21 @@ golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1962,7 +1922,6 @@ golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -2017,11 +1976,14 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
+golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
-golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
 golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
+golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY=
+golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2209,10 +2171,11 @@ honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 h1:gmJCKidOfjKDUHF1jjke+I+2iQIyE3HNNxu2OKO/FUI=
 inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252/go.mod h1:zq+R+tLcdHugi7Jt+FtIQY6m6wtX34lr2CdQVH2fhW0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks=
-k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/gengo v0.0.0-20240228010128-51d4e06bde70 h1:D9H6wq7PAmub2g4XUrekNWMFVI0JIz7s0F64HBPsPOw=
+k8s.io/gengo v0.0.0-20240228010128-51d4e06bde70/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo=
+k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
@@ -2261,10 +2224,12 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/knftables v0.0.14 h1:VzKQoDMCGBOH8c85sGrWSXSPCS0XrIpEfOlcCLBXiC0=
+sigs.k8s.io/knftables v0.0.14/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY=
 sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3 h1:vq2TtoDcQomhy7OxXLUOzSbHMuMYq0Bjn93cDtJEdKw=
@@ -2278,7 +2243,7 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
-tags.cncf.io/container-device-interface v0.6.2 h1:dThE6dtp/93ZDGhqaED2Pu374SOeUkBfuvkLuiTdwzg=
-tags.cncf.io/container-device-interface v0.6.2/go.mod h1:Shusyhjs1A5Na/kqPVLL0KqnHQHuunol9LFeUNkuGVE=
-tags.cncf.io/container-device-interface/specs-go v0.6.0 h1:V+tJJN6dqu8Vym6p+Ru+K5mJ49WL6Aoc5SJFSY0RLsQ=
-tags.cncf.io/container-device-interface/specs-go v0.6.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80=
+tags.cncf.io/container-device-interface v0.7.2 h1:MLqGnWfOr1wB7m08ieI4YJ3IoLKKozEnnNYBtacDPQU=
+tags.cncf.io/container-device-interface v0.7.2/go.mod h1:Xb1PvXv2BhfNb3tla4r9JL129ck1Lxv9KuU6eVOfKto=
+tags.cncf.io/container-device-interface/specs-go v0.7.0 h1:w/maMGVeLP6TIQJVYT5pbqTi8SCw/iHZ+n4ignuGHqg=
+tags.cncf.io/container-device-interface/specs-go v0.7.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80=

hack/crdgen.go

@@ -5,7 +5,7 @@ import (
 
 	k3scrd "github.com/k3s-io/k3s/pkg/crd"
 	_ "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/crd"
+	"github.com/rancher/wrangler/v3/pkg/crd"
 )
 
 func main() {

install.sh

@@ -474,12 +474,11 @@ installed_hash_matches() {
 
 # Use the GitHub API to identify the artifact associated with a given PR
 get_pr_artifact_url() {
-    GITHUB_API_URL=https://api.github.com/repos/k3s-io/k3s
+    github_api_url=https://api.github.com/repos/k3s-io/k3s
 
     # Check if jq is installed
     if ! [ -x "$(command -v jq)" ]; then
-        echo "jq is required to use INSTALL_K3S_PR. Please install jq and try again"
-        exit 1
+        fatal "Installing PR builds requires jq"
     fi
 
     if [ -z "${GITHUB_TOKEN}" ]; then
@@ -487,17 +486,17 @@ get_pr_artifact_url() {
     fi
 
     # GET request to the GitHub API to retrieve the latest commit SHA from the pull request
-    COMMIT_ID=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/pulls/$INSTALL_K3S_PR" | jq -r '.head.sha')
+    commit_id=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$github_api_url/pulls/$INSTALL_K3S_PR" | jq -r '.head.sha')
     
     # GET request to the GitHub API to retrieve the Build workflow associated with the commit
-    wf_raw=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/commits/$COMMIT_ID/check-runs")
+    wf_raw=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$github_api_url/commits/$commit_id/check-runs")
     build_workflow=$(printf "%s" "$wf_raw" | jq -r '.check_runs[] |  select(.name == "build / Build")')
     
     # Extract the Run ID from the build workflow and lookup artifacts associated with the run
-    RUN_ID=$(echo "$build_workflow" | jq -r ' .details_url' | awk -F'/' '{print $(NF-2)}')
+    run_id=$(echo "$build_workflow" | jq -r ' .details_url' | awk -F'/' '{print $(NF-2)}' | sort -rn | head -1)
 
-    # Extract the artifat ID for the "k3s" artifact    
-    artifacts=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/actions/runs/$RUN_ID/artifacts")
+    # Extract the artifact ID for the "k3s" artifact    
+    artifacts=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$github_api_url/actions/runs/$run_id/artifacts")
     artifacts_url=$(echo "$artifacts" | jq -r '.artifacts[] | select(.name == "k3s") | .archive_download_url')
     GITHUB_PR_URL=$artifacts_url
 }

install.sh.sha256sum

@@ -1 +1 @@
-3ce239d57d43b2d836d2b561043433e6decae8b9dc41f5d13908c0fafb0340cd  install.sh
+696c6a93262b3e1f06a78841b8a82c238a8f17755824c024baad652b18bc92bc  install.sh

main.go

@@ -48,6 +48,7 @@ func main() {
 			secretsencrypt.RotateKeys,
 		),
 		cmds.NewCertCommands(
+			cert.Check,
 			cert.Rotate,
 			cert.RotateCA,
 		),

manifests/local-storage.yaml

@@ -115,39 +115,13 @@ data:
     }
   setup: |-
     #!/bin/sh
-    while getopts "m:s:p:" opt
-    do
-        case $opt in
-            p)
-            absolutePath=$OPTARG
-            ;;
-            s)
-            sizeInBytes=$OPTARG
-            ;;
-            m)
-            volMode=$OPTARG
-            ;;
-        esac
-    done
-    mkdir -m 0777 -p ${absolutePath}
-    chmod 700 ${absolutePath}/..
+    set -eu
+    mkdir -m 0777 -p "${VOL_DIR}"
+    chmod 700 "${VOL_DIR}/.."
   teardown: |-
     #!/bin/sh
-    while getopts "m:s:p:" opt
-    do
-        case $opt in
-            p)
-            absolutePath=$OPTARG
-            ;;
-            s)
-            sizeInBytes=$OPTARG
-            ;;
-            m)
-            volMode=$OPTARG
-            ;;
-        esac
-    done
-    rm -rf ${absolutePath}
+    set -eu
+    rm -rf "${VOL_DIR}"
   helperPod.yaml: |-
     apiVersion: v1
     kind: Pod

package/Dockerfile

@@ -1,8 +1,8 @@
-FROM alpine:3.18 as base
-RUN apk add -U ca-certificates tar zstd tzdata
+FROM alpine:3.20 as base
+RUN apk add -U ca-certificates zstd tzdata
 COPY build/out/data.tar.zst /
 RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \
-    tar -xa -C /image -f /data.tar.zst && \
+    zstdcat -d /data.tar.zst | tar -xa -C /image && \
     echo "root:x:0:0:root:/:/bin/sh" > /image/etc/passwd && \
     echo "root:x:0:" > /image/etc/group && \
     cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt

pkg/agent/config/config.go

@@ -33,7 +33,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/pkg/errors"
 	"github.com/rancher/wharfie/pkg/registries"
-	"github.com/rancher/wrangler/pkg/slice"
+	"github.com/rancher/wrangler/v3/pkg/slice"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -200,7 +200,16 @@ func ensureNodePassword(nodePasswordFile string) (string, error) {
 		return "", err
 	}
 	nodePassword := hex.EncodeToString(password)
-	return nodePassword, os.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
+
+	if err = os.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600); err != nil {
+		return nodePassword, err
+	}
+
+	if err = configureACL(nodePassword); err != nil {
+		return nodePassword, err
+	}
+
+	return nodePassword, nil
 }
 
 func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string) {
@@ -307,19 +316,22 @@ func isValidResolvConf(resolvConfFile string) bool {
 
 	nameserver := regexp.MustCompile(`^nameserver\s+([^\s]*)`)
 	scanner := bufio.NewScanner(file)
+	foundNameserver := false
 	for scanner.Scan() {
 		ipMatch := nameserver.FindStringSubmatch(scanner.Text())
 		if len(ipMatch) == 2 {
 			ip := net.ParseIP(ipMatch[1])
 			if ip == nil || !ip.IsGlobalUnicast() {
 				return false
+			} else {
+				foundNameserver = true
 			}
 		}
 	}
 	if err := scanner.Err(); err != nil {
 		return false
 	}
-	return true
+	return foundNameserver
 }
 
 func locateOrGenerateResolvConf(envInfo *cmds.Agent) string {
@@ -512,12 +524,14 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		SELinux:                  envInfo.EnableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
 		ImageServiceEndpoint:     envInfo.ImageServiceEndpoint,
+		EnablePProf:              envInfo.EnablePProf,
 		EmbeddedRegistry:         controlConfig.EmbeddedRegistry,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
 		FlannelExternalIP:        controlConfig.FlannelExternalIP,
 		EgressSelectorMode:       controlConfig.EgressSelectorMode,
 		ServerHTTPSPort:          controlConfig.HTTPSPort,
+		SupervisorMetrics:        controlConfig.SupervisorMetrics,
 		Token:                    info.String(),
 	}
 	nodeConfig.FlannelIface = flannelIface
@@ -580,13 +594,18 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
 	nodeConfig.Certificate = servingCert
 
-	nodeConfig.AgentConfig.NodeIPs = nodeIPs
-	listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
+	if envInfo.BindAddress != "" {
+		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress
+	} else {
+		listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
+		}
+		nodeConfig.AgentConfig.ListenAddress = listenAddress
 	}
+
 	nodeConfig.AgentConfig.NodeIP = nodeIPs[0].String()
-	nodeConfig.AgentConfig.ListenAddress = listenAddress
+	nodeConfig.AgentConfig.NodeIPs = nodeIPs
 	nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs
 
 	// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
@@ -677,6 +696,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.ImageCredProvConfig = envInfo.ImageCredProvConfig
 	nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM
 	nodeConfig.AgentConfig.DisableNPC = controlConfig.DisableNPC
+	nodeConfig.AgentConfig.MinTLSVersion = controlConfig.MinTLSVersion
+	nodeConfig.AgentConfig.CipherSuites = controlConfig.CipherSuites
 	nodeConfig.AgentConfig.Rootless = envInfo.Rootless
 	nodeConfig.AgentConfig.PodManifests = filepath.Join(envInfo.DataDir, "agent", DefaultPodManifestPath)
 	nodeConfig.AgentConfig.ProtectKernelDefaults = envInfo.ProtectKernelDefaults

pkg/agent/config/config_linux.go

@@ -23,7 +23,7 @@ func applyCRIDockerdAddress(nodeConfig *config.Node) {
 }
 
 func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) {
-    containerdConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "containerd")
+	containerdConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "containerd")
 
 	blockioPath := filepath.Join(containerdConfigDir, "blockio_config.yaml")
 
@@ -45,3 +45,9 @@ func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdC
 		}
 	}
 }
+
+// configureACL will configure an Access Control List for the specified file.
+// On Linux, this function is a no-op
+func configureACL(file string) error {
+	return nil
+}

pkg/agent/config/config_windows.go

@@ -6,8 +6,11 @@ package config
 import (
 	"path/filepath"
 
+	"github.com/k3s-io/k3s/pkg/agent/util/acl"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
 )
 
 func applyContainerdStateAndAddress(nodeConfig *config.Node) {
@@ -22,3 +25,19 @@ func applyCRIDockerdAddress(nodeConfig *config.Node) {
 func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) {
 	// QoS-class resource management not supported on windows.
 }
+
+// configureACL will configure an Access Control List for the specified file,
+// ensuring that only the LocalSystem and Administrators Group have access to the file contents
+func configureACL(file string) error {
+	// by default Apply will use the current user (LocalSystem in the case of a Windows service)
+	// as the owner and current user group as the allowed group
+	// additionally, we define a DACL to permit access to the file to the local system and all administrators
+	if err := acl.Apply(file, nil, nil, []windows.EXPLICIT_ACCESS{
+		acl.GrantSid(windows.GENERIC_ALL, acl.LocalSystemSID()),
+		acl.GrantSid(windows.GENERIC_ALL, acl.BuiltinAdministratorsSID()),
+	}...); err != nil {
+		return errors.Wrapf(err, "failed to configure Access Control List For %s", file)
+	}
+
+	return nil
+}

pkg/agent/containerd/config_test.go

@@ -1471,6 +1471,17 @@ func Test_UnitGetHostConfigs(t *testing.T) {
 				t.Fatalf("failed to parse %s: %v\n", registriesFile, err)
 			}
 
+			nodeConfig := &config.Node{
+				Containerd: config.Containerd{
+					Registry: tempDir + "/hosts.d",
+				},
+				AgentConfig: config.Agent{
+					ImageServiceSocket: "containerd-stargz-grpc.sock",
+					Registry:           registry.Registry,
+					Snapshotter:        "stargz",
+				},
+			}
+
 			// set up embedded registry, if enabled for the test
 			if tt.args.mirrorAddr != "" {
 				conf := spegel.DefaultRegistry
@@ -1478,7 +1489,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
 				conf.ClientKeyFile = "client-key"
 				conf.ClientCertFile = "client-cert"
 				conf.InternalAddress, conf.RegistryPort, _ = net.SplitHostPort(tt.args.mirrorAddr)
-				conf.InjectMirror(&config.Node{AgentConfig: config.Agent{Registry: registry.Registry}})
+				conf.InjectMirror(nodeConfig)
 			}
 
 			// Generate config template struct for all hosts
@@ -1494,11 +1505,7 @@ func Test_UnitGetHostConfigs(t *testing.T) {
 
 			// Confirm that the main containerd config.toml renders properly
 			containerdConfig := templates.ContainerdConfig{
-				NodeConfig: &config.Node{
-					Containerd: config.Containerd{
-						Registry: tempDir + "/hosts.d",
-					},
-				},
+				NodeConfig:            nodeConfig,
 				PrivateRegistryConfig: registry.Registry,
 				Program:               "k3s",
 			}

pkg/agent/containerd/containerd.go

@@ -25,7 +25,7 @@ import (
 	"github.com/natefinch/lumberjack"
 	"github.com/pkg/errors"
 	"github.com/rancher/wharfie/pkg/tarfile"
-	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/merr"
 	"github.com/sirupsen/logrus"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
@@ -208,7 +208,7 @@ func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Clien
 		defer imageReader.Close()
 
 		logrus.Infof("Importing images from %s", filePath)
-		images, err = client.Import(ctx, imageReader, containerd.WithAllPlatforms(true))
+		images, err = client.Import(ctx, imageReader, containerd.WithAllPlatforms(true), containerd.WithSkipMissing())
 		if err != nil {
 			return errors.Wrap(err, "failed to import images from "+filePath)
 		}

pkg/agent/cridockerd/cridockerd.go

@@ -53,6 +53,7 @@ func getDockerCRIArgs(cfg *config.Node) []string {
 	argsMap := map[string]string{
 		"container-runtime-endpoint": cfg.CRIDockerd.Address,
 		"cri-dockerd-root-directory": cfg.CRIDockerd.Root,
+		"streaming-bind-addr":        "127.0.0.1:10010",
 	}
 
 	if dualNode, _ := utilsnet.IsDualStackIPs(cfg.AgentConfig.NodeIPs); dualNode {

pkg/agent/flannel/flannel.go

@@ -23,8 +23,9 @@ import (
 
 	"github.com/flannel-io/flannel/pkg/backend"
 	"github.com/flannel-io/flannel/pkg/ip"
-	"github.com/flannel-io/flannel/pkg/iptables"
 	"github.com/flannel-io/flannel/pkg/subnet/kube"
+	"github.com/flannel-io/flannel/pkg/trafficmngr/iptables"
+	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -80,49 +81,36 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 	if err != nil {
 		return errors.Wrap(err, "failed to register flannel network")
 	}
+	trafficMngr := &iptables.IPTablesManager{}
+	err = trafficMngr.Init(ctx, &sync.WaitGroup{})
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize flannel ipTables manager")
+	}
 
 	if netMode == (ipv4+ipv6) || netMode == ipv4 {
-		net, err := config.GetFlannelNetwork(&bn.Lease().Subnet)
-		if err != nil {
-			return errors.Wrap(err, "failed to get flannel network details")
+		if config.Network.Empty() {
+			return errors.New("ipv4 mode requested but no ipv4 network provided")
 		}
-		iptables.CreateIP4Chain("nat", "FLANNEL-POSTRTG")
-		iptables.CreateIP4Chain("filter", "FLANNEL-FWD")
-		getMasqRules := func() []iptables.IPTablesRule {
-			if config.HasNetworks() {
-				return iptables.MasqRules(config.Networks, bn.Lease())
-			}
-			return iptables.MasqRules([]ip.IP4Net{config.Network}, bn.Lease())
-		}
-		getFwdRules := func() []iptables.IPTablesRule {
-			return iptables.ForwardRules(net.String())
-		}
-		go iptables.SetupAndEnsureIP4Tables(getMasqRules, 60)
-		go iptables.SetupAndEnsureIP4Tables(getFwdRules, 50)
 	}
 
-	if config.IPv6Network.String() != emptyIPv6Network {
-		ip6net, err := config.GetFlannelIPv6Network(&bn.Lease().IPv6Subnet)
-		if err != nil {
-			return errors.Wrap(err, "failed to get ipv6 flannel network details")
-		}
-		if flannelIPv6Masq {
-			logrus.Debugf("Creating IPv6 masquerading iptables rules for %s network", config.IPv6Network.String())
-			iptables.CreateIP6Chain("nat", "FLANNEL-POSTRTG")
-			getRules := func() []iptables.IPTablesRule {
-				if config.HasIPv6Networks() {
-					return iptables.MasqIP6Rules(config.IPv6Networks, bn.Lease())
-				}
-				return iptables.MasqIP6Rules([]ip.IP6Net{config.IPv6Network}, bn.Lease())
-			}
-			go iptables.SetupAndEnsureIP6Tables(getRules, 60)
-		}
-		iptables.CreateIP6Chain("filter", "FLANNEL-FWD")
-		getRules := func() []iptables.IPTablesRule {
-			return iptables.ForwardRules(ip6net.String())
-		}
-		go iptables.SetupAndEnsureIP6Tables(getRules, 50)
+	//setup masq rules
+	prevNetwork := ReadCIDRFromSubnetFile(subnetFile, "FLANNEL_NETWORK")
+	prevSubnet := ReadCIDRFromSubnetFile(subnetFile, "FLANNEL_SUBNET")
+
+	prevIPv6Network := ReadIP6CIDRFromSubnetFile(subnetFile, "FLANNEL_IPV6_NETWORK")
+	prevIPv6Subnet := ReadIP6CIDRFromSubnetFile(subnetFile, "FLANNEL_IPV6_SUBNET")
+	if flannelIPv6Masq {
+		err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, config.IPv6Network, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60)
+	} else {
+		//set empty flannel ipv6 Network to prevent masquerading
+		err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, ip.IP6Net{}, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60)
 	}
+	if err != nil {
+		return errors.Wrap(err, "failed to setup masq rules")
+	}
+
+	//setup forward rules
+	trafficMngr.SetupAndEnsureForwardRules(ctx, config.Network, config.IPv6Network, 50)
 
 	if err := WriteSubnetFile(subnetFile, config.Network, config.IPv6Network, true, bn, netMode); err != nil {
 		// Continue, even though it failed.
@@ -237,3 +225,37 @@ func WriteSubnetFile(path string, nw ip.IP4Net, nwv6 ip.IP6Net, ipMasq bool, bn
 	return os.Rename(tempFile, path)
 	//TODO - is this safe? What if it's not on the same FS?
 }
+
+// ReadCIDRFromSubnetFile reads the flannel subnet file and extracts the value of IPv4 network CIDRKey
+func ReadCIDRFromSubnetFile(path string, CIDRKey string) ip.IP4Net {
+	var prevCIDR ip.IP4Net
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		prevSubnetVals, err := godotenv.Read(path)
+		if err != nil {
+			logrus.Errorf("Couldn't fetch previous %s from subnet file at %s: %v", CIDRKey, path, err)
+		} else if prevCIDRString, ok := prevSubnetVals[CIDRKey]; ok {
+			err = prevCIDR.UnmarshalJSON([]byte(prevCIDRString))
+			if err != nil {
+				logrus.Errorf("Couldn't parse previous %s from subnet file at %s: %v", CIDRKey, path, err)
+			}
+		}
+	}
+	return prevCIDR
+}
+
+// ReadIP6CIDRFromSubnetFile reads the flannel subnet file and extracts the value of IPv6 network CIDRKey
+func ReadIP6CIDRFromSubnetFile(path string, CIDRKey string) ip.IP6Net {
+	var prevCIDR ip.IP6Net
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		prevSubnetVals, err := godotenv.Read(path)
+		if err != nil {
+			logrus.Errorf("Couldn't fetch previous %s from subnet file at %s: %v", CIDRKey, path, err)
+		} else if prevCIDRString, ok := prevSubnetVals[CIDRKey]; ok {
+			err = prevCIDR.UnmarshalJSON([]byte(prevCIDRString))
+			if err != nil {
+				logrus.Errorf("Couldn't parse previous %s from subnet file at %s: %v", CIDRKey, path, err)
+			}
+		}
+	}
+	return prevCIDR
+}

pkg/agent/https/https.go

@@ -0,0 +1,110 @@
+package https
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"sync"
+
+	"github.com/gorilla/mux"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
+	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+)
+
+// RouterFunc provides a hook for components to register additional routes to a request router
+type RouterFunc func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error)
+
+var once sync.Once
+var router *mux.Router
+var err error
+
+// Start returns a router with authn/authz filters applied.
+// The first time it is called, the router is created and a new HTTPS listener is started if the handler is nil.
+// Subsequent calls will return the same router.
+func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.ControlRuntime) (*mux.Router, error) {
+	once.Do(func() {
+		router = mux.NewRouter().SkipClean(true)
+		config := server.Config{}
+
+		if runtime == nil {
+			// If we do not have an existing handler, set up a new listener
+			tcp, lerr := util.ListenWithLoopback(ctx, nodeConfig.AgentConfig.ListenAddress, strconv.Itoa(nodeConfig.ServerHTTPSPort))
+			if lerr != nil {
+				err = lerr
+				return
+			}
+
+			serving := options.NewSecureServingOptions()
+			serving.Listener = tcp
+			serving.CipherSuites = nodeConfig.AgentConfig.CipherSuites
+			serving.MinTLSVersion = nodeConfig.AgentConfig.MinTLSVersion
+			serving.ServerCert = options.GeneratableKeyCert{
+				CertKey: options.CertKey{
+					CertFile: nodeConfig.AgentConfig.ServingKubeletCert,
+					KeyFile:  nodeConfig.AgentConfig.ServingKubeletKey,
+				},
+			}
+			if aerr := serving.ApplyTo(&config.SecureServing); aerr != nil {
+				err = aerr
+				return
+			}
+		} else {
+			// If we have an existing handler, wrap it
+			router.NotFoundHandler = runtime.Handler
+			runtime.Handler = router
+		}
+
+		authn := options.NewDelegatingAuthenticationOptions()
+		authn.DisableAnonymous = true
+		authn.SkipInClusterLookup = true
+		authn.ClientCert = options.ClientCertAuthenticationOptions{
+			ClientCA: nodeConfig.AgentConfig.ClientCA,
+		}
+		authn.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet
+		if applyErr := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); applyErr != nil {
+			err = applyErr
+			return
+		}
+
+		authz := options.NewDelegatingAuthorizationOptions()
+		authz.AlwaysAllowPaths = []string{ // skip authz for paths that should not use SubjectAccessReview; basically everything that will use this router other than metrics
+			"/v1-" + version.Program + "/p2p", // spegel libp2p peer discovery
+			"/v2/*",                           // spegel registry mirror
+			"/debug/pprof/*",                  // profiling
+		}
+		authz.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet
+		if applyErr := authz.ApplyTo(&config.Authorization); applyErr != nil {
+			err = applyErr
+			return
+		}
+
+		router.Use(filterChain(config.Authentication.Authenticator, config.Authorization.Authorizer))
+
+		if config.SecureServing != nil {
+			_, _, err = config.SecureServing.Serve(router, 0, ctx.Done())
+		}
+	})
+
+	return router, err
+}
+
+// filterChain runs the kubernetes authn/authz filter chain using the mux middleware API
+func filterChain(authn authenticator.Request, authz authorizer.Authorizer) mux.MiddlewareFunc {
+	return func(handler http.Handler) http.Handler {
+		requestInfoResolver := &apirequest.RequestInfoFactory{}
+		failedHandler := genericapifilters.Unauthorized(scheme.Codecs)
+		handler = genericapifilters.WithAuthorization(handler, authz, scheme.Codecs)
+		handler = genericapifilters.WithAuthentication(handler, authn, failedHandler, nil, nil)
+		handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver)
+		handler = genericapifilters.WithCacheControl(handler)
+		return handler
+	}
+}

pkg/agent/loadbalancer/loadbalancer.go

@@ -158,6 +158,7 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, _ string) (net
 	lb.mutex.RLock()
 	defer lb.mutex.RUnlock()
 
+	var allChecksFailed bool
 	startIndex := lb.nextServerIndex
 	for {
 		targetServer := lb.currentServerAddress
@@ -165,7 +166,7 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, _ string) (net
 		server := lb.servers[targetServer]
 		if server == nil || targetServer == "" {
 			logrus.Debugf("Nil server for load balancer %s: %s", lb.serviceName, targetServer)
-		} else if server.healthCheck() {
+		} else if allChecksFailed || server.healthCheck() {
 			conn, err := server.dialContext(ctx, network, targetServer)
 			if err == nil {
 				return conn, nil
@@ -189,7 +190,11 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, _ string) (net
 			startIndex = maxIndex
 		}
 		if lb.nextServerIndex == startIndex {
-			return nil, errors.New("all servers failed")
+			if allChecksFailed {
+				return nil, errors.New("all servers failed")
+			}
+			logrus.Debugf("Health checks for all servers in load balancer %s have failed: retrying with health checks ignored", lb.serviceName)
+			allChecksFailed = true
 		}
 	}
 }

pkg/agent/loadbalancer/servers.go

@@ -227,13 +227,19 @@ func (lb *LoadBalancer) SetHealthCheck(address string, healthCheck func() bool)
 // runHealthChecks periodically health-checks all servers. Any servers that fail the health-check will have their
 // connections closed, to force clients to switch over to a healthy server.
 func (lb *LoadBalancer) runHealthChecks(ctx context.Context) {
+	previousStatus := map[string]bool{}
 	wait.Until(func() {
 		lb.mutex.RLock()
 		defer lb.mutex.RUnlock()
-		for _, server := range lb.servers {
-			if !server.healthCheck() {
+		for address, server := range lb.servers {
+			status := server.healthCheck()
+			if status == false && previousStatus[address] == true {
+				// Only close connections when the server transitions from healthy to unhealthy;
+				// we don't want to re-close all the connections every time as we might be ignoring
+				// health checks due to all servers being marked unhealthy.
 				defer server.closeAll()
 			}
+			previousStatus[address] = status
 		}
 	}, time.Second, ctx.Done())
 	logrus.Debugf("Stopped health checking for load balancer %s", lb.serviceName)

pkg/agent/netpol/netpol.go

@@ -19,25 +19,25 @@ import (
 
 	"github.com/cloudnativelabs/kube-router/v2/pkg/controllers/netpol"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/healthcheck"
-	"github.com/cloudnativelabs/kube-router/v2/pkg/metrics"
+	krmetrics "github.com/cloudnativelabs/kube-router/v2/pkg/metrics"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/options"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/utils"
 	"github.com/cloudnativelabs/kube-router/v2/pkg/version"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1core "k8s.io/api/core/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 func init() {
 	// ensure that kube-router exposes metrics through the same registry used by Kubernetes components
-	metrics.DefaultRegisterer = legacyregistry.Registerer()
-	metrics.DefaultGatherer = legacyregistry.DefaultGatherer
+	krmetrics.DefaultRegisterer = metrics.DefaultRegisterer
+	krmetrics.DefaultGatherer = metrics.DefaultGatherer
 }
 
 // Run creates and starts a new instance of the kube-router network policy controller
@@ -67,27 +67,26 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 		return err
 	}
 
-	// As kube-router netpol requires addresses to be available in the node object
-	// Wait until the node has ready addresses to avoid race conditions (max 1 minute).
+	// kube-router netpol requires addresses to be available in the node object.
+	// Wait until the uninitialized taint has been removed, at which point the addresses should be set.
 	// TODO: Replace with non-deprecated PollUntilContextTimeout when our and Kubernetes code migrate to it
-	if err := wait.PollImmediateWithContext(ctx, 2*time.Second, 60*time.Second, func(ctx context.Context) (bool, error) {
+	if err := wait.PollImmediateInfiniteWithContext(ctx, 2*time.Second, func(ctx context.Context) (bool, error) {
 		// Get the node object
 		node, err := client.CoreV1().Nodes().Get(ctx, nodeConfig.AgentConfig.NodeName, metav1.GetOptions{})
 		if err != nil {
-			logrus.Debugf("Network policy controller waiting to get Node %s: %v", nodeConfig.AgentConfig.NodeName, err)
+			logrus.Infof("Network policy controller waiting to get Node %s: %v", nodeConfig.AgentConfig.NodeName, err)
 			return false, nil
 		}
-		// Check for the uninitialized taint that should be removed by cloud-provider
-		// If there is no cloud-provider, the taint will not be there
+		// Check for the taint that should be removed by cloud-provider when the node has been initialized.
 		for _, taint := range node.Spec.Taints {
 			if taint.Key == cloudproviderapi.TaintExternalCloudProvider {
-				logrus.Debugf("Network policy controller waiting for removal of %s taint", cloudproviderapi.TaintExternalCloudProvider)
+				logrus.Infof("Network policy controller waiting for removal of %s taint", cloudproviderapi.TaintExternalCloudProvider)
 				return false, nil
 			}
 		}
 		return true, nil
 	}); err != nil {
-		return errors.Wrapf(err, "network policy controller timed out waiting for %s taint to be removed from Node %s", cloudproviderapi.TaintExternalCloudProvider, nodeConfig.AgentConfig.NodeName)
+		return errors.Wrapf(err, "network policy controller failed to wait for %s taint to be removed from Node %s", cloudproviderapi.TaintExternalCloudProvider, nodeConfig.AgentConfig.NodeName)
 	}
 
 	krConfig := options.NewKubeRouterConfig()
@@ -156,7 +155,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	}
 
 	// Start kube-router metrics controller to avoid complaints about metrics heartbeat missing
-	mc, err := metrics.NewMetricsController(krConfig)
+	mc, err := krmetrics.NewMetricsController(krConfig)
 	if err != nil {
 		return nil
 	}
@@ -188,13 +187,13 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 }
 
 // metricsRunCheck is a stub version of mc.Run() that doesn't start up a dedicated http server.
-func metricsRunCheck(mc *metrics.Controller, healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
+func metricsRunCheck(mc *krmetrics.Controller, healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
 	t := time.NewTicker(3 * time.Second)
 	defer wg.Done()
 
 	// register metrics for this controller
-	metrics.BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1)
-	metrics.DefaultRegisterer.MustRegister(metrics.BuildInfo)
+	krmetrics.BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1)
+	krmetrics.DefaultRegisterer.MustRegister(krmetrics.BuildInfo)
 
 	for {
 		healthcheck.SendHeartBeat(healthChan, "MC")

pkg/agent/run.go

@@ -27,7 +27,9 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
+	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/k3s-io/k3s/pkg/nodeconfig"
+	"github.com/k3s-io/k3s/pkg/profile"
 	"github.com/k3s-io/k3s/pkg/rootless"
 	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
@@ -113,6 +115,18 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		}
 	}
 
+	if nodeConfig.SupervisorMetrics {
+		if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to serve metrics")
+		}
+	}
+
+	if nodeConfig.EnablePProf {
+		if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to serve pprof")
+		}
+	}
+
 	if err := setupCriCtlConfig(cfg, nodeConfig); err != nil {
 		return err
 	}

pkg/agent/templates/templates_linux.go

@@ -44,19 +44,11 @@ cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket
 [plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
 enable_keychain = true
 {{end}}
+
+[plugins."io.containerd.snapshotter.v1.stargz".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
 {{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
 {{range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
 [plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth]
@@ -65,13 +57,6 @@ enable_keychain = true
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
 {{end}}
-{{ if $v.TLS }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-{{end}}
 {{end}}
 {{end}}
 {{end}}

pkg/agent/util/acl/acl_windows.go

@@ -0,0 +1,166 @@
+//go:build windows
+// +build windows
+
+package acl
+
+import (
+	"fmt"
+	"golang.org/x/sys/windows"
+	"unsafe"
+)
+
+// TODO: Remove in favor of the rancher/permissions repository once that is setup
+
+func BuiltinAdministratorsSID() *windows.SID {
+	return mustGetSid(windows.WinBuiltinAdministratorsSid)
+}
+
+func LocalSystemSID() *windows.SID {
+	return mustGetSid(windows.WinLocalSystemSid)
+}
+
+func mustGetSid(sidType windows.WELL_KNOWN_SID_TYPE) *windows.SID {
+	sid, err := windows.CreateWellKnownSid(sidType)
+	if err != nil {
+		panic(err)
+	}
+	return sid
+}
+
+// GrantSid creates an EXPLICIT_ACCESS instance granting permissions to the provided SID.
+func GrantSid(accessPermissions windows.ACCESS_MASK, sid *windows.SID) windows.EXPLICIT_ACCESS {
+	return windows.EXPLICIT_ACCESS{
+		AccessPermissions: accessPermissions,
+		AccessMode:        windows.GRANT_ACCESS,
+		Inheritance:       windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT,
+		Trustee: windows.TRUSTEE{
+			TrusteeForm:  windows.TRUSTEE_IS_SID,
+			TrusteeValue: windows.TrusteeValueFromSID(sid),
+		},
+	}
+}
+
+// Apply performs both Chmod and Chown at the same time, where the filemode's owner and group will correspond to
+// the provided owner and group (or the current owner and group, if they are set to nil)
+func Apply(path string, owner *windows.SID, group *windows.SID, access ...windows.EXPLICIT_ACCESS) error {
+	if path == "" {
+		return fmt.Errorf("path cannot be empty")
+	}
+	return apply(path, owner, group, access...)
+}
+
+// apply performs a Chmod (if owner and group are provided) and sets a custom ACL based on the provided EXPLICIT_ACCESS rules
+// To create EXPLICIT_ACCESS rules, see the helper functions in pkg/access
+func apply(path string, owner *windows.SID, group *windows.SID, access ...windows.EXPLICIT_ACCESS) error {
+	// assemble arguments
+	args := securityArgs{
+		path:   path,
+		owner:  owner,
+		group:  group,
+		access: access,
+	}
+
+	securityInfo := args.ToSecurityInfo()
+	if securityInfo == 0 {
+		// nothing to change
+		return nil
+	}
+	dacl, err := args.ToDACL()
+	if err != nil {
+		return err
+	}
+	return windows.SetNamedSecurityInfo(
+		path,
+		windows.SE_FILE_OBJECT,
+		securityInfo,
+		owner,
+		group,
+		dacl,
+		nil,
+	)
+}
+
+type securityArgs struct {
+	path string
+
+	owner *windows.SID
+	group *windows.SID
+
+	access []windows.EXPLICIT_ACCESS
+}
+
+func (a *securityArgs) ToSecurityInfo() windows.SECURITY_INFORMATION {
+	var securityInfo windows.SECURITY_INFORMATION
+
+	if a.owner != nil {
+		// override owner
+		securityInfo |= windows.OWNER_SECURITY_INFORMATION
+	}
+
+	if a.group != nil {
+		// override group
+		securityInfo |= windows.GROUP_SECURITY_INFORMATION
+	}
+
+	if len(a.access) != 0 {
+		// override DACL
+		securityInfo |= windows.DACL_SECURITY_INFORMATION
+		securityInfo |= windows.PROTECTED_DACL_SECURITY_INFORMATION
+	}
+
+	return securityInfo
+}
+
+func (a *securityArgs) ToSecurityAttributes() (*windows.SecurityAttributes, error) {
+	// define empty security descriptor
+	sd, err := windows.NewSecurityDescriptor()
+	if err != nil {
+		return nil, err
+	}
+	err = sd.SetOwner(a.owner, false)
+	if err != nil {
+		return nil, err
+	}
+	err = sd.SetGroup(a.group, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// define security attributes using descriptor
+	var sa windows.SecurityAttributes
+	sa.Length = uint32(unsafe.Sizeof(sa))
+	sa.SecurityDescriptor = sd
+
+	if len(a.access) == 0 {
+		// security attribute should simply inherit parent rules
+		sa.InheritHandle = 1
+		return &sa, nil
+	}
+
+	// apply provided access rules to the DACL
+	dacl, err := a.ToDACL()
+	if err != nil {
+		return nil, err
+	}
+	err = sd.SetDACL(dacl, true, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// set the protected DACL flag to prevent the DACL of the security descriptor from being modified by inheritable ACEs
+	// (i.e. prevent parent folders from modifying this ACL)
+	err = sd.SetControl(windows.SE_DACL_PROTECTED, windows.SE_DACL_PROTECTED)
+	if err != nil {
+		return nil, err
+	}
+
+	return &sa, nil
+}
+
+func (a *securityArgs) ToDACL() (*windows.ACL, error) {
+	if len(a.access) == 0 {
+		// No rules were specified
+		return nil, nil
+	}
+	return windows.ACLFromEntries(a.access, nil)
+}

pkg/certmonitor/certmonitor.go

@@ -11,29 +11,21 @@ import (
 
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/util/services"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/prometheus/client_golang/prometheus"
 	certutil "github.com/rancher/dynamiclistener/cert"
-	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/merr"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 var (
-	// DefaultRegisterer and DefaultGatherer are the implementations of the
-	// prometheus Registerer and Gatherer interfaces that all metrics operations
-	// will use. They are variables so that packages that embed this library can
-	// replace them at runtime, instead of having to pass around specific
-	// registries.
-	DefaultRegisterer = legacyregistry.Registerer()
-	DefaultGatherer   = legacyregistry.DefaultGatherer
-
 	// Check certificates twice an hour. Kubernetes events have a TTL of 1 hour by default,
 	// so similar events should be aggregated and refreshed by the event recorder as long
 	// as they are created within the TTL period.
@@ -50,7 +42,7 @@ var (
 // Setup starts the certificate expiration monitor
 func Setup(ctx context.Context, nodeConfig *daemonconfig.Node, dataDir string) error {
 	logrus.Debugf("Starting %s with monitoring period %s", controllerName, certCheckInterval)
-	DefaultRegisterer.MustRegister(certificateExpirationSeconds)
+	metrics.DefaultRegisterer.MustRegister(certificateExpirationSeconds)
 
 	client, err := util.GetClientSet(nodeConfig.AgentConfig.KubeConfigKubelet)
 	if err != nil {

pkg/cli/agent/agent.go

@@ -1,28 +1,29 @@
 package agent
 
 import (
+	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 
-	"github.com/erikdubbelboer/gspt"
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
-	"github.com/k3s-io/k3s/pkg/authenticator"
+	"github.com/k3s-io/k3s/pkg/agent/https"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/datadir"
+	k3smetrics "github.com/k3s-io/k3s/pkg/metrics"
+	"github.com/k3s-io/k3s/pkg/proctitle"
+	"github.com/k3s-io/k3s/pkg/profile"
 	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
-	"github.com/rancher/wrangler/pkg/signals"
+	"github.com/rancher/wrangler/v3/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
-	apiauth "k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
 func Run(ctx *cli.Context) error {
@@ -31,7 +32,7 @@ func Run(ctx *cli.Context) error {
 
 	// hide process arguments from ps output, since they may contain
 	// database credentials or other secrets.
-	gspt.SetProcTitle(os.Args[0] + " agent")
+	proctitle.SetProcTitle(os.Args[0] + " agent")
 
 	// Evacuate cgroup v2 before doing anything else that may fork.
 	if err := cmds.EvacuateCgroup2(); err != nil {
@@ -90,16 +91,16 @@ func Run(ctx *cli.Context) error {
 	contextCtx := signals.SetupSignalContext()
 
 	go cmds.WriteCoverage(contextCtx)
-	if cmds.AgentConfig.VPNAuthFile != "" {
-		cmds.AgentConfig.VPNAuth, err = util.ReadFile(cmds.AgentConfig.VPNAuthFile)
+	if cfg.VPNAuthFile != "" {
+		cfg.VPNAuth, err = util.ReadFile(cfg.VPNAuthFile)
 		if err != nil {
 			return err
 		}
 	}
 
 	// Starts the VPN in the agent if config was set up
-	if cmds.AgentConfig.VPNAuth != "" {
-		err := vpn.StartVPN(cmds.AgentConfig.VPNAuth)
+	if cfg.VPNAuth != "" {
+		err := vpn.StartVPN(cfg.VPNAuth)
 		if err != nil {
 			return err
 		}
@@ -108,33 +109,22 @@ func Run(ctx *cli.Context) error {
 	// Until the agent is run and retrieves config from the server, we won't know
 	// if the embedded registry is enabled. If it is not enabled, these are not
 	// used as the registry is never started.
-	conf := spegel.DefaultRegistry
-	conf.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir)
-	conf.HandlerFunc = func(conf *spegel.Config, router *mux.Router) error {
-		// Create and bind a new authenticator using the configured client CA
-		authArgs := []string{"--client-ca-file=" + conf.ClientCAFile}
-		auth, err := authenticator.FromArgs(authArgs)
-		if err != nil {
-			return err
-		}
-		conf.AuthFunc = func() apiauth.Request {
-			return auth
-		}
+	registry := spegel.DefaultRegistry
+	registry.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir)
+	registry.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, nil)
+	}
 
-		// Create a new server and listen on the configured port
-		server := &http.Server{
-			Handler: router,
-			Addr:    ":" + conf.RegistryPort,
-			TLSConfig: &tls.Config{
-				ClientAuth: tls.RequestClientCert,
-			},
-		}
-		go func() {
-			if err := server.ListenAndServeTLS(conf.ServerCertFile, conf.ServerKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) {
-				logrus.Fatalf("registry server failed: %v", err)
-			}
-		}()
-		return nil
+	// same deal for metrics - these are not used if the extra metrics listener is not enabled.
+	metrics := k3smetrics.DefaultMetrics
+	metrics.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, nil)
+	}
+
+	// and for pprof as well
+	pprof := profile.DefaultProfiler
+	pprof.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, nil)
 	}
 
 	return agent.Run(contextCtx, cfg)

pkg/cli/cert/cert.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/erikdubbelboer/gspt"
 	"github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/bootstrap"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
@@ -16,6 +15,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
 	"github.com/k3s-io/k3s/pkg/datadir"
+	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/util/services"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -27,7 +27,7 @@ import (
 )
 
 func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, error) {
-	gspt.SetProcTitle(os.Args[0])
+	proctitle.SetProcTitle(os.Args[0])
 
 	dataDir, err := datadir.Resolve(cfg.DataDir)
 	if err != nil {

pkg/cli/cmds/agent.go

@@ -20,6 +20,7 @@ type Agent struct {
 	LBServerPort             int
 	ResolvConf               string
 	DataDir                  string
+	BindAddress              string
 	NodeIP                   cli.StringSlice
 	NodeExternalIP           cli.StringSlice
 	NodeName                 string
@@ -36,6 +37,7 @@ type Agent struct {
 	VPNAuth                  string
 	VPNAuthFile              string
 	Debug                    bool
+	EnablePProf              bool
 	Rootless                 bool
 	RootlessAlreadyUnshared  bool
 	WithNodeID               bool
@@ -226,6 +228,16 @@ var (
 		Usage:       "(agent/containerd) Disables containerd's fallback default registry endpoint when a mirror is configured for that registry",
 		Destination: &AgentConfig.ContainerdNoDefault,
 	}
+	EnablePProfFlag = &cli.BoolFlag{
+		Name:        "enable-pprof",
+		Usage:       "(experimental) Enable pprof endpoint on supervisor port",
+		Destination: &AgentConfig.EnablePProf,
+	}
+	BindAddressFlag = &cli.StringFlag{
+		Name:        "bind-address",
+		Usage:       "(listener) " + version.Program + " bind address (default: 0.0.0.0)",
+		Destination: &AgentConfig.BindAddress,
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -278,6 +290,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			DisableDefaultRegistryEndpointFlag,
 			AirgapExtraRegistryFlag,
 			NodeIPFlag,
+			BindAddressFlag,
 			NodeExternalIPFlag,
 			ResolvConfFlag,
 			FlannelIfaceFlag,
@@ -286,6 +299,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			ExtraKubeletArgs,
 			ExtraKubeProxyArgs,
 			// Experimental flags
+			EnablePProfFlag,
 			&cli.BoolFlag{
 				Name:        "rootless",
 				Usage:       "(experimental) Run rootless",

pkg/cli/cmds/log_linux.go

@@ -11,7 +11,7 @@ import (
 	"syscall"
 
 	systemd "github.com/coreos/go-systemd/v22/daemon"
-	"github.com/erikdubbelboer/gspt"
+	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/natefinch/lumberjack"
 	"github.com/pkg/errors"
@@ -42,7 +42,7 @@ func forkIfLoggingOrReaping() error {
 	}
 
 	if enableLogRedirect || enableReaping {
-		gspt.SetProcTitle(os.Args[0] + " init")
+		proctitle.SetProcTitle(os.Args[0] + " init")
 
 		pwd, err := os.Getwd()
 		if err != nil {

pkg/cli/cmds/server.go

@@ -45,11 +45,10 @@ type Server struct {
 	DisableAgent             bool
 	KubeConfigOutput         string
 	KubeConfigMode           string
+	KubeConfigGroup          string
 	HelmJobImage             string
 	TLSSan                   cli.StringSlice
 	TLSSanSecurity           bool
-	BindAddress              string
-	EnablePProf              bool
 	ExtraAPIArgs             cli.StringSlice
 	ExtraEtcdArgs            cli.StringSlice
 	ExtraSchedulerArgs       cli.StringSlice
@@ -87,6 +86,7 @@ type Server struct {
 	EncryptSkip              bool
 	SystemDefaultRegistry    string
 	StartupHooks             []StartupHook
+	SupervisorMetrics        bool
 	EtcdSnapshotName         string
 	EtcdDisableSnapshots     bool
 	EtcdExposeMetrics        bool
@@ -178,11 +178,7 @@ var ServerFlags = []cli.Flag{
 	VModule,
 	LogFile,
 	AlsoLogToStderr,
-	&cli.StringFlag{
-		Name:        "bind-address",
-		Usage:       "(listener) " + version.Program + " bind address (default: 0.0.0.0)",
-		Destination: &ServerConfig.BindAddress,
-	},
+	BindAddressFlag,
 	&cli.IntFlag{
 		Name:        "https-listen-port",
 		Usage:       "(listener) HTTPS listen port",
@@ -255,6 +251,12 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.KubeConfigMode,
 		EnvVar:      version.ProgramUpper + "_KUBECONFIG_MODE",
 	},
+	&cli.StringFlag{
+		Name:        "write-kubeconfig-group",
+		Usage:       "(client) Write kubeconfig with this group",
+		Destination: &ServerConfig.KubeConfigGroup,
+		EnvVar:      version.ProgramUpper + "_KUBECONFIG_GROUP",
+	},
 	&cli.StringFlag{
 		Name:        "helm-job-image",
 		Usage:       "(helm) Default image to use for helm jobs",
@@ -493,9 +495,14 @@ var ServerFlags = []cli.Flag{
 	},
 	&cli.BoolFlag{
 		Name:        "embedded-registry",
-		Usage:       "(experimental/components) Enable embedded distributed container registry; requires use of embedded containerd",
+		Usage:       "(experimental/components) Enable embedded distributed container registry; requires use of embedded containerd; when enabled agents will also listen on the supervisor port",
 		Destination: &ServerConfig.EmbeddedRegistry,
 	},
+	&cli.BoolFlag{
+		Name:        "supervisor-metrics",
+		Usage:       "(experimental/components) Enable serving " + version.Program + " internal metrics on the supervisor port; when enabled agents will also listen on the supervisor port",
+		Destination: &ServerConfig.SupervisorMetrics,
+	},
 	NodeNameFlag,
 	WithNodeIDFlag,
 	NodeLabels,
@@ -534,11 +541,7 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.EncryptSecrets,
 	},
 	// Experimental flags
-	&cli.BoolFlag{
-		Name:        "enable-pprof",
-		Usage:       "(experimental) Enable pprof endpoint on supervisor port",
-		Destination: &ServerConfig.EnablePProf,
-	},
+	EnablePProfFlag,
 	&cli.BoolFlag{
 		Name:        "rootless",
 		Usage:       "(experimental) Run rootless",

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -2,6 +2,7 @@ package etcdsnapshot
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -11,12 +12,12 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/erikdubbelboer/gspt"
 	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
 	"github.com/k3s-io/k3s/pkg/etcd"
+	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/server"
 	util2 "github.com/k3s-io/k3s/pkg/util"
 	"github.com/pkg/errors"
@@ -26,12 +27,14 @@ import (
 	"k8s.io/cli-runtime/pkg/printers"
 )
 
+var timeout = 2 * time.Minute
+
 // commandSetup setups up common things needed
 // for each etcd command.
 func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *clientaccess.Info, error) {
 	// hide process arguments from ps output, since they may contain
 	// database credentials or other secrets.
-	gspt.SetProcTitle(os.Args[0] + " etcd-snapshot")
+	proctitle.SetProcTitle(os.Args[0] + " etcd-snapshot")
 
 	sr := &etcd.SnapshotRequest{}
 	// Operation and name are set by the command handler.
@@ -58,6 +61,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
 		sr.S3.SecretKey = cfg.EtcdS3SecretKey
 		sr.S3.SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
 		sr.S3.Timeout = metav1.Duration{Duration: cfg.EtcdS3Timeout}
+		// extend request timeout to allow the S3 operation to complete
+		timeout += cfg.EtcdS3Timeout
 	}
 
 	dataDir, err := server.ResolveDataDir(cfg.DataDir)
@@ -78,6 +83,11 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
 }
 
 func wrapServerError(err error) error {
+	if errors.Is(err, context.DeadlineExceeded) {
+		// if the request timed out the server log likely won't contain anything useful,
+		// since the operation may have actualy succeeded despite the client timing out the request.
+		return err
+	}
 	return errors.Wrap(err, "see server log for details")
 }
 
@@ -110,7 +120,7 @@ func save(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}
@@ -151,7 +161,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}
@@ -206,7 +216,7 @@ func list(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}
@@ -269,7 +279,7 @@ func prune(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -10,9 +10,9 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/erikdubbelboer/gspt"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
+	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -24,7 +24,7 @@ import (
 func commandPrep(cfg *cmds.Server) (*clientaccess.Info, error) {
 	// hide process arguments from ps output, since they may contain
 	// database credentials or other secrets.
-	gspt.SetProcTitle(os.Args[0] + " secrets-encrypt")
+	proctitle.SetProcTitle(os.Args[0] + " secrets-encrypt")
 
 	dataDir, err := server.ResolveDataDir(cfg.DataDir)
 	if err != nil {

pkg/cli/server/server.go

@@ -10,15 +10,18 @@ import (
 	"time"
 
 	systemd "github.com/coreos/go-systemd/v22/daemon"
-	"github.com/erikdubbelboer/gspt"
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
+	"github.com/k3s-io/k3s/pkg/agent/https"
 	"github.com/k3s-io/k3s/pkg/agent/loadbalancer"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/datadir"
 	"github.com/k3s-io/k3s/pkg/etcd"
+	k3smetrics "github.com/k3s-io/k3s/pkg/metrics"
+	"github.com/k3s-io/k3s/pkg/proctitle"
+	"github.com/k3s-io/k3s/pkg/profile"
 	"github.com/k3s-io/k3s/pkg/rootless"
 	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/spegel"
@@ -26,11 +29,10 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/signals"
+	"github.com/rancher/wrangler/v3/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/apiserver/pkg/authentication/authenticator"
 	kubeapiserverflag "k8s.io/component-base/cli/flag"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	utilsnet "k8s.io/utils/net"
@@ -55,7 +57,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 
 	// hide process arguments from ps output, since they may contain
 	// database credentials or other secrets.
-	gspt.SetProcTitle(os.Args[0] + " server")
+	proctitle.SetProcTitle(os.Args[0] + " server")
 
 	// If the agent is enabled, evacuate cgroup v2 before doing anything else that may fork.
 	// If the agent is disabled, we don't need to bother doing this as it is only the kubelet
@@ -131,17 +133,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.DataDir = cfg.DataDir
 	serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
 	serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
+	serverConfig.ControlConfig.KubeConfigGroup = cfg.KubeConfigGroup
 	serverConfig.ControlConfig.HelmJobImage = cfg.HelmJobImage
 	serverConfig.ControlConfig.Rootless = cfg.Rootless
 	serverConfig.ControlConfig.ServiceLBNamespace = cfg.ServiceLBNamespace
 	serverConfig.ControlConfig.SANs = util.SplitStringSlice(cfg.TLSSan)
 	serverConfig.ControlConfig.SANSecurity = cfg.TLSSanSecurity
-	serverConfig.ControlConfig.BindAddress = cfg.BindAddress
+	serverConfig.ControlConfig.BindAddress = cmds.AgentConfig.BindAddress
 	serverConfig.ControlConfig.SupervisorPort = cfg.SupervisorPort
 	serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.ControlConfig.APIServerPort = cfg.APIServerPort
 	serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress
-	serverConfig.ControlConfig.EnablePProf = cfg.EnablePProf
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
 	serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs
@@ -174,6 +176,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
 	serverConfig.ControlConfig.EtcdExposeMetrics = cfg.EtcdExposeMetrics
 	serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
+	serverConfig.ControlConfig.SupervisorMetrics = cfg.SupervisorMetrics
 	serverConfig.ControlConfig.VLevel = cmds.LogConfig.VLevel
 	serverConfig.ControlConfig.VModule = cmds.LogConfig.VModule
 
@@ -406,6 +409,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	}
 
 	tlsMinVersionArg := getArgValueFromList("tls-min-version", serverConfig.ControlConfig.ExtraAPIArgs)
+	serverConfig.ControlConfig.MinTLSVersion = tlsMinVersionArg
 	serverConfig.ControlConfig.TLSMinVersion, err = kubeapiserverflag.TLSVersion(tlsMinVersionArg)
 	if err != nil {
 		return errors.Wrap(err, "invalid tls-min-version")
@@ -435,6 +439,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		}
 		serverConfig.ControlConfig.ExtraAPIArgs = append(serverConfig.ControlConfig.ExtraAPIArgs, "tls-cipher-suites="+strings.Join(tlsCipherSuites, ","))
 	}
+	serverConfig.ControlConfig.CipherSuites = tlsCipherSuites
 	serverConfig.ControlConfig.TLSCipherSuites, err = kubeapiserverflag.TLSCipherSuites(tlsCipherSuites)
 	if err != nil {
 		return errors.Wrap(err, "invalid tls-cipher-suites")
@@ -556,28 +561,36 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		go getAPIAddressFromEtcd(ctx, serverConfig, agentConfig)
 	}
 
+	// Until the agent is run and retrieves config from the server, we won't know
+	// if the embedded registry is enabled. If it is not enabled, these are not
+	// used as the registry is never started.
+	registry := spegel.DefaultRegistry
+	registry.Bootstrapper = spegel.NewChainingBootstrapper(
+		spegel.NewServerBootstrapper(&serverConfig.ControlConfig),
+		spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir),
+		spegel.NewSelfBootstrapper(),
+	)
+	registry.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, serverConfig.ControlConfig.Runtime)
+	}
+
+	// same deal for metrics - these are not used if the extra metrics listener is not enabled.
+	metrics := k3smetrics.DefaultMetrics
+	metrics.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, serverConfig.ControlConfig.Runtime)
+	}
+
+	// and for pprof as well
+	pprof := profile.DefaultProfiler
+	pprof.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) {
+		return https.Start(ctx, nodeConfig, serverConfig.ControlConfig.Runtime)
+	}
+
 	if cfg.DisableAgent {
 		agentConfig.ContainerRuntimeEndpoint = "/dev/null"
 		return agent.RunStandalone(ctx, agentConfig)
 	}
 
-	if cfg.EmbeddedRegistry {
-		conf := spegel.DefaultRegistry
-		conf.Bootstrapper = spegel.NewChainingBootstrapper(
-			spegel.NewServerBootstrapper(&serverConfig.ControlConfig),
-			spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir),
-			spegel.NewSelfBootstrapper(),
-		)
-		conf.HandlerFunc = func(_ *spegel.Config, router *mux.Router) error {
-			router.NotFoundHandler = serverConfig.ControlConfig.Runtime.Handler
-			serverConfig.ControlConfig.Runtime.Handler = router
-			return nil
-		}
-		conf.AuthFunc = func() authenticator.Request {
-			return serverConfig.ControlConfig.Runtime.Authenticator
-		}
-	}
-
 	return agent.Run(ctx, agentConfig)
 }
 

pkg/cli/token/token.go

@@ -11,10 +11,10 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/erikdubbelboer/gspt"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/kubeadm"
+	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -171,7 +171,7 @@ func Rotate(app *cli.Context) error {
 
 func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) {
 	// hide process arguments from ps output, since they likely contain tokens.
-	gspt.SetProcTitle(os.Args[0] + " token")
+	proctitle.SetProcTitle(os.Args[0] + " token")
 
 	dataDir, err := server.ResolveDataDir("")
 	if err != nil {

pkg/cloudprovider/cloudprovider.go

@@ -7,15 +7,15 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/rancher/wrangler/pkg/apply"
-	"github.com/rancher/wrangler/pkg/generated/controllers/apps"
-	appsclient "github.com/rancher/wrangler/pkg/generated/controllers/apps/v1"
-	"github.com/rancher/wrangler/pkg/generated/controllers/core"
-	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
-	"github.com/rancher/wrangler/pkg/generated/controllers/discovery"
-	discoveryclient "github.com/rancher/wrangler/pkg/generated/controllers/discovery/v1"
-	"github.com/rancher/wrangler/pkg/generic"
-	"github.com/rancher/wrangler/pkg/start"
+	"github.com/rancher/wrangler/v3/pkg/apply"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/apps"
+	appsclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/discovery"
+	discoveryclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/discovery/v1"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"github.com/rancher/wrangler/v3/pkg/start"
 	"github.com/sirupsen/logrus"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -28,11 +28,12 @@ import (
 // Config describes externally-configurable cloud provider configuration.
 // This is normally unmarshalled from a JSON config file.
 type Config struct {
-	LBEnabled   bool   `json:"lbEnabled"`
-	LBImage     string `json:"lbImage"`
-	LBNamespace string `json:"lbNamespace"`
-	NodeEnabled bool   `json:"nodeEnabled"`
-	Rootless    bool   `json:"rootless"`
+	LBDefaultPriorityClassName string `json:"lbDefaultPriorityClassName"`
+	LBEnabled                  bool   `json:"lbEnabled"`
+	LBImage                    string `json:"lbImage"`
+	LBNamespace                string `json:"lbNamespace"`
+	NodeEnabled                bool   `json:"nodeEnabled"`
+	Rootless                   bool   `json:"rootless"`
 }
 
 type k3s struct {
@@ -56,10 +57,11 @@ func init() {
 		var err error
 		k := k3s{
 			Config: Config{
-				LBEnabled:   true,
-				LBImage:     DefaultLBImage,
-				LBNamespace: DefaultLBNS,
-				NodeEnabled: true,
+				LBDefaultPriorityClassName: DefaultLBPriorityClassName,
+				LBEnabled:                  true,
+				LBImage:                    DefaultLBImage,
+				LBNamespace:                DefaultLBNS,
+				NodeEnabled:                true,
 			},
 		}
 

pkg/cloudprovider/servicelb.go

@@ -10,11 +10,11 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/rancher/wrangler/pkg/condition"
-	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
-	discoveryclient "github.com/rancher/wrangler/pkg/generated/controllers/discovery/v1"
-	"github.com/rancher/wrangler/pkg/merr"
-	"github.com/rancher/wrangler/pkg/objectset"
+	"github.com/rancher/wrangler/v3/pkg/condition"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	discoveryclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/discovery/v1"
+	"github.com/rancher/wrangler/v3/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/objectset"
 	"github.com/sirupsen/logrus"
 	apps "k8s.io/api/apps/v1"
 	core "k8s.io/api/core/v1"
@@ -23,6 +23,7 @@ import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/util/retry"
@@ -40,12 +41,14 @@ var (
 	daemonsetNodeLabel     = "svccontroller." + version.Program + ".cattle.io/enablelb"
 	daemonsetNodePoolLabel = "svccontroller." + version.Program + ".cattle.io/lbpool"
 	nodeSelectorLabel      = "svccontroller." + version.Program + ".cattle.io/nodeselector"
+	priorityAnnotation     = "svccontroller." + version.Program + ".cattle.io/priorityclassname"
 	controllerName         = ccmapp.DefaultInitFuncConstructors["service"].InitContext.ClientName
 )
 
 const (
-	Ready       = condition.Cond("Ready")
-	DefaultLBNS = meta.NamespaceSystem
+	Ready                      = condition.Cond("Ready")
+	DefaultLBNS                = meta.NamespaceSystem
+	DefaultLBPriorityClassName = "system-node-critical"
 )
 
 var (
@@ -320,10 +323,8 @@ func (k *k3s) patchStatus(svc *core.Service, previousStatus, newStatus *core.Loa
 // If at least one node has External IPs available, only external IPs are returned.
 // If no nodes have External IPs set, the Internal IPs of all nodes running pods are returned.
 func (k *k3s) podIPs(pods []*core.Pod, svc *core.Service, readyNodes map[string]bool) ([]string, error) {
-	// Go doesn't have sets so we stuff things into a map of bools and then get lists of keys
-	// to determine the unique set of IPs in use by pods.
-	extIPs := map[string]bool{}
-	intIPs := map[string]bool{}
+	extIPs := sets.Set[string]{}
+	intIPs := sets.Set[string]{}
 
 	for _, pod := range pods {
 		if pod.Spec.NodeName == "" || pod.Status.PodIP == "" {
@@ -345,25 +346,18 @@ func (k *k3s) podIPs(pods []*core.Pod, svc *core.Service, readyNodes map[string]
 
 		for _, addr := range node.Status.Addresses {
 			if addr.Type == core.NodeExternalIP {
-				extIPs[addr.Address] = true
+				extIPs.Insert(addr.Address)
 			} else if addr.Type == core.NodeInternalIP {
-				intIPs[addr.Address] = true
+				intIPs.Insert(addr.Address)
 			}
 		}
 	}
 
-	keys := func(addrs map[string]bool) (ips []string) {
-		for k := range addrs {
-			ips = append(ips, k)
-		}
-		return ips
-	}
-
 	var ips []string
-	if len(extIPs) > 0 {
-		ips = keys(extIPs)
+	if extIPs.Len() > 0 {
+		ips = extIPs.UnsortedList()
 	} else {
-		ips = keys(intIPs)
+		ips = intIPs.UnsortedList()
 	}
 
 	ips, err := filterByIPFamily(ips, svc)
@@ -436,6 +430,7 @@ func (k *k3s) deleteDaemonSet(ctx context.Context, svc *core.Service) error {
 func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) {
 	name := generateName(svc)
 	oneInt := intstr.FromInt(1)
+	priorityClassName := k.getPriorityClassName(svc)
 	localTraffic := servicehelper.RequestsOnlyLocalTraffic(svc)
 	sourceRangesSet, err := servicehelper.GetLoadBalancerSourceRanges(svc)
 	if err != nil {
@@ -443,18 +438,11 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) {
 	}
 	sourceRanges := strings.Join(sourceRangesSet.StringSlice(), ",")
 
-	var sysctls []core.Sysctl
 	for _, ipFamily := range svc.Spec.IPFamilies {
-		switch ipFamily {
-		case core.IPv4Protocol:
-			sysctls = append(sysctls, core.Sysctl{Name: "net.ipv4.ip_forward", Value: "1"})
-		case core.IPv6Protocol:
-			sysctls = append(sysctls, core.Sysctl{Name: "net.ipv6.conf.all.forwarding", Value: "1"})
+		if ipFamily == core.IPv6Protocol && sourceRanges == "0.0.0.0/0" {
 			// The upstream default load-balancer source range only includes IPv4, even if the service is IPv6-only or dual-stack.
 			// If using the default range, and IPv6 is enabled, also allow IPv6.
-			if sourceRanges == "0.0.0.0/0" {
-				sourceRanges += ",::/0"
-			}
+			sourceRanges += ",::/0"
 		}
 	}
 
@@ -487,10 +475,14 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) {
 					},
 				},
 				Spec: core.PodSpec{
+					PriorityClassName:            priorityClassName,
 					ServiceAccountName:           "svclb",
 					AutomountServiceAccountToken: utilsptr.To(false),
 					SecurityContext: &core.PodSecurityContext{
-						Sysctls: sysctls,
+						Sysctls: []core.Sysctl{
+							{Name: "net.ipv4.ip_forward", Value: "1"},
+							{Name: "net.ipv6.conf.all.forwarding", Value: "1"},
+						},
 					},
 					Tolerations: []core.Toleration{
 						{
@@ -694,6 +686,17 @@ func (k *k3s) removeFinalizer(ctx context.Context, svc *core.Service) (*core.Ser
 	return svc, nil
 }
 
+// getPriorityClassName returns the value of the priority class name annotation on the service,
+// or the system default priority class name.
+func (k *k3s) getPriorityClassName(svc *core.Service) string {
+	if svc != nil {
+		if v, ok := svc.Annotations[priorityAnnotation]; ok {
+			return v
+		}
+	}
+	return k.LBDefaultPriorityClassName
+}
+
 // generateName generates a distinct name for the DaemonSet based on the service name and UID
 func generateName(svc *core.Service) string {
 	return fmt.Sprintf("svclb-%s-%s", svc.Name, svc.UID[:8])

pkg/cluster/address_controller.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 
 	"github.com/k3s-io/k3s/pkg/util"
-	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 )

pkg/cluster/https.go

@@ -4,27 +4,25 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
-	"fmt"
 	"io"
 	"log"
 	"net"
 	"net/http"
-	"net/http/pprof"
 	"os"
 	"path/filepath"
+	"strconv"
 
-	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/rancher/dynamiclistener"
 	"github.com/rancher/dynamiclistener/factory"
 	"github.com/rancher/dynamiclistener/storage/file"
 	"github.com/rancher/dynamiclistener/storage/kubernetes"
 	"github.com/rancher/dynamiclistener/storage/memory"
-	"github.com/rancher/wrangler/pkg/generated/controllers/core"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilsnet "k8s.io/utils/net"
 )
 
 // newListener returns a new TCP listener and HTTP request handler using dynamiclistener.
@@ -43,11 +41,7 @@ func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler,
 			os.Remove(filepath.Join(c.config.DataDir, "tls/dynamic-cert.json"))
 		}
 	}
-	ip := c.config.BindAddress
-	if utilsnet.IsIPv6String(ip) {
-		ip = fmt.Sprintf("[%s]", ip)
-	}
-	tcp, err := dynamiclistener.NewTCPListener(ip, c.config.SupervisorPort)
+	tcp, err := util.ListenWithLoopback(ctx, c.config.BindAddress, strconv.Itoa(c.config.SupervisorPort))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -114,17 +108,6 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {
 		return err
 	}
 
-	if c.config.EnablePProf {
-		mux := mux.NewRouter().SkipClean(true)
-		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		mux.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index)
-		mux.NotFoundHandler = handler
-		handler = mux
-	}
-
 	// Create a HTTP server with the registered request handlers, using logrus for logging
 	server := http.Server{
 		Handler: handler,

pkg/cluster/managed.go

@@ -91,7 +91,9 @@ func (c *Cluster) start(ctx context.Context) error {
 	return c.managedDB.Start(ctx, c.clientAccessInfo)
 }
 
-// registerDBHandlers registers routes for database info with the http request handler
+// registerDBHandlers registers managed-datastore-specific callbacks, and installs additional HTTP route handlers.
+// Note that for etcd, controllers only run on nodes with a local apiserver, in order to provide stable external
+// management of etcd cluster membership without being disrupted when a member is removed from the cluster.
 func (c *Cluster) registerDBHandlers(handler http.Handler) (http.Handler, error) {
 	if c.managedDB == nil {
 		return handler, nil

pkg/cluster/router.go

@@ -1,7 +1,10 @@
 package cluster
 
 import (
+	"fmt"
 	"net/http"
+
+	"github.com/k3s-io/k3s/pkg/util"
 )
 
 // getHandler returns a basic request handler that processes requests through
@@ -19,11 +22,10 @@ func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) {
 // if no additional handlers are available.
 func (c *Cluster) router() http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		if c.config.Runtime.Handler == nil {
-			http.Error(rw, "starting", http.StatusServiceUnavailable)
-			return
+		if c.config.Runtime.Handler != nil {
+			c.config.Runtime.Handler.ServeHTTP(rw, req)
+		} else {
+			util.SendError(fmt.Errorf("starting"), rw, req, http.StatusServiceUnavailable)
 		}
-
-		c.config.Runtime.Handler.ServeHTTP(rw, req)
 	})
 }

pkg/codegen/cleanup/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/rancher/wrangler/pkg/cleanup"
+	"github.com/rancher/wrangler/v3/pkg/cleanup"
 	"github.com/sirupsen/logrus"
 )
 

pkg/codegen/main.go

@@ -5,8 +5,8 @@ import (
 
 	bindata "github.com/go-bindata/go-bindata"
 	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	controllergen "github.com/rancher/wrangler/pkg/controller-gen"
-	"github.com/rancher/wrangler/pkg/controller-gen/args"
+	controllergen "github.com/rancher/wrangler/v3/pkg/controller-gen"
+	"github.com/rancher/wrangler/v3/pkg/controller-gen/args"
 	"github.com/sirupsen/logrus"
 )
 

pkg/configfilearg/parser.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/agent/util"
-	"github.com/rancher/wrangler/pkg/data/convert"
+	"github.com/rancher/wrangler/v3/pkg/data/convert"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"gopkg.in/yaml.v2"

pkg/crd/crds.go

@@ -2,7 +2,7 @@ package crd
 
 import (
 	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/crd"
+	"github.com/rancher/wrangler/v3/pkg/crd"
 )
 
 func List() []crd.CRD {

pkg/daemons/agent/agent_linux.go

@@ -112,9 +112,6 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 			argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
 		}
 	}
-	if cfg.PauseImage != "" {
-		argsMap["pod-infra-container-image"] = cfg.PauseImage
-	}
 	if cfg.ImageServiceSocket != "" {
 		if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) {
 			argsMap["image-service-endpoint"] = cfg.ImageServiceSocket

pkg/daemons/agent/agent_windows.go

@@ -81,9 +81,6 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 			argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
 		}
 	}
-	if cfg.PauseImage != "" {
-		argsMap["pod-infra-container-image"] = cfg.PauseImage
-	}
 	if cfg.ListenAddress != "" {
 		argsMap["address"] = cfg.ListenAddress
 	}

pkg/daemons/config/types.go

@@ -13,8 +13,8 @@ import (
 	"github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io"
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/wharfie/pkg/registries"
-	"github.com/rancher/wrangler/pkg/generated/controllers/core"
-	"github.com/rancher/wrangler/pkg/leader"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
+	"github.com/rancher/wrangler/v3/pkg/leader"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/client-go/tools/record"
@@ -41,6 +41,8 @@ type Node struct {
 	ImageServiceEndpoint     string
 	NoFlannel                bool
 	SELinux                  bool
+	EnablePProf              bool
+	SupervisorMetrics        bool
 	EmbeddedRegistry         bool
 	FlannelBackend           string
 	FlannelConfFile          string
@@ -128,6 +130,8 @@ type Agent struct {
 	AirgapExtraRegistry     []string
 	DisableCCM              bool
 	DisableNPC              bool
+	MinTLSVersion           string
+	CipherSuites            []string
 	Rootless                bool
 	ProtectKernelDefaults   bool
 	DisableServiceLB        bool
@@ -159,6 +163,7 @@ type CriticalControlArgs struct {
 	EgressSelectorMode    string       `cli:"egress-selector-mode"`
 	ServiceIPRange        *net.IPNet   `cli:"service-cidr"`
 	ServiceIPRanges       []*net.IPNet `cli:"service-cidr"`
+	SupervisorMetrics     bool         `cli:"supervisor-metrics"`
 }
 
 type Control struct {
@@ -177,6 +182,7 @@ type Control struct {
 	ServiceNodePortRange     *utilnet.PortRange
 	KubeConfigOutput         string
 	KubeConfigMode           string
+	KubeConfigGroup          string
 	HelmJobImage             string
 	DataDir                  string
 	KineTLS                  bool
@@ -191,7 +197,6 @@ type Control struct {
 	DisableServiceLB         bool
 	Rootless                 bool
 	ServiceLBNamespace       string
-	EnablePProf              bool
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
 	ExtraCloudControllerArgs []string
@@ -208,8 +213,10 @@ type Control struct {
 	ClusterResetRestorePath  string
 	EncryptForce             bool
 	EncryptSkip              bool
-	TLSMinVersion            uint16
-	TLSCipherSuites          []uint16
+	MinTLSVersion            string
+	CipherSuites             []string
+	TLSMinVersion            uint16        `json:"-"`
+	TLSCipherSuites          []uint16      `json:"-"`
 	EtcdSnapshotName         string        `json:"-"`
 	EtcdDisableSnapshots     bool          `json:"-"`
 	EtcdExposeMetrics        bool          `json:"-"`

pkg/daemons/control/deps/deps.go

@@ -30,7 +30,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/apis/apiserver"
-	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/client-go/util/keyutil"
 )
@@ -829,11 +829,12 @@ func genEgressSelectorConfig(controlConfig *config.Control) error {
 
 func genCloudConfig(controlConfig *config.Control) error {
 	cloudConfig := cloudprovider.Config{
-		LBEnabled:   !controlConfig.DisableServiceLB,
-		LBNamespace: controlConfig.ServiceLBNamespace,
-		LBImage:     cloudprovider.DefaultLBImage,
-		Rootless:    controlConfig.Rootless,
-		NodeEnabled: !controlConfig.DisableCCM,
+		LBDefaultPriorityClassName: cloudprovider.DefaultLBPriorityClassName,
+		LBEnabled:                  !controlConfig.DisableServiceLB,
+		LBNamespace:                controlConfig.ServiceLBNamespace,
+		LBImage:                    cloudprovider.DefaultLBImage,
+		Rootless:                   controlConfig.Rootless,
+		NodeEnabled:                !controlConfig.DisableCCM,
 	}
 	if controlConfig.SystemDefaultRegistry != "" {
 		cloudConfig.LBImage = controlConfig.SystemDefaultRegistry + "/" + cloudConfig.LBImage

pkg/daemons/control/tunnel.go

@@ -29,8 +29,7 @@ var defaultDialer = net.Dialer{}
 
 func loggingErrorWriter(rw http.ResponseWriter, req *http.Request, code int, err error) {
 	logrus.Debugf("Tunnel server error: %d %v", code, err)
-	rw.WriteHeader(code)
-	rw.Write([]byte(err.Error()))
+	util.SendError(err, rw, req, code)
 }
 
 func setupTunnel(ctx context.Context, cfg *config.Control) (http.Handler, error) {

pkg/datadir/datadir.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/resolvehome"
+	"github.com/rancher/wrangler/v3/pkg/resolvehome"
 )
 
 var (

pkg/deploy/controller.go

@@ -20,10 +20,10 @@ import (
 	controllersv1 "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
 	pkgutil "github.com/k3s-io/k3s/pkg/util"
 	errors2 "github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/apply"
-	"github.com/rancher/wrangler/pkg/kv"
-	"github.com/rancher/wrangler/pkg/merr"
-	"github.com/rancher/wrangler/pkg/objectset"
+	"github.com/rancher/wrangler/v3/pkg/apply"
+	"github.com/rancher/wrangler/v3/pkg/kv"
+	"github.com/rancher/wrangler/v3/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/objectset"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -119,6 +119,26 @@ func (w *watcher) listFilesIn(base string, force bool) error {
 		if err != nil {
 			return err
 		}
+		// Descend into symlinked directories, however, only top-level links are followed
+		if info.Mode()&os.ModeSymlink != 0 {
+			linkInfo, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+			if linkInfo.IsDir() {
+				evalPath, err := filepath.EvalSymlinks(path)
+				if err != nil {
+					return err
+				}
+				filepath.Walk(evalPath, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						return err
+					}
+					files[path] = info
+					return nil
+				})
+			}
+		}
 		files[path] = info
 		return nil
 	}); err != nil {

pkg/deploy/zz_generated_bindata.go

@@ -132,7 +132,7 @@ func corednsYaml() (*asset, error) {
 	return a, nil
 }
 
-var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\xdf\x6f\xdb\xb6\x13\x7f\xd7\x5f\x71\x5f\x7d\x9b\x87\x0d\xa5\x9d\x6c\x40\x33\xb0\xd8\x83\x9b\x38\x69\x80\xc4\x36\x6c\xb7\x43\x51\x14\x06\x2d\x9d\x6d\x36\x14\x49\x90\x94\x5b\x35\xcb\xff\x3e\x90\x94\x1d\x29\x71\x12\x07\xdb\xde\xa6\x17\x81\xe4\xfd\xe2\xe7\x3e\x77\x47\xa6\xf9\x47\x34\x96\x2b\x49\x61\x7d\x94\x5c\x73\x99\x53\x98\xa0\x59\xf3\x0c\x7b\x59\xa6\x4a\xe9\x92\x02\x1d\xcb\x99\x63\x34\x01\x90\xac\x40\x0a\x42\x65\x4c\x10\xcd\xdc\x8a\x68\xa3\xd6\xdc\xeb\xa3\x21\x36\xea\x11\x56\x2b\x46\x71\xab\x59\x86\x14\xae\xcb\x39\x12\x5b\x59\x87\x45\x42\x08\x49\x9a\x9e\xcd\x9c\x65\x1d\x56\xba\x95\x32\xfc\x07\x73\x5c\xc9\xce\xf5\x6f\xb6\xc3\x55\x77\x1b\xd3\x89\x28\xad\x43\x33\x56\x02\xf7\x0f\xc8\x78\x69\x53\x0a\xb4\x34\x21\xc0\x34\x3f\x37\xaa\xd4\x96\xc2\xe7\x34\xfd\x92\x00\x18\xb4\xaa\x34\x19\x86\x1d\xa9\x72\xb4\xe9\x6b\x48\xb5\x0f\xcb\x3a\x94\x6e\xad\x44\x59\x60\x26\x18\x2f\xc2\x49\xa6\xe4\x82\x2f\x0b\xa6\xa3\x9c\xca\x6d\x57\xa8\x65\x30\xb5\x46\x33\x0f\x66\x96\xe8\xfc\xa1\xe0\x36\xfc\xbf\x31\x97\xad\xd2\x2f\xcf\xbb\x47\x99\x6b\xc5\xa5\xdb\x19\xc2\xd6\x5f\xdb\xd7\xcf\x7b\x19\x5e\xa3\xb7\xda\x52\xcc\x0c\x32\x87\xc1\xe8\xee\xf8\xac\x53\x86\x2d\xb1\x4e\xc3\x43\xa3\xf5\x79\x26\x98\xb5\x68\xf7\x43\xe0\x6f\x25\xfd\x1d\x97\x39\x97\xcb\xfd\x73\x3f\xe7\x32\x4f\x3c\x01\xc6\xb8\xf0\xc2\x9b\xeb\x3d\xe1\x38\x01\x78\x48\xb6\x7d\x28\x66\xcb\xf9\x57\xcc\x5c\x60\xd9\xce\x12\xfa\xb7\x0a\x87\x69\x6d\xef\xe0\x3a\x45\x2d\x54\x55\xe0\x0b\x6a\xf6\x71\x57\x56\x63\x46\x43\xda\xa3\xec\x7b\xee\x73\x5e\x5d\xf2\x82\x3b\x0a\x87\x09\x80\x75\x86\x39\x5c\x56\x5e\x0a\xc0\x55\x1a\x29\x8c\x95\x10\x5c\x2e\x3f\xe8\x9c\x39\x0c\xfb\xa6\xb9\x13\x45\x01\x0a\xf6\xfd\x83\x64\x6b\xc6\x05\x9b\x0b\xa4\x70\xe4\xcd\xa1\xc0\xcc\x29\x13\x65\x0a\xcf\x9a\x4b\x36\x47\x61\x37\x4a\x4c\xeb\x27\xae\xe1\xb0\xd0\x62\xeb\xa2\x79\x7f\xff\x89\x96\xa5\xe7\x6c\x01\x6c\x6e\xef\x3f\x6d\xb8\x32\xdc\x55\x27\x9e\xec\x83\x00\x66\x1a\x41\x22\xbe\x67\x90\xcc\x70\xc7\x33\x26\xd2\x5a\xde\xb6\x72\x3f\x78\x59\xe2\x03\x94\x4a\xa0\x09\xc4\x6c\x44\x0c\x40\xe0\x1a\x2b\x0a\xe9\x49\xed\xaf\x97\xe7\x4a\xda\xa1\x14\x55\xda\x90\x02\x50\xda\x6b\x2b\x43\x21\xed\x7f\xe7\xd6\xd9\x74\x87\x91\x10\xb9\x27\x6f\xc7\x27\xdd\x48\x74\x18\x6a\x2f\x53\xd2\x19\x25\x88\x16\x4c\xe2\x0b\xec\x02\xe0\x62\x81\x99\xa3\x90\x0e\xd4\x24\x5b\x61\x5e\x0a\x7c\x89\xe3\x82\xf9\x92\xfb\xa7\x3c\xfa\x6b\x30\x2e\xd1\x6c\x11\x24\xcf\xd5\x41\xfc\x78\xc1\x96\x3e\xc1\x07\x37\x93\x4f\x93\x69\xff\x6a\x76\xda\x3f\xeb\x7d\xb8\x9c\xce\xc6\xfd\xf3\x8b\xc9\x74\xfc\xe9\xf6\xc0\x30\x99\xad\xd0\x74\x77\x5b\xa2\xeb\xc3\xce\x61\xe7\x97\x37\x69\xdb\xe4\xa8\x14\x62\xa4\x04\xcf\x2a\x0a\x17\x8b\x81\x72\x23\x83\x16\xb7\x29\xf7\x11\x17\x05\x93\xf9\x5d\xc2\xc9\x73\xa1\x12\xb0\x8e\x19\xd7\x58\x13\x12\x27\x54\x63\xab\x8b\x2e\xeb\xc6\xdd\xfa\xd7\xf9\x6a\x95\xdc\x4a\xc4\xf9\x72\xe5\xd9\x67\x9b\xbe\x23\x58\x51\x83\x44\xa1\x06\xf6\x85\x97\x1f\x31\xb7\xa2\x2d\x07\x5b\x09\x94\xeb\x87\xc6\x46\xc3\xd3\xd9\xa0\x77\xd5\x9f\x8c\x7a\x27\xfd\x86\xb1\x35\x13\x25\x9e\x19\x55\xd0\x56\x76\x17\x1c\x45\x5e\x37\xef\x07\xfb\xd1\xf7\xa6\xca\x3b\xdb\x1e\x96\x34\x6f\xf5\x82\x0b\xc5\xfd\x2b\xa6\xdb\xde\x1e\x50\xa6\xc6\xf7\x7e\x1f\x6e\x8f\xcb\xbb\x8e\x3c\x89\xfb\xa1\x73\x3c\xd9\x93\xfd\x80\x92\x52\xb9\x66\xd5\xe7\xb8\x60\xa5\x70\x1f\x43\xac\xd3\xd0\x5e\xd3\xa0\x11\xa9\xd5\x1c\xc1\xf7\x6a\x89\x5b\x52\x2b\x93\x70\x4c\x21\x75\xa6\xc4\x34\x69\xf2\x14\x6a\x1e\x7b\x85\x46\x20\x11\x9a\x7a\xdc\x5e\xa9\x1c\x29\xfc\xc1\xb8\x3b\x53\xe6\x8c\x1b\xeb\x4e\x94\xb4\x65\x81\x26\x31\xf1\x5d\xb4\xe1\xf4\x29\x0a\x74\x18\x80\xa9\x67\xe8\x06\xd1\xe4\xde\x1b\xf3\xc9\xd1\xb4\xe5\xef\x23\x53\x69\xa3\xd8\xa0\x32\x85\x3f\x49\x00\xe4\xa6\x4e\x5d\x68\x31\x9e\x20\x57\x4c\xa7\xf4\x73\xbd\x7b\xb3\x4d\x6c\x38\x4f\x69\xba\xa9\xec\x51\x6f\xfa\x7e\x76\x36\x1c\xcf\x06\xc3\xc1\xec\xf2\x62\x32\xed\x9f\xce\x06\xc3\xd3\xfe\x24\x7d\x7d\xa7\xe3\xa3\xb3\x29\xfd\x9c\x1e\xdc\x6c\xf4\x2e\x87\x27\xbd\xcb\xd9\x64\x3a\x1c\xf7\xce\xfb\xc1\xca\xed\x41\x78\x09\xf9\xef\xb6\xfe\xc7\xf5\x6d\x98\x6f\xce\xbf\x3e\xea\x60\xff\xff\xbf\xee\x9c\xcb\xae\x5d\x85\xd5\xb7\x15\x17\x08\x4b\x74\x4a\x3b\x0b\x69\x41\x2d\xd5\x34\x05\xa5\x63\x75\xe7\xea\xae\x4d\x30\x8b\xf0\x4a\x69\x07\x5c\xb6\xa8\xaa\x7f\x6a\x2d\xd9\xdc\x2a\x51\xba\x80\xc3\xef\xaf\x86\xa3\x69\x6f\x7c\xde\x12\x78\xfb\xb6\xb5\xb4\x6d\x75\xcb\x7f\xe0\x85\x7c\x57\x39\xb4\xfb\x68\x17\x6d\xed\xb5\x12\x9e\x39\xcf\x69\xa2\x65\x59\x7d\x3f\x19\x8b\xb1\xb8\xce\xb9\x01\x52\xc0\xe1\xf1\xf1\x31\x10\x0d\xaf\x6e\x9a\x17\x89\xa0\x66\xab\x42\xe5\x70\x7c\x78\x78\xff\xb4\xdb\xe9\x84\x87\x00\x33\xb9\xfa\x26\xff\x83\xfa\x49\xa8\x4d\x01\xc4\x2c\x76\x00\xbc\x42\xa1\xd1\x8c\x54\xde\xa9\x58\x21\xb6\x28\xde\xab\x62\xbf\x15\x0b\x7d\xa4\xf2\x9d\x4f\xae\x58\xdb\xd1\x1a\xd1\xb5\x50\xf3\x5d\xf5\xf8\x8c\xbe\xa7\x04\x2f\x9b\xcb\x05\x37\x46\x19\xcc\x89\xe0\x73\xc3\x4c\x45\xe6\xa5\xad\xe6\xea\x3b\x3d\xea\xfc\xfa\xa6\x73\xb4\xef\x60\xfe\x2b\x00\x00\xff\xff\x23\x2c\xa0\x6c\x1b\x0f\x00\x00")
+var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\xdf\x6f\xdb\xb6\x13\x7f\xd7\x5f\x71\x5f\x7d\x97\x97\xa1\x94\x93\x0d\x68\x06\xbe\x79\xb1\xd3\x06\x70\x6c\xc3\x76\x3b\x14\x45\x61\xd0\xd4\xd9\x66\x43\x91\x04\x49\xb9\xf5\xb2\xfc\xef\x03\x49\xd9\x91\x93\x34\x71\xb0\x4d\x2f\x82\x8e\x77\x9f\x3b\xde\xe7\x7e\x88\x19\xf1\x11\xad\x13\x5a\x51\xd8\x9c\x65\x37\x42\x95\x14\xa6\x68\x37\x82\x63\x97\x73\x5d\x2b\x9f\x55\xe8\x59\xc9\x3c\xa3\x19\x80\x62\x15\x52\x90\x9a\x33\x49\x0c\xf3\x6b\x62\xac\xde\x88\x60\x8f\x96\xb8\x64\x47\x58\x63\x98\xd4\x9d\x61\x1c\x29\xdc\xd4\x0b\x24\x6e\xeb\x3c\x56\x19\x21\x24\x6b\x7b\xb6\x0b\xc6\x0b\x56\xfb\xb5\xb6\xe2\x4f\xe6\x85\x56\xc5\xcd\x6f\xae\x10\xba\xb3\x8f\xe9\x42\xd6\xce\xa3\x9d\x68\x89\xc7\x07\x64\x83\xb6\xad\x25\x3a\x9a\x11\x60\x46\xbc\xb3\xba\x36\x8e\xc2\xe7\x3c\xff\x92\x01\x58\x74\xba\xb6\x1c\xa3\x44\xe9\x12\x5d\xfe\x06\x72\x13\xc2\x72\x1e\x95\xdf\x68\x59\x57\xc8\x25\x13\x55\x3c\xe1\x5a\x2d\xc5\xaa\x62\x26\xe9\xe9\xd2\x75\xa4\x5e\x45\xa8\x0d\xda\x45\x84\x59\xa1\x0f\x87\x52\xb8\xf8\xfe\xc6\x3c\x5f\xe7\x5f\x5e\x76\x8f\xaa\x34\x5a\x28\xff\x64\x08\x7b\x7f\x87\xbe\x7e\x3e\x0a\x78\x83\x01\xf5\xc0\x90\x5b\x64\x1e\x23\xe8\xd3\xf1\x39\xaf\x2d\x5b\x61\x43\xc3\x63\xd0\xe6\x9c\x4b\xe6\x1c\xba\xe3\x32\xf0\x8f\x48\xff\x5d\xa8\x52\xa8\xd5\xf1\xdc\x2f\x84\x2a\xb3\x50\x00\x13\x5c\x06\xe5\xdd\xf5\x9e\x71\x9c\x01\x3c\x2e\xb6\x63\x4a\xcc\xd5\x8b\xaf\xc8\x7d\xac\xb2\x27\x5b\xe8\xbf\x6a\x1c\x66\x8c\xbb\x4f\x57\x0f\x8d\xd4\xdb\x0a\x5f\xd1\xb3\x3f\x76\xe5\x0c\x72\x1a\x69\x4f\xba\xef\x45\xe0\x7c\x3b\x10\x95\xf0\x14\x4e\x33\x00\xe7\x2d\xf3\xb8\xda\x06\x2d\x00\xbf\x35\x48\x61\xa2\xa5\x14\x6a\xf5\xc1\x94\xcc\x63\x94\xdb\xb6\x24\xa9\x02\x54\xec\xfb\x07\xc5\x36\x4c\x48\xb6\x90\x48\xe1\x2c\xc0\xa1\x44\xee\xb5\x4d\x3a\x55\xa8\x9a\x01\x5b\xa0\x74\x3b\x23\x66\xcc\x33\xd7\xf0\x58\x19\xb9\x77\xd1\xbe\x7f\x78\xe4\x01\xd2\x4b\x58\x00\xbb\xdb\x87\xc7\x58\xa1\xad\xf0\xdb\x8b\x50\xec\xc3\x98\xcc\x3c\x25\x89\x84\x99\x41\xb8\x15\x5e\x70\x26\xf3\x46\xdf\x1d\x70\x3f\x7c\x1d\xf1\x31\x95\x5a\xa2\x8d\x85\xd9\x8a\x18\x80\xc0\x0d\x6e\x29\xe4\x17\x8d\xbf\x6e\x59\x6a\xe5\x46\x4a\x6e\xf3\x96\x16\x80\x36\xc1\x5a\x5b\x0a\x79\xff\xbb\x70\xde\xe5\x4f\x80\xc4\xc8\x43\xf1\x16\x81\x74\xab\xd0\x63\xec\x3d\xae\x95\xb7\x5a\x12\x23\x99\xc2\x57\xe0\x02\xe0\x72\x89\xdc\x53\xc8\x87\x7a\xca\xd7\x58\xd6\x12\x5f\xe3\xb8\x62\xa1\xe5\xfe\x2d\x8f\xe1\x1a\x4c\x28\xb4\xfb\x0c\x92\x97\xfa\x20\x3d\xa2\x62\xab\x40\xf0\xc9\xed\xf4\xd3\x74\xd6\xbf\x9e\xf7\xfa\x97\xdd\x0f\x83\xd9\x7c\xd2\x7f\x77\x35\x9d\x4d\x3e\xdd\x9d\x58\xa6\xf8\x1a\x6d\xe7\x69\x24\xba\x39\x2d\x4e\x8b\x5f\xde\xe6\x87\x90\xe3\x5a\xca\xb1\x96\x82\x6f\x29\x5c\x2d\x87\xda\x8f\x2d\x3a\xdc\x53\x1e\x22\xae\x2a\xa6\xca\x7b\xc2\xc9\x4b\xa1\x12\x70\x9e\x59\xdf\xfa\x26\x24\x6d\xa8\x96\xa8\x83\x9e\x77\x92\xb4\x79\x15\x5f\x9d\x56\x7b\x8d\xb4\x5f\xae\x43\xf5\xb9\xb6\xef\x94\xac\x64\x41\x92\x52\x2b\xf7\x55\xd0\x1f\x33\xbf\xa6\x07\x0e\xf6\x1a\xa8\x36\x8f\xc1\xc6\xa3\xde\x7c\xd8\xbd\xee\x4f\xc7\xdd\x8b\x7e\x0b\x6c\xc3\x64\x8d\x97\x56\x57\xf4\x80\xdd\xa5\x40\x59\x36\xc3\xfb\x91\x3c\xf9\xde\x75\x79\xb1\x9f\x61\x59\xfb\x56\xaf\xb8\x50\x92\x5f\x33\x73\xe8\xed\x51\xc9\x34\xf9\x7d\x38\x87\x0f\xd7\xe5\xfd\x44\x9e\x26\x79\x9c\x1c\xcf\xce\xe4\xb0\xa0\x94\xd2\xbe\xdd\xf5\x25\x2e\x59\x2d\xfd\xc7\x18\xeb\x2c\x8e\xd7\x3c\x5a\xa4\xd2\x6a\xaf\xe0\x07\xbd\x24\x1c\x69\x8c\x49\x3c\xa6\x90\x7b\x5b\x63\x9e\xb5\xeb\x14\x9a\x3a\x0e\x06\xad\x40\x52\x6a\x9a\x75\x7b\xad\x4b\xa4\xf0\x07\x13\xfe\x52\xdb\x4b\x61\x9d\xbf\xd0\xca\xd5\x15\xda\xcc\xa6\xff\xa2\x5d\x4d\xf7\x50\xa2\xc7\x98\x98\x66\x87\xee\x32\x9a\x3d\xf8\xc7\x7c\x76\x35\xed\xeb\xf7\x07\x5b\x69\x67\xd8\x2a\x65\x0a\x7f\x91\x98\x90\xdb\x86\xba\x38\x62\x42\x81\x5c\x33\x93\xd3\xcf\x8d\xf4\x76\x4f\x6c\x3c\xcf\x69\xbe\xeb\xec\x71\x77\xf6\x7e\x7e\x39\x9a\xcc\x87\xa3\xe1\x7c\x70\x35\x9d\xf5\x7b\xf3\xe1\xa8\xd7\x9f\xe6\x6f\xee\x6d\x42\x74\x2e\xa7\x9f\xf3\x93\xdb\x9d\xdd\x60\x74\xd1\x1d\xcc\xa7\xb3\xd1\xa4\xfb\xae\x1f\x51\xee\x4e\xe2\x9f\x50\x78\xee\x9a\x77\xfa\xbe\x8b\xfb\xcd\x87\xbf\x8f\x26\xd8\xff\xff\xaf\xb3\x10\xaa\xe3\xd6\x89\x4b\xf4\x40\xb0\x4e\xab\xeb\xa6\x14\x16\x48\x05\xa7\xe7\xe7\xe7\x40\x0c\xe4\x3f\xdd\x7e\x1c\x0d\xe6\xbd\xab\xc9\x5d\x62\x9e\xaf\x2b\x5d\xc2\xf9\xe9\x69\xfb\xa8\x53\x14\x79\x5c\x83\xcc\x96\xfa\x9b\x3a\xc2\x91\xad\x80\xd8\xe5\x43\xf8\x35\x4a\x83\x76\xac\xcb\x62\xcb\x2a\xb9\x87\x79\x40\x62\x10\x25\x9e\xc7\xba\x7c\x72\xe3\x26\x6a\x13\x1a\x31\x8d\x52\x7b\xad\xfe\x78\x44\x3f\x30\x82\xd7\x8d\xe5\x4a\x58\xab\x2d\x96\x44\x8a\x85\x65\x76\x4b\x16\xb5\xdb\x2e\xf4\x77\x7a\x56\xfc\xfa\xb6\x38\x3b\x76\x2e\xff\x1d\x00\x00\xff\xff\x33\x50\x2d\x30\x1a\x0d\x00\x00")
 
 func localStorageYamlBytes() ([]byte, error) {
 	return bindataRead(

pkg/etcd/etcd.go

@@ -33,7 +33,8 @@ import (
 	cp "github.com/otiai10/copy"
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
-	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/v3/pkg/start"
 	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/etcd/api/v3/etcdserverpb"
@@ -619,6 +620,12 @@ func (e *ETCD) Register(handler http.Handler) (http.Handler, error) {
 			registerEndpointsHandlers(ctx, e)
 			registerMemberHandlers(ctx, e)
 			registerSnapshotHandlers(ctx, e)
+
+			// Re-run informer factory startup after core and leader-elected controllers have started.
+			// Additional caches may need to start for the newly added OnChange/OnRemove callbacks.
+			if err := start.All(ctx, 5, e.config.Runtime.K3s, e.config.Runtime.Core); err != nil {
+				panic(errors.Wrap(err, "failed to start wrangler controllers"))
+			}
 		}
 	}
 
@@ -664,12 +671,18 @@ func (e *ETCD) setName(force bool) error {
 
 // handler wraps the handler with routes for database info
 func (e *ETCD) handler(next http.Handler) http.Handler {
-	mux := mux.NewRouter().SkipClean(true)
-	mux.Use(auth.Middleware(e.config, version.Program+":server"))
-	mux.Handle("/db/info", e.infoHandler())
-	mux.Handle("/db/snapshot", e.snapshotHandler())
-	mux.NotFoundHandler = next
-	return mux
+	r := mux.NewRouter().SkipClean(true)
+	r.NotFoundHandler = next
+
+	ir := r.Path("/db/info").Subrouter()
+	ir.Use(auth.IsLocalOrHasRole(e.config, version.Program+":server"))
+	ir.Handle("", e.infoHandler())
+
+	sr := r.Path("/db/snapshot").Subrouter()
+	sr.Use(auth.HasRole(e.config, version.Program+":server"))
+	sr.Handle("", e.snapshotHandler())
+
+	return r
 }
 
 // infoHandler returns etcd cluster information. This is used by new members when joining the cluster.
@@ -748,7 +761,7 @@ func getEndpoints(control *config.Control) []string {
 // for use by etcd.
 func toTLSConfig(runtime *config.ControlRuntime) (*tls.Config, error) {
 	if runtime.ClientETCDCert == "" || runtime.ClientETCDKey == "" || runtime.ETCDServerCA == "" {
-		return nil, errors.New("runtime is not ready yet")
+		return nil, util.ErrCoreNotReady
 	}
 
 	clientCert, err := tls.LoadX509KeyPair(runtime.ClientETCDCert, runtime.ClientETCDKey)
@@ -1164,7 +1177,7 @@ func (e *ETCD) manageLearners(ctx context.Context) {
 
 func (e *ETCD) getETCDNodes() ([]*v1.Node, error) {
 	if e.config.Runtime.Core == nil {
-		return nil, errors.New("runtime core not ready")
+		return nil, util.ErrCoreNotReady
 	}
 
 	nodes := e.config.Runtime.Core.Core().V1().Node()

pkg/etcd/etcdproxy.go

@@ -130,7 +130,7 @@ func (e etcdproxy) createHealthCheck(ctx context.Context, address string) func()
 			statusCode = resp.StatusCode
 		}
 		if err != nil || statusCode != http.StatusOK {
-			logrus.Debugf("Health check %s failed: %v (StatusCode: %d)", url, err, statusCode)
+			logrus.Debugf("Health check %s failed: %v (StatusCode: %d)", address, err, statusCode)
 			connected = false
 		} else {
 			connected = true

pkg/etcd/member_controller.go

@@ -8,7 +8,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
 	v1 "k8s.io/api/core/v1"

pkg/etcd/metadata_controller.go

@@ -7,13 +7,13 @@ import (
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/util"
-	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/util/retry"
-	nodeUtil "k8s.io/kubernetes/pkg/controller/util/node"
+	nodeutil "k8s.io/kubernetes/pkg/controller/util/node"
 )
 
 func registerMetadataHandlers(ctx context.Context, etcd *ETCD) {
@@ -109,7 +109,7 @@ func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
 			node.Labels = map[string]string{}
 		}
 
-		if find, _ := nodeUtil.GetNodeCondition(&node.Status, etcdStatusType); find >= 0 {
+		if find, _ := nodeutil.GetNodeCondition(&node.Status, etcdStatusType); find >= 0 {
 			node.Status.Conditions = append(node.Status.Conditions[:find], node.Status.Conditions[find+1:]...)
 		}
 

pkg/etcd/snapshot_controller.go

@@ -13,7 +13,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"

pkg/etcd/snapshot_handler.go

@@ -170,14 +170,15 @@ func (e *ETCD) withRequest(sr *SnapshotRequest) *ETCD {
 	}
 	if sr.S3 != nil {
 		re.config.EtcdS3 = true
-		re.config.EtcdS3BucketName = sr.S3.Bucket
 		re.config.EtcdS3AccessKey = sr.S3.AccessKey
-		re.config.EtcdS3SecretKey = sr.S3.SecretKey
+		re.config.EtcdS3BucketName = sr.S3.Bucket
 		re.config.EtcdS3Endpoint = sr.S3.Endpoint
 		re.config.EtcdS3EndpointCA = sr.S3.EndpointCA
-		re.config.EtcdS3SkipSSLVerify = sr.S3.SkipSSLVerify
+		re.config.EtcdS3Folder = sr.S3.Folder
 		re.config.EtcdS3Insecure = sr.S3.Insecure
 		re.config.EtcdS3Region = sr.S3.Region
+		re.config.EtcdS3SecretKey = sr.S3.SecretKey
+		re.config.EtcdS3SkipSSLVerify = sr.S3.SkipSSLVerify
 		re.config.EtcdS3Timeout = sr.S3.Timeout.Duration
 	}
 	return re

pkg/generated/controllers/k3s.cattle.io/factory.go

@@ -20,7 +20,7 @@ package k3s
 
 import (
 	"github.com/rancher/lasso/pkg/controller"
-	"github.com/rancher/wrangler/pkg/generic"
+	"github.com/rancher/wrangler/v3/pkg/generic"
 	"k8s.io/client-go/rest"
 )
 

pkg/generated/controllers/k3s.cattle.io/v1/addon.go

@@ -20,7 +20,7 @@ package v1
 
 import (
 	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/generic"
+	"github.com/rancher/wrangler/v3/pkg/generic"
 )
 
 // AddonController interface for managing Addon resources.

pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go

@@ -20,13 +20,14 @@ package v1
 
 import (
 	"context"
+	"sync"
 	"time"
 
 	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/apply"
-	"github.com/rancher/wrangler/pkg/condition"
-	"github.com/rancher/wrangler/pkg/generic"
-	"github.com/rancher/wrangler/pkg/kv"
+	"github.com/rancher/wrangler/v3/pkg/apply"
+	"github.com/rancher/wrangler/v3/pkg/condition"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"github.com/rancher/wrangler/v3/pkg/kv"
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -48,10 +49,14 @@ type ETCDSnapshotFileCache interface {
 	generic.NonNamespacedCacheInterface[*v1.ETCDSnapshotFile]
 }
 
+// ETCDSnapshotFileStatusHandler is executed for every added or modified ETCDSnapshotFile. Should return the new status to be updated
 type ETCDSnapshotFileStatusHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error)
 
+// ETCDSnapshotFileGeneratingHandler is the top-level handler that is executed for every ETCDSnapshotFile event. It extends ETCDSnapshotFileStatusHandler by a returning a slice of child objects to be passed to apply.Apply
 type ETCDSnapshotFileGeneratingHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) ([]runtime.Object, v1.ETCDSnapshotStatus, error)
 
+// RegisterETCDSnapshotFileStatusHandler configures a ETCDSnapshotFileController to execute a ETCDSnapshotFileStatusHandler for every events observed.
+// If a non-empty condition is provided, it will be updated in the status conditions for every handler execution
 func RegisterETCDSnapshotFileStatusHandler(ctx context.Context, controller ETCDSnapshotFileController, condition condition.Cond, name string, handler ETCDSnapshotFileStatusHandler) {
 	statusHandler := &eTCDSnapshotFileStatusHandler{
 		client:    controller,
@@ -61,6 +66,8 @@ func RegisterETCDSnapshotFileStatusHandler(ctx context.Context, controller ETCDS
 	controller.AddGenericHandler(ctx, name, generic.FromObjectHandlerToHandler(statusHandler.sync))
 }
 
+// RegisterETCDSnapshotFileGeneratingHandler configures a ETCDSnapshotFileController to execute a ETCDSnapshotFileGeneratingHandler for every events observed, passing the returned objects to the provided apply.Apply.
+// If a non-empty condition is provided, it will be updated in the status conditions for every handler execution
 func RegisterETCDSnapshotFileGeneratingHandler(ctx context.Context, controller ETCDSnapshotFileController, apply apply.Apply,
 	condition condition.Cond, name string, handler ETCDSnapshotFileGeneratingHandler, opts *generic.GeneratingHandlerOptions) {
 	statusHandler := &eTCDSnapshotFileGeneratingHandler{
@@ -82,6 +89,7 @@ type eTCDSnapshotFileStatusHandler struct {
 	handler   ETCDSnapshotFileStatusHandler
 }
 
+// sync is executed on every resource addition or modification. Executes the configured handlers and sends the updated status to the Kubernetes API
 func (a *eTCDSnapshotFileStatusHandler) sync(key string, obj *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) {
 	if obj == nil {
 		return obj, nil
@@ -127,8 +135,10 @@ type eTCDSnapshotFileGeneratingHandler struct {
 	opts  generic.GeneratingHandlerOptions
 	gvk   schema.GroupVersionKind
 	name  string
+	seen  sync.Map
 }
 
+// Remove handles the observed deletion of a resource, cascade deleting every associated resource previously applied
 func (a *eTCDSnapshotFileGeneratingHandler) Remove(key string, obj *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) {
 	if obj != nil {
 		return obj, nil
@@ -138,12 +148,17 @@ func (a *eTCDSnapshotFileGeneratingHandler) Remove(key string, obj *v1.ETCDSnaps
 	obj.Namespace, obj.Name = kv.RSplit(key, "/")
 	obj.SetGroupVersionKind(a.gvk)
 
+	if a.opts.UniqueApplyForResourceVersion {
+		a.seen.Delete(key)
+	}
+
 	return nil, generic.ConfigureApplyForObject(a.apply, obj, &a.opts).
 		WithOwner(obj).
 		WithSetID(a.name).
 		ApplyObjects()
 }
 
+// Handle executes the configured ETCDSnapshotFileGeneratingHandler and pass the resulting objects to apply.Apply, finally returning the new status of the resource
 func (a *eTCDSnapshotFileGeneratingHandler) Handle(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error) {
 	if !obj.DeletionTimestamp.IsZero() {
 		return status, nil
@@ -153,9 +168,41 @@ func (a *eTCDSnapshotFileGeneratingHandler) Handle(obj *v1.ETCDSnapshotFile, sta
 	if err != nil {
 		return newStatus, err
 	}
+	if !a.isNewResourceVersion(obj) {
+		return newStatus, nil
+	}
 
-	return newStatus, generic.ConfigureApplyForObject(a.apply, obj, &a.opts).
+	err = generic.ConfigureApplyForObject(a.apply, obj, &a.opts).
 		WithOwner(obj).
 		WithSetID(a.name).
 		ApplyObjects(objs...)
+	if err != nil {
+		return newStatus, err
+	}
+	a.storeResourceVersion(obj)
+	return newStatus, nil
+}
+
+// isNewResourceVersion detects if a specific resource version was already successfully processed.
+// Only used if UniqueApplyForResourceVersion is set in generic.GeneratingHandlerOptions
+func (a *eTCDSnapshotFileGeneratingHandler) isNewResourceVersion(obj *v1.ETCDSnapshotFile) bool {
+	if !a.opts.UniqueApplyForResourceVersion {
+		return true
+	}
+
+	// Apply once per resource version
+	key := obj.Namespace + "/" + obj.Name
+	previous, ok := a.seen.Load(key)
+	return !ok || previous != obj.ResourceVersion
+}
+
+// storeResourceVersion keeps track of the latest resource version of an object for which Apply was executed
+// Only used if UniqueApplyForResourceVersion is set in generic.GeneratingHandlerOptions
+func (a *eTCDSnapshotFileGeneratingHandler) storeResourceVersion(obj *v1.ETCDSnapshotFile) {
+	if !a.opts.UniqueApplyForResourceVersion {
+		return
+	}
+
+	key := obj.Namespace + "/" + obj.Name
+	a.seen.Store(key, obj.ResourceVersion)
 }

pkg/generated/controllers/k3s.cattle.io/v1/interface.go

@@ -21,8 +21,8 @@ package v1
 import (
 	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
 	"github.com/rancher/lasso/pkg/controller"
-	"github.com/rancher/wrangler/pkg/generic"
-	"github.com/rancher/wrangler/pkg/schemes"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"github.com/rancher/wrangler/v3/pkg/schemes"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 

pkg/kubectl/main.go

@@ -54,7 +54,8 @@ func checkReadConfigPermissions(configFile string) error {
 	if err != nil {
 		if os.IsPermission(err) {
 			return fmt.Errorf("Unable to read %s, please start server "+
-				"with --write-kubeconfig-mode to modify kube config permissions", configFile)
+				"with --write-kubeconfig-mode or --write-kubeconfig-group "+
+				"to modify kube config permissions", configFile)
 		}
 	}
 	file.Close()

pkg/metrics/metrics.go

@@ -0,0 +1,45 @@
+package metrics
+
+import (
+	"context"
+	"errors"
+
+	"github.com/gorilla/mux"
+	"github.com/k3s-io/k3s/pkg/agent/https"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// DefaultRegisterer is the implementation of the
+// prometheus Registerer interface that all metrics operations
+// will use.
+var DefaultRegisterer = legacyregistry.Registerer()
+
+// DefaultGatherer  is the implementation of the
+// prometheus Gatherere interface that all metrics operations
+// will use.
+var DefaultGatherer = legacyregistry.DefaultGatherer
+
+// DefaultMetrics is the default instance of a Metrics server
+var DefaultMetrics = &Config{
+	Router: func(context.Context, *config.Node) (*mux.Router, error) {
+		return nil, errors.New("not implemented")
+	},
+}
+
+// Config holds fields for the metrics listener
+type Config struct {
+	// Router will be called to add the metrics API handler to an existing router.
+	Router https.RouterFunc
+}
+
+// Start starts binds the metrics API to an existing HTTP router.
+func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
+	mRouter, err := c.Router(ctx, nodeConfig)
+	if err != nil {
+		return err
+	}
+	mRouter.Handle("/metrics", promhttp.HandlerFor(DefaultGatherer, promhttp.HandlerOpts{}))
+	return nil
+}

pkg/node/controller.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/nodepassword"
 	"github.com/pkg/errors"
-	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	core "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"

pkg/nodepassword/nodepassword.go

@@ -10,7 +10,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/passwd"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"

pkg/nodepassword/nodepassword_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/golang/mock/gomock"
-	"github.com/rancher/wrangler/pkg/generic/fake"
+	"github.com/rancher/wrangler/v3/pkg/generic/fake"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

pkg/proctitle/proctile.go

@@ -0,0 +1,12 @@
+//go:build linux
+// +build linux
+
+package proctitle
+
+import (
+	"github.com/erikdubbelboer/gspt"
+)
+
+func SetProcTitle(cmd string) {
+	gspt.SetProcTitle(cmd)
+}

pkg/proctitle/proctile_windows.go

@@ -0,0 +1,6 @@
+//go:build windows
+// +build windows
+
+package proctitle
+
+func SetProcTitle(cmd string) {}

pkg/profile/profile.go

@@ -0,0 +1,38 @@
+package profile
+
+import (
+	"context"
+	"errors"
+	"net/http/pprof"
+
+	"github.com/gorilla/mux"
+	"github.com/k3s-io/k3s/pkg/agent/https"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+)
+
+// DefaultProfiler the default instance of a performance profiling server
+var DefaultProfiler = &Config{
+	Router: func(context.Context, *config.Node) (*mux.Router, error) {
+		return nil, errors.New("not implemented")
+	},
+}
+
+// Config holds fields for the pprof listener
+type Config struct {
+	// Router will be called to add the pprof API handler to an existing router.
+	Router https.RouterFunc
+}
+
+// Start starts binds the pprof API to an existing HTTP router.
+func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
+	mRouter, err := c.Router(ctx, nodeConfig)
+	if err != nil {
+		return err
+	}
+	mRouter.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mRouter.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mRouter.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mRouter.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	mRouter.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index)
+	return nil
+}

pkg/rootlessports/controller.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/rootless"
-	coreClients "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreClients "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/rootless-containers/rootlesskit/pkg/api/client"
 	"github.com/rootless-containers/rootlesskit/pkg/port"
 	"github.com/sirupsen/logrus"

pkg/rootlessports/controller_windows.go

@@ -3,7 +3,7 @@ package rootlessports
 import (
 	"context"
 
-	coreClients "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreClients "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 )
 
 func Register(ctx context.Context, serviceController coreClients.ServiceController, enabled bool, httpsPort int) error {

pkg/secretsencrypt/config.go

@@ -21,7 +21,7 @@ import (
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 
 	"k8s.io/client-go/rest"
 )

pkg/secretsencrypt/controller.go

@@ -8,7 +8,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/cluster"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
-	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"

pkg/server/auth/auth.go

@@ -1,6 +1,7 @@
 package auth
 
 import (
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -22,6 +23,7 @@ func hasRole(mustRoles []string, roles []string) bool {
 	return false
 }
 
+// doAuth calls the cluster's authenticator to validate that the client has at least one of the listed roles
 func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
 	switch {
 	case serverConfig == nil:
@@ -51,10 +53,27 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw
 	next.ServeHTTP(rw, req)
 }
 
-func Middleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
+// HasRole returns a middleware function that validates that the request
+// is being made with at least one of the listed roles.
+func HasRole(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 			doAuth(roles, serverConfig, next, rw, req)
 		})
 	}
 }
+
+// IsLocalOrHasRole returns a middleware function that validates that the request
+// is from a local client or has at least one of the listed roles.
+func IsLocalOrHasRole(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+			client, _, _ := net.SplitHostPort(req.RemoteAddr)
+			if client == "127.0.0.1" || client == "::1" {
+				next.ServeHTTP(rw, req)
+			} else {
+				doAuth(roles, serverConfig, next, rw, req)
+			}
+		})
+	}
+}

pkg/server/cert.go

@@ -23,15 +23,15 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
-	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/merr"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/util/keyutil"
 )
 
 func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil || req.Method != http.MethodPut {
-			resp.WriteHeader(http.StatusNotFound)
+		if req.Method != http.MethodPut {
+			util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed)
 			return
 		}
 		force, _ := strconv.ParseBool(req.FormValue("force"))

pkg/server/context.go

@@ -10,12 +10,12 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/crd"
-	"github.com/rancher/wrangler/pkg/generated/controllers/apps"
-	"github.com/rancher/wrangler/pkg/generated/controllers/batch"
-	"github.com/rancher/wrangler/pkg/generated/controllers/core"
-	"github.com/rancher/wrangler/pkg/generated/controllers/rbac"
-	"github.com/rancher/wrangler/pkg/start"
+	"github.com/rancher/wrangler/v3/pkg/crd"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/apps"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/batch"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/rbac"
+	"github.com/rancher/wrangler/v3/pkg/start"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"

pkg/server/router.go

@@ -25,7 +25,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
-	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -52,7 +52,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 
 	prefix := "/v1-" + version.Program
 	authed := mux.NewRouter().SkipClean(true)
-	authed.Use(auth.Middleware(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
+	authed.Use(auth.HasRole(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
 	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
@@ -71,12 +71,12 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 
 	nodeAuthed := mux.NewRouter().SkipClean(true)
 	nodeAuthed.NotFoundHandler = authed
-	nodeAuthed.Use(auth.Middleware(serverConfig, user.NodesGroup))
+	nodeAuthed.Use(auth.HasRole(serverConfig, user.NodesGroup))
 	nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel)
 
 	serverAuthed := mux.NewRouter().SkipClean(true)
 	serverAuthed.NotFoundHandler = nodeAuthed
-	serverAuthed.Use(auth.Middleware(serverConfig, version.Program+":server"))
+	serverAuthed.Use(auth.HasRole(serverConfig, version.Program+":server"))
 	serverAuthed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
 	serverAuthed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
 	serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
@@ -86,7 +86,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	systemAuthed := mux.NewRouter().SkipClean(true)
 	systemAuthed.NotFoundHandler = serverAuthed
 	systemAuthed.MethodNotAllowedHandler = serverAuthed
-	systemAuthed.Use(auth.Middleware(serverConfig, user.SystemPrivilegedGroup))
+	systemAuthed.Use(auth.HasRole(serverConfig, user.SystemPrivilegedGroup))
 	systemAuthed.Methods(http.MethodConnect).Handler(serverConfig.Runtime.Tunnel)
 
 	staticDir := filepath.Join(serverConfig.DataDir, "static")
@@ -200,11 +200,6 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer
 
 func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
-
 		nodeName, errCode, err := auth(req)
 		if err != nil {
 			util.SendError(err, resp, req, errCode)
@@ -256,11 +251,6 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo
 
 func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
-
 		nodeName, errCode, err := auth(req)
 		if err != nil {
 			util.SendError(err, resp, req, errCode)
@@ -296,10 +286,6 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot
 
 func fileHandler(fileName ...string) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
 		resp.Header().Set("Content-Type", "text/plain")
 
 		if len(fileName) == 1 {
@@ -310,8 +296,7 @@ func fileHandler(fileName ...string) http.Handler {
 		for _, f := range fileName {
 			bytes, err := os.ReadFile(f)
 			if err != nil {
-				logrus.Errorf("Failed to read %s: %v", f, err)
-				resp.WriteHeader(http.StatusInternalServerError)
+				util.SendError(errors.Wrapf(err, "failed to read %s", f), resp, req, http.StatusInternalServerError)
 				return
 			}
 			resp.Write(bytes)
@@ -336,18 +321,13 @@ func apiserversHandler(server *config.Control) http.Handler {
 
 		resp.Header().Set("content-type", "application/json")
 		if err := json.NewEncoder(resp).Encode(endpoints); err != nil {
-			logrus.Errorf("Failed to encode apiserver endpoints: %v", err)
-			resp.WriteHeader(http.StatusInternalServerError)
+			util.SendError(errors.Wrap(err, "failed to encode apiserver endpoints"), resp, req, http.StatusInternalServerError)
 		}
 	})
 }
 
 func configHandler(server *config.Control, cfg *cmds.Server) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
 		// Startup hooks may read and modify cmds.Server in a goroutine, but as these are copied into
 		// config.Control before the startup hooks are called, any modifications need to be sync'd back
 		// into the struct before it is sent to agents.
@@ -355,23 +335,21 @@ func configHandler(server *config.Control, cfg *cmds.Server) http.Handler {
 		server.DisableKubeProxy = cfg.DisableKubeProxy
 		resp.Header().Set("content-type", "application/json")
 		if err := json.NewEncoder(resp).Encode(server); err != nil {
-			logrus.Errorf("Failed to encode agent config: %v", err)
-			resp.WriteHeader(http.StatusInternalServerError)
+			util.SendError(errors.Wrap(err, "failed to encode agent config"), resp, req, http.StatusInternalServerError)
 		}
 	})
 }
 
 func readyzHandler(server *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		code := http.StatusOK
-		data := []byte("ok")
 		if server.Runtime.Core == nil {
-			code = http.StatusInternalServerError
-			data = []byte("runtime core not ready")
+			util.SendError(util.ErrCoreNotReady, resp, req, http.StatusServiceUnavailable)
+			return
 		}
-		resp.WriteHeader(code)
+		data := []byte("ok")
+		resp.WriteHeader(http.StatusOK)
 		resp.Header().Set("Content-Type", "text/plain")
-		resp.Header().Set("Content-length", strconv.Itoa(len(data)))
+		resp.Header().Set("Content-Length", strconv.Itoa(len(data)))
 		resp.Write(data)
 	})
 }
@@ -379,6 +357,7 @@ func readyzHandler(server *config.Control) http.Handler {
 func ping() http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		data := []byte("pong")
+		resp.WriteHeader(http.StatusOK)
 		resp.Header().Set("Content-Type", "text/plain")
 		resp.Header().Set("Content-Length", strconv.Itoa(len(data)))
 		resp.Write(data)
@@ -432,7 +411,7 @@ func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper
 				return verifyRemotePassword(ctx, config, &mu, deferredNodes, node)
 			} else {
 				// Otherwise, reject the request until the core is ready.
-				return "", http.StatusServiceUnavailable, errors.New("runtime core not ready")
+				return "", http.StatusServiceUnavailable, util.ErrCoreNotReady
 			}
 		}
 

pkg/server/secrets-encrypt.go

@@ -17,11 +17,11 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/rancher/wrangler/pkg/generated/controllers/core"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/utils/ptr"
 )
@@ -56,10 +56,6 @@ func getEncryptionRequest(req *http.Request) (*EncryptionRequest, error) {
 
 func encryptionStatusHandler(server *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
 		status, err := encryptionStatus(server)
 		if err != nil {
 			util.SendErrorWithID(err, "secret-encrypt", resp, req, http.StatusInternalServerError)
@@ -160,18 +156,13 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool)
 
 func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
 		if req.Method != http.MethodPut {
-			resp.WriteHeader(http.StatusBadRequest)
+			util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed)
 			return
 		}
 		encryptReq, err := getEncryptionRequest(req)
 		if err != nil {
-			resp.WriteHeader(http.StatusBadRequest)
-			resp.Write([]byte(err.Error()))
+			util.SendError(err, resp, req, http.StatusBadRequest)
 			return
 		}
 		if encryptReq.Stage != nil {

pkg/server/server.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"runtime/debug"
 	"strconv"
@@ -28,10 +27,10 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/apply"
-	v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
-	"github.com/rancher/wrangler/pkg/leader"
-	"github.com/rancher/wrangler/pkg/resolvehome"
+	"github.com/rancher/wrangler/v3/pkg/apply"
+	v1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/v3/pkg/leader"
+	"github.com/rancher/wrangler/v3/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -167,8 +166,8 @@ func apiserverControllers(ctx context.Context, sc *Context, config *Config) {
 		}
 	}
 
-	// Re-run context startup after core and leader-elected controllers have started. Additional
-	// informer caches may need to start for the newly added OnChange callbacks.
+	// Re-run informer factory startup after core and leader-elected controllers have started.
+	// Additional caches may need to start for the newly added OnChange/OnRemove callbacks.
 	if err := sc.Start(ctx); err != nil {
 		panic(errors.Wrap(err, "failed to start wranger controllers"))
 	}
@@ -220,7 +219,7 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 			return err
 		}
 
-		apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup()
+		apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup().WithSetOwnerReference(false, false)
 		helm := sc.Helm.WithAgent(restConfig.UserAgent)
 		batch := sc.Batch.WithAgent(restConfig.UserAgent)
 		auth := sc.Auth.WithAgent(restConfig.UserAgent)
@@ -282,10 +281,6 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
 	}
 
 	skip := controlConfig.Skips
-	if !skip["traefik"] && isHelmChartTraefikV1(sc) {
-		logrus.Warn("Skipping Traefik v2 deployment due to existing Traefik v1 installation")
-		skip["traefik"] = true
-	}
 	if err := deploy.Stage(dataDir, templateVars, skip); err != nil {
 		return err
 	}
@@ -332,23 +327,6 @@ func addrTypesPrioTemplate(flannelExternal bool) string {
 	return "InternalIP,ExternalIP,Hostname"
 }
 
-// isHelmChartTraefikV1 checks for an existing HelmChart resource with spec.chart containing traefik-1,
-// as deployed by the legacy chart (https://%{KUBERNETES_API}%/static/charts/traefik-1.81.0.tgz)
-func isHelmChartTraefikV1(sc *Context) bool {
-	prefix := "traefik-1."
-	helmChart, err := sc.Helm.Helm().V1().HelmChart().Get(metav1.NamespaceSystem, "traefik", metav1.GetOptions{})
-	if err != nil {
-		logrus.WithError(err).Info("Failed to get existing traefik HelmChart")
-		return false
-	}
-	chart := path.Base(helmChart.Spec.Chart)
-	if strings.HasPrefix(chart, prefix) {
-		logrus.WithField("chart", chart).Info("Found existing traefik v1 HelmChart")
-		return true
-	}
-	return false
-}
-
 func HomeKubeConfig(write, rootless bool) (string, error) {
 	if write {
 		if os.Getuid() == 0 && !rootless {
@@ -465,6 +443,13 @@ func writeKubeConfig(certs string, config *Config) error {
 		util.SetFileModeForPath(kubeConfig, os.FileMode(0600))
 	}
 
+	if config.ControlConfig.KubeConfigGroup != "" {
+		err := util.SetFileGroupForPath(kubeConfig, config.ControlConfig.KubeConfigGroup)
+		if err != nil {
+			logrus.Errorf("Failed to set %s to group %s: %v", kubeConfig, config.ControlConfig.KubeConfigGroup, err)
+		}
+	}
+
 	if kubeConfigSymlink != kubeConfig {
 		if err := writeConfigSymlink(kubeConfig, kubeConfigSymlink); err != nil {
 			logrus.Errorf("Failed to write kubeconfig symlink: %v", err)

pkg/server/token.go

@@ -32,16 +32,15 @@ func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) {
 
 func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil || req.Method != http.MethodPut {
-			resp.WriteHeader(http.StatusBadRequest)
+		if req.Method != http.MethodPut {
+			util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed)
 			return
 		}
 		var err error
 		sTokenReq, err := getServerTokenRequest(req)
 		logrus.Debug("Received token request")
 		if err != nil {
-			resp.WriteHeader(http.StatusBadRequest)
-			resp.Write([]byte(err.Error()))
+			util.SendError(err, resp, req, http.StatusBadRequest)
 			return
 		}
 		if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil {

pkg/spegel/bootstrap.go

@@ -10,10 +10,11 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/merr"
 	"github.com/sirupsen/logrus"
 	"github.com/spegel-org/spegel/pkg/routing"
 	v1 "k8s.io/api/core/v1"
@@ -133,7 +134,7 @@ func (s *serverBootstrapper) Run(_ context.Context, id string) error {
 
 func (s *serverBootstrapper) Get() (addrInfo *peer.AddrInfo, err error) {
 	if s.controlConfig.Runtime.Core == nil {
-		return nil, errors.New("runtime core not ready")
+		return nil, util.ErrCoreNotReady
 	}
 	nodeName := os.Getenv("NODE_NAME")
 	if nodeName == "" {

pkg/spegel/spegel.go

@@ -13,13 +13,12 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/k3s-io/k3s/pkg/agent/https"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/rancher/dynamiclistener/cert"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/utils/ptr"
 
 	"github.com/go-logr/logr"
@@ -43,11 +42,8 @@ import (
 // DefaultRegistry is the default instance of a Spegel distributed registry
 var DefaultRegistry = &Config{
 	Bootstrapper: NewSelfBootstrapper(),
-	HandlerFunc: func(_ *Config, _ *mux.Router) error {
-		return errors.New("not implemented")
-	},
-	AuthFunc: func() authenticator.Request {
-		return union.New(nil)
+	Router: func(context.Context, *config.Node) (*mux.Router, error) {
+		return nil, errors.New("not implemented")
 	},
 }
 
@@ -60,9 +56,6 @@ var (
 	resolveLatestTag = false
 )
 
-type authFunc func() authenticator.Request
-type handlerFunc func(config *Config, router *mux.Router) error
-
 // Config holds fields for a distributed registry
 type Config struct {
 	ClientCAFile   string
@@ -89,10 +82,7 @@ type Config struct {
 	Bootstrapper routing.Bootstrapper
 
 	// HandlerFunc will be called to add the registry API handler to an existing router.
-	HandlerFunc handlerFunc
-
-	// Authenticator will be called to retrieve an authenticator used to validate the request to the registry API.
-	AuthFunc authFunc
+	Router https.RouterFunc
 }
 
 // These values are not currently configurable
@@ -147,7 +137,8 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
 	ipfslog.SetAllLoggers(level)
 
 	// Get containerd client
-	ociClient, err := oci.NewContainerd(nodeConfig.Containerd.Address, registryNamespace, nodeConfig.Containerd.Registry, urls)
+	ociOpts := []oci.Option{oci.WithContentPath(filepath.Join(nodeConfig.Containerd.Root, "io.containerd.content.v1.content"))}
+	ociClient, err := oci.NewContainerd(nodeConfig.Containerd.Address, registryNamespace, nodeConfig.Containerd.Registry, urls, ociOpts...)
 	if err != nil {
 		return errors.Wrap(err, "failed to create OCI client")
 	}
@@ -222,9 +213,10 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
 		registry.WithResolveRetries(resolveRetries),
 		registry.WithResolveTimeout(resolveTimeout),
 		registry.WithTransport(client.Transport),
+		registry.WithLogger(logr.FromContextOrDiscard(ctx)),
 	}
 	reg := registry.NewRegistry(ociClient, router, registryOpts...)
-	regSvr := reg.Server(":"+c.RegistryPort, logr.FromContextOrDiscard(ctx))
+	regSvr := reg.Server(":" + c.RegistryPort)
 
 	// Close router on shutdown
 	go func() {
@@ -235,13 +227,12 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
 	// Track images available in containerd and publish via p2p router
 	go state.Track(ctx, ociClient, router, resolveLatestTag)
 
-	mRouter := mux.NewRouter().SkipClean(true)
-	mRouter.Use(c.authMiddleware())
-	mRouter.PathPrefix("/v2").Handler(regSvr.Handler)
-	mRouter.PathPrefix("/v1-" + version.Program + "/p2p").Handler(c.peerInfo())
-	if err := c.HandlerFunc(c, mRouter); err != nil {
+	mRouter, err := c.Router(ctx, nodeConfig)
+	if err != nil {
 		return err
 	}
+	mRouter.PathPrefix("/v2").Handler(regSvr.Handler)
+	mRouter.PathPrefix("/v1-" + version.Program + "/p2p").Handler(c.peerInfo())
 
 	// Wait up to 5 seconds for the p2p network to find peers. This will return
 	// immediately if the node is bootstrapping from itself.
@@ -267,16 +258,3 @@ func (c *Config) peerInfo() http.HandlerFunc {
 		fmt.Fprintf(resp, "%s/p2p/%s", info.Addrs[0].String(), info.ID.String())
 	})
 }
-
-// authMiddleware calls the configured authenticator to gate access to the registry API
-func (c *Config) authMiddleware() mux.MiddlewareFunc {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			if _, ok, err := c.AuthFunc().AuthenticateRequest(req); !ok || err != nil {
-				http.Error(resp, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-			next.ServeHTTP(resp, req)
-		})
-	}
-}

pkg/util/api.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/merr"
-	"github.com/rancher/wrangler/pkg/schemes"
+	"github.com/rancher/wrangler/v3/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/schemes"
 	"github.com/sirupsen/logrus"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	v1 "k8s.io/api/core/v1"

pkg/util/apierrors.go

@@ -17,6 +17,7 @@ import (
 
 var ErrAPINotReady = errors.New("apiserver not ready")
 var ErrAPIDisabled = errors.New("apiserver disabled")
+var ErrCoreNotReady = errors.New("runtime core not ready")
 
 // SendErrorWithID sends and logs a random error ID so that logs can be correlated
 // between the REST API (which does not provide any detailed error output, to avoid

pkg/util/file.go

@@ -2,7 +2,9 @@ package util
 
 import (
 	"os"
+	"os/user"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -14,6 +16,27 @@ func SetFileModeForPath(name string, mode os.FileMode) error {
 	return os.Chmod(name, mode)
 }
 
+func SetFileGroupForPath(name string, group string) error {
+	// Try to use as group id
+	gid, err := strconv.Atoi(group)
+	if err == nil {
+		return os.Chown(name, -1, gid)
+	}
+
+	// Otherwise, it must be a group name
+	g, err := user.LookupGroup(group)
+	if err != nil {
+		return err
+	}
+
+	gid, err = strconv.Atoi(g.Gid)
+	if err != nil {
+		return err
+	}
+
+	return os.Chown(name, -1, gid)
+}
+
 func SetFileModeForFile(file *os.File, mode os.FileMode) error {
 	return file.Chmod(mode)
 }

pkg/util/net.go

@@ -1,12 +1,15 @@
 package util
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net"
 	"os"
 	"strings"
+	"time"
 
+	"github.com/rancher/wrangler/v3/pkg/merr"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	apinet "k8s.io/apimachinery/pkg/util/net"
@@ -319,3 +322,111 @@ func getIPFromInterface(ifaceName string) (string, error) {
 
 	return "", fmt.Errorf("can't find ip for interface %s", ifaceName)
 }
+
+type multiListener struct {
+	listeners []net.Listener
+	closing   chan struct{}
+	conns     chan acceptRes
+}
+
+type acceptRes struct {
+	conn net.Conn
+	err  error
+}
+
+// explicit interface check
+var _ net.Listener = &multiListener{}
+
+var loopbacks = []string{"127.0.0.1", "::1"}
+
+// ListenWithLoopback listens on the given address, as well as on IPv4 and IPv6 loopback addresses.
+// If the address is a wildcard, the listener is return unwrapped.
+func ListenWithLoopback(ctx context.Context, addr string, port string) (net.Listener, error) {
+	lc := &net.ListenConfig{
+		KeepAlive: 3 * time.Minute,
+		Control:   permitReuse,
+	}
+	l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(addr, port))
+	if err != nil {
+		return nil, err
+	}
+
+	// If we're listening on a wildcard address, we don't need to wrap with the other loopback addresses
+	switch addr {
+	case "", "::", "0.0.0.0":
+		return l, nil
+	}
+
+	ml := &multiListener{
+		listeners: []net.Listener{l},
+		closing:   make(chan struct{}),
+		conns:     make(chan acceptRes),
+	}
+
+	for _, laddr := range loopbacks {
+		if laddr == addr {
+			continue
+		}
+		if l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(laddr, port)); err == nil {
+			ml.listeners = append(ml.listeners, l)
+		} else {
+			logrus.Debugf("Failed to listen on %s: %v", net.JoinHostPort(laddr, port), err)
+		}
+	}
+
+	for i := range ml.listeners {
+		go ml.accept(ml.listeners[i])
+	}
+
+	return ml, nil
+}
+
+// Addr returns the address of the non-loopback address that this multiListener is listening on
+func (ml *multiListener) Addr() net.Addr {
+	return ml.listeners[0].Addr()
+}
+
+// Close closes all the listeners
+func (ml *multiListener) Close() error {
+	close(ml.closing)
+	var errs merr.Errors
+	for i := range ml.listeners {
+		err := ml.listeners[i].Close()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return merr.NewErrors(errs)
+}
+
+// Accept returns a Conn/err pair from one of the waiting listeners
+func (ml *multiListener) Accept() (net.Conn, error) {
+	select {
+	case res, ok := <-ml.conns:
+		if ok {
+			return res.conn, res.err
+		}
+		return nil, fmt.Errorf("connection channel closed")
+	case <-ml.closing:
+		return nil, fmt.Errorf("listener closed")
+	}
+}
+
+// accept runs a loop, accepting connections and trying to send on the result channel
+func (ml *multiListener) accept(listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		r := acceptRes{
+			conn: conn,
+			err:  err,
+		}
+		select {
+		case ml.conns <- r:
+		case <-ml.closing:
+			if r.err == nil {
+				r.conn.Close()
+			}
+			return
+		}
+	}
+}

pkg/util/net_unix.go

@@ -0,0 +1,18 @@
+//go:build !windows
+// +build !windows
+
+package util
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// permitReuse enables port and address sharing on the socket
+func permitReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1)
+	})
+}

pkg/util/net_windows.go

@@ -0,0 +1,11 @@
+//go:build windows
+// +build windows
+
+package util
+
+import "syscall"
+
+// permitReuse is a no-op; port and address reuse is not supported on Windows
+func permitReuse(network, addr string, conn syscall.RawConn) error {
+	return nil
+}