Mirror/k3s
1
0
Fork 0
mirror of https://github.com/k3s-io/k3s.git synced 2024-06-07 19:41:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for IPv6 only mode

Browse Source 
Automatically switch to IPv6 only mode if first node-ip is IPv6 address

Signed-off-by: Olli Janatuinen <olli.janatuinen@gmail.com>
This commit is contained in:
Olli Janatuinen 2021-11-10 21:23:05 +02:00
parent 830c330aad
commit 966f4d6a01
6 changed files with 187 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/agent/config/config.go
View File

@ -425,7 +425,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
@ -466,18 +465,20 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Certificate = servingCert
nodeConfig.AgentConfig.NodeIPs = nodeIPs
nodeIP, err := util.GetFirst4(nodeIPs)
nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
}
nodeConfig.AgentConfig.NodeIP = nodeIP.String()
nodeConfig.AgentConfig.ListenAddress = listenAddress
nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs
// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
// unless only IPv6 address given
if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs)
nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip")
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
}
nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
}

4
pkg/agent/run.go
View File

@ -58,8 +58,10 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
if err != nil {
return errors.Wrap(err, "failed to validate node-ip")
}
serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)
enableIPv6 := dualCluster || dualService || dualNode
enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6
conntrackConfig, err := getConntrackConfig(nodeConfig)
if err != nil {
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")

52
pkg/cli/server/server.go
View File

@ -198,17 +198,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.PrivateIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}
// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeExternalIP)
}
// if not set, try setting advertise-ip from agent node-ip
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
}
// if we ended up with any advertise-ips, ensure they're added to the SAN list;
@ -226,14 +226,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
return err
}
serverConfig.ControlConfig.ServerNodeName = nodeName
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
for _, ip := range nodeIPs {
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
}
// configure ClusterIPRanges
_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
clusterCIDR := "10.42.0.0/16"
if IPv6only {
clusterCIDR = "fd:42::/56"
}
cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
}
for _, cidr := range cmds.ServerConfig.ClusterCIDR {
for _, v := range strings.Split(cidr, ",") {
@ -246,15 +251,20 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
// set ClusterIPRange to the first IPv4 block, for legacy clients
clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges)
// unless only IPv6 range given
clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
}
serverConfig.ControlConfig.ClusterIPRange = clusterIPRange
// configure ServiceIPRanges
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
serviceCIDR := "10.43.0.0/16"
if IPv6only {
serviceCIDR = "fd:43::/112"
}
cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
}
for _, cidr := range cmds.ServerConfig.ServiceCIDR {
for _, v := range strings.Split(cidr, ",") {
@ -267,9 +277,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
// set ServiceIPRange to the first IPv4 block, for legacy clients
serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges)
// unless only IPv6 range given
serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 service-cidr")
return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
}
serverConfig.ControlConfig.ServiceIPRange = serviceIPRange
@ -287,7 +298,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
// If there are no IPv4 ServiceCIDRs, an error will be raised.
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
// If neither of IPv4 or IPv6 are found an error is raised.
if len(cmds.ServerConfig.ClusterDNS) == 0 {
clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
if err != nil {
@ -306,9 +318,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs)
// unless only IPv6 range given
clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4 cluster-dns address")
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
}
@ -457,6 +470,9 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
ip := serverConfig.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
if IPv6only {
ip = "[::1]"
}
}
url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
@ -517,6 +533,16 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
return errors.New("dual-stack cluster-dns is not supported")
}
IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serverConfig.ControlConfig.ServiceIPRanges)
if IPv6OnlyService {
if serverConfig.ControlConfig.DisableNPC == false {
return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy")
}
if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone {
return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none")
}
}
return nil
}

14
pkg/daemons/agent/agent_linux.go
View File

@ -39,9 +39,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}
func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
"conntrack-max-per-core": "0",
@ -55,8 +60,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}
func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,

14
pkg/daemons/agent/agent_windows.go
View File

@ -27,9 +27,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
}
func kubeProxyArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"proxy-mode": "kernelspace",
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
}
@ -45,8 +50,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
}
func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
if IPv6only {
bindAddress = "::1"
}
argsMap := map[string]string{
"healthz-bind-address": "127.0.0.1",
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,

115
pkg/util/net.go
View File

@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}
// GetFirst6 returns the first IPv6 address from the list of IP addresses.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem == nil || elem.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 address found")
}
// GetFirst6Net returns the first IPv4 network from the list of IP networks.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem == nil || elem.IP.To16() == nil {
continue
}
return elem, nil
}
return nil, errors.New("no IPv6 CIDRs found")
}
// GetFirst6String returns the first IPv6 address from a list of IP address strings.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6String(elems []string) (string, error) {
ips := []net.IP{}
for _, elem := range elems {
for _, v := range strings.Split(elem, ",") {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst6(ips)
if err != nil {
return "", err
}
return ip.String(), nil
}
// JoinIP6Nets stringifies and joins a list of IPv6 networks with commas.
func JoinIP6Nets(elems []*net.IPNet) string {
var strs []string
@ -141,3 +181,78 @@ func ParseStringSliceToIPs(s cli.StringSlice) ([]net.IP, error) {
return ips, nil
}
// GetFirstIP returns the first IPv4 address from the list of IP addresses.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
// Additionally matching listen address and IP version is returned.
func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
nodeIP, err := GetFirst4(nodeIPs)
ListenAddress := "0.0.0.0"
IPv6only := false
if err != nil {
nodeIP, err = GetFirst6(nodeIPs)
if err != nil {
return nil, "", false, err
}
ListenAddress = "::"
IPv6only = true
}
return nodeIP, ListenAddress, IPv6only, nil
}
// GetFirstNet returns the first IPv4 network from the list of IP networks.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
serviceIPRange, err := GetFirst4Net(elems)
if err != nil {
serviceIPRange, err = GetFirst6Net(elems)
if err != nil {
return nil, err
}
}
return serviceIPRange, nil
}
// GetFirstString returns the first IP4 address from a list of IP address strings.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstString(elems []string) (string, bool, error) {
ip, err := GetFirst4String(elems)
IPv6only := false
if err != nil {
ip, err = GetFirst6String(elems)
if err != nil {
return "", false, err
}
IPv6only = true
}
return ip, IPv6only, nil
}
// IsIPv6OnlyCIDRs returns if
// - all are valid cidrs
// - at least one cidr from v6 family is found
// - v4 family cidr is not found
func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
v4Found := false
v6Found := false
for _, cidr := range cidrs {
if cidr == nil {
return false, fmt.Errorf("cidr %v is invalid", cidr)
}
if v4Found && v6Found {
continue
}
if cidr.IP != nil && cidr.IP.To4() == nil {
v6Found = true
continue
}
v4Found = true
}
return !v4Found && v6Found, nil
}