Kubelet resolv.conf DNS update

Allow the kubelet resolv-conf flag to be set, or automatically
discovered from /etc/resolv.conf & /run/systemd/resolve/resolv.conf if
no loopback devices are present, or create our own which points to
nameserver 8.8.8.8

manifests/coredns.yaml

@@ -61,7 +61,7 @@ data:
           fallthrough in-addr.arpa ip6.arpa
         }
         prometheus :9153
-        proxy . 1.1.1.1
+        proxy . /etc/resolv.conf
         cache 30
         loop
         reload

pkg/agent/config/config.go

@@ -1,15 +1,18 @@
 package config
 
 import (
+	"bufio"
 	"context"
 	"crypto/tls"
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	sysnet "net"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -115,6 +118,49 @@ func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig
 	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
 }
 
+func isValidResolvConf(resolvConfFile string) bool {
+	file, err := os.Open(resolvConfFile)
+	if err != nil {
+		return false
+	}
+	defer file.Close()
+
+	nameserver := regexp.MustCompile(`^nameserver\s+([^\s]*)`)
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		ipMatch := nameserver.FindStringSubmatch(scanner.Text())
+		if len(ipMatch) == 2 {
+			ip := sysnet.ParseIP(ipMatch[1])
+			if ip == nil || !ip.IsGlobalUnicast() {
+				return false
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return false
+	}
+	return true
+}
+
+func locateOrGenerateResolvConf(envInfo *cmds.Agent) string {
+	if envInfo.ResolvConf != "" {
+		return envInfo.ResolvConf
+	}
+	resolvConfs := []string{"/etc/resolv.conf", "/run/systemd/resolve/resolv.conf"}
+	for _, conf := range resolvConfs {
+		if isValidResolvConf(conf) {
+			return conf
+		}
+	}
+
+	tmpConf := filepath.Join(os.TempDir(), "k3s-resolv.conf")
+	if err := ioutil.WriteFile(tmpConf, []byte("nameserver 8.8.8.8\n"), 0444); err != nil {
+		logrus.Error(err)
+		return ""
+	}
+	return tmpConf
+}
+
 func get(envInfo *cmds.Agent) (*config.Node, error) {
 	if envInfo.Debug {
 		logrus.SetLevel(logrus.DebugLevel)
@@ -170,6 +216,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.NodeIP = nodeIP
 	nodeConfig.AgentConfig.NodeName = nodeName
 	nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
+	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
 	nodeConfig.AgentConfig.CACertPath = clientCA
 	nodeConfig.AgentConfig.ListenAddress = "127.0.0.1"
 	nodeConfig.AgentConfig.KubeConfig = kubeConfig

pkg/cli/cmds/agent.go

@@ -11,6 +11,7 @@ type Agent struct {
 	Token                    string
 	TokenFile                string
 	ServerURL                string
+	ResolvConf               string
 	DataDir                  string
 	NodeIP                   string
 	NodeName                 string
@@ -55,6 +56,12 @@ var (
 		Usage:       "(agent) Disable embedded containerd and use alternative CRI implementation",
 		Destination: &AgentConfig.ContainerRuntimeEndpoint,
 	}
+	ResolvConfFlag = cli.StringFlag{
+		Name:        "resolv-conf",
+		Usage:       "Kubelet resolv.conf file",
+		EnvVar:      "K3S_RESOLV_CONF",
+		Destination: &AgentConfig.ResolvConf,
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -99,6 +106,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			NodeNameFlag,
 			NodeIPFlag,
 			CRIEndpointFlag,
+			ResolvConfFlag,
 		},
 	}
 }

pkg/cli/cmds/server.go

@@ -105,6 +105,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			DockerFlag,
 			FlannelFlag,
 			CRIEndpointFlag,
+			ResolvConfFlag,
 		},
 	}
 }

pkg/daemons/agent/agent.go

@@ -76,6 +76,9 @@ func kubelet(cfg *config.Agent) {
 	if len(cfg.ClusterDNS) > 0 {
 		args = append(args, "--cluster-dns", cfg.ClusterDNS.String())
 	}
+	if cfg.ResolvConf != "" {
+		args = append(args, "--resolv-conf", cfg.ResolvConf)
+	}
 	if cfg.RuntimeSocket != "" {
 		args = append(args, "--container-runtime", "remote")
 		args = append(args, "--container-runtime-endpoint", cfg.RuntimeSocket)

pkg/daemons/config/types.go

@@ -36,6 +36,7 @@ type Agent struct {
 	NodeName           string
 	ClusterCIDR        net.IPNet
 	ClusterDNS         net.IP
+	ResolvConf         string
 	RootDir            string
 	KubeConfig         string
 	NodeIP             string