k3s/pkg/agent/flannel/setup.go

307 lines
9.6 KiB
Go
Raw Normal View History

2019-01-01 08:23:01 +00:00
package flannel
import (
"context"
2019-09-03 23:41:54 +00:00
"fmt"
"net"
2019-09-06 00:39:18 +00:00
"os"
2019-01-09 16:54:15 +00:00
"path/filepath"
2019-01-01 08:23:01 +00:00
"strings"
"github.com/k3s-io/k3s/pkg/agent/util"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/version"
"github.com/pkg/errors"
2019-01-01 08:23:01 +00:00
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
2019-01-01 08:23:01 +00:00
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/cache"
toolswatch "k8s.io/client-go/tools/watch"
utilsnet "k8s.io/utils/net"
2019-01-01 08:23:01 +00:00
)
const (
cniConf = `{
"name":"cbr0",
"cniVersion":"1.0.0",
2019-01-01 08:23:01 +00:00
"plugins":[
{
"type":"flannel",
"delegate":{
2019-08-11 05:50:21 +00:00
"hairpinMode":true,
2019-01-01 08:23:01 +00:00
"forceAddress":true,
"isDefaultGateway":true
}
},
{
"type":"portmap",
"capabilities":{
"portMappings":true
}
}
]
}
`
2019-09-03 23:41:54 +00:00
flannelConf = `{
"Network": "%CIDR%",
"EnableIPv6": %IPV6_ENABLED%,
"EnableIPv4": %IPV4_ENABLED%,
"IPv6Network": "%CIDR_IPV6%",
2019-09-03 23:41:54 +00:00
"Backend": %backend%
2019-01-01 08:23:01 +00:00
}
`
2019-09-03 23:41:54 +00:00
vxlanBackend = `{
"Type": "vxlan"
}`
2019-11-27 17:25:55 +00:00
hostGWBackend = `{
"Type": "host-gw"
}`
2019-09-03 23:41:54 +00:00
ipsecBackend = `{
"Type": "ipsec",
"UDPEncap": true,
"PSK": "%psk%"
}`
wireguardBackend = `{
"Type": "extension",
"PreStartupCommand": "wg genkey | tee %flannelConfDir%/privatekey | wg pubkey",
"PostStartupCommand": "export SUBNET_IP=$(echo $SUBNET | cut -d'/' -f 1); ip link del flannel.1 2>/dev/null; echo $PATH >&2; wg-add.sh flannel.1 && wg set flannel.1 listen-port 51820 private-key %flannelConfDir%/privatekey && ip addr add $SUBNET_IP/32 dev flannel.1 && ip link set flannel.1 up && ip route add $NETWORK dev flannel.1",
2019-09-03 23:41:54 +00:00
"ShutdownCommand": "ip link del flannel.1",
"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET persistent-keepalive 25",
2019-09-03 23:41:54 +00:00
"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
}`
wireguardNativeBackend = `{
"Type": "wireguard",
"PersistentKeepaliveInterval": %PersistentKeepaliveInterval%,
"Mode": "%Mode%"
}`
emptyIPv6Network = "::/0"
ipv4 = iota
ipv6
2019-01-01 08:23:01 +00:00
)
2019-09-03 23:41:54 +00:00
func Prepare(ctx context.Context, nodeConfig *config.Node) error {
if err := createCNIConf(nodeConfig.AgentConfig.CNIConfDir, nodeConfig); err != nil {
2019-01-09 16:54:15 +00:00
return err
}
2019-09-03 23:41:54 +00:00
return createFlannelConf(nodeConfig)
2019-01-09 16:54:15 +00:00
}
func Run(ctx context.Context, nodeConfig *config.Node, nodes typedcorev1.NodeInterface) error {
logrus.Infof("Starting flannel with backend %s", nodeConfig.FlannelBackend)
if err := waitForPodCIDR(ctx, nodeConfig.AgentConfig.NodeName, nodes); err != nil {
return errors.Wrap(err, "flannel failed to wait for PodCIDR assignment")
2019-01-01 08:23:01 +00:00
}
netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
if err != nil {
return errors.Wrap(err, "failed to check netMode for flannel")
}
2019-01-09 16:54:15 +00:00
go func() {
err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, nodeConfig.AgentConfig.KubeConfigKubelet, nodeConfig.FlannelIPv6Masq, netMode)
if err != nil && !errors.Is(err, context.Canceled) {
logrus.Fatalf("flannel exited: %v", err)
}
2019-01-09 16:54:15 +00:00
}()
2019-01-01 08:23:01 +00:00
return nil
2019-01-01 08:23:01 +00:00
}
// waitForPodCIDR watches nodes with this node's name, and returns when the PodCIDR has been set.
func waitForPodCIDR(ctx context.Context, nodeName string, nodes typedcorev1.NodeInterface) error {
fieldSelector := fields.Set{metav1.ObjectNameField: nodeName}.String()
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) {
options.FieldSelector = fieldSelector
return nodes.List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) {
options.FieldSelector = fieldSelector
return nodes.Watch(ctx, options)
},
}
condition := func(ev watch.Event) (bool, error) {
if n, ok := ev.Object.(*v1.Node); ok {
return n.Spec.PodCIDR != "", nil
}
return false, errors.New("event object not of type v1.Node")
}
if _, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition); err != nil {
return errors.Wrap(err, "failed to wait for PodCIDR assignment")
}
logrus.Info("Flannel found PodCIDR assigned for node " + nodeName)
return nil
}
func createCNIConf(dir string, nodeConfig *config.Node) error {
logrus.Debugf("Creating the CNI conf in directory %s", dir)
2019-01-09 16:54:15 +00:00
if dir == "" {
return nil
}
p := filepath.Join(dir, "10-flannel.conflist")
if nodeConfig.AgentConfig.FlannelCniConfFile != "" {
logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile)
return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p)
}
2019-01-09 16:54:15 +00:00
return util.WriteFile(p, cniConf)
2019-01-01 08:23:01 +00:00
}
2019-09-03 23:41:54 +00:00
func createFlannelConf(nodeConfig *config.Node) error {
var ipv4Enabled string
logrus.Debugf("Creating the flannel configuration for backend %s in file %s", nodeConfig.FlannelBackend, nodeConfig.FlannelConfFile)
if nodeConfig.FlannelConfFile == "" {
return errors.New("Flannel configuration not defined")
2019-01-09 16:54:15 +00:00
}
2019-09-03 23:41:54 +00:00
if nodeConfig.FlannelConfOverride {
logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConfFile)
2019-08-08 05:56:09 +00:00
return nil
}
netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
if err != nil {
logrus.Fatalf("Flannel error checking netMode: %v", err)
return err
}
if netMode == ipv4 || netMode == (ipv4+ipv6) {
ipv4Enabled = "true"
} else {
ipv4Enabled = "false"
}
confJSON := strings.ReplaceAll(flannelConf, "%IPV4_ENABLED%", ipv4Enabled)
if netMode == ipv4 {
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "false")
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", emptyIPv6Network)
} else if netMode == (ipv4 + ipv6) {
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "true")
for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
if utilsnet.IsIPv6(cidr.IP) {
// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
}
}
} else {
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", "0.0.0.0/0")
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "true")
for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
if utilsnet.IsIPv6(cidr.IP) {
// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
}
}
}
2019-09-03 23:41:54 +00:00
var backendConf string
parts := strings.SplitN(nodeConfig.FlannelBackend, "=", 2)
backend := parts[0]
backendOptions := make(map[string]string)
if len(parts) > 1 {
logrus.Warnf("The additional options through flannel-backend are deprecated and will be removed in k3s v1.27, use flannel-conf instead")
options := strings.Split(parts[1], ",")
for _, o := range options {
p := strings.SplitN(o, "=", 2)
if len(p) == 1 {
backendOptions[p[0]] = ""
} else {
backendOptions[p[0]] = p[1]
}
}
}
2019-09-03 23:41:54 +00:00
switch backend {
2019-09-03 23:41:54 +00:00
case config.FlannelBackendVXLAN:
backendConf = vxlanBackend
2019-11-27 17:25:55 +00:00
case config.FlannelBackendHostGW:
backendConf = hostGWBackend
2019-09-03 23:41:54 +00:00
case config.FlannelBackendIPSEC:
backendConf = strings.ReplaceAll(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK)
2019-09-06 00:39:18 +00:00
if err := setupStrongSwan(nodeConfig); err != nil {
return err
}
2019-09-03 23:41:54 +00:00
case config.FlannelBackendWireguard:
backendConf = strings.ReplaceAll(wireguardBackend, "%flannelConfDir%", filepath.Dir(nodeConfig.FlannelConfFile))
logrus.Warnf("The wireguard backend is deprecated and will be removed in k3s v1.26, please switch to wireguard-native. Check our docs for information about how to migrate")
case config.FlannelBackendWireguardNative:
mode, ok := backendOptions["Mode"]
if !ok {
mode = "separate"
}
keepalive, ok := backendOptions["PersistentKeepaliveInterval"]
if !ok {
keepalive = "25"
}
backendConf = strings.ReplaceAll(wireguardNativeBackend, "%Mode%", mode)
backendConf = strings.ReplaceAll(backendConf, "%PersistentKeepaliveInterval%", keepalive)
2019-09-03 23:41:54 +00:00
default:
return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend)
}
confJSON = strings.ReplaceAll(confJSON, "%backend%", backendConf)
2019-09-03 23:41:54 +00:00
logrus.Debugf("The flannel configuration is %s", confJSON)
return util.WriteFile(nodeConfig.FlannelConfFile, confJSON)
2019-01-01 08:23:01 +00:00
}
2019-09-06 00:39:18 +00:00
func setupStrongSwan(nodeConfig *config.Node) error {
// if data dir env is not set point to root
dataDir := os.Getenv(version.ProgramUpper + "_DATA_DIR")
2019-09-06 00:39:18 +00:00
if dataDir == "" {
dataDir = "/"
2019-09-06 00:39:18 +00:00
}
2020-04-25 07:22:48 +00:00
dataDir = filepath.Join(dataDir, "etc", "strongswan")
2019-09-06 00:39:18 +00:00
info, err := os.Lstat(nodeConfig.AgentConfig.StrongSwanDir)
// something exists but is not a symlink, return
if err == nil && info.Mode()&os.ModeSymlink == 0 {
return nil
}
if err == nil {
target, err := os.Readlink(nodeConfig.AgentConfig.StrongSwanDir)
// current link is the same, return
if err == nil && target == dataDir {
return nil
}
}
2019-09-06 00:39:18 +00:00
// clean up strongswan old link
os.Remove(nodeConfig.AgentConfig.StrongSwanDir)
// make new strongswan link
return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir)
2019-11-26 20:30:12 +00:00
}
// fundNetMode returns the mode (ipv4, ipv6 or dual-stack) in which flannel is operating
func findNetMode(cidrs []*net.IPNet) (int, error) {
dualStack, err := utilsnet.IsDualStackCIDRs(cidrs)
if err != nil {
return 0, err
}
if dualStack {
return ipv4 + ipv6, nil
}
for _, cidr := range cidrs {
if utilsnet.IsIPv4CIDR(cidr) {
return ipv4, nil
}
if utilsnet.IsIPv6CIDR(cidr) {
return ipv6, nil
}
}
return 0, errors.New("Failed checking netMode")
}