mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
2f5ee914f9
In k3s today the kubernetes API and the /v1-k3s API are combined into one http server. In rke2 we are running unmodified, non-embedded Kubernetes and as such it is preferred to run k8s and the /v1-k3s API on different ports. The /v1-k3s API port is called the SupervisorPort in the code. To support this separation of ports a new shim was added on the client in then pkg/agent/proxy package that will launch two load balancers instead of just one load balancer. One load balancer for 6443 and the other for 9345 (which is the supervisor port).
145 lines
3.3 KiB
Go
145 lines
3.3 KiB
Go
package loadbalancer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net"
|
|
"path/filepath"
|
|
"sync"
|
|
|
|
"github.com/google/tcpproxy"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
type LoadBalancer struct {
|
|
mutex sync.Mutex
|
|
dialer *net.Dialer
|
|
proxy *tcpproxy.Proxy
|
|
|
|
configFile string
|
|
localAddress string
|
|
localServerURL string
|
|
originalServerAddress string
|
|
ServerURL string
|
|
ServerAddresses []string
|
|
randomServers []string
|
|
currentServerAddress string
|
|
nextServerIndex int
|
|
}
|
|
|
|
const (
|
|
SupervisorServiceName = "k3s-agent-load-balancer"
|
|
APIServerServiceName = "k3s-api-server-agent-load-balancer"
|
|
)
|
|
|
|
func New(dataDir, serviceName, serverURL string) (_lb *LoadBalancer, _err error) {
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
defer func() {
|
|
if _err != nil {
|
|
logrus.Warnf("Error starting load balancer: %s", _err)
|
|
if listener != nil {
|
|
listener.Close()
|
|
}
|
|
}
|
|
}()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
localAddress := listener.Addr().String()
|
|
|
|
originalServerAddress, localServerURL, err := parseURL(serverURL, localAddress)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
lb := &LoadBalancer{
|
|
dialer: &net.Dialer{},
|
|
configFile: filepath.Join(dataDir, "etc", serviceName+".json"),
|
|
localAddress: localAddress,
|
|
localServerURL: localServerURL,
|
|
originalServerAddress: originalServerAddress,
|
|
ServerURL: serverURL,
|
|
}
|
|
|
|
lb.setServers([]string{lb.originalServerAddress})
|
|
|
|
lb.proxy = &tcpproxy.Proxy{
|
|
ListenFunc: func(string, string) (net.Listener, error) {
|
|
return listener, nil
|
|
},
|
|
}
|
|
lb.proxy.AddRoute(serviceName, &tcpproxy.DialProxy{
|
|
Addr: serviceName,
|
|
DialContext: lb.dialContext,
|
|
OnDialError: onDialError,
|
|
})
|
|
|
|
if err := lb.updateConfig(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := lb.proxy.Start(); err != nil {
|
|
return nil, err
|
|
}
|
|
logrus.Infof("Running load balancer %s -> %v", lb.localAddress, lb.randomServers)
|
|
|
|
return lb, nil
|
|
}
|
|
|
|
func (lb *LoadBalancer) Update(serverAddresses []string) {
|
|
if lb == nil {
|
|
return
|
|
}
|
|
if !lb.setServers(serverAddresses) {
|
|
return
|
|
}
|
|
logrus.Infof("Updating load balancer server addresses -> %v", lb.randomServers)
|
|
|
|
if err := lb.writeConfig(); err != nil {
|
|
logrus.Warnf("Error updating load balancer config: %s", err)
|
|
}
|
|
}
|
|
|
|
func (lb *LoadBalancer) LoadBalancerServerURL() string {
|
|
if lb == nil {
|
|
return ""
|
|
}
|
|
return lb.localServerURL
|
|
}
|
|
|
|
func (lb *LoadBalancer) dialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
|
startIndex := lb.nextServerIndex
|
|
for {
|
|
targetServer := lb.currentServerAddress
|
|
|
|
conn, err := lb.dialer.DialContext(ctx, network, targetServer)
|
|
if err == nil {
|
|
return conn, nil
|
|
}
|
|
logrus.Debugf("Dial error from load balancer: %s", err)
|
|
|
|
newServer, err := lb.nextServer(targetServer)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if targetServer != newServer {
|
|
logrus.Debugf("Dial server in load balancer failed over to %s", newServer)
|
|
}
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
maxIndex := len(lb.randomServers)
|
|
if startIndex > maxIndex {
|
|
startIndex = maxIndex
|
|
}
|
|
if lb.nextServerIndex == startIndex {
|
|
return nil, errors.New("all servers failed")
|
|
}
|
|
}
|
|
}
|
|
|
|
func onDialError(src net.Conn, dstDialErr error) {
|
|
logrus.Debugf("Incoming conn %v, error dialing load balancer servers: %v", src.RemoteAddr().String(), dstDialErr)
|
|
src.Close()
|
|
}
|