Mirror/k3s
1
0
Fork 0
mirror of https://github.com/k3s-io/k3s.git synced 2024-06-07 19:41:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add rootless support

Browse Source
This commit is contained in:
Darren Shepherd 2019-03-08 15:47:44 -07:00
parent 6f01e13fc4
commit 046a817818
17 changed files with 445 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/agent/containerd/containerd.go
View File

@ -15,6 +15,7 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/namespaces"
"github.com/natefinch/lumberjack"
"github.com/opencontainers/runc/libcontainer/system"
util2 "github.com/rancher/k3s/pkg/agent/util"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/sirupsen/logrus"
@ -27,10 +28,15 @@ const (
maxMsgSize = 1024 * 1024 * 16
configToml = `
[plugins.opt]
path = "%OPT%"
path = "%OPT%"
[plugins.cri]
stream_server_address = "%NODE%"
stream_server_port = "10010"
`
configUserNSToml = `
disable_cgroup = true
disable_apparmor = true
restrict_oom_score_adj = true
`
configCNIToml = `
[plugins.cri.cni]
@ -49,6 +55,9 @@ func Run(ctx context.Context, cfg *config.Node) error {
}
template := configToml
if system.RunningInUserNS() {
template += configUserNSToml
}
if !cfg.NoFlannel {
template += configCNIToml
}

7
pkg/agent/run.go
View File

@ -17,6 +17,7 @@ import (
"github.com/rancher/k3s/pkg/agent/tunnel"
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/daemons/agent"
"github.com/rancher/k3s/pkg/rootless"
"github.com/rancher/norman/pkg/clientaccess"
"github.com/sirupsen/logrus"
)
@ -69,6 +70,12 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
return err
}
if cfg.Rootless {
if err := rootless.Rootless(cfg.DataDir); err != nil {
return err
}
}
cfg.DataDir = filepath.Join(cfg.DataDir, "agent")
if cfg.ClusterSecret != "" {

4
pkg/cli/agent/agent.go
View File

@ -10,7 +10,7 @@ import (
"github.com/rancher/k3s/pkg/agent"
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/norman/pkg/resolvehome"
"github.com/rancher/k3s/pkg/datadir"
"github.com/rancher/norman/signal"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -57,7 +57,7 @@ func Run(ctx *cli.Context) error {
logrus.Infof("Starting k3s agent %s", ctx.App.Version)
dataDir, err := resolvehome.Resolve(cmds.AgentConfig.DataDir)
dataDir, err := datadir.LocalHome(cmds.AgentConfig.DataDir, cmds.AgentConfig.Rootless)
if err != nil {
return err
}

6
pkg/cli/cmds/agent.go
View File

@ -20,6 +20,7 @@ type Agent struct {
ContainerRuntimeEndpoint string
NoFlannel bool
Debug bool
Rootless bool
AgentShared
ExtraKubeletArgs cli.StringSlice
ExtraKubeProxyArgs cli.StringSlice
@ -113,6 +114,11 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
Destination: &AgentConfig.ClusterSecret,
EnvVar: "K3S_CLUSTER_SECRET",
},
cli.BoolFlag{
Name: "rootless",
Usage: "(experimental) Run rootless",
Destination: &AgentConfig.Rootless,
},
DockerFlag,
FlannelFlag,
NodeNameFlag,

6
pkg/cli/cmds/server.go
View File

@ -21,6 +21,7 @@ type Server struct {
ExtraAPIArgs cli.StringSlice
ExtraSchedulerArgs cli.StringSlice
ExtraControllerArgs cli.StringSlice
Rootless bool
}
var ServerConfig Server
@ -124,6 +125,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
Usage: "Customized flag for kube-controller-manager process",
Value: &ServerConfig.ExtraControllerArgs,
},
cli.BoolFlag{
Name: "rootless",
Usage: "(experimental) Run rootless",
Destination: &ServerConfig.Rootless,
},
NodeIPFlag,
NodeNameFlag,
DockerFlag,

20
pkg/cli/server/server.go
View File

@ -15,12 +15,14 @@ import (
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/agent"
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/datadir"
"github.com/rancher/k3s/pkg/rootless"
"github.com/rancher/k3s/pkg/server"
"github.com/rancher/norman/signal"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"k8s.io/apimachinery/pkg/util/net"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/csi"
_ "github.com/mattn/go-sqlite3" // ensure we have sqlite
)
@ -67,18 +69,30 @@ func run(app *cli.Context, cfg *cmds.Server) error {
setupLogging(app)
if !cfg.DisableAgent && os.Getuid() != 0 {
if !cfg.DisableAgent && os.Getuid() != 0 && !cfg.Rootless {
return fmt.Errorf("must run as root unless --disable-agent is specified")
}
if cfg.Rootless {
dataDir, err := datadir.LocalHome(cfg.DataDir, true)
if err != nil {
return err
}
cfg.DataDir = dataDir
if err := rootless.Rootless(dataDir); err != nil {
return err
}
}
// If running agent in server, set this so that CSI initializes properly
volume.WaitForValidHost = !cfg.DisableAgent
csi.WaitForValidHostName = !cfg.DisableAgent
serverConfig := server.Config{}
serverConfig.ControlConfig.ClusterSecret = cfg.ClusterSecret
serverConfig.ControlConfig.DataDir = cfg.DataDir
serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
serverConfig.Rootless = cfg.Rootless
serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)

7
pkg/daemons/agent/agent.go
View File

@ -9,12 +9,14 @@ import (
"strings"
"time"
"github.com/opencontainers/runc/libcontainer/system"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/net"
"k8s.io/component-base/logs"
app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
"k8s.io/kubernetes/cmd/kubelet/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)
@ -107,8 +109,11 @@ func kubelet(cfg *config.Agent) {
argsMap["runtime-cgroups"] = root
argsMap["kubelet-cgroups"] = root
}
args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
if system.RunningInUserNS() {
argsMap["feature-gates"] = "DevicePlugins=false"
}
args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
command.SetArgs(args)
go func() {

6
pkg/datadir/datadir.go
View File

@ -15,8 +15,12 @@ const (
)
func Resolve(dataDir string) (string, error) {
return LocalHome(dataDir, false)
}
func LocalHome(dataDir string, forceLocal bool) (string, error) {
if dataDir == "" {
if os.Getuid() == 0 {
if os.Getuid() == 0 && !forceLocal {
dataDir = DefaultDataDir
} else {
dataDir = DefaultHomeDataDir

2
pkg/kubectl/main.go
View File

@ -17,7 +17,7 @@ import (
func Main() {
kubenv := os.Getenv("KUBECONFIG")
if kubenv == "" {
config, err := server.HomeKubeConfig(false)
config, err := server.HomeKubeConfig(false, false)
if _, serr := os.Stat(config); err == nil && serr == nil {
os.Setenv("KUBECONFIG", config)
}

64
pkg/rootless/mounts.go Normal file
View File

@ -0,0 +1,64 @@
package rootless
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
func setupMounts(stateDir string) error {
mountMap := [][]string{
{"/run", ""},
{"/var/run", ""},
{"/var/log", filepath.Join(stateDir, "logs")},
{"/var/lib/cni", filepath.Join(stateDir, "cni")},
}
for _, v := range mountMap {
if err := setupMount(v[0], v[1]); err != nil {
return errors.Wrapf(err, "failed to setup mount %s => %s", v[0], v[1])
}
}
return nil
}
func setupMount(target, dir string) error {
toCreate := target
for {
if toCreate == "/" {
return fmt.Errorf("missing /%s on the root filesystem", strings.Split(target, "/")[0])
}
if err := os.MkdirAll(toCreate, 0700); err == nil {
break
}
toCreate = filepath.Base(toCreate)
}
logrus.Debug("Mounting none ", toCreate, " tmpfs")
if err := unix.Mount("none", toCreate, "tmpfs", 0, ""); err != nil {
return errors.Wrapf(err, "failed to mount tmpfs to %s", toCreate)
}
if err := os.MkdirAll(target, 0700); err != nil {
return errors.Wrapf(err, "failed to create directory %s")
}
if dir == "" {
return nil
}
if err := os.MkdirAll(dir, 0700); err != nil {
return errors.Wrapf(err, "failed to create directory %s")
}
logrus.Debug("Mounting ", dir, target, " none bind")
return unix.Mount(dir, target, "none", unix.MS_BIND, "")
}

133
pkg/rootless/rootless.go Normal file
View File

@ -0,0 +1,133 @@
package rootless
import (
"io/ioutil"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/rootless-containers/rootlesskit/pkg/child"
"github.com/rootless-containers/rootlesskit/pkg/copyup/tmpfssymlink"
"github.com/rootless-containers/rootlesskit/pkg/network/slirp4netns"
"github.com/rootless-containers/rootlesskit/pkg/parent"
"github.com/rootless-containers/rootlesskit/pkg/port/socat"
"github.com/sirupsen/logrus"
)
var (
pipeFD = "_K3S_ROOTLESS_FD"
childEnv = "_K3S_ROOTLESS_SOCK"
Sock = ""
)
func Rootless(stateDir string) error {
defer func() {
os.Unsetenv(pipeFD)
os.Unsetenv(childEnv)
}()
hasFD := os.Getenv(pipeFD) != ""
hasChildEnv := os.Getenv(childEnv) != ""
if hasFD {
logrus.Debug("Running rootless child")
childOpt, err := createChildOpt()
if err != nil {
logrus.Fatal(err)
}
if err := child.Child(*childOpt); err != nil {
logrus.Fatal("child died", err)
}
}
if hasChildEnv {
Sock = os.Getenv(childEnv)
logrus.Debug("Running rootless process")
return setupMounts(stateDir)
}
logrus.Debug("Running rootless parent")
parentOpt, err := createParentOpt(filepath.Join(stateDir, "rootless"))
if err != nil {
logrus.Fatal(err)
}
os.Setenv(childEnv, filepath.Join(parentOpt.StateDir, parent.StateFileAPISock))
if err := parent.Parent(*parentOpt); err != nil {
logrus.Fatal(err)
}
os.Exit(0)
return nil
}
func parseCIDR(s string) (*net.IPNet, error) {
if s == "" {
return nil, nil
}
ip, ipnet, err := net.ParseCIDR(s)
if err != nil {
return nil, err
}
if !ip.Equal(ipnet.IP) {
return nil, errors.Errorf("cidr must be like 10.0.2.0/24, not like 10.0.2.100/24")
}
return ipnet, nil
}
func createParentOpt(stateDir string) (*parent.Opt, error) {
if err := os.MkdirAll(stateDir, 0755); err != nil {
return nil, errors.Wrapf(err, "failed to mkdir %s", stateDir)
}
stateDir, err := ioutil.TempDir("", "rootless")
if err != nil {
return nil, err
}
opt := &parent.Opt{
StateDir: stateDir,
}
mtu := 0
ipnet, err := parseCIDR("10.41.0.0/16")
if err != nil {
return nil, err
}
disableHostLoopback := true
binary := "slirp4netns"
if _, err := exec.LookPath(binary); err != nil {
return nil, err
}
opt.NetworkDriver = slirp4netns.NewParentDriver(binary, mtu, ipnet, disableHostLoopback, "")
opt.PortDriver, err = socat.NewParentDriver(&logrusDebugWriter{})
if err != nil {
return nil, err
}
opt.PipeFDEnvKey = pipeFD
return opt, nil
}
type logrusDebugWriter struct {
}
func (w *logrusDebugWriter) Write(p []byte) (int, error) {
s := strings.TrimSuffix(string(p), "\n")
logrus.Debug(s)
return len(p), nil
}
func createChildOpt() (*child.Opt, error) {
opt := &child.Opt{}
opt.TargetCmd = os.Args
opt.PipeFDEnvKey = pipeFD
opt.NetworkDriver = slirp4netns.NewChildDriver()
opt.CopyUpDirs = []string{"/etc", "/run"}
opt.CopyUpDriver = tmpfssymlink.NewChildDriver()
return opt, nil
}

150
pkg/rootlessports/controller.go Normal file
View File

@ -0,0 +1,150 @@
package rootlessports
import (
"context"
"time"
"github.com/rancher/k3s/pkg/rootless"
coreClients "github.com/rancher/k3s/types/apis/core/v1"
"github.com/rootless-containers/rootlesskit/pkg/api/client"
"github.com/rootless-containers/rootlesskit/pkg/port"
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
)
var (
all = "_all_"
)
func Register(ctx context.Context, httpsPort int) error {
var (
err error
rootlessClient client.Client
)
if rootless.Sock == "" {
return nil
}
coreClients := coreClients.ClientsFrom(ctx)
for i := 0; i < 30; i++ {
rootlessClient, err = client.New(rootless.Sock)
if err == nil {
break
} else {
logrus.Infof("waiting for rootless API socket %s: %v", rootless.Sock, err)
time.Sleep(1 * time.Second)
}
}
if err != nil {
return err
}
h := &handler{
rootlessClient: rootlessClient,
serviceClient: coreClients.Service,
serviceCache: coreClients.Service.Cache(),
httpsPort: httpsPort,
ctx: ctx,
}
coreClients.Service.Interface().Controller().AddHandler(ctx, "rootlessports", h.serviceChanged)
coreClients.Service.Enqueue("", all)
return nil
}
type handler struct {
rootlessClient client.Client
serviceClient coreClients.ServiceClient
serviceCache coreClients.ServiceClientCache
httpsPort int
ctx context.Context
}
func (h *handler) serviceChanged(key string, svc *v1.Service) (runtime.Object, error) {
if key != all {
h.serviceClient.Enqueue("", all)
return svc, nil
}
ports, err := h.rootlessClient.PortManager().ListPorts(h.ctx)
if err != nil {
return svc, err
}
boundPorts := map[int]int{}
for _, port := range ports {
boundPorts[port.Spec.ParentPort] = port.ID
}
toBindPort, err := h.toBindPorts()
if err != nil {
return svc, err
}
for bindPort, childBindPort := range toBindPort {
if _, ok := boundPorts[bindPort]; ok {
logrus.Debugf("Parent port %d to child already bound", bindPort)
delete(boundPorts, bindPort)
continue
}
status, err := h.rootlessClient.PortManager().AddPort(h.ctx, port.Spec{
Proto: "tcp",
ParentPort: bindPort,
ChildPort: childBindPort,
})
if err != nil {
return svc, err
}
logrus.Infof("Bound parent port %s:%d to child namespace port %d", status.Spec.ParentIP,
status.Spec.ParentPort, status.Spec.ChildPort)
}
for bindPort, id := range boundPorts {
if err := h.rootlessClient.PortManager().RemovePort(h.ctx, id); err != nil {
return svc, err
}
logrus.Infof("Removed parent port %d to child namespace", bindPort)
}
return svc, nil
}
func (h *handler) toBindPorts() (map[int]int, error) {
svcs, err := h.serviceCache.List("", labels.Everything())
if err != nil {
return nil, err
}
toBindPorts := map[int]int{
h.httpsPort: h.httpsPort,
}
for _, svc := range svcs {
for _, ingress := range svc.Status.LoadBalancer.Ingress {
if ingress.IP == "" {
continue
}
for _, port := range svc.Spec.Ports {
if port.Protocol != v1.ProtocolTCP {
continue
}
if port.Port != 0 {
if port.Port <= 1024 {
toBindPorts[10000+int(port.Port)] = int(port.Port)
} else {
toBindPorts[int(port.Port)] = int(port.Port)
}
}
}
}
}
return toBindPorts, nil
}

44
pkg/server/server.go
View File

@ -18,6 +18,7 @@ import (
"github.com/rancher/k3s/pkg/datadir"
"github.com/rancher/k3s/pkg/deploy"
"github.com/rancher/k3s/pkg/helm"
"github.com/rancher/k3s/pkg/rootlessports"
"github.com/rancher/k3s/pkg/servicelb"
"github.com/rancher/k3s/pkg/static"
"github.com/rancher/k3s/pkg/tls"
@ -36,16 +37,8 @@ import (
)
func resolveDataDir(dataDir string) (string, error) {
if dataDir == "" {
if os.Getuid() == 0 {
dataDir = "/var/lib/rancher/k3s"
} else {
dataDir = "${HOME}/.rancher/k3s"
}
}
dataDir = filepath.Join(dataDir, "server")
return resolvehome.Resolve(dataDir)
dataDir, err := datadir.Resolve(dataDir)
return filepath.Join(dataDir, "server"), err
}
func StartServer(ctx context.Context, config *Config) (string, error) {
@ -72,7 +65,7 @@ func StartServer(ctx context.Context, config *Config) (string, error) {
}
printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)
writeKubeConfig(certs, &config.TLSConfig, &config.ControlConfig)
writeKubeConfig(certs, &config.TLSConfig, config)
return certs, nil
}
@ -121,7 +114,8 @@ func startNorman(ctx context.Context, config *Config) (string, error) {
MasterControllers: []norman.ControllerRegister{
helm.Register,
func(ctx context.Context) error {
return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB)
return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB,
config.Rootless)
},
func(ctx context.Context) error {
dataDir := filepath.Join(controlConfig.DataDir, "static")
@ -138,6 +132,12 @@ func startNorman(ctx context.Context, config *Config) (string, error) {
}
return nil
},
func(ctx context.Context) error {
if !config.DisableServiceLB && config.Rootless {
return rootlessports.Register(ctx, config.TLSConfig.HTTPSPort)
}
return nil
},
},
}
@ -156,9 +156,9 @@ func startNorman(ctx context.Context, config *Config) (string, error) {
}
}
func HomeKubeConfig(write bool) (string, error) {
func HomeKubeConfig(write, rootless bool) (string, error) {
if write {
if os.Getuid() == 0 {
if os.Getuid() == 0 && !rootless {
return datadir.GlobalConfig, nil
}
return resolvehome.Resolve(datadir.HomeConfig)
@ -194,30 +194,30 @@ func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfi
}
func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
clientToken := FormatToken(config.Runtime.ClientToken, certs)
func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) {
clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
ip := tlsConfig.BindAddress
if ip == "" {
ip = "localhost"
}
url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort)
kubeConfig, err := HomeKubeConfig(true)
kubeConfig, err := HomeKubeConfig(true, config.Rootless)
def := true
if err != nil {
kubeConfig = filepath.Join(config.DataDir, "kubeconfig-k3s.yaml")
kubeConfig = filepath.Join(config.ControlConfig.DataDir, "kubeconfig-k3s.yaml")
def = false
}
if config.KubeConfigOutput != "" {
kubeConfig = config.KubeConfigOutput
if config.ControlConfig.KubeConfigOutput != "" {
kubeConfig = config.ControlConfig.KubeConfigOutput
}
if err = clientaccess.AgentAccessInfoToKubeConfig(kubeConfig, url, clientToken); err != nil {
logrus.Errorf("Failed to generate kubeconfig: %v", err)
}
if config.KubeConfigMode != "" {
mode, err := strconv.ParseInt(config.KubeConfigMode, 8, 0)
if config.ControlConfig.KubeConfigMode != "" {
mode, err := strconv.ParseInt(config.ControlConfig.KubeConfigMode, 8, 0)
if err == nil {
os.Chmod(kubeConfig, os.FileMode(mode))
} else {

1
pkg/server/types.go
View File

@ -10,4 +10,5 @@ type Config struct {
DisableServiceLB bool
TLSConfig dynamiclistener.UserConfig
ControlConfig config.Control
Rootless bool
}

9
pkg/servicelb/controller.go
View File

@ -34,11 +34,12 @@ var (
trueVal = true
)
func Register(ctx context.Context, kubernetes kubernetes.Interface, enabled bool) error {
func Register(ctx context.Context, kubernetes kubernetes.Interface, enabled, rootless bool) error {
clients := coreclient.ClientsFrom(ctx)
appClients := appclient.ClientsFrom(ctx)
h := &handler{
rootless: rootless,
enabled: enabled,
nodeCache: clients.Node.Cache(),
podCache: clients.Pod.Cache(),
@ -59,6 +60,7 @@ func Register(ctx context.Context, kubernetes kubernetes.Interface, enabled bool
}
type handler struct {
rootless bool
enabled bool
nodeCache coreclient.NodeClientCache
podCache coreclient.PodClientCache
@ -189,6 +191,11 @@ func (h *handler) podIPs(pods []*core.Pod) ([]string, error) {
for k := range ips {
ipList = append(ipList, k)
}
if len(ipList) > 0 && h.rootless {
return []string{"127.0.0.1"}, nil
}
return ipList, nil
}

7
scripts/dev-server.sh
View File

@ -5,4 +5,9 @@ mkdir -p $(dirname $0)/../bin
cd $(dirname $0)/../bin
echo Running
go run -tags "apparmor" ../main.go --debug server --disable-agent "$@"
ARGS="--disable-agent"
if echo -- "$@" | grep -q rootless; then
ARGS=""
PATH=$(pwd):$PATH
fi
go run -tags "apparmor" ../main.go server $ARGS "$@"

2
scripts/download
View File

@ -4,7 +4,7 @@ source $(dirname $0)/version.sh
cd $(dirname $0)/..
ROOT_VERSION=v0.0.1
ROOT_VERSION=v0.1.1
TRAEFIK_VERSION=1.64.0
CHARTS_DIR=build/static/charts