From 5ccc880ddb2e6489d3af310f7035c1b90f607fdf Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Tue, 8 Oct 2019 01:04:58 +0200 Subject: [PATCH] Add private registry to containerd --- pkg/agent/config/config.go | 1 + pkg/agent/containerd/containerd.go | 26 ++++++++++++-- pkg/agent/templates/registry.go | 54 ++++++++++++++++++++++++++++++ pkg/agent/templates/templates.go | 30 +++++++++++++++-- pkg/cli/cmds/agent.go | 8 +++++ pkg/cli/cmds/server.go | 1 + pkg/daemons/config/types.go | 1 + 7 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 pkg/agent/templates/registry.go diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index b4296f360a..33b63db4e6 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -397,6 +397,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.AgentConfig.NodeTaints = envInfo.Taints nodeConfig.AgentConfig.NodeLabels = envInfo.Labels + nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry return nodeConfig, nil } diff --git a/pkg/agent/containerd/containerd.go b/pkg/agent/containerd/containerd.go index 0799208467..9d40462a79 100644 --- a/pkg/agent/containerd/containerd.go +++ b/pkg/agent/containerd/containerd.go @@ -20,6 +20,7 @@ import ( "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" "google.golang.org/grpc" + yaml "gopkg.in/yaml.v2" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/util" ) @@ -158,10 +159,15 @@ func preloadImages(cfg *config.Node) error { } func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { + privRegistries, err := getPrivateRegistries(ctx, cfg) + if err != nil { + return err + } var containerdTemplate string containerdConfig := templates.ContainerdConfig{ - NodeConfig: cfg, - IsRunningInUserNS: system.RunningInUserNS(), + NodeConfig: cfg, + IsRunningInUserNS: system.RunningInUserNS(), + PrivateRegistryConfig: privRegistries, } containerdTemplateBytes, err := ioutil.ReadFile(cfg.Containerd.Template) @@ -180,3 +186,19 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) } + +func getPrivateRegistries(ctx context.Context, cfg *config.Node) (*templates.Registry, error) { + privRegistries := &templates.Registry{} + privRegistryFile, err := ioutil.ReadFile(cfg.AgentConfig.PrivateRegistry) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + logrus.Infof("Using registry config file at %s", cfg.AgentConfig.PrivateRegistry) + if err := yaml.Unmarshal([]byte(privRegistryFile), &privRegistries); err != nil { + return nil, err + } + return privRegistries, nil +} diff --git a/pkg/agent/templates/registry.go b/pkg/agent/templates/registry.go new file mode 100644 index 0000000000..8a2cf96a92 --- /dev/null +++ b/pkg/agent/templates/registry.go @@ -0,0 +1,54 @@ +package templates + +// Mirror contains the config related to the registry mirror +type Mirror struct { + // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints + // one by one until a working one is found. The endpoint must be a valid url + // with host specified. + // The scheme, host and path from the endpoint URL will be used. + Endpoints []string `toml:"endpoint" yaml:"endpoint"` +} + +// AuthConfig contains the config related to authentication to a specific registry +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username" yaml:"username"` + // Password is the password to login the registry. + Password string `toml:"password" yaml:"password"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth" yaml:"auth"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken" yaml:"identity_token"` +} + +// TLSConfig contains the CA/Cert/Key used for a registry +type TLSConfig struct { + CAFile string `toml:"ca_file" yaml:"ca_file"` + CertFile string `toml:"cert_file" yaml:"cert_file"` + KeyFile string `toml:"key_file" yaml:"key_file"` +} + +// Registry is registry settings configured +type Registry struct { + // Mirrors are namespace to mirror mapping for all namespaces. + Mirrors map[string]Mirror `toml:"mirrors" yaml:"mirrors"` + // Configs are configs for each registry. + // The key is the FDQN or IP of the registry. + Configs map[string]RegistryConfig `toml:"configs" yaml:"configs"` + + // Auths are registry endpoint to auth config mapping. The registry endpoint must + // be a valid url with host specified. + // DEPRECATED: Use Configs instead. Remove in containerd 1.4. + Auths map[string]AuthConfig `toml:"auths" yaml:"auths"` +} + +// RegistryConfig contains configuration used to communicate with the registry. +type RegistryConfig struct { + // Auth contains information to authenticate to the registry. + Auth *AuthConfig `toml:"auth" yaml:"auth"` + // TLS is a pair of CA/Cert/Key which then are used when creating the transport + // that communicates with the registry. + TLS *TLSConfig `toml:"tls" yaml:"tls"` +} diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index d25c83b774..b7c2f6a2e4 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -8,8 +8,9 @@ import ( ) type ContainerdConfig struct { - NodeConfig *config.Node - IsRunningInUserNS bool + NodeConfig *config.Node + IsRunningInUserNS bool + PrivateRegistryConfig *Registry } const ContainerdConfigTemplate = ` @@ -35,6 +36,31 @@ sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}" bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}" conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}" {{ end -}} + +{{ if .PrivateRegistryConfig }} +{{ if .PrivateRegistryConfig.Mirrors }} +[plugins.cri.registry.mirrors]{{end}} +{{range $k, $v := .PrivateRegistryConfig.Mirrors }} +[plugins.cri.registry.mirrors."{{$k}}"] + endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] +{{end}} + +{{range $k, $v := .PrivateRegistryConfig.Configs }} +{{ if $v.Auth }} +[plugins.cri.registry.configs."{{$k}}".auth] + {{ if $v.Auth.Username }}username = "{{ $v.Auth.Username }}"{{end}} + {{ if $v.Auth.Password }}password = "{{ $v.Auth.Password }}"{{end}} + {{ if $v.Auth.Auth }}auth = "{{ $v.Auth.Auth }}"{{end}} + {{ if $v.Auth.IdentityToken }}identity_token = "{{ $v.Auth.IdentityToken }}"{{end}} +{{end}} +{{ if $v.TLS }} +[plugins.cri.registry.configs."{{$k}}".tls] + {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} + {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} + {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} +{{end}} +{{end}} +{{end}} ` func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 91ed00b0c4..a2499634df 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -30,6 +30,7 @@ type Agent struct { ExtraKubeProxyArgs cli.StringSlice Labels cli.StringSlice Taints cli.StringSlice + PrivateRegistry string } type AgentShared struct { @@ -106,6 +107,12 @@ var ( Usage: "(agent) Registering kubelet with set of labels", Value: &AgentConfig.Labels, } + PrivateRegistryFlag = cli.StringFlag{ + Name: "private-registry", + Usage: "(agent) Private registry configuration file", + Destination: &AgentConfig.PrivateRegistry, + Value: "/etc/rancher/k3s/registries.yaml", + } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -167,6 +174,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { ExtraKubeProxyArgs, NodeLabels, NodeTaints, + PrivateRegistryFlag, }, } } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 3d1e22945c..0d57f47dd8 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -218,6 +218,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { ExtraKubeProxyArgs, NodeLabels, NodeTaints, + PrivateRegistryFlag, }, } } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 328aaaa452..57ac2e5df2 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -76,6 +76,7 @@ type Agent struct { NodeLabels []string IPSECPSK string StrongSwanDir string + PrivateRegistry string } type Control struct {