From c45524e66237bdb491ebb0afc633a5bd6d3a76c8 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 30 Nov 2023 02:14:01 +0000 Subject: [PATCH] Add support for containerd cri registry config_path Render cri registry mirrors.x.endpoints and configs.x.tls into config_path; keep using mirrors.x.rewrites and configs.x.auth those do not yet have an equivalent in the new format. The new config file format allows disabling containerd's fallback to the default endpoint when using mirror endpoints; a new CLI flag is added to control that behavior. This also re-shares some code that was unnecessarily split into parallel implementations for linux/windows versions. There is probably more work to be done on this front but it's a good start. Signed-off-by: Brad Davidson --- pkg/agent/config/config.go | 2 + pkg/agent/containerd/config.go | 109 +++++++++++++++++++++++ pkg/agent/containerd/config_linux.go | 17 +--- pkg/agent/containerd/config_windows.go | 19 +--- pkg/agent/templates/templates.go | 67 ++++++++++++++ pkg/agent/templates/templates_linux.go | 38 ++------ pkg/agent/templates/templates_windows.go | 71 +++++---------- pkg/cli/cmds/agent.go | 7 ++ pkg/cli/cmds/server.go | 1 + pkg/daemons/config/types.go | 2 + 10 files changed, 224 insertions(+), 109 deletions(-) create mode 100644 pkg/agent/containerd/config.go diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 47e625b2f8..e8df928c14 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -559,6 +559,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd") nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log") + nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d") + nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault nodeConfig.Containerd.Debug = envInfo.Debug applyContainerdStateAndAddress(nodeConfig) applyCRIDockerdAddress(nodeConfig) diff --git a/pkg/agent/containerd/config.go b/pkg/agent/containerd/config.go new file mode 100644 index 0000000000..9f7358470b --- /dev/null +++ b/pkg/agent/containerd/config.go @@ -0,0 +1,109 @@ +package containerd + +import ( + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/remotes/docker" + "github.com/k3s-io/k3s/pkg/agent/templates" + util2 "github.com/k3s-io/k3s/pkg/agent/util" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/version" + "github.com/sirupsen/logrus" +) + +// writeContainerdConfig renders and saves config.toml from the filled template +func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { + var containerdTemplate string + containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) + if err == nil { + logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) + containerdTemplate = string(containerdTemplateBytes) + } else if os.IsNotExist(err) { + containerdTemplate = templates.ContainerdConfigTemplate + } else { + return err + } + parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) + if err != nil { + return err + } + + return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) +} + +// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template +func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { + registry := containerdConfig.PrivateRegistryConfig + hosts := map[string]templates.HostConfig{} + + for host, mirror := range registry.Mirrors { + defaultHost, _ := docker.DefaultHost(host) + config := templates.HostConfig{ + Host: defaultHost, + Program: version.Program, + } + if host == "*" { + host = "_default" + config.Host = "" + } else if containerdConfig.NoDefaultEndpoint { + config.Host = "" + } + // TODO: rewrites are currently copied from the mirror settings into each endpoint. + // In the future, we should allow for per-endpoint rewrites, instead of expecting + // all mirrors to have the same structure. This will require changes to the registries.yaml + // structure, which is defined in rancher/wharfie. + for _, endpoint := range mirror.Endpoints { + if endpointURL, err := url.Parse(endpoint); err == nil { + config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{ + OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"), + Config: registry.Configs[endpointURL.Host], + Rewrites: mirror.Rewrites, + URI: endpoint, + }) + } + } + hosts[host] = config + } + + for host, registry := range registry.Configs { + config, ok := hosts[host] + if !ok { + config = templates.HostConfig{ + Program: version.Program, + } + } + if len(config.Endpoints) == 0 { + config.Endpoints = []templates.RegistryEndpoint{ + { + Config: registry, + URI: "https://" + host, + }, + } + } + hosts[host] = config + } + + // Clean up previous configuration templates + os.RemoveAll(cfg.Containerd.Registry) + + // Write out new templates + for host, config := range hosts { + hostDir := filepath.Join(cfg.Containerd.Registry, host) + hostsFile := filepath.Join(hostDir, "hosts.toml") + hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) + if err != nil { + return err + } + if err := os.MkdirAll(hostDir, 0700); err != nil { + return err + } + if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 4a5cd21dfe..5750844c0c 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -13,7 +13,6 @@ import ( stargz "github.com/containerd/stargz-snapshotter/service" "github.com/docker/docker/pkg/parsers/kernel" "github.com/k3s-io/k3s/pkg/agent/templates" - util2 "github.com/k3s-io/k3s/pkg/agent/util" "github.com/k3s-io/k3s/pkg/cgroups" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/version" @@ -67,7 +66,6 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime) } - var containerdTemplate string containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, DisableCgroup: disableCgroup, @@ -77,6 +75,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { PrivateRegistryConfig: privRegistries.Registry, ExtraRuntimes: extraRuntimes, Program: version.Program, + NoDefaultEndpoint: cfg.Containerd.NoDefault, } selEnabled, selConfigured, err := selinuxStatus() @@ -90,21 +89,11 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { logrus.Warnf("SELinux is enabled for "+version.Program+" but process is not running in context '%s', "+version.Program+"-selinux policy may need to be applied", SELinuxContextType) } - containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) - if err == nil { - logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) - containerdTemplate = string(containerdTemplateBytes) - } else if os.IsNotExist(err) { - containerdTemplate = templates.ContainerdConfigTemplate - } else { - return err - } - parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) - if err != nil { + if err := writeContainerdConfig(cfg, containerdConfig); err != nil { return err } - return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) + return writeContainerdHosts(cfg, containerdConfig) } func Client(address string) (*containerd.Client, error) { diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index 6efbb7a148..f4826391e8 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -5,11 +5,9 @@ package containerd import ( "context" - "os" "github.com/containerd/containerd" "github.com/k3s-io/k3s/pkg/agent/templates" - util2 "github.com/k3s-io/k3s/pkg/agent/util" "github.com/k3s-io/k3s/pkg/daemons/config" util3 "github.com/k3s-io/k3s/pkg/util" "github.com/pkg/errors" @@ -38,31 +36,20 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { logrus.Warn("SELinux isn't supported on windows") } - var containerdTemplate string - containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, DisableCgroup: true, SystemdCgroup: false, IsRunningInUserNS: false, PrivateRegistryConfig: privRegistries.Registry, + NoDefaultEndpoint: cfg.Containerd.NoDefault, } - containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) - if err == nil { - logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) - containerdTemplate = string(containerdTemplateBytes) - } else if os.IsNotExist(err) { - containerdTemplate = templates.ContainerdConfigTemplate - } else { - return err - } - parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) - if err != nil { + if err := writeContainerdConfig(cfg, containerdConfig); err != nil { return err } - return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) + return writeContainerdHosts(cfg, containerdConfig) } func Client(address string) (*containerd.Client, error) { diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index a9317ea8f1..d920693d35 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -1,6 +1,9 @@ package templates import ( + "bytes" + "text/template" + "github.com/rancher/wharfie/pkg/registries" "github.com/k3s-io/k3s/pkg/daemons/config" @@ -17,7 +20,71 @@ type ContainerdConfig struct { SystemdCgroup bool IsRunningInUserNS bool EnableUnprivileged bool + NoDefaultEndpoint bool PrivateRegistryConfig *registries.Registry ExtraRuntimes map[string]ContainerdRuntimeConfig Program string } + +type RegistryEndpoint struct { + OverridePath bool + URI string + Rewrites map[string]string + Config registries.RegistryConfig +} + +type HostConfig struct { + Host string + Program string + Endpoints []RegistryEndpoint +} + +const HostsTomlTemplate = ` +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. +{{ if .Host }}server = "https://{{ .Host }}"{{ end }} + +{{ range $e := .Endpoints -}} +[host."{{ $e.URI }}"] + capabilities = ["pull", "resolve"] + {{- if $e.OverridePath }} + override_path = true + {{- end }} +{{- if $e.Config.TLS }} + {{- if $e.Config.TLS.CAFile }} + ca = [{{ printf "%q" $e.Config.TLS.CAFile }}] + {{- end }} + {{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }} + client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]] + {{- end }} + {{- if $e.Config.TLS.InsecureSkipVerify }} + skip_verify = true + {{- end }} +{{ end }} +{{- if $e.Rewrites }} + [host."{{ $e.URI }}".rewrite] + {{- range $pattern, $replace := $e.Rewrites }} + "{{ $pattern }}" = "{{ $replace }}" + {{- end }} +{{ end }} +{{ end -}} +` + +func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { + out := new(bytes.Buffer) + t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer)) + template.Must(t.New("base").Parse(ContainerdConfigTemplate)) + if err := t.Execute(out, config); err != nil { + return "", err + } + return out.String(), nil +} + +func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { + out := new(bytes.Buffer) + t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer)) + if err := t.Execute(out, config); err != nil { + return "", err + } + return out.String(), nil +} diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 66f8ee1080..0df107abaa 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -3,11 +3,11 @@ package templates import ( - "bytes" "text/template" ) const ContainerdConfigTemplate = ` +{{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. version = 2 @@ -95,20 +95,10 @@ enable_keychain = true [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] SystemdCgroup = {{ .SystemdCgroup }} -{{ if .PrivateRegistryConfig }} -{{ if .PrivateRegistryConfig.Mirrors }} -[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}} -{{range $k, $v := .PrivateRegistryConfig.Mirrors }} -[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"] - endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] -{{if $v.Rewrites}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite] -{{range $pattern, $replace := $v.Rewrites}} - "{{$pattern}}" = "{{$replace}}" -{{end}} -{{end}} -{{end}} +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "{{ .NodeConfig.Containerd.Registry }}" +{{ if .PrivateRegistryConfig }} {{range $k, $v := .PrivateRegistryConfig.Configs }} {{ if $v.Auth }} [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth] @@ -117,13 +107,6 @@ enable_keychain = true {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} {{end}} -{{ if $v.TLS }} -[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls] - {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} - {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} - {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} - {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} -{{end}} {{end}} {{end}} @@ -136,12 +119,9 @@ enable_keychain = true {{end}} ` -func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { - out := new(bytes.Buffer) - t := template.Must(template.New("compiled_template").Parse(templateBuffer)) - template.Must(t.New("base").Parse(ContainerdConfigTemplate)) - if err := t.Execute(out, config); err != nil { - return "", err - } - return out.String(), nil +// Linux config templates do not need fixups +var templateFuncs = template.FuncMap{ + "deschemify": func(s string) string { + return s + }, } diff --git a/pkg/agent/templates/templates_windows.go b/pkg/agent/templates/templates_windows.go index f52b5edbd7..5cccd7e43d 100644 --- a/pkg/agent/templates/templates_windows.go +++ b/pkg/agent/templates/templates_windows.go @@ -4,16 +4,17 @@ package templates import ( - "bytes" "net/url" "strings" "text/template" ) const ContainerdConfigTemplate = ` +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. version = 2 -root = "{{ replace .NodeConfig.Containerd.Root }}" -state = "{{ replace .NodeConfig.Containerd.State }}" +root = {{ printf "%q" .NodeConfig.Containerd.Root }} +state = {{ printf "%q" .NodeConfig.Containerd.State }} plugin_dir = "" disabled_plugins = [] required_plugins = [] @@ -107,14 +108,15 @@ oom_score = 0 privileged_without_host_devices = false base_runtime_spec = "" [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "{{ replace .NodeConfig.AgentConfig.CNIBinDir }}" - conf_dir = "{{ replace .NodeConfig.AgentConfig.CNIConfDir }}" + bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }} + conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }} max_conf_num = 1 conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "" + config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} + + {{ if .PrivateRegistryConfig }} {{range $k, $v := .PrivateRegistryConfig.Configs }} - [plugins."io.containerd.grpc.v1.cri".registry.auths] {{ if $v.Auth }} [plugins."io.containerd.grpc.v1.cri".registry.configs.auth."{{$k}}"] {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} @@ -122,35 +124,15 @@ oom_score = 0 {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} {{end}} - [plugins."io.containerd.grpc.v1.cri".registry.configs] - {{ if $v.TLS }} - [plugins."io.containerd.grpc.v1.cri".registry.configs.tls."{{$k}}"] - {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} - {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} - {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} - {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} {{end}} {{end}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - {{ if .PrivateRegistryConfig.Mirrors }} - {{range $k, $v := .PrivateRegistryConfig.Mirrors }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"] - endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] - {{if $v.Rewrites}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite] - {{range $pattern, $replace := $v.Rewrites}} - "{{$pattern}}" = "{{$replace}}" - {{end}} - {{end}} - {{end}} - {{end}} [plugins."io.containerd.grpc.v1.cri".image_decryption] key_model = "" [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] tls_cert_file = "" tls_key_file = "" [plugins."io.containerd.internal.v1.opt"] - path = "{{ replace .NodeConfig.Containerd.Opt }}" + path = {{ printf "%q" .NodeConfig.Containerd.Opt }} [plugins."io.containerd.internal.v1.restart"] interval = "10s" [plugins."io.containerd.metadata.v1.bolt"] @@ -161,27 +143,16 @@ oom_score = 0 default = ["windows", "windows-lcow"] ` -func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { - out := new(bytes.Buffer) - funcs := template.FuncMap{ - "replace": func(s string) string { - return strings.ReplaceAll(s, "\\", "\\\\") - }, - "deschemify": func(s string) string { - if strings.HasPrefix(s, "npipe:") { - u, err := url.Parse(s) - if err != nil { - return "" - } - return u.Path +// Windows config templates need named pipe addresses fixed up +var templateFuncs = template.FuncMap{ + "deschemify": func(s string) string { + if strings.HasPrefix(s, "npipe:") { + u, err := url.Parse(s) + if err != nil { + return "" } - return s - }, - } - t := template.Must(template.New("compiled_template").Funcs(funcs).Parse(templateBuffer)) - template.Must(t.New("base").Parse(ContainerdConfigTemplate)) - if err := t.Execute(out, config); err != nil { - return "", err - } - return out.String(), nil + return u.Path + } + return s + }, } diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index b36f22fe9d..83c04d84ff 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -26,6 +26,7 @@ type Agent struct { PauseImage string Snapshotter string Docker bool + ContainerdNoDefault bool ContainerRuntimeEndpoint string DefaultRuntime string ImageServiceEndpoint string @@ -220,6 +221,11 @@ var ( Usage: "(agent/networking) (experimental) Disable the agent's client-side load-balancer and connect directly to the configured server address", Destination: &AgentConfig.DisableLoadBalancer, } + DisableDefaultRegistryEndpointFlag = &cli.BoolFlag{ + Name: "disable-default-registry-endpoint", + Usage: "(agent/containerd) Disables containerd's fallback default registry endpoint when a mirror is configured for that registry", + Destination: &AgentConfig.ContainerdNoDefault, + } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -269,6 +275,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, + DisableDefaultRegistryEndpointFlag, AirgapExtraRegistryFlag, NodeIPFlag, NodeExternalIPFlag, diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index b8330d629b..a79ece1751 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -493,6 +493,7 @@ var ServerFlags = []cli.Flag{ CRIEndpointFlag, DefaultRuntimeFlag, ImageServiceEndpointFlag, + DisableDefaultRegistryEndpointFlag, PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 5a7fbcf050..00204a787f 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -67,6 +67,8 @@ type Containerd struct { Template string BlockIOConfig string RDTConfig string + Registry string + NoDefault bool SELinux bool Debug bool }