Add support for containerd cri registry config_path

Render cri registry mirrors.x.endpoints and configs.x.tls into config_path; keep
using mirrors.x.rewrites and configs.x.auth those do not yet have an
equivalent in the new format.

The new config file format allows disabling containerd's fallback to the
default endpoint when using mirror endpoints; a new CLI flag is added to
control that behavior.

This also re-shares some code that was unnecessarily split into parallel
implementations for linux/windows versions. There is probably more work
to be done on this front but it's a good start.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/agent/config/config.go

@@ -559,6 +559,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log")
+	nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d")
+	nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault
 	nodeConfig.Containerd.Debug = envInfo.Debug
 	applyContainerdStateAndAddress(nodeConfig)
 	applyCRIDockerdAddress(nodeConfig)

pkg/agent/containerd/config.go

@@ -0,0 +1,109 @@
+package containerd
+
+import (
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/k3s-io/k3s/pkg/agent/templates"
+	util2 "github.com/k3s-io/k3s/pkg/agent/util"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+)
+
+// writeContainerdConfig renders and saves config.toml from the filled template
+func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	var containerdTemplate string
+	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
+	if err == nil {
+		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
+		containerdTemplate = string(containerdTemplateBytes)
+	} else if os.IsNotExist(err) {
+		containerdTemplate = templates.ContainerdConfigTemplate
+	} else {
+		return err
+	}
+	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
+	if err != nil {
+		return err
+	}
+
+	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+}
+
+// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
+func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	registry := containerdConfig.PrivateRegistryConfig
+	hosts := map[string]templates.HostConfig{}
+
+	for host, mirror := range registry.Mirrors {
+		defaultHost, _ := docker.DefaultHost(host)
+		config := templates.HostConfig{
+			Host:    defaultHost,
+			Program: version.Program,
+		}
+		if host == "*" {
+			host = "_default"
+			config.Host = ""
+		} else if containerdConfig.NoDefaultEndpoint {
+			config.Host = ""
+		}
+		// TODO: rewrites are currently copied from the mirror settings into each endpoint.
+		// In the future, we should allow for per-endpoint rewrites, instead of expecting
+		// all mirrors to have the same structure. This will require changes to the registries.yaml
+		// structure, which is defined in rancher/wharfie.
+		for _, endpoint := range mirror.Endpoints {
+			if endpointURL, err := url.Parse(endpoint); err == nil {
+				config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
+					OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
+					Config:       registry.Configs[endpointURL.Host],
+					Rewrites:     mirror.Rewrites,
+					URI:          endpoint,
+				})
+			}
+		}
+		hosts[host] = config
+	}
+
+	for host, registry := range registry.Configs {
+		config, ok := hosts[host]
+		if !ok {
+			config = templates.HostConfig{
+				Program: version.Program,
+			}
+		}
+		if len(config.Endpoints) == 0 {
+			config.Endpoints = []templates.RegistryEndpoint{
+				{
+					Config: registry,
+					URI:    "https://" + host,
+				},
+			}
+		}
+		hosts[host] = config
+	}
+
+	// Clean up previous configuration templates
+	os.RemoveAll(cfg.Containerd.Registry)
+
+	// Write out new templates
+	for host, config := range hosts {
+		hostDir := filepath.Join(cfg.Containerd.Registry, host)
+		hostsFile := filepath.Join(hostDir, "hosts.toml")
+		hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
+		if err != nil {
+			return err
+		}
+		if err := os.MkdirAll(hostDir, 0700); err != nil {
+			return err
+		}
+		if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/agent/containerd/config_linux.go

@@ -13,7 +13,6 @@ import (
 	stargz "github.com/containerd/stargz-snapshotter/service"
 	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/k3s-io/k3s/pkg/agent/templates"
-	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/cgroups"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -67,7 +66,6 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime)
 	}
 
-	var containerdTemplate string
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         disableCgroup,
@@ -77,6 +75,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		PrivateRegistryConfig: privRegistries.Registry,
 		ExtraRuntimes:         extraRuntimes,
 		Program:               version.Program,
+		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}
 
 	selEnabled, selConfigured, err := selinuxStatus()
@@ -90,21 +89,11 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		logrus.Warnf("SELinux is enabled for "+version.Program+" but process is not running in context '%s', "+version.Program+"-selinux policy may need to be applied", SELinuxContextType)
 	}
 
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
-	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
-	if err != nil {
+	if err := writeContainerdConfig(cfg, containerdConfig); err != nil {
 		return err
 	}
 
-	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+	return writeContainerdHosts(cfg, containerdConfig)
 }
 
 func Client(address string) (*containerd.Client, error) {

pkg/agent/containerd/config_windows.go

@@ -5,11 +5,9 @@ package containerd
 
 import (
 	"context"
-	"os"
 
 	"github.com/containerd/containerd"
 	"github.com/k3s-io/k3s/pkg/agent/templates"
-	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	util3 "github.com/k3s-io/k3s/pkg/util"
 	"github.com/pkg/errors"
@@ -38,31 +36,20 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		logrus.Warn("SELinux isn't supported on windows")
 	}
 
-	var containerdTemplate string
-
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         true,
 		SystemdCgroup:         false,
 		IsRunningInUserNS:     false,
 		PrivateRegistryConfig: privRegistries.Registry,
+		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}
 
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
-	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
-	if err != nil {
+	if err := writeContainerdConfig(cfg, containerdConfig); err != nil {
 		return err
 	}
 
-	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+	return writeContainerdHosts(cfg, containerdConfig)
 }
 
 func Client(address string) (*containerd.Client, error) {

pkg/agent/templates/templates.go

@@ -1,6 +1,9 @@
 package templates
 
 import (
+	"bytes"
+	"text/template"
+
 	"github.com/rancher/wharfie/pkg/registries"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
@@ -17,7 +20,71 @@ type ContainerdConfig struct {
 	SystemdCgroup         bool
 	IsRunningInUserNS     bool
 	EnableUnprivileged    bool
+	NoDefaultEndpoint     bool
 	PrivateRegistryConfig *registries.Registry
 	ExtraRuntimes         map[string]ContainerdRuntimeConfig
 	Program               string
 }
+
+type RegistryEndpoint struct {
+	OverridePath bool
+	URI          string
+	Rewrites     map[string]string
+	Config       registries.RegistryConfig
+}
+
+type HostConfig struct {
+	Host      string
+	Program   string
+	Endpoints []RegistryEndpoint
+}
+
+const HostsTomlTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT.
+{{ if .Host }}server = "https://{{ .Host }}"{{ end }}
+
+{{ range $e := .Endpoints -}}
+[host."{{ $e.URI }}"]
+  capabilities = ["pull", "resolve"]
+  {{- if $e.OverridePath }}
+  override_path = true
+  {{- end }}
+{{- if $e.Config.TLS }}
+  {{- if $e.Config.TLS.CAFile }}
+  ca = [{{ printf "%q" $e.Config.TLS.CAFile }}]
+  {{- end }}
+  {{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }}
+  client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]]
+  {{- end }}
+  {{- if $e.Config.TLS.InsecureSkipVerify }}
+  skip_verify = true
+  {{- end }}
+{{ end }}
+{{- if $e.Rewrites }}
+  [host."{{ $e.URI }}".rewrite]
+  {{- range $pattern, $replace := $e.Rewrites }}
+    "{{ $pattern }}" = "{{ $replace }}"
+  {{- end }}
+{{ end }}
+{{ end -}}
+`
+
+func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}

pkg/agent/templates/templates_linux.go

@@ -3,11 +3,11 @@
 package templates
 
 import (
-	"bytes"
 	"text/template"
 )
 
 const ContainerdConfigTemplate = `
+{{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
 version = 2
 
@@ -95,20 +95,10 @@ enable_keychain = true
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
   SystemdCgroup = {{ .SystemdCgroup }}
 
-{{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
 
+{{ if .PrivateRegistryConfig }}
 {{range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
 [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
@@ -117,13 +107,6 @@ enable_keychain = true
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
 {{end}}
-{{ if $v.TLS }}
-[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-{{end}}
 {{end}}
 {{end}}
 
@@ -136,12 +119,9 @@ enable_keychain = true
 {{end}}
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
+// Linux config templates do not need fixups
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		return s
+	},
 }

pkg/agent/templates/templates_windows.go

@@ -4,16 +4,17 @@
 package templates
 
 import (
-	"bytes"
 	"net/url"
 	"strings"
 	"text/template"
 )
 
 const ContainerdConfigTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
 version = 2
-root = "{{ replace .NodeConfig.Containerd.Root }}"
-state = "{{ replace .NodeConfig.Containerd.State }}"
+root = {{ printf "%q" .NodeConfig.Containerd.Root }}
+state = {{ printf "%q" .NodeConfig.Containerd.State }}
 plugin_dir = ""
 disabled_plugins = []
 required_plugins = []
@@ -107,14 +108,15 @@ oom_score = 0
           privileged_without_host_devices = false
           base_runtime_spec = ""
     [plugins."io.containerd.grpc.v1.cri".cni]
-      bin_dir = "{{ replace .NodeConfig.AgentConfig.CNIBinDir }}"
-      conf_dir = "{{ replace .NodeConfig.AgentConfig.CNIConfDir }}"
+      bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }}
+      conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }}
       max_conf_num = 1
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = ""
+      config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }}
+
+      {{ if .PrivateRegistryConfig }}
       {{range $k, $v := .PrivateRegistryConfig.Configs }}
-      [plugins."io.containerd.grpc.v1.cri".registry.auths]
       {{ if $v.Auth }}
         [plugins."io.containerd.grpc.v1.cri".registry.configs.auth."{{$k}}"]
           {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
@@ -122,35 +124,15 @@ oom_score = 0
           {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
           {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
       {{end}}
-      [plugins."io.containerd.grpc.v1.cri".registry.configs]
-      {{ if $v.TLS }}
-        [plugins."io.containerd.grpc.v1.cri".registry.configs.tls."{{$k}}"]
-          {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-          {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-          {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-          {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
       {{end}}
       {{end}}
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-      {{ if .PrivateRegistryConfig.Mirrors }}
-      {{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
-	        endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-	    {{if $v.Rewrites}}
-	      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
-	      {{range $pattern, $replace := $v.Rewrites}}
-	        "{{$pattern}}" = "{{$replace}}"
-	      {{end}}
-	    {{end}}
-      {{end}}
-    {{end}}
     [plugins."io.containerd.grpc.v1.cri".image_decryption]
       key_model = ""
     [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
       tls_cert_file = ""
       tls_key_file = ""
   [plugins."io.containerd.internal.v1.opt"]
-    path = "{{ replace .NodeConfig.Containerd.Opt }}"
+    path = {{ printf "%q" .NodeConfig.Containerd.Opt }}
   [plugins."io.containerd.internal.v1.restart"]
     interval = "10s"
   [plugins."io.containerd.metadata.v1.bolt"]
@@ -161,27 +143,16 @@ oom_score = 0
     default = ["windows", "windows-lcow"]
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	funcs := template.FuncMap{
-		"replace": func(s string) string {
-			return strings.ReplaceAll(s, "\\", "\\\\")
-		},
-		"deschemify": func(s string) string {
-			if strings.HasPrefix(s, "npipe:") {
-				u, err := url.Parse(s)
-				if err != nil {
-					return ""
-				}
-				return u.Path
+// Windows config templates need named pipe addresses fixed up
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		if strings.HasPrefix(s, "npipe:") {
+			u, err := url.Parse(s)
+			if err != nil {
+				return ""
 			}
-			return s
-		},
-	}
-	t := template.Must(template.New("compiled_template").Funcs(funcs).Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
+			return u.Path
+		}
+		return s
+	},
 }

pkg/cli/cmds/agent.go

@@ -26,6 +26,7 @@ type Agent struct {
 	PauseImage               string
 	Snapshotter              string
 	Docker                   bool
+	ContainerdNoDefault      bool
 	ContainerRuntimeEndpoint string
 	DefaultRuntime           string
 	ImageServiceEndpoint     string
@@ -220,6 +221,11 @@ var (
 		Usage:       "(agent/networking) (experimental) Disable the agent's client-side load-balancer and connect directly to the configured server address",
 		Destination: &AgentConfig.DisableLoadBalancer,
 	}
+	DisableDefaultRegistryEndpointFlag = &cli.BoolFlag{
+		Name:        "disable-default-registry-endpoint",
+		Usage:       "(agent/containerd) Disables containerd's fallback default registry endpoint when a mirror is configured for that registry",
+		Destination: &AgentConfig.ContainerdNoDefault,
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -269,6 +275,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			PauseImageFlag,
 			SnapshotterFlag,
 			PrivateRegistryFlag,
+			DisableDefaultRegistryEndpointFlag,
 			AirgapExtraRegistryFlag,
 			NodeIPFlag,
 			NodeExternalIPFlag,

pkg/cli/cmds/server.go

@@ -493,6 +493,7 @@ var ServerFlags = []cli.Flag{
 	CRIEndpointFlag,
 	DefaultRuntimeFlag,
 	ImageServiceEndpointFlag,
+	DisableDefaultRegistryEndpointFlag,
 	PauseImageFlag,
 	SnapshotterFlag,
 	PrivateRegistryFlag,

pkg/daemons/config/types.go

@@ -67,6 +67,8 @@ type Containerd struct {
 	Template      string
 	BlockIOConfig string
 	RDTConfig     string
+	Registry      string
+	NoDefault     bool
 	SELinux       bool
 	Debug         bool
 }