Add embedded registry implementation

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

go.mod

@@ -20,6 +20,8 @@ replace (
 	github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78
 	github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.10.1
 	github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d
+	github.com/ugorji/go => github.com/ugorji/go v1.2.11
+	github.com/xenitab/spegel => github.com/k3s-io/spegel v0.0.17-0.20240109004735-9466a5529330 // k3s-main
 	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1
 	go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1
 	go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1
@@ -42,9 +44,9 @@ replace (
 	go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.13.0
 	go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.13.0
 	go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.19.0
-	golang.org/x/crypto => golang.org/x/crypto v0.1.0
+	golang.org/x/crypto => golang.org/x/crypto v0.14.0
 	golang.org/x/net => golang.org/x/net v0.17.0
-	golang.org/x/sys => golang.org/x/sys v0.6.0
+	golang.org/x/sys => golang.org/x/sys v0.13.0
 	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230525234035-dd9d682886f9
 	google.golang.org/grpc => google.golang.org/grpc v1.58.3
 	gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.6.0
@@ -92,7 +94,7 @@ require (
 	github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000
 	github.com/containerd/aufs v1.0.0
 	github.com/containerd/cgroups/v3 v3.0.2
-	github.com/containerd/containerd v1.7.3
+	github.com/containerd/containerd v1.7.11
 	github.com/containerd/fuse-overlayfs-snapshotter v1.0.5
 	github.com/containerd/stargz-snapshotter v0.14.4-0.20230913082252-7275d45b185c
 	github.com/containerd/zfs v1.1.0
@@ -102,6 +104,8 @@ require (
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
 	github.com/flannel-io/flannel v0.24.0
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
+	github.com/go-logr/logr v1.4.1
+	github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/go-test/deep v1.0.7
 	github.com/golang/mock v1.6.0
@@ -109,12 +113,15 @@ require (
 	github.com/google/uuid v1.4.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
+	github.com/ipfs/go-ds-leveldb v0.5.0
+	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/json-iterator/go v1.1.12
 	github.com/k3s-io/helm-controller v0.15.4
 	github.com/k3s-io/kine v0.11.0
 	github.com/klauspost/compress v1.17.2
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
 	github.com/lib/pq v1.10.2
+	github.com/libp2p/go-libp2p v0.30.0
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/minio/minio-go/v7 v7.0.33
 	github.com/mwitkow/go-http-dialer v0.0.0-20161116154839-378f744fb2b8
@@ -128,7 +135,7 @@ require (
 	github.com/rancher/dynamiclistener v0.3.6
 	github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29
 	github.com/rancher/remotedialer v0.3.0
-	github.com/rancher/wharfie v0.5.3
+	github.com/rancher/wharfie v0.6.4
 	github.com/rancher/wrangler v1.1.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rootless-containers/rootlesskit v1.0.1
@@ -137,16 +144,17 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli v1.22.14
 	github.com/vishvananda/netlink v1.2.1-beta.2
+	github.com/xenitab/spegel v0.0.16
 	github.com/yl2chen/cidranger v1.0.2
 	go.etcd.io/etcd/api/v3 v3.5.10
 	go.etcd.io/etcd/client/pkg/v3 v3.5.10
 	go.etcd.io/etcd/client/v3 v3.5.10
 	go.etcd.io/etcd/etcdutl/v3 v3.5.9
 	go.etcd.io/etcd/server/v3 v3.5.10
-	go.uber.org/zap v1.24.0
+	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.17.0
 	golang.org/x/net v0.19.0
-	golang.org/x/sync v0.4.0
+	golang.org/x/sync v0.6.0
 	golang.org/x/sys v0.15.0
 	google.golang.org/grpc v1.59.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -169,7 +177,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
@@ -194,14 +202,17 @@ require (
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/avast/retry-go/v4 v4.3.2 // indirect
+	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/canonical/go-dqlite v1.5.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
 	github.com/container-orchestrated-devices/container-device-interface v0.5.4 // indirect
 	github.com/container-storage-interface/spec v1.8.0 // indirect
@@ -230,6 +241,8 @@ require (
 	github.com/danwinship/knftables v0.0.13 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/daviddengcn/go-colortext v1.0.0 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
@@ -239,6 +252,7 @@ require (
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect
@@ -246,16 +260,23 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/gin-gonic/gin v1.9.1 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
@@ -263,16 +284,18 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/cel-go v0.17.7 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-containerregistry v0.14.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
@@ -284,31 +307,63 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
+	github.com/huin/goupnp v1.2.0 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/intel/goresctrl v0.3.0 // indirect
+	github.com/ipfs/boxo v0.10.0 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipld/go-ipld-prime v0.20.0 // indirect
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.4.2 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/karrick/godirwalk v1.17.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libopenstorage/openstorage v1.0.0 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.25.2 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
+	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.7.2 // indirect
+	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/libp2p/go-nat v0.2.0 // indirect
+	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-reuseport v0.4.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
 	github.com/mdlayher/netlink v1.7.2 // indirect
 	github.com/mdlayher/socket v0.4.1 // indirect
+	github.com/miekg/dns v1.1.55 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
 	github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -326,7 +381,18 @@ require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/mrunalp/fileutils v0.5.1 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.12.0 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-multistream v0.4.1 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee // indirect
@@ -336,35 +402,53 @@ require (
 	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
+	github.com/quic-go/quic-go v0.38.0 // indirect
+	github.com/quic-go/webtransport-go v0.5.3 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
 	github.com/shengdoushi/base58 v1.0.0 // indirect
+	github.com/slok/go-http-metrics v0.10.0 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/urfave/cli/v2 v2.26.0 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	github.com/vmware/govmomi v0.30.6 // indirect
+	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	github.com/xenitab/pkg/channels v0.0.2 // indirect
+	github.com/xenitab/pkg/gin v0.0.9 // indirect
 	github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
@@ -376,7 +460,7 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
@@ -386,22 +470,25 @@ require (
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
-	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/dig v1.17.0 // indirect
+	go.uber.org/fx v1.20.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/oauth2 v0.11.0 // indirect
+	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.4.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect
-	google.golang.org/api v0.138.0 // indirect
+	gonum.org/v1/gonum v0.13.0 // indirect
+	google.golang.org/api v0.152.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect
+	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/gcfg.v1 v1.2.3 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -428,6 +515,7 @@ require (
 	k8s.io/metrics v0.0.0 // indirect
 	k8s.io/mount-utils v0.29.0 // indirect
 	k8s.io/pod-security-admission v0.0.0 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect

pkg/agent/config/config.go

@@ -16,6 +16,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
@@ -682,6 +684,29 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 	nodeConfig.AgentConfig.Registry = privRegistries.Registry
 
+	if nodeConfig.EmbeddedRegistry {
+		psk, err := hex.DecodeString(controlConfig.IPSECPSK)
+		if err != nil {
+			return nil, err
+		}
+		if len(psk) < 32 {
+			return nil, errors.New("insufficient PSK bytes")
+		}
+
+		conf := spegel.DefaultRegistry
+		conf.ExternalAddress = nodeConfig.AgentConfig.NodeIP
+		conf.InternalAddress = controlConfig.Loopback(false)
+		conf.RegistryPort = strconv.Itoa(controlConfig.SupervisorPort)
+		conf.ClientCAFile = clientCAFile
+		conf.ClientCertFile = clientK3sControllerCert
+		conf.ClientKeyFile = clientK3sControllerKey
+		conf.ServerCAFile = serverCAFile
+		conf.ServerCertFile = servingKubeletCert
+		conf.ServerKeyFile = servingKubeletKey
+		conf.PSK = psk[:32]
+		conf.InjectMirror(nodeConfig)
+	}
+
 	if err := validateNetworkConfig(nodeConfig); err != nil {
 		return nil, err
 	}

pkg/agent/containerd/config.go

@@ -1,6 +1,7 @@
 package containerd
 
 import (
+	"net"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -10,6 +11,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/templates"
 	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
 )
@@ -36,6 +38,7 @@ func writeContainerdConfig(cfg *config.Node, containerdConfig templates.Containe
 
 // writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
 func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	mirrorAddr := net.JoinHostPort(spegel.DefaultRegistry.InternalAddress, spegel.DefaultRegistry.RegistryPort)
 	registry := containerdConfig.PrivateRegistryConfig
 	hosts := map[string]templates.HostConfig{}
 
@@ -57,12 +60,17 @@ func writeContainerdHosts(cfg *config.Node, containerdConfig templates.Container
 		// structure, which is defined in rancher/wharfie.
 		for _, endpoint := range mirror.Endpoints {
 			if endpointURL, err := url.Parse(endpoint); err == nil {
-				config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
+				re := templates.RegistryEndpoint{
 					OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
 					Config:       registry.Configs[endpointURL.Host],
 					Rewrites:     mirror.Rewrites,
 					URI:          endpoint,
-				})
+				}
+				// Do not apply rewrites to the embedded registry endpoint
+				if endpointURL.Host == mirrorAddr {
+					re.Rewrites = nil
+				}
+				config.Endpoints = append(config.Endpoints, re)
 			}
 		}
 		hosts[host] = config

pkg/agent/run.go

@@ -29,6 +29,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
 	"github.com/k3s-io/k3s/pkg/nodeconfig"
 	"github.com/k3s-io/k3s/pkg/rootless"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -102,6 +103,16 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	nodeConfig.AgentConfig.EnableIPv4 = enableIPv4
 	nodeConfig.AgentConfig.EnableIPv6 = enableIPv6
 
+	if nodeConfig.EmbeddedRegistry {
+		if nodeConfig.Docker || nodeConfig.ContainerRuntimeEndpoint != "" {
+			return errors.New("embedded registry mirror requires embedded containerd")
+		}
+
+		if err := spegel.DefaultRegistry.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to start embedded registry")
+		}
+	}
+
 	if err := setupCriCtlConfig(cfg, nodeConfig); err != nil {
 		return err
 	}

pkg/cli/agent/agent.go

@@ -2,21 +2,27 @@ package agent
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 
 	"github.com/erikdubbelboer/gspt"
+	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
+	"github.com/k3s-io/k3s/pkg/authenticator"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/datadir"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
+	apiauth "k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
 func Run(ctx *cli.Context) error {
@@ -99,5 +105,37 @@ func Run(ctx *cli.Context) error {
 		}
 	}
 
+	// Until the agent is run and retrieves config from the server, we won't know
+	// if the embedded registry is enabled. If it is not enabled, these are not
+	// used as the registry is never started.
+	conf := spegel.DefaultRegistry
+	conf.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir)
+	conf.HandlerFunc = func(conf *spegel.Config, router *mux.Router) error {
+		// Create and bind a new authenticator using the configured client CA
+		authArgs := []string{"--client-ca-file=" + conf.ClientCAFile}
+		auth, err := authenticator.FromArgs(authArgs)
+		if err != nil {
+			return err
+		}
+		conf.AuthFunc = func() apiauth.Request {
+			return auth
+		}
+
+		// Create a new server and listen on the configured port
+		server := &http.Server{
+			Handler: router,
+			Addr:    ":" + conf.RegistryPort,
+			TLSConfig: &tls.Config{
+				ClientAuth: tls.RequestClientCert,
+			},
+		}
+		go func() {
+			if err := server.ListenAndServeTLS(conf.ServerCertFile, conf.ServerKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				logrus.Fatalf("registry server failed: %v", err)
+			}
+		}()
+		return nil
+	}
+
 	return agent.Run(contextCtx, cfg)
 }

pkg/cli/server/server.go

@@ -11,6 +11,7 @@ import (
 
 	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/erikdubbelboer/gspt"
+	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
 	"github.com/k3s-io/k3s/pkg/agent/loadbalancer"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
@@ -20,6 +21,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/k3s/pkg/rootless"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
@@ -28,6 +30,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
 	kubeapiserverflag "k8s.io/component-base/cli/flag"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	utilsnet "k8s.io/utils/net"
@@ -548,6 +551,23 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		return agent.RunStandalone(ctx, agentConfig)
 	}
 
+	if cfg.EmbeddedRegistry {
+		conf := spegel.DefaultRegistry
+		conf.Bootstrapper = spegel.NewChainingBootstrapper(
+			spegel.NewServerBootstrapper(&serverConfig.ControlConfig),
+			spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir),
+			spegel.NewSelfBootstrapper(),
+		)
+		conf.HandlerFunc = func(_ *spegel.Config, router *mux.Router) error {
+			router.NotFoundHandler = serverConfig.ControlConfig.Runtime.Handler
+			serverConfig.ControlConfig.Runtime.Handler = router
+			return nil
+		}
+		conf.AuthFunc = func() authenticator.Request {
+			return serverConfig.ControlConfig.Runtime.Authenticator
+		}
+	}
+
 	return agent.Run(ctx, agentConfig)
 }
 

pkg/daemons/config/types.go

@@ -152,6 +152,7 @@ type CriticalControlArgs struct {
 	DisableNPC            bool         `cli:"disable-network-policy"`
 	DisableServiceLB      bool         `cli:"disable-service-lb"`
 	EncryptSecrets        bool         `cli:"secrets-encryption"`
+	EmbeddedRegistry      bool         `cli:"embedded-registry"`
 	FlannelBackend        string       `cli:"flannel-backend"`
 	FlannelIPv6Masq       bool         `cli:"flannel-ipv6-masq"`
 	FlannelExternalIP     bool         `cli:"flannel-external-ip"`
@@ -186,7 +187,6 @@ type Control struct {
 	DisableKubeProxy         bool
 	DisableScheduler         bool
 	DisableServiceLB         bool
-	EmbeddedRegistry         bool
 	Rootless                 bool
 	ServiceLBNamespace       string
 	EnablePProf              bool

pkg/spegel/bootstrap.go

@@ -0,0 +1,200 @@
+package spegel
+
+import (
+	"context"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/k3s-io/k3s/pkg/clientaccess"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/pkg/errors"
+	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/sirupsen/logrus"
+	"github.com/xenitab/spegel/pkg/routing"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	nodeutil "k8s.io/kubernetes/pkg/controller/util/node"
+)
+
+// explicit interface checks
+var _ routing.Bootstrapper = &selfBootstrapper{}
+var _ routing.Bootstrapper = &agentBootstrapper{}
+var _ routing.Bootstrapper = &serverBootstrapper{}
+var _ routing.Bootstrapper = &chainingBootstrapper{}
+
+type selfBootstrapper struct {
+	id string
+}
+
+// NewSelfBootstrapper returns a stub p2p bootstrapper that just returns its own ID
+func NewSelfBootstrapper() routing.Bootstrapper {
+	return &selfBootstrapper{}
+}
+
+func (s *selfBootstrapper) Run(_ context.Context, id string) error {
+	s.id = id
+	return nil
+}
+
+func (s *selfBootstrapper) GetAddress() (*peer.AddrInfo, error) {
+	return peer.AddrInfoFromString(s.id)
+}
+
+type agentBootstrapper struct {
+	server     string
+	token      string
+	clientCert string
+	clientKey  string
+}
+
+// NewAgentBootstrapper returns a p2p bootstrapper that retrieves a peer address from its server
+func NewAgentBootstrapper(server, token, dataDir string) routing.Bootstrapper {
+	return &agentBootstrapper{
+		clientCert: filepath.Join(dataDir, "agent", "client-kubelet.crt"),
+		clientKey:  filepath.Join(dataDir, "agent", "client-kubelet.key"),
+		server:     server,
+		token:      token,
+	}
+}
+
+func (c *agentBootstrapper) Run(_ context.Context, _ string) error {
+	return nil
+}
+
+func (c *agentBootstrapper) GetAddress() (*peer.AddrInfo, error) {
+	if c.server == "" || c.token == "" {
+		return nil, errors.New("cannot get addresses without server and token")
+	}
+
+	withCert := clientaccess.WithClientCertificate(c.clientCert, c.clientKey)
+	info, err := clientaccess.ParseAndValidateToken(c.server, c.token, withCert)
+	if err != nil {
+		return nil, err
+	}
+
+	addr, err := info.Get("/v1-" + version.Program + "/p2p")
+	if err != nil {
+		return nil, err
+	}
+
+	addrInfo, err := peer.AddrInfoFromString(string(addr))
+	return addrInfo, err
+}
+
+type serverBootstrapper struct {
+	controlConfig *config.Control
+}
+
+// NewServerBootstrapper returns a p2p bootstrapper that returns an address from a random other cluster member.
+func NewServerBootstrapper(controlConfig *config.Control) routing.Bootstrapper {
+	return &serverBootstrapper{
+		controlConfig: controlConfig,
+	}
+}
+
+func (s *serverBootstrapper) Run(_ context.Context, id string) error {
+	s.controlConfig.Runtime.ClusterControllerStarts["spegel-p2p"] = func(ctx context.Context) {
+		nodes := s.controlConfig.Runtime.Core.Core().V1().Node()
+		wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (bool, error) {
+			nodeName := os.Getenv("NODE_NAME")
+			if nodeName == "" {
+				return false, nil
+			}
+			node, err := nodes.Get(nodeName, metav1.GetOptions{})
+			if err != nil {
+				return false, nil
+			}
+
+			if node.Annotations == nil {
+				node.Annotations = map[string]string{}
+			}
+			node.Annotations[P2pAddressAnnotation] = id
+			if node.Labels == nil {
+				node.Labels = map[string]string{}
+			}
+			node.Labels[P2pEnabledLabel] = "true"
+
+			if _, err = nodes.Update(node); err != nil {
+				return false, nil
+			}
+			logrus.Infof("Node P2P address annotations and labels added: %s", id)
+			return true, nil
+		})
+	}
+	return nil
+}
+
+func (s *serverBootstrapper) GetAddress() (addrInfo *peer.AddrInfo, err error) {
+	if s.controlConfig.Runtime.Core == nil {
+		return nil, errors.New("runtime core not ready")
+	}
+	nodeName := os.Getenv("NODE_NAME")
+	if nodeName == "" {
+		return nil, errors.New("node name not set")
+	}
+	nodes := s.controlConfig.Runtime.Core.Core().V1().Node()
+	labelSelector := labels.Set{P2pEnabledLabel: "true"}.String()
+	nodeList, err := nodes.List(metav1.ListOptions{LabelSelector: labelSelector})
+	if err != nil {
+		return nil, err
+	}
+	for _, i := range rand.Perm(len(nodeList.Items)) {
+		node := nodeList.Items[i]
+		if node.Name == nodeName {
+			// don't return our own address
+			continue
+		}
+		if find, condition := nodeutil.GetNodeCondition(&node.Status, v1.NodeReady); find == -1 || condition.Status != v1.ConditionTrue {
+			// don't return the address of a not-ready node
+			continue
+		}
+		if val, ok := node.Annotations[P2pAddressAnnotation]; ok {
+			for _, addr := range strings.Split(val, ",") {
+				if info, err := peer.AddrInfoFromString(addr); err == nil {
+					return info, nil
+				}
+			}
+		}
+	}
+	return nil, errors.New("no ready p2p peers found")
+}
+
+type chainingBootstrapper struct {
+	bootstrappers []routing.Bootstrapper
+}
+
+// NewChainingBootstrapper returns a p2p bootstrapper that passes through to a list of bootstrappers.
+func NewChainingBootstrapper(bootstrappers ...routing.Bootstrapper) routing.Bootstrapper {
+	return &chainingBootstrapper{
+		bootstrappers: bootstrappers,
+	}
+}
+
+func (c *chainingBootstrapper) Run(ctx context.Context, id string) error {
+	errs := merr.Errors{}
+	for _, b := range c.bootstrappers {
+		if err := b.Run(ctx, id); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return merr.NewErrors(errs...)
+}
+
+func (c *chainingBootstrapper) GetAddress() (*peer.AddrInfo, error) {
+	errs := merr.Errors{}
+	for _, b := range c.bootstrappers {
+		addr, err := b.GetAddress()
+		if err == nil {
+			return addr, nil
+		}
+		errs = append(errs, err)
+	}
+	return nil, merr.NewErrors(errs...)
+}

pkg/spegel/registry.go

@@ -0,0 +1,38 @@
+package spegel
+
+import (
+	"net"
+
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/rancher/wharfie/pkg/registries"
+)
+
+// InjectMirror configures TLS for the registry mirror client, and  adds the mirror address as an endpoint
+// to all configured registries.
+func (c *Config) InjectMirror(nodeConfig *config.Node) error {
+	mirrorAddr := net.JoinHostPort(c.InternalAddress, c.RegistryPort)
+	registry := nodeConfig.AgentConfig.Registry
+
+	if registry.Configs == nil {
+		registry.Configs = map[string]registries.RegistryConfig{}
+	}
+	registry.Configs[mirrorAddr] = registries.RegistryConfig{
+		TLS: &registries.TLSConfig{
+			CAFile:   c.ServerCAFile,
+			CertFile: c.ClientCertFile,
+			KeyFile:  c.ClientKeyFile,
+		},
+	}
+
+	if registry.Mirrors == nil {
+		registry.Mirrors = map[string]registries.Mirror{}
+	}
+	for host, mirror := range registry.Mirrors {
+		if host != "*" {
+			mirror.Endpoints = append([]string{"https://" + mirrorAddr}, mirror.Endpoints...)
+			registry.Mirrors[host] = mirror
+		}
+	}
+
+	return nil
+}

pkg/spegel/spegel.go

@@ -0,0 +1,261 @@
+package spegel
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/k3s-io/k3s/pkg/clientaccess"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/rancher/dynamiclistener/cert"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/request/union"
+	"k8s.io/utils/pointer"
+
+	"github.com/go-logr/logr"
+	"github.com/go-logr/stdr"
+	"github.com/gorilla/mux"
+	leveldb "github.com/ipfs/go-ds-leveldb"
+	ipfslog "github.com/ipfs/go-log/v2"
+	"github.com/libp2p/go-libp2p"
+	"github.com/libp2p/go-libp2p/core/crypto"
+	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoreds"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/xenitab/spegel/pkg/metrics"
+	"github.com/xenitab/spegel/pkg/oci"
+	"github.com/xenitab/spegel/pkg/registry"
+	"github.com/xenitab/spegel/pkg/routing"
+	"github.com/xenitab/spegel/pkg/state"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// DefaultRegistry is the default instance of a Spegel distributed registry
+var DefaultRegistry = &Config{
+	Bootstrapper: NewSelfBootstrapper(),
+	HandlerFunc: func(_ *Config, _ *mux.Router) error {
+		return errors.New("not implemented")
+	},
+	AuthFunc: func() authenticator.Request {
+		return union.New(nil)
+	},
+}
+
+var (
+	P2pAddressAnnotation = "p2p." + version.Program + ".cattle.io/node-address"
+	P2pEnabledLabel      = "p2p." + version.Program + ".cattle.io/enabled"
+	P2pPortEnv           = version.ProgramUpper + "_P2P_PORT"
+)
+
+type authFunc func() authenticator.Request
+type handlerFunc func(config *Config, router *mux.Router) error
+
+// Config holds fields for a distributed registry
+type Config struct {
+	ClientCAFile   string
+	ClientCertFile string
+	ClientKeyFile  string
+
+	ServerCAFile   string
+	ServerCertFile string
+	ServerKeyFile  string
+
+	// ExternalAddress is the address for other nodes to connect to the registry API.
+	ExternalAddress string
+
+	// InternalAddress is the address for the local containerd instance to connect to the registry API.
+	InternalAddress string
+
+	// RegistryPort is the port for the registry API.
+	RegistryPort string
+
+	// PSK is the preshared key required to join the p2p network.
+	PSK []byte
+
+	// Bootstrapper is the bootstrapper that will be used to discover p2p peers.
+	Bootstrapper routing.Bootstrapper
+
+	// HandlerFunc will be called to add the registry API handler to an existing router.
+	HandlerFunc handlerFunc
+
+	// Authenticator will be called to retrieve an authenticator used to validate the request to the registry API.
+	AuthFunc authFunc
+}
+
+// These values are not currently configurable
+const (
+	resolveLatestTag  = false
+	resolveRetries    = 0
+	resolveTimeout    = time.Second * 5
+	registryScheme    = "https"
+	registryNamespace = "k8s.io"
+	defaultRouterPort = "5001"
+)
+
+func init() {
+	// ensure that spegel exposes metrics through the same registry used by Kubernetes components
+	metrics.DefaultRegisterer = legacyregistry.Registerer()
+	metrics.DefaultGatherer = legacyregistry.DefaultGatherer
+}
+
+// Start starts the embedded p2p router, and binds the registry API to an existing HTTP router.
+func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
+	// distribute images for all configured mirrors. there doesn't need to be a
+	// configured endpoint, just having a key for the registry will do.
+	urls := []url.URL{}
+	registries := []string{}
+	for host := range nodeConfig.AgentConfig.Registry.Mirrors {
+		if u, err := url.Parse("https://" + host); err != nil || host == "*" {
+			logrus.Errorf("Distributed registry mirror skipping unsupported registry: %s", host)
+		} else {
+			urls = append(urls, *u)
+			registries = append(registries, host)
+		}
+	}
+
+	if len(registries) == 0 {
+		logrus.Errorf("Not starting distributed registry mirror: no registries configured for distributed mirroring")
+		return nil
+	}
+
+	logrus.Infof("Starting distributed registry mirror at https://%s:%s/v2 for registries %v",
+		c.ExternalAddress, c.RegistryPort, registries)
+
+	// set up the various logging logging frameworks
+	level := ipfslog.LevelInfo
+	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+		level = ipfslog.LevelDebug
+		stdlog := log.New(logrus.StandardLogger().Writer(), "spegel ", log.LstdFlags)
+		logger := stdr.NewWithOptions(stdlog, stdr.Options{Verbosity: pointer.Int(10)})
+		ctx = logr.NewContext(ctx, logger)
+	}
+	ipfslog.SetAllLoggers(level)
+
+	// Get containerd client
+	ociClient, err := oci.NewContainerd(nodeConfig.Containerd.Address, registryNamespace, nodeConfig.Containerd.Registry, urls)
+	if err != nil {
+		return errors.Wrap(err, "failed to create OCI client")
+	}
+
+	// create or load persistent private key
+	keyFile := filepath.Join(nodeConfig.Containerd.Opt, "peer.key")
+	keyBytes, _, err := cert.LoadOrGenerateKeyFile(keyFile, false)
+	if err != nil {
+		return errors.Wrap(err, "failed to load or generate p2p private key")
+	}
+	privKey, err := cert.ParsePrivateKeyPEM(keyBytes)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse p2p private key")
+	}
+	p2pKey, _, err := crypto.KeyPairFromStdKey(privKey)
+	if err != nil {
+		return errors.Wrap(err, "failed to convert p2p private key")
+	}
+
+	// create a peerstore to allow persisting nodes across restarts
+	peerFile := filepath.Join(nodeConfig.Containerd.Opt, "peerstore.db")
+	ds, err := leveldb.NewDatastore(peerFile, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to create peerstore datastore")
+	}
+	ps, err := pstoreds.NewPeerstore(ctx, ds, pstoreds.DefaultOpts())
+	if err != nil {
+		return errors.Wrap(err, "failed to create peerstore")
+	}
+
+	// get port and start p2p router
+	routerPort := defaultRouterPort
+	if env := os.Getenv(P2pPortEnv); env != "" {
+		if i, err := strconv.Atoi(env); i == 0 || err != nil {
+			logrus.Warnf("Invalid P2P node port; using default")
+		} else {
+			routerPort = env
+		}
+	}
+	routerAddr := net.JoinHostPort(c.ExternalAddress, routerPort)
+
+	logrus.Infof("Starting distributed registry P2P node at %s", routerAddr)
+	opts := []libp2p.Option{
+		libp2p.Identity(p2pKey),
+		libp2p.Peerstore(ps),
+		libp2p.PrivateNetwork(c.PSK),
+		libp2p.PrometheusRegisterer(metrics.DefaultRegisterer),
+	}
+	router, err := routing.NewP2PRouter(ctx, routerAddr, c.Bootstrapper, c.RegistryPort, registryScheme, opts...)
+	if err != nil {
+		return errors.Wrap(err, "failed to create p2p router")
+	}
+
+	caCert, err := os.ReadFile(c.ServerCAFile)
+	if err != nil {
+		return errors.Wrap(err, "failed to read server CA")
+	}
+	localAddr := net.JoinHostPort(c.InternalAddress, c.RegistryPort)
+	client := clientaccess.GetHTTPClient(caCert, c.ClientCertFile, c.ClientKeyFile)
+	metrics.Register()
+	reg := registry.NewRegistry(ociClient, router, localAddr, resolveRetries, resolveTimeout, resolveLatestTag, client.Transport)
+	regSvr := reg.Server(":"+c.RegistryPort, logr.FromContextOrDiscard(ctx))
+
+	// Close router on shutdown
+	go func() {
+		<-ctx.Done()
+		router.Close()
+	}()
+
+	// Track images available in containerd and publish via p2p router
+	go state.Track(ctx, ociClient, router, resolveLatestTag)
+
+	mRouter := mux.NewRouter().SkipClean(true)
+	mRouter.Use(c.authMiddleware())
+	mRouter.PathPrefix("/v2").Handler(regSvr.Handler)
+	mRouter.PathPrefix("/v1-" + version.Program + "/p2p").Handler(c.peerInfo())
+	if err := c.HandlerFunc(c, mRouter); err != nil {
+		return err
+	}
+
+	// Wait up to 5 seconds for the p2p network to find peers. This will return
+	// immediately if the node is bootstrapping from itself.
+	wait.PollImmediateWithContext(ctx, time.Second, resolveTimeout, func(_ context.Context) (bool, error) {
+		return router.HasMirrors()
+	})
+
+	return nil
+}
+
+// peerInfo sends a peer address retrieved from the bootstrapper via HTTP
+func (c *Config) peerInfo() http.HandlerFunc {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		client, _, _ := net.SplitHostPort(req.RemoteAddr)
+		info, err := c.Bootstrapper.GetAddress()
+		if err != nil {
+			http.Error(resp, "Internal Error", http.StatusInternalServerError)
+			return
+		}
+		logrus.Debugf("Serving p2p peer addr %s to client at %s", info, client)
+		resp.WriteHeader(http.StatusOK)
+		resp.Header().Set("Content-Type", "text/plain")
+		fmt.Fprintf(resp, "%s/p2p/%s", info.Addrs[0].String(), info.ID.Pretty())
+	})
+}
+
+// authMiddleware calls the configured authenticator to gate access to the registry API
+func (c *Config) authMiddleware() mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+			if _, ok, err := c.AuthFunc().AuthenticateRequest(req); !ok || err != nil {
+				http.Error(resp, "Unauthorized", http.StatusUnauthorized)
+				return
+			}
+			next.ServeHTTP(resp, req)
+		})
+	}
+}