Add the ability to prune etcd snapshots (#3310)

* add prune subcommand to force rentention policy enforcement

cmd/etcdsnapshot/main.go

@@ -13,7 +13,12 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete, etcdsnapshot.List)),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run,
+			cmds.NewEtcdSnapshotSubcommands(
+				etcdsnapshot.Delete,
+				etcdsnapshot.List,
+				etcdsnapshot.Prune),
+		),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

cmd/k3s/main.go

@@ -44,7 +44,12 @@ func main() {
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
 		cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand, cmds.NewEtcdSnapshotSubcommands(etcdsnapshotCommand, etcdsnapshotCommand)),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand,
+			cmds.NewEtcdSnapshotSubcommands(
+				etcdsnapshotCommand,
+				etcdsnapshotCommand,
+				etcdsnapshotCommand),
+		),
 	}
 
 	if err := app.Run(os.Args); err != nil {

cmd/server/main.go

@@ -43,7 +43,12 @@ func main() {
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
 		cmds.NewCtrCommand(ctr.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete, etcdsnapshot.List)),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run,
+			cmds.NewEtcdSnapshotSubcommands(
+				etcdsnapshot.Delete,
+				etcdsnapshot.List,
+				etcdsnapshot.Prune),
+		),
 	}
 
 	err := app.Run(configfilearg.MustParse(os.Args))

main.go

@@ -27,7 +27,12 @@ func main() {
 		cmds.NewAgentCommand(agent.Run),
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete, etcdsnapshot.List)),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run,
+			cmds.NewEtcdSnapshotSubcommands(
+				etcdsnapshot.Delete,
+				etcdsnapshot.List,
+				etcdsnapshot.Prune),
+		),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

pkg/cli/cmds/etcd_snapshot.go

@@ -95,7 +95,7 @@ func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.C
 	}
 }
 
-func NewEtcdSnapshotSubcommands(delete, list func(ctx *cli.Context) error) []cli.Command {
+func NewEtcdSnapshotSubcommands(delete, list, prune func(ctx *cli.Context) error) []cli.Command {
 	return []cli.Command{
 		{
 			Name:            "delete",
@@ -108,11 +108,24 @@ func NewEtcdSnapshotSubcommands(delete, list func(ctx *cli.Context) error) []cli
 		{
 			Name:            "ls",
 			Aliases:         []string{"list", "l"},
-			Usage:           "List etcd snapshots",
+			Usage:           "List snapshots",
 			SkipFlagParsing: false,
 			SkipArgReorder:  true,
 			Action:          list,
 			Flags:           EtcdSnapshotFlags,
 		},
+		{
+			Name:            "prune",
+			Usage:           "Remove snapshots that exceed the configured retention count",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          prune,
+			Flags: append(EtcdSnapshotFlags, &cli.IntFlag{
+				Name:        "snapshot-retention",
+				Usage:       "(db) Number of snapshots to retain",
+				Destination: &ServerConfig.EtcdSnapshotRetention,
+				Value:       defaultSnapshotRentention,
+			}),
+		},
 	}
 }

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -181,3 +181,28 @@ func list(app *cli.Context, cfg *cmds.Server) error {
 
 	return nil
 }
+
+func Prune(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	return prune(app, &cmds.ServerConfig)
+}
+
+func prune(app *cli.Context, cfg *cmds.Server) error {
+	var serverConfig server.Config
+
+	dataDir, err := commandSetup(app, cfg, &serverConfig)
+	if err != nil {
+		return err
+	}
+
+	serverConfig.ControlConfig.DataDir = dataDir
+	serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
+
+	ctx := signals.SetupSignalHandler(context.Background())
+	e := etcd.NewETCD()
+	e.SetControlConfig(&serverConfig.ControlConfig)
+
+	return e.PruneSnapshots(ctx)
+}

pkg/etcd/etcd.go

@@ -970,6 +970,17 @@ func (e *ETCD) initS3IfNil(ctx context.Context) error {
 	return nil
 }
 
+// PruneSnapshots perfrorms a retention run with the given
+// retention duration and removes expired snapshots.
+func (e *ETCD) PruneSnapshots(ctx context.Context) error {
+	snapshotDir, err := snapshotDir(e.config)
+	if err != nil {
+		return errors.Wrap(err, "failed to get the snapshot dir")
+	}
+
+	return snapshotRetention(e.config.EtcdSnapshotRetention, snapshotDir)
+}
+
 // ListSnapshots is an exported wrapper method that wraps an
 // unexported method of the same name.
 func (e *ETCD) ListSnapshots(ctx context.Context) ([]SnapshotFile, error) {