From 6ee28214fac2b1d88b6ee34ca48afab51527315e Mon Sep 17 00:00:00 2001 From: Brian Downs Date: Thu, 13 May 2021 13:36:33 -0700 Subject: [PATCH] Add the ability to prune etcd snapshots (#3310) * add prune subcommand to force rentention policy enforcement --- cmd/etcdsnapshot/main.go | 7 ++++++- cmd/k3s/main.go | 7 ++++++- cmd/server/main.go | 7 ++++++- main.go | 7 ++++++- pkg/cli/cmds/etcd_snapshot.go | 17 +++++++++++++++-- pkg/cli/etcdsnapshot/etcd_snapshot.go | 25 +++++++++++++++++++++++++ pkg/etcd/etcd.go | 11 +++++++++++ 7 files changed, 75 insertions(+), 6 deletions(-) diff --git a/cmd/etcdsnapshot/main.go b/cmd/etcdsnapshot/main.go index 2fd55364a2..38c0ed47d8 100644 --- a/cmd/etcdsnapshot/main.go +++ b/cmd/etcdsnapshot/main.go @@ -13,7 +13,12 @@ import ( func main() { app := cmds.NewApp() app.Commands = []cli.Command{ - cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete, etcdsnapshot.List)), + cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, + cmds.NewEtcdSnapshotSubcommands( + etcdsnapshot.Delete, + etcdsnapshot.List, + etcdsnapshot.Prune), + ), } if err := app.Run(configfilearg.MustParse(os.Args)); err != nil { diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 2efc0920d9..bbf13850c1 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -44,7 +44,12 @@ func main() { cmds.NewCRICTL(externalCLIAction("crictl", dataDir)), cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)), cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)), - cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand, cmds.NewEtcdSnapshotSubcommands(etcdsnapshotCommand, etcdsnapshotCommand)), + cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand, + cmds.NewEtcdSnapshotSubcommands( + etcdsnapshotCommand, + etcdsnapshotCommand, + etcdsnapshotCommand), + ), } if err := app.Run(os.Args); err != nil { diff --git a/cmd/server/main.go b/cmd/server/main.go index 6b24460beb..17fc818838 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -43,7 +43,12 @@ func main() { cmds.NewKubectlCommand(kubectl.Run), cmds.NewCRICTL(crictl.Run), cmds.NewCtrCommand(ctr.Run), - cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete, etcdsnapshot.List)), + cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, + cmds.NewEtcdSnapshotSubcommands( + etcdsnapshot.Delete, + etcdsnapshot.List, + etcdsnapshot.Prune), + ), } err := app.Run(configfilearg.MustParse(os.Args)) diff --git a/main.go b/main.go index 5c02c9aad5..a1c09d7fed 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,12 @@ func main() { cmds.NewAgentCommand(agent.Run), cmds.NewKubectlCommand(kubectl.Run), cmds.NewCRICTL(crictl.Run), - cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete, etcdsnapshot.List)), + cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, + cmds.NewEtcdSnapshotSubcommands( + etcdsnapshot.Delete, + etcdsnapshot.List, + etcdsnapshot.Prune), + ), } if err := app.Run(configfilearg.MustParse(os.Args)); err != nil { diff --git a/pkg/cli/cmds/etcd_snapshot.go b/pkg/cli/cmds/etcd_snapshot.go index 220a34535c..b76a3b3ba1 100644 --- a/pkg/cli/cmds/etcd_snapshot.go +++ b/pkg/cli/cmds/etcd_snapshot.go @@ -95,7 +95,7 @@ func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.C } } -func NewEtcdSnapshotSubcommands(delete, list func(ctx *cli.Context) error) []cli.Command { +func NewEtcdSnapshotSubcommands(delete, list, prune func(ctx *cli.Context) error) []cli.Command { return []cli.Command{ { Name: "delete", @@ -108,11 +108,24 @@ func NewEtcdSnapshotSubcommands(delete, list func(ctx *cli.Context) error) []cli { Name: "ls", Aliases: []string{"list", "l"}, - Usage: "List etcd snapshots", + Usage: "List snapshots", SkipFlagParsing: false, SkipArgReorder: true, Action: list, Flags: EtcdSnapshotFlags, }, + { + Name: "prune", + Usage: "Remove snapshots that exceed the configured retention count", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: prune, + Flags: append(EtcdSnapshotFlags, &cli.IntFlag{ + Name: "snapshot-retention", + Usage: "(db) Number of snapshots to retain", + Destination: &ServerConfig.EtcdSnapshotRetention, + Value: defaultSnapshotRentention, + }), + }, } } diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index 850b433781..72038862a5 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -181,3 +181,28 @@ func list(app *cli.Context, cfg *cmds.Server) error { return nil } + +func Prune(app *cli.Context) error { + if err := cmds.InitLogging(); err != nil { + return err + } + return prune(app, &cmds.ServerConfig) +} + +func prune(app *cli.Context, cfg *cmds.Server) error { + var serverConfig server.Config + + dataDir, err := commandSetup(app, cfg, &serverConfig) + if err != nil { + return err + } + + serverConfig.ControlConfig.DataDir = dataDir + serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention + + ctx := signals.SetupSignalHandler(context.Background()) + e := etcd.NewETCD() + e.SetControlConfig(&serverConfig.ControlConfig) + + return e.PruneSnapshots(ctx) +} diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 13f462385a..58dd1c9e9e 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -970,6 +970,17 @@ func (e *ETCD) initS3IfNil(ctx context.Context) error { return nil } +// PruneSnapshots perfrorms a retention run with the given +// retention duration and removes expired snapshots. +func (e *ETCD) PruneSnapshots(ctx context.Context) error { + snapshotDir, err := snapshotDir(e.config) + if err != nil { + return errors.Wrap(err, "failed to get the snapshot dir") + } + + return snapshotRetention(e.config.EtcdSnapshotRetention, snapshotDir) +} + // ListSnapshots is an exported wrapper method that wraps an // unexported method of the same name. func (e *ETCD) ListSnapshots(ctx context.Context) ([]SnapshotFile, error) {