2019-01-09 16:54:15 +00:00
package server
import (
"context"
"fmt"
2021-02-01 19:11:17 +00:00
"net"
2019-01-09 16:54:15 +00:00
"os"
"path/filepath"
2019-01-22 21:14:58 +00:00
"strings"
2021-02-12 15:35:57 +00:00
"time"
2019-01-09 16:54:15 +00:00
2019-03-07 03:49:17 +00:00
systemd "github.com/coreos/go-systemd/daemon"
2020-07-27 19:09:21 +00:00
"github.com/erikdubbelboer/gspt"
2019-03-04 05:25:02 +00:00
"github.com/pkg/errors"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/agent"
"github.com/rancher/k3s/pkg/cli/cmds"
2021-06-22 20:42:34 +00:00
"github.com/rancher/k3s/pkg/clientaccess"
2019-03-08 22:47:44 +00:00
"github.com/rancher/k3s/pkg/datadir"
2021-02-12 15:35:57 +00:00
"github.com/rancher/k3s/pkg/etcd"
2019-07-26 21:54:44 +00:00
"github.com/rancher/k3s/pkg/netutil"
2019-03-08 22:47:44 +00:00
"github.com/rancher/k3s/pkg/rootless"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/server"
2019-10-27 05:53:25 +00:00
"github.com/rancher/k3s/pkg/token"
2021-04-21 22:56:20 +00:00
"github.com/rancher/k3s/pkg/util"
2020-05-05 22:09:04 +00:00
"github.com/rancher/k3s/pkg/version"
2019-05-09 22:05:51 +00:00
"github.com/rancher/wrangler/pkg/signals"
2019-01-09 16:54:15 +00:00
"github.com/sirupsen/logrus"
2020-08-29 19:46:55 +00:00
"github.com/urfave/cli"
2021-02-01 19:11:17 +00:00
utilnet "k8s.io/apimachinery/pkg/util/net"
2020-05-06 17:43:15 +00:00
kubeapiserverflag "k8s.io/component-base/cli/flag"
2020-12-01 01:06:26 +00:00
"k8s.io/kubernetes/pkg/controlplane"
2021-04-21 22:56:20 +00:00
utilsnet "k8s.io/utils/net"
2019-01-22 21:14:58 +00:00
2019-05-15 23:05:24 +00:00
_ "github.com/go-sql-driver/mysql" // ensure we have mysql
_ "github.com/lib/pq" // ensure we have postgres
_ "github.com/mattn/go-sqlite3" // ensure we have sqlite
2019-01-09 16:54:15 +00:00
)
func Run ( app * cli . Context ) error {
2020-08-29 19:46:55 +00:00
if err := cmds . InitLogging ( ) ; err != nil {
return err
}
2021-03-11 18:39:00 +00:00
return run ( app , & cmds . ServerConfig , server . CustomControllers { } , server . CustomControllers { } )
2019-01-09 16:54:15 +00:00
}
2021-03-11 18:39:00 +00:00
func RunWithControllers ( app * cli . Context , leaderControllers server . CustomControllers , controllers server . CustomControllers ) error {
if err := cmds . InitLogging ( ) ; err != nil {
return err
}
return run ( app , & cmds . ServerConfig , leaderControllers , controllers )
}
func run ( app * cli . Context , cfg * cmds . Server , leaderControllers server . CustomControllers , controllers server . CustomControllers ) error {
2019-03-04 05:25:02 +00:00
var (
err error
)
2020-07-27 19:09:21 +00:00
// hide process arguments from ps output, since they may contain
// database credentials or other secrets.
gspt . SetProcTitle ( os . Args [ 0 ] + " server" )
2019-03-08 22:47:44 +00:00
if ! cfg . DisableAgent && os . Getuid ( ) != 0 && ! cfg . Rootless {
2019-01-09 16:54:15 +00:00
return fmt . Errorf ( "must run as root unless --disable-agent is specified" )
}
2019-03-08 22:47:44 +00:00
if cfg . Rootless {
dataDir , err := datadir . LocalHome ( cfg . DataDir , true )
if err != nil {
return err
}
cfg . DataDir = dataDir
if err := rootless . Rootless ( dataDir ) ; err != nil {
return err
}
}
2019-11-14 19:42:42 +00:00
if cfg . Token == "" && cfg . ClusterSecret != "" {
cfg . Token = cfg . ClusterSecret
}
2019-01-09 16:54:15 +00:00
serverConfig := server . Config { }
2019-10-28 05:43:11 +00:00
serverConfig . DisableAgent = cfg . DisableAgent
2019-10-27 05:53:25 +00:00
serverConfig . ControlConfig . Token = cfg . Token
serverConfig . ControlConfig . AgentToken = cfg . AgentToken
serverConfig . ControlConfig . JoinURL = cfg . ServerURL
if cfg . AgentTokenFile != "" {
serverConfig . ControlConfig . AgentToken , err = token . ReadFile ( cfg . AgentTokenFile )
if err != nil {
return err
}
}
if cfg . TokenFile != "" {
serverConfig . ControlConfig . Token , err = token . ReadFile ( cfg . TokenFile )
if err != nil {
return err
}
}
2019-01-09 16:54:15 +00:00
serverConfig . ControlConfig . DataDir = cfg . DataDir
2019-01-22 21:14:58 +00:00
serverConfig . ControlConfig . KubeConfigOutput = cfg . KubeConfigOutput
serverConfig . ControlConfig . KubeConfigMode = cfg . KubeConfigMode
2019-03-08 22:47:44 +00:00
serverConfig . Rootless = cfg . Rootless
2019-10-27 05:53:25 +00:00
serverConfig . ControlConfig . SANs = knownIPs ( cfg . TLSSan )
serverConfig . ControlConfig . BindAddress = cfg . BindAddress
2020-04-28 22:00:30 +00:00
serverConfig . ControlConfig . SupervisorPort = cfg . SupervisorPort
2019-05-29 18:53:51 +00:00
serverConfig . ControlConfig . HTTPSPort = cfg . HTTPSPort
2020-04-28 22:00:30 +00:00
serverConfig . ControlConfig . APIServerPort = cfg . APIServerPort
serverConfig . ControlConfig . APIServerBindAddress = cfg . APIServerBindAddress
2019-04-05 00:43:00 +00:00
serverConfig . ControlConfig . ExtraAPIArgs = cfg . ExtraAPIArgs
serverConfig . ControlConfig . ExtraControllerArgs = cfg . ExtraControllerArgs
serverConfig . ControlConfig . ExtraSchedulerAPIArgs = cfg . ExtraSchedulerArgs
2019-04-12 06:06:35 +00:00
serverConfig . ControlConfig . ClusterDomain = cfg . ClusterDomain
2019-11-16 00:12:27 +00:00
serverConfig . ControlConfig . Datastore . Endpoint = cfg . DatastoreEndpoint
serverConfig . ControlConfig . Datastore . CAFile = cfg . DatastoreCAFile
serverConfig . ControlConfig . Datastore . CertFile = cfg . DatastoreCertFile
serverConfig . ControlConfig . Datastore . KeyFile = cfg . DatastoreKeyFile
2019-05-29 18:53:51 +00:00
serverConfig . ControlConfig . AdvertiseIP = cfg . AdvertiseIP
serverConfig . ControlConfig . AdvertisePort = cfg . AdvertisePort
2019-09-03 23:41:54 +00:00
serverConfig . ControlConfig . FlannelBackend = cfg . FlannelBackend
2019-10-15 21:17:26 +00:00
serverConfig . ControlConfig . ExtraCloudControllerArgs = cfg . ExtraCloudControllerArgs
serverConfig . ControlConfig . DisableCCM = cfg . DisableCCM
2019-10-17 21:46:15 +00:00
serverConfig . ControlConfig . DisableNPC = cfg . DisableNPC
2021-06-25 18:54:36 +00:00
serverConfig . ControlConfig . DisableHelmController = cfg . DisableHelmController
2020-04-27 16:31:25 +00:00
serverConfig . ControlConfig . DisableKubeProxy = cfg . DisableKubeProxy
2021-02-12 15:35:57 +00:00
serverConfig . ControlConfig . DisableETCD = cfg . DisableETCD
serverConfig . ControlConfig . DisableAPIServer = cfg . DisableAPIServer
serverConfig . ControlConfig . DisableScheduler = cfg . DisableScheduler
serverConfig . ControlConfig . DisableControllerManager = cfg . DisableControllerManager
2019-10-27 05:53:25 +00:00
serverConfig . ControlConfig . ClusterInit = cfg . ClusterInit
2019-12-12 22:41:10 +00:00
serverConfig . ControlConfig . EncryptSecrets = cfg . EncryptSecrets
2021-07-09 17:22:49 +00:00
serverConfig . ControlConfig . EtcdExposeMetrics = cfg . EtcdExposeMetrics
serverConfig . ControlConfig . EtcdDisableSnapshots = cfg . EtcdDisableSnapshots
2021-06-04 16:18:16 +00:00
2021-07-09 17:22:49 +00:00
if ! cfg . EtcdDisableSnapshots {
serverConfig . ControlConfig . EtcdSnapshotName = cfg . EtcdSnapshotName
serverConfig . ControlConfig . EtcdSnapshotCron = cfg . EtcdSnapshotCron
serverConfig . ControlConfig . EtcdSnapshotDir = cfg . EtcdSnapshotDir
serverConfig . ControlConfig . EtcdSnapshotRetention = cfg . EtcdSnapshotRetention
serverConfig . ControlConfig . EtcdS3 = cfg . EtcdS3
serverConfig . ControlConfig . EtcdS3Endpoint = cfg . EtcdS3Endpoint
serverConfig . ControlConfig . EtcdS3EndpointCA = cfg . EtcdS3EndpointCA
serverConfig . ControlConfig . EtcdS3SkipSSLVerify = cfg . EtcdS3SkipSSLVerify
serverConfig . ControlConfig . EtcdS3AccessKey = cfg . EtcdS3AccessKey
serverConfig . ControlConfig . EtcdS3SecretKey = cfg . EtcdS3SecretKey
serverConfig . ControlConfig . EtcdS3BucketName = cfg . EtcdS3BucketName
serverConfig . ControlConfig . EtcdS3Region = cfg . EtcdS3Region
serverConfig . ControlConfig . EtcdS3Folder = cfg . EtcdS3Folder
} else {
2021-06-04 16:18:16 +00:00
logrus . Info ( "ETCD snapshots are disabled" )
}
Galal hussein etcd backup restore (#2154)
* Add etcd snapshot and restore
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* fix error logs
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* goimports
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* fix flag describtion
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* Add disable snapshot and retention
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* use creation time for snapshot retention
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* unexport method, update var name
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* adjust snapshot flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var name, string concat
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* revert previous change, create constants
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* type assertion error checking
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* simplify logic, remove unneeded function
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add comment
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update disable snapshots flag and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* move function
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update defaultSnapshotIntervalMinutes to 12 like rke
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update directory perms
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update etc-snapshot-dir usage
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update interval to 12 hours
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* fix usage typo
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update deps target to work, add build/data target for creation, and generate
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* remove dead make targets
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* error handling, cluster reset functionality
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* error handling, cluster reset functionality
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* remove intermediate dapper file
Signed-off-by: Brian Downs <brian.downs@gmail.com>
Co-authored-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2020-08-28 23:57:40 +00:00
if cfg . ClusterResetRestorePath != "" && ! cfg . ClusterReset {
2021-04-21 22:56:20 +00:00
return errors . New ( "invalid flag use; --cluster-reset required with --cluster-reset-restore-path" )
Galal hussein etcd backup restore (#2154)
* Add etcd snapshot and restore
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* fix error logs
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* goimports
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* fix flag describtion
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* Add disable snapshot and retention
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* use creation time for snapshot retention
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* unexport method, update var name
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* adjust snapshot flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var name, string concat
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* revert previous change, create constants
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* type assertion error checking
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* simplify logic, remove unneeded function
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add comment
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update disable snapshots flag and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* move function
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update defaultSnapshotIntervalMinutes to 12 like rke
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update directory perms
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update etc-snapshot-dir usage
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update interval to 12 hours
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* fix usage typo
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update deps target to work, add build/data target for creation, and generate
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* remove dead make targets
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* error handling, cluster reset functionality
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* error handling, cluster reset functionality
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* remove intermediate dapper file
Signed-off-by: Brian Downs <brian.downs@gmail.com>
Co-authored-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2020-08-28 23:57:40 +00:00
}
2021-03-03 18:14:12 +00:00
// make sure components are disabled so we only perform a restore
// and bail out
if cfg . ClusterResetRestorePath != "" && cfg . ClusterReset {
serverConfig . ControlConfig . ClusterInit = true
serverConfig . ControlConfig . DisableAPIServer = true
serverConfig . ControlConfig . DisableControllerManager = true
serverConfig . ControlConfig . DisableScheduler = true
serverConfig . ControlConfig . DisableCCM = true
}
Galal hussein etcd backup restore (#2154)
* Add etcd snapshot and restore
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* fix error logs
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* goimports
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* fix flag describtion
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* Add disable snapshot and retention
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* use creation time for snapshot retention
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
* unexport method, update var name
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* adjust snapshot flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var name, string concat
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* revert previous change, create constants
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* type assertion error checking
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* pr remediation
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* updates
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* simplify logic, remove unneeded function
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update flags
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add comment
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* exit on restore completion, update flag names, move retention check
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update disable snapshots flag and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* move function
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update var and field names
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update defaultSnapshotIntervalMinutes to 12 like rke
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update directory perms
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update etc-snapshot-dir usage
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update interval to 12 hours
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* fix usage typo
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* add cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* wire in cron
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update deps target to work, add build/data target for creation, and generate
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* remove dead make targets
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* error handling, cluster reset functionality
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* error handling, cluster reset functionality
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* update
Signed-off-by: Brian Downs <brian.downs@gmail.com>
* remove intermediate dapper file
Signed-off-by: Brian Downs <brian.downs@gmail.com>
Co-authored-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2020-08-28 23:57:40 +00:00
serverConfig . ControlConfig . ClusterReset = cfg . ClusterReset
serverConfig . ControlConfig . ClusterResetRestorePath = cfg . ClusterResetRestorePath
2021-05-10 22:58:41 +00:00
serverConfig . ControlConfig . SystemDefaultRegistry = cfg . SystemDefaultRegistry
2019-05-29 18:53:51 +00:00
2020-04-28 22:00:30 +00:00
if serverConfig . ControlConfig . SupervisorPort == 0 {
serverConfig . ControlConfig . SupervisorPort = serverConfig . ControlConfig . HTTPSPort
}
2021-03-06 10:29:57 +00:00
if serverConfig . ControlConfig . DisableETCD && serverConfig . ControlConfig . JoinURL == "" {
2021-04-21 22:56:20 +00:00
return errors . New ( "invalid flag use; --server is required with --disable-etcd" )
2021-03-06 10:29:57 +00:00
}
2021-02-12 15:35:57 +00:00
if serverConfig . ControlConfig . DisableAPIServer {
2021-03-06 10:29:57 +00:00
// Servers without a local apiserver need to connect to the apiserver via the proxy load-balancer.
serverConfig . ControlConfig . APIServerPort = cmds . AgentConfig . LBServerPort
// If the supervisor and externally-facing apiserver are not on the same port, the proxy will
// have a separate load-balancer for the apiserver that we need to use instead.
2021-02-12 15:35:57 +00:00
if serverConfig . ControlConfig . SupervisorPort != serverConfig . ControlConfig . HTTPSPort {
2021-03-06 10:29:57 +00:00
serverConfig . ControlConfig . APIServerPort = cmds . AgentConfig . LBServerPort - 1
2021-02-12 15:35:57 +00:00
}
}
2021-04-21 22:56:20 +00:00
if cmds . AgentConfig . FlannelIface != "" && len ( cmds . AgentConfig . NodeIP ) == 0 {
cmds . AgentConfig . NodeIP . Set ( netutil . GetIPFromInterface ( cmds . AgentConfig . FlannelIface ) )
2019-07-08 23:02:06 +00:00
}
2021-04-21 22:56:20 +00:00
if serverConfig . ControlConfig . PrivateIP == "" && len ( cmds . AgentConfig . NodeIP ) != 0 {
// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
serverConfig . ControlConfig . PrivateIP , _ = util . GetFirst4String ( cmds . AgentConfig . NodeIP )
2020-10-28 19:55:10 +00:00
}
2021-04-21 22:56:20 +00:00
// if not set, try setting advertise-ip from agent node-external-ip
if serverConfig . ControlConfig . AdvertiseIP == "" && len ( cmds . AgentConfig . NodeExternalIP ) != 0 {
serverConfig . ControlConfig . AdvertiseIP , _ = util . GetFirst4String ( cmds . AgentConfig . NodeExternalIP )
2019-10-27 05:53:25 +00:00
}
2021-04-21 22:56:20 +00:00
// if not set, try setting advertise-up from agent node-ip
if serverConfig . ControlConfig . AdvertiseIP == "" && len ( cmds . AgentConfig . NodeIP ) != 0 {
serverConfig . ControlConfig . AdvertiseIP , _ = util . GetFirst4String ( cmds . AgentConfig . NodeIP )
2019-05-29 18:53:51 +00:00
}
2021-04-21 22:56:20 +00:00
// if we ended up with any advertise-ips, ensure they're added to the SAN list;
// note that kube-apiserver does not support dual-stack advertise-ip as of 1.21.0:
/// https://github.com/kubernetes/kubeadm/issues/1612#issuecomment-772583989
2019-05-29 18:53:51 +00:00
if serverConfig . ControlConfig . AdvertiseIP != "" {
2019-10-27 05:53:25 +00:00
serverConfig . ControlConfig . SANs = append ( serverConfig . ControlConfig . SANs , serverConfig . ControlConfig . AdvertiseIP )
2019-05-29 18:53:51 +00:00
}
2019-01-09 16:54:15 +00:00
2021-04-21 22:56:20 +00:00
// configure ClusterIPRanges
if len ( cmds . ServerConfig . ClusterCIDR ) == 0 {
cmds . ServerConfig . ClusterCIDR . Set ( "10.42.0.0/16" )
}
for _ , cidr := range cmds . ServerConfig . ClusterCIDR {
for _ , v := range strings . Split ( cidr , "," ) {
_ , parsed , err := net . ParseCIDR ( v )
if err != nil {
return errors . Wrapf ( err , "invalid cluster-cidr %s" , v )
}
serverConfig . ControlConfig . ClusterIPRanges = append ( serverConfig . ControlConfig . ClusterIPRanges , parsed )
}
}
// set ClusterIPRange to the first IPv4 block, for legacy clients
clusterIPRange , err := util . GetFirst4Net ( serverConfig . ControlConfig . ClusterIPRanges )
2019-03-04 05:25:02 +00:00
if err != nil {
2021-04-21 22:56:20 +00:00
return errors . Wrap ( err , "cannot configure IPv4 cluster-cidr" )
2019-03-04 05:25:02 +00:00
}
2021-04-21 22:56:20 +00:00
serverConfig . ControlConfig . ClusterIPRange = clusterIPRange
// configure ServiceIPRanges
if len ( cmds . ServerConfig . ServiceCIDR ) == 0 {
cmds . ServerConfig . ServiceCIDR . Set ( "10.43.0.0/16" )
}
for _ , cidr := range cmds . ServerConfig . ServiceCIDR {
for _ , v := range strings . Split ( cidr , "," ) {
_ , parsed , err := net . ParseCIDR ( v )
if err != nil {
return errors . Wrapf ( err , "invalid service-cidr %s" , v )
}
serverConfig . ControlConfig . ServiceIPRanges = append ( serverConfig . ControlConfig . ServiceIPRanges , parsed )
}
}
// set ServiceIPRange to the first IPv4 block, for legacy clients
serviceIPRange , err := util . GetFirst4Net ( serverConfig . ControlConfig . ServiceIPRanges )
2019-03-06 10:37:03 +00:00
if err != nil {
2021-04-21 22:56:20 +00:00
return errors . Wrap ( err , "cannot configure IPv4 service-cidr" )
2019-03-06 10:37:03 +00:00
}
2021-04-21 22:56:20 +00:00
serverConfig . ControlConfig . ServiceIPRange = serviceIPRange
2019-03-06 18:41:07 +00:00
2021-02-01 19:11:17 +00:00
serverConfig . ControlConfig . ServiceNodePortRange , err = utilnet . ParsePortRange ( cfg . ServiceNodePortRange )
if err != nil {
2021-04-21 22:56:20 +00:00
return errors . Wrapf ( err , "invalid port range %s" , cfg . ServiceNodePortRange )
2021-02-01 19:11:17 +00:00
}
2021-04-21 22:56:20 +00:00
// the apiserver service does not yet support dual-stack operation
2020-12-01 01:06:26 +00:00
_ , apiServerServiceIP , err := controlplane . ServiceIPRange ( * serverConfig . ControlConfig . ServiceIPRange )
2019-05-29 18:53:51 +00:00
if err != nil {
return err
}
2019-10-27 05:53:25 +00:00
serverConfig . ControlConfig . SANs = append ( serverConfig . ControlConfig . SANs , apiServerServiceIP . String ( ) )
2019-05-29 18:53:51 +00:00
2021-04-21 22:56:20 +00:00
// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
2019-03-06 18:41:07 +00:00
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
2021-04-21 22:56:20 +00:00
// If there are no IPv4 ServiceCIDRs, an error will be raised.
if len ( cmds . ServerConfig . ClusterDNS ) == 0 {
clusterDNS , err := utilsnet . GetIndexedIP ( serverConfig . ControlConfig . ServiceIPRange , 10 )
if err != nil {
return errors . Wrap ( err , "cannot configure default cluster-dns address" )
}
serverConfig . ControlConfig . ClusterDNS = clusterDNS
serverConfig . ControlConfig . ClusterDNSs = [ ] net . IP { serverConfig . ControlConfig . ClusterDNS }
2019-03-06 18:41:07 +00:00
} else {
2021-04-21 22:56:20 +00:00
for _ , ip := range cmds . ServerConfig . ClusterDNS {
for _ , v := range strings . Split ( ip , "," ) {
parsed := net . ParseIP ( v )
if parsed == nil {
return fmt . Errorf ( "invalid cluster-dns address %s" , v )
}
serverConfig . ControlConfig . ClusterDNSs = append ( serverConfig . ControlConfig . ClusterDNSs , parsed )
}
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
clusterDNS , err := util . GetFirst4 ( serverConfig . ControlConfig . ClusterDNSs )
if err != nil {
return errors . Wrap ( err , "cannot configure IPv4 cluster-dns address" )
}
serverConfig . ControlConfig . ClusterDNS = clusterDNS
}
if err := validateNetworkConfiguration ( serverConfig ) ; err != nil {
return err
2019-03-06 18:41:07 +00:00
}
2019-09-27 00:18:37 +00:00
if cfg . DefaultLocalStoragePath == "" {
dataDir , err := datadir . LocalHome ( cfg . DataDir , false )
if err != nil {
return err
}
serverConfig . ControlConfig . DefaultLocalStoragePath = filepath . Join ( dataDir , "/storage" )
} else {
serverConfig . ControlConfig . DefaultLocalStoragePath = cfg . DefaultLocalStoragePath
}
2020-01-29 23:40:49 +00:00
serverConfig . ControlConfig . Skips = map [ string ] bool { }
2019-01-22 21:14:58 +00:00
for _ , noDeploy := range app . StringSlice ( "no-deploy" ) {
2020-01-29 23:40:49 +00:00
for _ , v := range strings . Split ( noDeploy , "," ) {
2020-04-27 16:45:51 +00:00
v = strings . TrimSpace ( v )
2020-01-29 23:40:49 +00:00
serverConfig . ControlConfig . Skips [ v ] = true
2019-10-01 15:27:17 +00:00
}
}
2020-01-29 23:40:49 +00:00
serverConfig . ControlConfig . Disables = map [ string ] bool { }
for _ , disable := range app . StringSlice ( "disable" ) {
for _ , v := range strings . Split ( disable , "," ) {
2020-04-27 16:45:51 +00:00
v = strings . TrimSpace ( v )
2020-01-29 23:40:49 +00:00
serverConfig . ControlConfig . Skips [ v ] = true
serverConfig . ControlConfig . Disables [ v ] = true
2019-02-02 05:09:11 +00:00
}
2020-01-29 23:40:49 +00:00
}
if serverConfig . ControlConfig . Skips [ "servicelb" ] {
serverConfig . DisableServiceLB = true
2019-01-22 21:14:58 +00:00
}
2020-04-29 06:08:22 +00:00
if serverConfig . ControlConfig . DisableCCM {
serverConfig . ControlConfig . Skips [ "ccm" ] = true
serverConfig . ControlConfig . Disables [ "ccm" ] = true
}
2020-08-18 23:44:10 +00:00
tlsMinVersionArg := getArgValueFromList ( "tls-min-version" , cfg . ExtraAPIArgs )
serverConfig . ControlConfig . TLSMinVersion , err = kubeapiserverflag . TLSVersion ( tlsMinVersionArg )
2020-05-06 17:43:15 +00:00
if err != nil {
2021-04-21 22:56:20 +00:00
return errors . Wrap ( err , "invalid tls-min-version" )
2020-05-06 17:43:15 +00:00
}
2020-08-19 21:30:53 +00:00
serverConfig . StartupHooks = append ( serverConfig . StartupHooks , cfg . StartupHooks ... )
2020-08-19 20:30:51 +00:00
2021-03-11 18:39:00 +00:00
serverConfig . LeaderControllers = append ( serverConfig . LeaderControllers , leaderControllers ... )
serverConfig . Controllers = append ( serverConfig . Controllers , controllers ... )
2020-05-13 13:34:45 +00:00
// TLS config based on mozilla ssl-config generator
// https://ssl-config.mozilla.org/#server=golang&version=1.13.6&config=intermediate&guideline=5.4
// Need to disable the TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 Cipher for TLS1.2
2020-08-18 23:44:10 +00:00
tlsCipherSuitesArg := getArgValueFromList ( "tls-cipher-suites" , cfg . ExtraAPIArgs )
tlsCipherSuites := strings . Split ( tlsCipherSuitesArg , "," )
for i := range tlsCipherSuites {
tlsCipherSuites [ i ] = strings . TrimSpace ( tlsCipherSuites [ i ] )
}
if len ( tlsCipherSuites ) == 0 || tlsCipherSuites [ 0 ] == "" {
tlsCipherSuites = [ ] string {
2020-05-13 13:34:45 +00:00
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" ,
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" ,
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" ,
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" ,
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305" ,
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" ,
2020-05-06 17:43:15 +00:00
}
}
2020-08-18 23:44:10 +00:00
serverConfig . ControlConfig . TLSCipherSuites , err = kubeapiserverflag . TLSCipherSuites ( tlsCipherSuites )
2020-05-13 13:34:45 +00:00
if err != nil {
2021-04-21 22:56:20 +00:00
return errors . Wrap ( err , "invalid tls-cipher-suites" )
2020-05-13 13:34:45 +00:00
}
2020-05-06 17:43:15 +00:00
2020-09-21 16:56:03 +00:00
logrus . Info ( "Starting " + version . Program + " " + app . App . Version )
2019-03-07 03:49:17 +00:00
2019-05-09 22:05:51 +00:00
ctx := signals . SetupSignalHandler ( context . Background ( ) )
2021-02-12 15:35:57 +00:00
2019-10-27 05:53:25 +00:00
if err := server . StartServer ( ctx , & serverConfig ) ; err != nil {
2019-01-09 16:54:15 +00:00
return err
}
2020-04-28 22:44:05 +00:00
go func ( ) {
2021-03-01 21:50:50 +00:00
if ! serverConfig . ControlConfig . DisableAPIServer {
<- serverConfig . ControlConfig . Runtime . APIServerReady
logrus . Info ( "Kube API server is now running" )
} else {
<- serverConfig . ControlConfig . Runtime . ETCDReady
logrus . Info ( "ETCD server is now running" )
}
2021-06-15 11:20:26 +00:00
2020-09-21 16:56:03 +00:00
logrus . Info ( version . Program + " is up and running" )
2021-07-29 21:42:19 +00:00
if ( cfg . DisableAgent || cfg . DisableAPIServer ) && os . Getenv ( "NOTIFY_SOCKET" ) != "" {
2020-04-28 22:44:05 +00:00
systemd . SdNotify ( true , "READY=1\n" )
}
} ( )
2019-02-08 04:12:49 +00:00
2019-01-09 16:54:15 +00:00
if cfg . DisableAgent {
<- ctx . Done ( )
return nil
}
2019-10-27 05:53:25 +00:00
ip := serverConfig . ControlConfig . BindAddress
2019-03-31 00:10:23 +00:00
if ip == "" {
2019-08-22 18:56:00 +00:00
ip = "127.0.0.1"
2019-03-31 00:10:23 +00:00
}
2019-10-27 05:53:25 +00:00
2020-04-28 22:00:30 +00:00
url := fmt . Sprintf ( "https://%s:%d" , ip , serverConfig . ControlConfig . SupervisorPort )
2021-06-22 20:42:34 +00:00
token , err := clientaccess . FormatToken ( serverConfig . ControlConfig . Runtime . AgentToken , serverConfig . ControlConfig . Runtime . ServerCA )
2019-10-27 05:53:25 +00:00
if err != nil {
return err
}
2019-01-09 16:54:15 +00:00
agentConfig := cmds . AgentConfig
2020-09-01 17:43:19 +00:00
agentConfig . Debug = app . GlobalBool ( "debug" )
2019-01-22 21:14:58 +00:00
agentConfig . DataDir = filepath . Dir ( serverConfig . ControlConfig . DataDir )
2019-01-09 16:54:15 +00:00
agentConfig . ServerURL = url
agentConfig . Token = token
2021-02-12 15:35:57 +00:00
agentConfig . DisableLoadBalancer = ! serverConfig . ControlConfig . DisableAPIServer
agentConfig . ETCDAgent = serverConfig . ControlConfig . DisableAPIServer
2021-05-05 15:40:04 +00:00
agentConfig . ClusterReset = serverConfig . ControlConfig . ClusterReset
2021-02-12 15:35:57 +00:00
2019-10-19 10:18:51 +00:00
agentConfig . Rootless = cfg . Rootless
2021-02-12 15:35:57 +00:00
2019-10-19 10:18:51 +00:00
if agentConfig . Rootless {
// let agent specify Rootless kubelet flags, but not unshare twice
agentConfig . RootlessAlreadyUnshared = true
}
2019-01-09 16:54:15 +00:00
2021-02-12 15:35:57 +00:00
if serverConfig . ControlConfig . DisableAPIServer {
// initialize the apiAddress Channel for receiving the api address from etcd
agentConfig . APIAddressCh = make ( chan string , 1 )
setAPIAddressChannel ( ctx , & serverConfig , & agentConfig )
defer close ( agentConfig . APIAddressCh )
}
2019-01-09 16:54:15 +00:00
return agent . Run ( ctx , agentConfig )
}
2019-01-31 23:57:40 +00:00
2021-04-21 22:56:20 +00:00
// validateNetworkConfig ensures that the network configuration values make sense.
func validateNetworkConfiguration ( serverConfig server . Config ) error {
// Dual-stack operation requires fairly extensive manual configuration at the moment - do some
// preflight checks to make sure that the user isn't trying to use flannel/npc, or trying to
// enable dual-stack DNS (which we don't currently support since it's not easy to template)
dualCluster , err := utilsnet . IsDualStackCIDRs ( serverConfig . ControlConfig . ClusterIPRanges )
if err != nil {
return errors . Wrap ( err , "failed to validate cluster-cidr" )
}
dualService , err := utilsnet . IsDualStackCIDRs ( serverConfig . ControlConfig . ServiceIPRanges )
if err != nil {
return errors . Wrap ( err , "failed to validate service-cidr" )
}
dualDNS , err := utilsnet . IsDualStackIPs ( serverConfig . ControlConfig . ClusterDNSs )
if err != nil {
return errors . Wrap ( err , "failed to validate cluster-dns" )
}
if ( serverConfig . ControlConfig . FlannelBackend != "none" || serverConfig . ControlConfig . DisableNPC == false ) && ( dualCluster || dualService ) {
return errors . New ( "flannel CNI and network policy enforcement are not compatible with dual-stack operation; server must be restarted with --flannel-backend=none --disable-network-policy and an alternative CNI plugin deployed" )
}
if dualDNS == true {
return errors . New ( "dual-stack cluster-dns is not supported" )
}
return nil
}
2019-03-23 17:34:55 +00:00
func knownIPs ( ips [ ] string ) [ ] string {
ips = append ( ips , "127.0.0.1" )
2021-02-01 19:11:17 +00:00
ip , err := utilnet . ChooseHostInterface ( )
2019-01-31 23:57:40 +00:00
if err == nil {
ips = append ( ips , ip . String ( ) )
}
return ips
}
2020-05-06 17:43:15 +00:00
func getArgValueFromList ( searchArg string , argList [ ] string ) string {
var value string
for _ , arg := range argList {
splitArg := strings . SplitN ( arg , "=" , 2 )
if splitArg [ 0 ] == searchArg {
value = splitArg [ 1 ]
// break if we found our value
break
}
}
return value
}
2021-02-12 15:35:57 +00:00
// setAPIAddressChannel will try to get the api address key from etcd and when it succeed it will
// set the APIAddressCh channel with its value, the function works for both k3s and rke2 in case
// of k3s we block returning back to the agent.Run until we get the api address, however in rke2
// the code will not block operation and will run the operation in a goroutine
func setAPIAddressChannel ( ctx context . Context , serverConfig * server . Config , agentConfig * cmds . Agent ) {
// start a goroutine to check for the server ip if set from etcd in case of rke2
if serverConfig . ControlConfig . HTTPSPort != serverConfig . ControlConfig . SupervisorPort {
go getAPIAddressFromEtcd ( ctx , serverConfig , agentConfig )
return
}
getAPIAddressFromEtcd ( ctx , serverConfig , agentConfig )
}
func getAPIAddressFromEtcd ( ctx context . Context , serverConfig * server . Config , agentConfig * cmds . Agent ) {
t := time . NewTicker ( 5 * time . Second )
defer t . Stop ( )
for range t . C {
serverAddress , err := etcd . GetAPIServerURLFromETCD ( ctx , & serverConfig . ControlConfig )
if err == nil {
agentConfig . ServerURL = "https://" + serverAddress
agentConfig . APIAddressCh <- agentConfig . ServerURL
break
}
logrus . Warn ( err )
}
}