From 7a2a2d075c3e4b47a38ffe01d201a612608a4eae Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 11 Mar 2024 22:18:55 +0000 Subject: [PATCH] Move error response generation code into util Signed-off-by: Brad Davidson --- pkg/daemons/control/tunnel.go | 31 ++------------- pkg/server/auth.go | 39 +++---------------- pkg/server/router.go | 50 ++++++------------------ pkg/util/apierrors.go | 71 +++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 100 deletions(-) create mode 100644 pkg/util/apierrors.go diff --git a/pkg/daemons/control/tunnel.go b/pkg/daemons/control/tunnel.go index e363b20e95..3c4b2d54ce 100644 --- a/pkg/daemons/control/tunnel.go +++ b/pkg/daemons/control/tunnel.go @@ -13,7 +13,6 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/control/proxy" - "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" "github.com/k3s-io/k3s/pkg/nodeconfig" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" @@ -22,10 +21,6 @@ import ( "github.com/sirupsen/logrus" "github.com/yl2chen/cidranger" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/kubernetes" ) @@ -173,29 +168,20 @@ func (t *TunnelServer) onChangePod(podName string, pod *v1.Pod) (*v1.Pod, error) func (t *TunnelServer) serveConnect(resp http.ResponseWriter, req *http.Request) { bconn, err := t.dialBackend(req.Context(), req.Host) if err != nil { - responsewriters.ErrorNegotiated( - newBadGateway(err.Error()), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(err, resp, req, http.StatusBadGateway) return } hijacker, ok := resp.(http.Hijacker) if !ok { - responsewriters.ErrorNegotiated( - apierrors.NewInternalError(errors.New("hijacking not supported")), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(errors.New("hijacking not supported"), resp, req, http.StatusInternalServerError) return } resp.WriteHeader(http.StatusOK) rconn, bufrw, err := hijacker.Hijack() if err != nil { - responsewriters.ErrorNegotiated( - apierrors.NewInternalError(err), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(err, resp, req, http.StatusInternalServerError) return } @@ -301,14 +287,3 @@ func (crw *connReadWriteCloser) Close() (err error) { crw.once.Do(func() { err = crw.conn.Close() }) return } - -func newBadGateway(message string) *apierrors.StatusError { - return &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Status: metav1.StatusFailure, - Code: http.StatusBadGateway, - Reason: metav1.StatusReasonInternalError, - Message: message, - }, - } -} diff --git a/pkg/server/auth.go b/pkg/server/auth.go index d0795e12fa..306b2d77b2 100644 --- a/pkg/server/auth.go +++ b/pkg/server/auth.go @@ -5,12 +5,9 @@ import ( "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/daemons/config" - "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" + "github.com/k3s-io/k3s/pkg/util" + "github.com/pkg/errors" "github.com/sirupsen/logrus" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" ) @@ -29,23 +26,23 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw switch { case serverConfig == nil: logrus.Errorf("Authenticate not initialized: serverConfig is nil") - unauthorized(rw, req) + util.SendError(errors.New("not authorized"), rw, req, http.StatusUnauthorized) return case serverConfig.Runtime.Authenticator == nil: logrus.Errorf("Authenticate not initialized: serverConfig.Runtime.Authenticator is nil") - unauthorized(rw, req) + util.SendError(errors.New("not authorized"), rw, req, http.StatusUnauthorized) return } resp, ok, err := serverConfig.Runtime.Authenticator.AuthenticateRequest(req) if err != nil { logrus.Errorf("Failed to authenticate request from %s: %v", req.RemoteAddr, err) - unauthorized(rw, req) + util.SendError(errors.New("not authorized"), rw, req, http.StatusUnauthorized) return } if !ok || !hasRole(roles, resp.User.GetGroups()) { - forbidden(rw, req) + util.SendError(errors.New("forbidden"), rw, req, http.StatusForbidden) return } @@ -61,27 +58,3 @@ func authMiddleware(serverConfig *config.Control, roles ...string) mux.Middlewar }) } } - -func unauthorized(resp http.ResponseWriter, req *http.Request) { - responsewriters.ErrorNegotiated( - &apierrors.StatusError{ErrStatus: metav1.Status{ - Status: metav1.StatusFailure, - Code: http.StatusUnauthorized, - Reason: metav1.StatusReasonUnauthorized, - Message: "not authorized", - }}, - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) -} - -func forbidden(resp http.ResponseWriter, req *http.Request) { - responsewriters.ErrorNegotiated( - &apierrors.StatusError{ErrStatus: metav1.Status{ - Status: metav1.StatusFailure, - Code: http.StatusForbidden, - Reason: metav1.StatusReasonForbidden, - Message: "forbidden", - }}, - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) -} diff --git a/pkg/server/router.go b/pkg/server/router.go index e336e3ca05..984f9d71a7 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -19,7 +19,6 @@ import ( "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/daemons/config" - "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" "github.com/k3s-io/k3s/pkg/nodepassword" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" @@ -28,14 +27,11 @@ import ( coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" bootstrapapi "k8s.io/cluster-bootstrap/token/api" "k8s.io/kubernetes/pkg/auth/nodeidentifier" @@ -108,20 +104,14 @@ func apiserver(runtime *config.ControlRuntime) http.Handler { if runtime != nil && runtime.APIServer != nil { runtime.APIServer.ServeHTTP(resp, req) } else { - responsewriters.ErrorNegotiated( - apierrors.NewServiceUnavailable("apiserver not ready"), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(util.ErrNotReady, resp, req, http.StatusServiceUnavailable) } }) } func apiserverDisabled() http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - responsewriters.ErrorNegotiated( - apierrors.NewServiceUnavailable("apiserver disabled"), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(errors.New("apiserver disabled"), resp, req, http.StatusServiceUnavailable) }) } @@ -131,10 +121,7 @@ func bootstrapHandler(runtime *config.ControlRuntime) http.Handler { } return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr) - responsewriters.ErrorNegotiated( - apierrors.NewBadRequest("etcd disabled"), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) + util.SendError(errors.New("etcd disabled"), resp, req, http.StatusBadRequest) }) } @@ -145,7 +132,7 @@ func cacerts(serverCA string) http.Handler { var err error ca, err = os.ReadFile(serverCA) if err != nil { - sendError(err, resp, req) + util.SendError(err, resp, req) return } } @@ -220,13 +207,13 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo nodeName, errCode, err := auth(req) if err != nil { - sendError(err, resp, req, errCode) + util.SendError(err, resp, req, errCode) return } caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey) if err != nil { - sendError(err, resp, req) + util.SendError(err, resp, req) return } @@ -236,7 +223,7 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo for _, v := range strings.Split(nodeIP, ",") { ip := net.ParseIP(v) if ip == nil { - sendError(fmt.Errorf("invalid node IP address %s", ip), resp, req) + util.SendError(fmt.Errorf("invalid node IP address %s", ip), resp, req) return } ips = append(ips, ip) @@ -252,7 +239,7 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo }, }, key, caCerts[0], caKey) if err != nil { - sendError(err, resp, req) + util.SendError(err, resp, req) return } @@ -276,13 +263,13 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot nodeName, errCode, err := auth(req) if err != nil { - sendError(err, resp, req, errCode) + util.SendError(err, resp, req, errCode) return } caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey) if err != nil { - sendError(err, resp, req) + util.SendError(err, resp, req) return } @@ -292,7 +279,7 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, }, key, caCerts[0], caKey) if err != nil { - sendError(err, resp, req) + util.SendError(err, resp, req) return } @@ -402,21 +389,6 @@ func serveStatic(urlPrefix, staticDir string) http.Handler { return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir))) } -func sendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) { - var code int - if len(status) == 1 { - code = status[0] - } - if code == 0 || code == http.StatusOK { - code = http.StatusInternalServerError - } - logrus.Errorf("Sending HTTP %d response to %s: %v", code, req.RemoteAddr, err) - responsewriters.ErrorNegotiated( - apierrors.NewGenericServerResponse(code, req.Method, schema.GroupResource{}, req.URL.Path, err.Error(), 0, true), - scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, - ) -} - // nodePassBootstrapper returns a node name, or http error code and error type nodePassBootstrapper func(req *http.Request) (string, int, error) diff --git a/pkg/util/apierrors.go b/pkg/util/apierrors.go new file mode 100644 index 0000000000..7666a835f7 --- /dev/null +++ b/pkg/util/apierrors.go @@ -0,0 +1,71 @@ +package util + +import ( + "net/http" + + "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" +) + +var ErrNotReady = errors.New("apiserver not ready") + +// SendError sends a properly formatted error response +func SendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) { + var code int + if len(status) == 1 { + code = status[0] + } + if code == 0 || code == http.StatusOK { + code = http.StatusInternalServerError + } + + // Don't log "apiserver not ready" errors, they are frequent during startup + if !errors.Is(err, ErrNotReady) { + logrus.Errorf("Sending HTTP %d response to %s: %v", code, req.RemoteAddr, err) + } + + var serr *apierrors.StatusError + switch code { + case http.StatusBadRequest: + serr = apierrors.NewBadRequest(err.Error()) + case http.StatusUnauthorized: + serr = apierrors.NewUnauthorized(err.Error()) + case http.StatusForbidden: + serr = newForbidden(err) + case http.StatusInternalServerError: + serr = apierrors.NewInternalError(err) + case http.StatusBadGateway: + serr = newBadGateway(err) + case http.StatusServiceUnavailable: + serr = apierrors.NewServiceUnavailable(err.Error()) + default: + serr = apierrors.NewGenericServerResponse(code, req.Method, schema.GroupResource{}, req.URL.Path, err.Error(), 0, true) + } + + responsewriters.ErrorNegotiated(serr, scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req) +} + +func newForbidden(err error) *apierrors.StatusError { + return &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, + Message: err.Error(), + }} +} + +func newBadGateway(err error) *apierrors.StatusError { + return &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusBadGateway, + Reason: metav1.StatusReasonInternalError, + Message: err.Error(), + }} +}