diff --git a/pkg/agent/proxy/apiproxy.go b/pkg/agent/proxy/apiproxy.go
index f1c7a916ee..2d0503fcb4 100644
--- a/pkg/agent/proxy/apiproxy.go
+++ b/pkg/agent/proxy/apiproxy.go
@@ -119,7 +119,7 @@ func (p *proxy) SetAPIServerPort(ctx context.Context, port int) error {
 	p.apiServerURL = u.String()
 	p.apiServerEnabled = true
 
-	if p.lbEnabled {
+	if p.lbEnabled && p.apiServerLB == nil {
 		lbServerPort := p.lbServerPort
 		if lbServerPort != 0 {
 			lbServerPort = lbServerPort - 1