diff --git a/pkg/authentication/bearertoken/bearertoken.go b/pkg/authentication/bearertoken/bearertoken.go
new file mode 100644
index 0000000000..0f6f7dc1d1
--- /dev/null
+++ b/pkg/authentication/bearertoken/bearertoken.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bearertoken
+
+import (
+	"net/http"
+	"strings"
+
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/warning"
+)
+
+const (
+	invalidTokenWithSpaceWarning = "the provided Authorization header contains extra space before the bearer token, and is ignored"
+)
+
+type Authenticator struct {
+	auth authenticator.Token
+}
+
+func New(auth authenticator.Token) *Authenticator {
+	return &Authenticator{auth}
+}
+
+func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
+	auth := strings.TrimSpace(req.Header.Get("Authorization"))
+	if auth == "" {
+		return nil, false, nil
+	}
+	parts := strings.SplitN(auth, " ", 3)
+	if len(parts) < 2 || strings.ToLower(parts[0]) != "bearer" {
+		return nil, false, nil
+	}
+
+	token := parts[1]
+
+	// Empty bearer tokens aren't valid
+	if len(token) == 0 {
+		// The space before the token case
+		if len(parts) == 3 {
+			warning.AddWarning(req.Context(), "", invalidTokenWithSpaceWarning)
+		}
+		return nil, false, nil
+	}
+
+	resp, ok, err := a.auth.AuthenticateToken(req.Context(), token)
+	// if we authenticated successfully, go ahead and remove the bearer token so that no one
+	// is ever tempted to use it inside of the API server
+	if ok {
+		req.Header.Del("Authorization")
+	}
+
+	return resp, ok, err
+}
diff --git a/pkg/authentication/delegatingauthenticator/delegatingauthenticator.go b/pkg/authentication/delegatingauthenticator/delegatingauthenticator.go
index bf3cab889d..419d33f9a1 100644
--- a/pkg/authentication/delegatingauthenticator/delegatingauthenticator.go
+++ b/pkg/authentication/delegatingauthenticator/delegatingauthenticator.go
@@ -2,14 +2,13 @@ package delegatingauthenticator
 
 import (
 	"context"
-	"errors"
 	"time"
 
 	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/loft-sh/vcluster/pkg/authentication/bearertoken"
 	"github.com/loft-sh/vcluster/pkg/util/clienthelper"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -50,7 +49,7 @@ func (d *delegatingAuthenticator) AuthenticateToken(ctx context.Context, token s
 	if err != nil {
 		return nil, false, err
 	} else if !tokReview.Status.Authenticated {
-		return nil, false, errors.New(tokReview.Status.Error)
+		return nil, false, nil
 	}
 
 	response := &authenticator.Response{
diff --git a/pkg/server/server.go b/pkg/server/server.go
index ea89f2fde6..6e8e41d9fd 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -31,6 +31,7 @@ import (
 	webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
 	unionauthentication "k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/apiserver/pkg/authorization/union"
 	"k8s.io/apiserver/pkg/endpoints/filterlatency"
@@ -51,6 +52,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+// ExtraAuthenticators are extra authenticators that should be added to the server
+var ExtraAuthenticators []authenticator.Request
+
 // Server is a http.Handler which proxies Kubernetes APIs to remote API server.
 type Server struct {
 	uncachedVirtualClient  client.Client
@@ -222,8 +226,15 @@ func (s *Server) ServeOnListenerTLS(address string, port int, stopChan <-chan st
 		return err
 	}
 
-	// make sure the tokens are correctly authenticated
-	serverConfig.Authentication.Authenticator = unionauthentication.NewFailOnError(delegatingauthenticator.New(s.uncachedVirtualClient), serverConfig.Authentication.Authenticator)
+	// make sure the tokens are correctly authenticated. We use the following order:
+	// 1. try the service account token one first since it's cheap to check this.
+	// 2. try the extra authenticators like platform that might take longer
+	// 3. last is the certificate authenticator
+	authenticators := []authenticator.Request{}
+	authenticators = append(authenticators, delegatingauthenticator.New(s.uncachedVirtualClient))
+	authenticators = append(authenticators, ExtraAuthenticators...)
+	authenticators = append(authenticators, serverConfig.Authentication.Authenticator)
+	serverConfig.Authentication.Authenticator = unionauthentication.NewFailOnError(authenticators...)
 
 	// create server
 	klog.Info("Starting tls proxy server at " + address + ":" + strconv.Itoa(port))