diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index eae8e28c056d..2af926896c22 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -33,6 +33,7 @@ import (
 	"github.com/docker/machine/libmachine"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"k8s.io/client-go/util/homedir"
 	"k8s.io/klog/v2"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/drivers/kic"
@@ -53,6 +54,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/sshagent"
 	"k8s.io/minikube/pkg/minikube/style"
+	"k8s.io/minikube/pkg/util"
 )
 
 var (
@@ -233,6 +235,14 @@ func runDelete(_ *cobra.Command, args []string) {
 	defer cancel()
 
 	if deleteAll {
+		// use a background goroutine to run this delete
+		// so that this will not block for a long time
+		deleteKnownHostsChannel := make(chan struct{})
+		go func() {
+			deleteKnownHosts()
+			deleteKnownHostsChannel <- struct{}{}
+		}()
+
 		deleteContainersAndVolumes(delCtx, oci.Docker)
 		deleteContainersAndVolumes(delCtx, oci.Podman)
 
@@ -242,8 +252,11 @@ func runDelete(_ *cobra.Command, args []string) {
 		if len(errs) > 0 {
 			HandleDeletionErrors(errs)
 		} else {
+			// if deleteKnownHosts hasn't finished, then wait
+			<-deleteKnownHostsChannel
 			out.Step(style.DeletingHost, "Successfully deleted all profiles")
 		}
+
 	} else {
 		if len(args) > 0 {
 			exit.Message(reason.Usage, "usage: minikube delete")
@@ -723,3 +736,37 @@ func getPids(path string) ([]int, error) {
 
 	return pids, nil
 }
+
+func deleteKnownHosts() {
+	// remove this line from known_hosts file if it exists
+	knownHosts := filepath.Join(homedir.HomeDir(), ".ssh", "known_hosts")
+	machinesDir := filepath.Join(localpath.MiniPath(), "machines")
+	fileInfo, err := os.ReadDir(machinesDir)
+	if err != nil {
+		klog.Warningf("error reading machines in minikube home: %v", err)
+		return
+	}
+	for _, file := range fileInfo {
+		if file.IsDir() {
+			nodeName := file.Name()
+
+			knowHostPath := filepath.Join(localpath.MiniPath(), "machines", nodeName, "known_host")
+			if _, err := os.Stat(knowHostPath); err == nil {
+				// if this file exists, remove this line from known_hosts
+				key, err := os.ReadFile(knowHostPath)
+				if err != nil {
+					klog.Warningf("error reading keys from %s: %v", knowHostPath, err)
+					continue
+				}
+				if err := util.RemoveLineFromFile(string(key), knownHosts); err != nil {
+					klog.Warningf("failed to remove key: %v", err)
+				}
+				// and, remove the file which stores this key
+				if err := os.Remove(knowHostPath); err != nil {
+					klog.Warningf("failed to remove key: %v", err)
+				}
+			}
+		}
+	}
+
+}
diff --git a/cmd/minikube/cmd/ssh-host.go b/cmd/minikube/cmd/ssh-host.go
index 51d408d74cff..73e4190f7ab5 100644
--- a/cmd/minikube/cmd/ssh-host.go
+++ b/cmd/minikube/cmd/ssh-host.go
@@ -27,6 +27,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/mustload"
 	"k8s.io/minikube/pkg/minikube/node"
@@ -122,6 +123,15 @@ func appendKnownHelper(nodeName string, appendKnown bool) {
 
 		fmt.Fprintf(os.Stderr, "Host added: %s (%s)\n", knownHosts, host)
 
+		// then store it into minikube home folder
+		// so that when we execute `minikube delete`
+		// these keys can be removed properly
+		_, cc := mustload.Partial(ClusterFlagValue())
+		knownHostPath := filepath.Join(localpath.MiniPath(), "machines", config.MachineName(*cc, *n), "known_host")
+		if err := os.WriteFile(knownHostPath, []byte(keys), 0666); err != nil {
+			out.ErrLn("WriteString to %s: %v", knownHostPath, err)
+			os.Exit(1)
+		}
 		return
 	}
 }
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index a30b9e08e555..b0a7790c0933 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -17,11 +17,13 @@ limitations under the License.
 package util
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/blang/semver/v4"
 	units "github.com/docker/go-units"
@@ -123,3 +125,35 @@ func RemoveDuplicateStrings(initial []string) []string {
 	}
 	return result
 }
+
+// remove the specified line from the given file
+func RemoveLineFromFile(knownHostLine string, filePath string) error {
+	fd, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to open %s: %v", filePath, err)
+	}
+	defer fd.Close()
+
+	// read each line from known_hosts and find theline we want to delete
+	scanner := bufio.NewScanner(fd)
+	newLines := make([]string, 0)
+	for scanner.Scan() {
+		line := string(scanner.Bytes())
+		if strings.TrimSpace(line) != strings.TrimSpace(knownHostLine) {
+			// remove the line which is identical with the key
+			newLines = append(newLines, line)
+		}
+	}
+	// remove the contents and move to the head of this file
+	fd.Truncate(0)
+	fd.Seek(0, 0)
+
+	// write the new content into the file
+
+	for _, line := range newLines {
+		if _, err := fmt.Fprintln(fd, line); err != nil {
+			return fmt.Errorf("failed to write to %s: %v", filePath, err)
+		}
+	}
+	return nil
+}