diff --git a/pkg/machine/gvproxy.go b/pkg/machine/gvproxy.go
index fc236038eb..2c083cb20b 100644
--- a/pkg/machine/gvproxy.go
+++ b/pkg/machine/gvproxy.go
@@ -27,5 +27,5 @@ func CleanupGVProxy(f define.VMFile) error {
 	if err := waitOnProcess(proxyPid); err != nil {
 		return err
 	}
-	return f.Delete()
+	return removeGVProxyPIDFile(f)
 }
diff --git a/pkg/machine/gvproxy_unix.go b/pkg/machine/gvproxy_unix.go
index 20869d583e..60f4399654 100644
--- a/pkg/machine/gvproxy_unix.go
+++ b/pkg/machine/gvproxy_unix.go
@@ -8,6 +8,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containers/podman/v5/pkg/machine/define"
 	psutil "github.com/shirou/gopsutil/v3/process"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -72,3 +73,9 @@ func waitOnProcess(processID int) error {
 	}
 	return backoffForProcess(p)
 }
+
+// removeGVProxyPIDFile is just a wrapper to vmfile delete so we handle differently
+// on windows
+func removeGVProxyPIDFile(f define.VMFile) error {
+	return f.Delete()
+}
diff --git a/pkg/machine/gvproxy_windows.go b/pkg/machine/gvproxy_windows.go
index d0d5f1b725..8d572c42b1 100644
--- a/pkg/machine/gvproxy_windows.go
+++ b/pkg/machine/gvproxy_windows.go
@@ -1,11 +1,14 @@
 package machine
 
 import (
+	"errors"
 	"os"
 	"time"
 
+	"github.com/containers/podman/v5/pkg/machine/define"
 	"github.com/containers/winquit/pkg/winquit"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/windows"
 )
 
 func waitOnProcess(processID int) error {
@@ -44,3 +47,13 @@ func waitOnProcess(processID int) error {
 
 	return nil
 }
+
+// removeGVProxyPIDFile special wrapper for deleting the GVProxyPIDFile on windows in case
+// the file has an open handle which we will ignore.  unix does not have this problem
+func removeGVProxyPIDFile(f define.VMFile) error {
+	err := f.Delete()
+	if err != nil && !errors.Is(err, windows.ERROR_SHARING_VIOLATION) {
+		return err
+	}
+	return nil
+}