diff --git a/src/Commands/MultipleVotesPollHandler.php b/src/Commands/MultipleVotesPollHandler.php
index c5d3b4c7..84d4e94e 100644
--- a/src/Commands/MultipleVotesPollHandler.php
+++ b/src/Commands/MultipleVotesPollHandler.php
@@ -239,7 +239,8 @@ protected function isChangingVotes(array $optionIds, array $myVotes): bool
         // Check the arrays have the same values
         $same = (count(array_diff($optionIds, $myVotes)) === 0 && count(array_diff($myVotes, $optionIds)) === 0);
 
-        return !$same;
+        // Only when we have already voted that we are changing votes
+        return count($myVotes) !== 0 && !$same;
     }
 
     protected function validateInput(?array $optionIds, int $maxVotes, Collection $options): void