diff --git a/src/value/map.rs b/src/value/map.rs
index 2c100c024..fd840caa4 100644
--- a/src/value/map.rs
+++ b/src/value/map.rs
@@ -62,7 +62,14 @@ impl Map {
 
     /// Removes an element by its `key`.
     pub fn remove(&mut self, key: &Value) -> Option<Value> {
-        self.0.remove(key)
+        #[cfg(feature = "indexmap")]
+        {
+            self.0.shift_remove(key)
+        }
+        #[cfg(not(feature = "indexmap"))]
+        {
+            self.0.remove(key)
+        }
     }
 
     /// Iterate all key-value pairs.