diff --git a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
index 4c965a12558f0..108aafd8c7771 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
@@ -125,7 +125,9 @@ public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
         if (sortBuilders != null) {
             newSortBuilders = new ArrayList<>(sortBuilders.size());
             for (var sort : sortBuilders) {
-                changed |= newSortBuilders.add(sort.rewrite(ctx));
+                var newSort = sort.rewrite(ctx);
+                newSortBuilders.add(newSort);
+                changed = newSort != sort;
             }
         }
         var rewrittenFilters = rewritePreFilters(ctx);