diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
index 7a202df6fa744..ba81e9bc74b4f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
@@ -491,7 +491,12 @@ private void loadModel(String modelId, Consumer consumer) {
                 handleLoadFailure(modelId, failure);
             }));
         }, failure -> {
-            logger.warn(() -> "[" + modelId + "] failed to load model configuration", failure);
+            if (consumer != Consumer.PIPELINE) {
+                // The model loading was triggered by an ingest pipeline change
+                // referencing a model that cannot be found. This is not an error
+                // as the model may be put later
+                logger.warn(() -> "[" + modelId + "] failed to load model configuration ", failure);
+            }
             handleLoadFailure(modelId, failure);
         }));
     }
@@ -745,14 +750,13 @@ private void cacheEvictionListener(RemovalNotification<String, ModelAndConsumer>
 
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
-        final boolean prefetchModels = event.state().nodes().getLocalNode().isIngestNode();
-        // If we are not prefetching models and there were no model alias changes, don't bother handling the changes
-        if ((prefetchModels == false)
-            && (event.changedCustomMetadataSet().contains(IngestMetadata.TYPE) == false)
-            && (event.changedCustomMetadataSet().contains(ModelAliasMetadata.NAME) == false)) {
+        // If no changes to ingest pipelines and no model alias changes, nothing to do
+        if (event.changedCustomMetadataSet().contains(IngestMetadata.TYPE) == false
+            && event.changedCustomMetadataSet().contains(ModelAliasMetadata.NAME) == false) {
             return;
         }
 
+        final boolean prefetchModels = event.state().nodes().getLocalNode().isIngestNode();
         ClusterState state = event.state();
         IngestMetadata currentIngestMetadata = state.metadata().custom(IngestMetadata.TYPE);
         Set<String> allReferencedModelKeys = event.changedCustomMetadataSet().contains(IngestMetadata.TYPE)
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java
index 2f4640cfa38dc..8a494b094445c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java
@@ -656,6 +656,26 @@ public void testAliasesGetUpdatedEvenWhenNotIngestNode() throws IOException {
         assertThat(modelLoadingService.getModelId("loaded_model_again"), equalTo(model1));
     }
 
+    public void testEarlyExitOnIrrelevantClusterChangeEvent() {
+        ModelLoadingService modelLoadingService = new ModelLoadingService(
+            trainedModelProvider,
+            auditor,
+            threadPool,
+            clusterService,
+            trainedModelStatsService,
+            Settings.EMPTY,
+            "test-node",
+            circuitBreaker,
+            mock(XPackLicenseState.class)
+        );
+
+        ClusterChangedEvent event = mock(ClusterChangedEvent.class);
+        when(event.changedCustomMetadataSet()).thenReturn(Set.of());
+        modelLoadingService.clusterChanged(event);
+
+        verify(event, never()).state();
+    }
+
     @SuppressWarnings("unchecked")
     private void withTrainedModel(String modelId, long size) {
         InferenceDefinition definition = mock(InferenceDefinition.class);