diff --git a/src/transformers/models/longt5/modeling_longt5.py b/src/transformers/models/longt5/modeling_longt5.py
index 303755ae4338dc..d08ed83af07ea1 100644
--- a/src/transformers/models/longt5/modeling_longt5.py
+++ b/src/transformers/models/longt5/modeling_longt5.py
@@ -2152,6 +2152,7 @@ def _reorder_cache(self, past_key_values, beam_idx):
 )
 class LongT5EncoderModel(LongT5PreTrainedModel):
     _tied_weights_keys = ["encoder.embed_tokens.weight"]
+    _keys_to_ignore_on_load_unexpected = [r"decoder"]
 
     def __init__(self, config: LongT5Config):
         super().__init__(config)
diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py
index 33f29298dcca5c..e6d9deefa14639 100644
--- a/src/transformers/models/t5/modeling_t5.py
+++ b/src/transformers/models/t5/modeling_t5.py
@@ -1866,6 +1866,7 @@ def _reorder_cache(self, past_key_values, beam_idx):
 )
 class T5EncoderModel(T5PreTrainedModel):
     _tied_weights_keys = ["encoder.embed_tokens.weight"]
+    _keys_to_ignore_on_load_unexpected = [r"decoder"]
 
     def __init__(self, config: T5Config):
         super().__init__(config)