Merge remote-tracking branch 'dan/master' into fix-decoder-model-onnx…

…-export
k2-fsa · Sep 21, 2023 · b5cc7ca · b5cc7ca
2 parents d20d0e4 + f5dc957
commit b5cc7ca
Show file tree

Hide file tree

Showing 19 changed files with 64 additions and 2 deletions.
diff --git a/egs/aishell/ASR/pruned_transducer_stateless7/onnx_pretrained.py b/egs/aishell/ASR/pruned_transducer_stateless7/onnx_pretrained.py
@@ -151,12 +151,14 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -170,6 +172,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7/onnx_pretrained.py b/egs/commonvoice/ASR/pruned_transducer_stateless7/onnx_pretrained.py
@@ -152,12 +152,14 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -171,6 +173,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/onnx_pretrained.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/onnx_pretrained.py
@@ -136,6 +136,7 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
         self.init_encoder_states()
 
@@ -184,6 +185,7 @@ def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -197,6 +199,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/lstm_transducer_stateless2/onnx_pretrained.py b/egs/librispeech/ASR/lstm_transducer_stateless2/onnx_pretrained.py
@@ -129,6 +129,7 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
         self.init_encoder_states()
 
@@ -166,6 +167,7 @@ def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -179,6 +181,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/lstm_transducer_stateless2/streaming-onnx-decode.py b/egs/librispeech/ASR/lstm_transducer_stateless2/streaming-onnx-decode.py
@@ -172,30 +172,35 @@ def init_encoder(self, args):
         self.encoder = ort.InferenceSession(
             args.encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_decoder(self, args):
         self.decoder = ort.InferenceSession(
             args.decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_joiner(self, args):
         self.joiner = ort.InferenceSession(
             args.joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_joiner_encoder_proj(self, args):
         self.joiner_encoder_proj = ort.InferenceSession(
             args.joiner_encoder_proj_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_joiner_decoder_proj(self, args):
         self.joiner_decoder_proj = ort.InferenceSession(
             args.joiner_decoder_proj_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def run_encoder(self, x, h0, c0) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless3/onnx_pretrained.py b/egs/librispeech/ASR/pruned_transducer_stateless3/onnx_pretrained.py
@@ -150,12 +150,14 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -169,6 +171,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless3/test_onnx.py b/egs/librispeech/ASR/pruned_transducer_stateless3/test_onnx.py
@@ -78,6 +78,7 @@ def test_conv2d_subsampling():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -133,6 +134,7 @@ def test_rel_pos():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -220,6 +222,7 @@ def test_conformer_encoder_layer():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -304,6 +307,7 @@ def test_conformer_encoder():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -359,6 +363,7 @@ def test_conformer():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/onnx_pretrained-streaming.py b/egs/librispeech/ASR/pruned_transducer_stateless5/onnx_pretrained-streaming.py
@@ -138,6 +138,7 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
         self.init_encoder_states()
 
@@ -185,6 +186,7 @@ def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -198,6 +200,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/test_onnx.py b/egs/librispeech/ASR/pruned_transducer_stateless7/test_onnx.py
@@ -74,6 +74,7 @@ def test_conv2d_subsampling():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -128,6 +129,7 @@ def test_rel_pos():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -204,6 +206,7 @@ def test_zipformer_encoder_layer():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -284,6 +287,7 @@ def test_zipformer_encoder():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()
@@ -338,6 +342,7 @@ def test_zipformer():
     session = ort.InferenceSession(
         filename,
         sess_options=options,
+        providers=["CPUExecutionProvider"],
     )
 
     input_nodes = session.get_inputs()

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/onnx_pretrained.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/onnx_pretrained.py
@@ -326,41 +326,49 @@ def main():
     encoder = ort.InferenceSession(
         args.encoder_model_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     decoder = ort.InferenceSession(
         args.decoder_model_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     joiner = ort.InferenceSession(
         args.joiner_model_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     joiner_encoder_proj = ort.InferenceSession(
         args.joiner_encoder_proj_model_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     joiner_decoder_proj = ort.InferenceSession(
         args.joiner_decoder_proj_model_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     lconv = ort.InferenceSession(
         args.lconv_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     frame_reducer = ort.InferenceSession(
         args.frame_reducer_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     ctc_output = ort.InferenceSession(
         args.ctc_output_filename,
         sess_options=session_opts,
+        providers=["CPUExecutionProvider"],
     )
 
     sp = spm.SentencePieceProcessor()

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/onnx_pretrained.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/onnx_pretrained.py
@@ -130,6 +130,7 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
         self.init_encoder_states()
 
@@ -229,6 +230,7 @@ def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -242,6 +244,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py
@@ -865,7 +865,7 @@ def get_dynamic_dropout_rate(self):
             return final_dropout_rate
         else:
             return initial_dropout_rate - (
-                initial_dropout_rate * final_dropout_rate
+                initial_dropout_rate - final_dropout_rate
             ) * (self.batch_count / warmup_period)
 
     def forward(

diff --git a/egs/librispeech/ASR/streaming_conformer_ctc/conformer.py b/egs/librispeech/ASR/streaming_conformer_ctc/conformer.py
@@ -230,7 +230,7 @@ def train_run_encoder(
                 x, pos_emb, mask=mask, src_key_padding_mask=src_key_padding_mask
             )  # (T, B, F)
         else:
-            x = self.encoder(x, pos_emb, src_key_padding_mask=mask)  # (T, B, F)
+            x = self.encoder(x, pos_emb, src_key_padding_mask=src_key_padding_mask)  # (T, B, F)
 
         if self.normalize_before:
             x = self.after_norm(x)

diff --git a/egs/librispeech/ASR/zipformer/onnx_pretrained-streaming.py b/egs/librispeech/ASR/zipformer/onnx_pretrained-streaming.py
@@ -146,6 +146,7 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
         self.init_encoder_states()
 
@@ -236,6 +237,7 @@ def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -249,6 +251,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map

diff --git a/egs/librispeech/ASR/zipformer/onnx_pretrained.py b/egs/librispeech/ASR/zipformer/onnx_pretrained.py
@@ -151,12 +151,14 @@ def init_encoder(self, encoder_model_filename: str):
         self.encoder = ort.InferenceSession(
             encoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
     def init_decoder(self, decoder_model_filename: str):
         self.decoder = ort.InferenceSession(
             decoder_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         decoder_meta = self.decoder.get_modelmeta().custom_metadata_map
@@ -170,6 +172,7 @@ def init_joiner(self, joiner_model_filename: str):
         self.joiner = ort.InferenceSession(
             joiner_model_filename,
             sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
         )
 
         joiner_meta = self.joiner.get_modelmeta().custom_metadata_map