From d9e8fe2260bc071ebf22e39c00a5a99896b8ccfa Mon Sep 17 00:00:00 2001 From: dorotat-nv <115542912+dorotat-nv@users.noreply.github.com> Date: Thu, 24 Aug 2023 23:41:24 +0200 Subject: [PATCH] migrated class CSVFieldsMemmapDataset from BioNeMo (#7314) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * migrated class Signed-off-by: dorotat * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: dorotat * added unit test Signed-off-by: dorotat * memmap worker arg (#7062) * memmap worker arg Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update Signed-off-by: arendu * update Signed-off-by: arendu --------- Signed-off-by: arendu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix caching bug in causal convolutions for cache-aware ASR models (#7034) (#7082) Co-authored-by: Vahid Noroozi Signed-off-by: dorotat * Fast Conformer global token fix (#7085) * old way Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * remove extra Signed-off-by: sam1373 * clean Signed-off-by: sam1373 * clean Signed-off-by: sam1373 * clean Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: sam1373 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Refined export_config (#7053) (#7066) * Refined export_config * Rolling back hierarchy change --------- Signed-off-by: Boris Fomitchev Co-authored-by: Boris Fomitchev Signed-off-by: dorotat * small Bugfix (#7081) * small Bugfix (#7079) * fix branch Signed-off-by: fayejf * fix typo Signed-off-by: fayejf * fix link Signed-off-by: fayejf --------- Signed-off-by: fayejf * Update tutorials/nlp/SpellMapper_English_ASR_Customization.ipynb Signed-off-by: Somshubra Majumdar * Update tutorials/nlp/SpellMapper_English_ASR_Customization.ipynb Signed-off-by: Somshubra Majumdar --------- Signed-off-by: fayejf Signed-off-by: Somshubra Majumdar Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Signed-off-by: dorotat * Added script to extract ASR CTC and RNNT models from ASR hybrid models (#7092) * Added script to extract ctc and rnnt models from hybrid models Signed-off-by: Daniel Egert * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated hybrid extraction script for review request 1 Signed-off-by: Daniel Egert * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated hybrid convert script to remove --cuda flag Signed-off-by: Daniel Egert --------- Signed-off-by: Daniel Egert Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Signed-off-by: dorotat * Adding docs and models for multiple lookahead cache-aware ASR (#7067) (#7094) Signed-off-by: dorotat * update TTS readme (#7088) * update TTS readme Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --------- Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: dorotat * Fix absolute path in path join call (#7099) Signed-off-by: Jan Beckmann Signed-off-by: dorotat * Disable distopt contiguous param buffer by default (#7095) Signed-off-by: Tim Moon Signed-off-by: dorotat * microphone demo (#7110) Signed-off-by: Linnea Pari Leaver Co-authored-by: Linnea Pari Leaver Signed-off-by: dorotat * [Fix] load_state_dict in nlp_model.py (#7086) * Fix load_state_dict in nlp_model.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix plot function in vad_utils.py (#7113) Fix plot function in vad_utils.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: dorotat * Fixed small bug with NoisePerturbationWithNormalization (#7118) Signed-off-by: Daniel Egert Signed-off-by: dorotat * Fix import guard checks (#7124) Signed-off-by: smajumdar Signed-off-by: dorotat * Revert "Fix import guard checks (#7124)" (#7125) This reverts commit ae7624da7d773a6b9436ff61903dc4b99c7c27cb. Signed-off-by: dorotat * Fix import guard checks (#7126) * Fix import guard checks Signed-off-by: smajumdar * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: smajumdar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Add updated fc ctc and rnnt xxl models (#7128) (#7130) Signed-off-by: dorotat * [TTS] Create EnCodec training recipe (#6852) * [TTS] Create EnCodec training recipe Signed-off-by: Ryan * [TTS] Update encodec recipe Signed-off-by: Ryan * [TTS] Rename EnCodec to AudioCodec Signed-off-by: Ryan * [TTS] Add EnCodec unit tests Signed-off-by: Ryan * [TTS] Add copyright header to distributed.py Signed-off-by: Ryan --------- Signed-off-by: Ryan Signed-off-by: dorotat * Fix rank where torch.distributed may not be initialized yet and would not wait for tokenizer file caching (#7061) Signed-off-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Co-authored-by: David Signed-off-by: dorotat * fix default attention size (#7141) (#7143) Signed-off-by: dorotat * fix evaluator.py for various exceptions by ast (#7150) Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: dorotat * [TTS][ZH] add Chinese TTS recipes based on IPA symbol sets. (#6893) * [TTS] add Chinese TTS recipe based on IPA. * add new pinyin and ipa dictionaries with 36 finals. * add yaml configs for 24-final pinyin and ipa. * add copyright header * add a directory level 24finals to discriminate from 36 finals. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * unify configs into a single one and add detailed comments providing supported candidates. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * choose 36-final IPA as default phoneme dict Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --------- Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: dorotat * [TTS] Add output audio format to preprocessing (#6889) * [TTS] Add output audio format to preprocessing Signed-off-by: Ryan * [TTS] Add format validation Signed-off-by: Ryan * [TTS] Fix data tutorial Signed-off-by: Ryan --------- Signed-off-by: Ryan Signed-off-by: dorotat * freeze (#7152) Signed-off-by: arendu Signed-off-by: dorotat * make sure any empty segments are removed (#7155) Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * Update RIR generation scripts (#6547) - fix: reduce room size if evaluation of params fails - added randomized mic placement - added diffuse noise generation - added an option to specify the format and subtype for saved audio Signed-off-by: Ante Jukić Signed-off-by: dorotat * A quickstart speech enhancement tutorial (#6492) A simple example of training a model for speech enhancement task Signed-off-by: Ante Jukić Signed-off-by: dorotat * NFA subtitle file config - specify colors and vertical alignment (#7160) * allow specifying colors of text in ASS subtitle file Signed-off-by: Elena Rastorgueva * specify vertical_alignment instead of marginv in ass_file_config Signed-off-by: Elena Rastorgueva * add documentation of CTMFileConfig and ASSFileConfig to NFA README Signed-off-by: Elena Rastorgueva --------- Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * Eagerly accumulate embedding grads into fp32 buffer (#6958) (#7153) Signed-off-by: Tim Moon Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Signed-off-by: dorotat * TE bug fix (#7027) (#7036) Signed-off-by: Dmytro Pykhtar Co-authored-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Signed-off-by: dorotat * [TTS] Remove nested TTS configs (#7154) * [TTS] Remove nested TTS configs Signed-off-by: Ryan * [TTS] Modify tutorial to support multiple sampling rates Signed-off-by: Ryan * [TTS] Clarify min_duration unit Signed-off-by: Ryan * [TTS] Default 22.05kHz highfreq to null Signed-off-by: Ryan --------- Signed-off-by: Ryan Signed-off-by: dorotat * Merge release r1.20.0 to main (#7167) * update package info Signed-off-by: ericharper * Add ASR with TTS Tutorial. Fix enhancer usage. (#6955) * Add ASR with TTS Tutorial * Fix enhancer usage Signed-off-by: Vladimir Bataev * install_bs (#7019) Signed-off-by: Nikolay Karpov * Fix typo and branch in tutorial (#7048) Signed-off-by: Vladimir Bataev * fix syntax error introduced in PR-7079 (#7102) * fix syntax error introduced in PR-7079 Signed-off-by: Alexandra Antonova * fixes for pr review Signed-off-by: Alexandra Antonova --------- Signed-off-by: Alexandra Antonova * fix links for TN (#7117) Signed-off-by: Evelina * update branch (#7135) Signed-off-by: ericharper * Fixed main and merging this to r1.20 (#7127) * Fixed main and merging this to r1.20 Signed-off-by: Taejin Park * Update vad_utils.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> --------- Signed-off-by: Taejin Park Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> * update branch Signed-off-by: ericharper * fix version Signed-off-by: ericharper * resolve conflict the other way Signed-off-by: ericharper * keep both Signed-off-by: ericharper * revert keep both Signed-off-by: ericharper --------- Signed-off-by: ericharper Signed-off-by: Vladimir Bataev Signed-off-by: Nikolay Karpov Signed-off-by: Alexandra Antonova Signed-off-by: Evelina Signed-off-by: Taejin Park Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: Vladimir Bataev Co-authored-by: Nikolay Karpov Co-authored-by: bene-ges Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Taejin Park Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: dorotat * Upgrade to pytorch lightning 2.0 (#6433) * Upgrade pytorch lightning version in requirements Signed-off-by: Abhishree * Initial fixes for PTL2.0 Signed-off-by: Abhishree * Add further fixes to support lightning 2.0 Signed-off-by: Abhishree * Add replacements for replace_sampler_ddp, resume_from_checkpoint_fit_path and few occurances of validation_epoch_end Signed-off-by: Abhishree * Replace all occurances of validation_epoch_end to on_validation_epoch_end Signed-off-by: Abhishree * Replace training_epoch_end, test_epoch_end with on_train_epoch_end and on_test_epoch_end respectively Signed-off-by: Abhishree * Change logger=None to logger=False in Trainer object Signed-off-by: Abhishree * Remove PTL2.0 deprecated Trainer args from TrainerConfig dataclass Signed-off-by: Abhishree * Modify trainer.precision check and other small edits Signed-off-by: Abhishree * Replace logger=None with logger=False in test_ptl_stateless_timer.py Trainer Signed-off-by: Abhishree * Add default values for args to fix Attribute Error Signed-off-by: Abhishree * Add the following modifications 1) Remove outputs arg from on_validation_epoch_end, on_test_epoch_end and make it an arg of the class 2) Replace resume_from_checkpoint with ckpt_path as needed 3) Explicitly add accelerator as 'CPU' in UTs being run on CPU Signed-off-by: Abhishree * Remove outputs arg from on_validation_epoch_end, on_test_epoch_end Signed-off-by: Abhishree * Remove outputs arg in on_validation_epoch_end in MultiBinaryAccuracy docstrings Signed-off-by: Abhishree * Add val, test outputs as instance vars in PunctuationCapitalizationModel and TokenClassificationModel Signed-off-by: Abhishree * Replace trainer.fit_loop.max_steps with trainer.fit_loop.epoch_loop.max_steps in test_optimizers_schedulers.py Signed-off-by: Abhishree * Revert an extra space that was mistakenly added Signed-off-by: Abhishree * Use self.validation_step_outputs and self.test_step_outputs in test_ema.py for uniformity Signed-off-by: Abhishree * Use self.validation_step_outputs and self.test_step_outputs in test_ptl_stateless_timer.py and check_for_ranks.py for uniformity Signed-off-by: Abhishree * Add self.validation_step_outputs.clear() and self.test_step_outputs.clear() wherever missing Signed-off-by: Abhishree * Remove outputs arg from on_train_epoch_end Signed-off-by: Abhishree * Remove outputs from on_validation_epoch_end in multi_binary_acc.py Signed-off-by: Abhishree * Remove output args from on_validation_epoch_end in the docstrings of some ASR files Signed-off-by: Abhishree * Remove output args from on_validation_epoch_end and clear memory from validation_step_outputs Signed-off-by: Abhishree * Add on_validation_epoch_end and remove outputs args for nlp models Signed-off-by: Abhishree * Append output of validation_step to validation_step_outputs in EncDecClassificationModel Signed-off-by: Abhishree * Add the following changes 1) Index self.validation_step_outputs and self.test_step.outputs with dataloader_idx wherever needed 2) Initialize self.validation_step_outputs and self.test_step.outputs as empty lists and add support for multi dataloaders if they exist 3) Remove self.pre_configure_ddp from NLPDDPStrategy class as its removed in PTL 2.0 Signed-off-by: Abhishree * Add default value dataloader_idx=0 for on_validation_batch_end() in megatron_base_model.py Signed-off-by: Abhishree * TypeCast precision to str in attention.py and utils_funcs.py to avoid TypeError Signed-off-by: Abhishree * Add if condition check for multiple dataloaders when appending to validation outputs Signed-off-by: Abhishree * Separate validation pass to be used with both validation_step and test_step Signed-off-by: Abhishree * Add if condition check for multiple dataloader while appending to test_step_outputs in punctuation_capitalization_model.py Signed-off-by: Abhishree * Add condition check for multiple dataloaders based on type of trainer.val/test_dataloaders or self._validation/test_dl instead of len Signed-off-by: Abhishree * Comment Megatron T5 IA3 PP=2 in CI pipeline due to dataloader_iter issue with PTL 2.0 Signed-off-by: Abhishree * Modify precision checks to account for 16-mixed and bf16-mixed Signed-off-by: Abhishree * Append output of validation/test_step to self.validation/test_step_outputs in CTCG2PModel Signed-off-by: Abhishree * Modify find_unused_parameters=True in g2p_heteronym model 1) Add find_unused_parameters=True for DDP strategy in g2p_heteronym_classification_train_and_evaluate.py 2) Remove args output in validation/test_step and add instance variables instead for heteronym_classification.py Signed-off-by: Abhishree * Remove outputs from on_test_epoch_end in DialogueGPTClassificationModel Signed-off-by: Abhishree * Add validation/test outputs in sgdqa_model and modify dialogue_config.yaml Signed-off-by: Abhishree * Add split arg self.test_step_outputs to TextClassificationModel Signed-off-by: Abhishree * Add test_step_outputs to dialogue and text classification models Signed-off-by: Abhishree * Change condition check for multiple dataloaders: 1) Replace ds_item as list in dialogue_config.yaml 2) Check for len of val/test_dataloaders or validation/test_dl along with type check of list in sgdqa_model.py while appending outputs of validation/test_step 3) Check for len of _validation/test_dl for creating self.validation/test_step_outputs in ModelPT and punctuation_cpitalization_model.py Signed-off-by: Abhishree * Add additional condition for multi dataloaders Check len(self.trainer.val/test_dataloaders) > 1 along with type(self.trainer.val/test_dataloaders) == list for multi dataloaders in validation/test_step Signed-off-by: Abhishree * Add val step outputs and default val for dataloader_idx 1) Append validation_step outout to self.validation_step_outputs in MultiLabelIntentSlotClassificationMode 2) Add default val for dataloader_idx for on_test_batch_start/end in TimingCallback 3) Add self.validation/test_step_outputs in BERTQAModel and remove outputs arg Signed-off-by: Abhishree * Add val/test_step_outputs to S2SQAModel and GPTQAModel Signed-off-by: Abhishree * Edit JenkinsFile for bert_pretrainig.py Edit Jenkinsfile for this test to disable validation as a workaround for trainer.val_dataloader None error Signed-off-by: Abhishree * Modify precision to support 16-mixed, bf16-mixed in megatron_gpt_pretraining.py Signed-off-by: Abhishree * Add ddp_find_unused_parameters_true and remove output args 1) Add ddp_find_unused_parameters_true fro trainer.strategy in self_alignment_pretraining.py as it has unused parameters 2) Remove output args and add self.validation/test_step_outputs to validation/test_step in mt_enc_dec_model.py 3) Comment tests in JenkinsFile that need to be fixed Signed-off-by: Abhishree * Precision fix in megatron_nmt_training.py for 16-mixed, bf16-mixed Signed-off-by: Abhishree * Precision fix for megatron_bert_pretraining.py and megatron_bert_model.py Signed-off-by: Abhishree * Precision fix and validation/test_step_outputs 1) Add fix to account for 16-mixed and bf16-mixed in megatron_retro_mutransfer_pretrain.py, megatron_retro_pretraining.py 2) Reset ckpt_path for test in enc_dec_nmt.py 3) Remove outputs args and add validation/test_step_outputs in megatron_retrieval_model.py 4) Comment Megatron Bert Pretraining and Resume Training with Pipeline Paralleism and add back NMT Training Post-LN Signed-off-by: Abhishree * Precision fix and skip few failing tests Signed-off-by: Abhishree * Add missing comment lines in JenkinsFile Signed-off-by: Abhishree * Comment jenkin tests and super().on_validation_epoch_end() in megatron_gpt_sft_model.py Signed-off-by: Abhishree * Minor edit JenkinsFile Signed-off-by: Abhishree * Minor edit in jenkins file Signed-off-by: Abhishree * Edit in Jenkins file Signed-off-by: Abhishree * Comment missed lines in Jenkins file Signed-off-by: Abhishree * Fix precision and validation/test outputs 1) Add precision fix to account for 16-mixed and bf16-mixed in megatron_t5_pretraining.py 2) Remove outputs args and add append loss to self.validation/test_step_outputs in megatron_lm_encoder_decoder_model.py 3) Add back resume_from_checkpoint in the megatron_t5_config.yaml 4) Comment out certain tests in Jenkins file Signed-off-by: Abhishree * Fix precision and validation/test/predict errors in megatron_t5_prompt_learning.py Signed-off-by: Abhishree * Precision fix and edit precision typo in all files 1) Account for 16-mixed and bf16-mixed in megatron_bart_pretraining.py and megatron_t5_seq2seq_finetune.py 2) Fix precision typo in all files Signed-off-by: Abhishree * Fix all CI TTS tests and comment few Jenkins tests Signed-off-by: Abhishree * Combine xx_epoch_end and on_xx_epoch_end Add on_inference_epoch_end to inference_epoch_end function and have a single on_validation/test_epoch_end in megatron_finetune_model.py and megatron_gpt_sft_model.py Signed-off-by: Abhishree * Add a missing comment in JenkinsFile Signed-off-by: Abhishree * Add try except StopIteration in validation_step for models with dataloader_iter Signed-off-by: Abhishree * Remove pyyaml from requirements Signed-off-by: Abhishree * Add try except for inference_step in megatron_finetune_model.py Signed-off-by: Abhishree * Remove limit_val_batches for mockGPTDataset test Signed-off-by: Abhishree * Add new self.validation_step_outputs for MegatronGPTSFTModel Signed-off-by: Abhishree * Minor edit Jenkinsfile Signed-off-by: Abhishree * Initialize self.validation/test_step_outputs in megatron_gpt_sft_model.py Initialize self.validation/test_step_outputs in setup of MegatronGPTSFTModel to take care of cases when datalaoders are not setup in ModelPT for example while restoring the model. Signed-off-by: Abhishree * Remove resume_from_checkpoint if trainer arg in conf yaml files Signed-off-by: Abhishree * Remove resume_from_checkpoint as trainer arg in GPT, T5 configs Signed-off-by: Abhishree * Remove resume_from_checkpoint in duplex_tn_config.yaml Signed-off-by: Abhishree * Fix typos, unused imports and refactor code to remove redundant funcs Signed-off-by: Abhishree * Remove commented code in megatron_nmt_model.py Signed-off-by: Abhishree * Fix overriden functions to match parent class functions Signed-off-by: Abhishree * Prefetch dataloader_iter to prevent hang for PP>1 Signed-off-by: Abhishree * Override setup() in NLPDDPStrategy to avoid hang during predict with PP>1 Signed-off-by: Abhishree * Uncomment tests in JenkinsFile Signed-off-by: Abhishree * Add '16' to precision checks and other minor fixes Signed-off-by: Abhishree * Clear validation/test_step_outputs with dataloader_idx for multi dataloaders Signed-off-by: Abhishree * Minor edits Signed-off-by: Abhishree * Modify precision checks to avoid indexing Signed-off-by: Abhishree * Remove self.validation_step_outputs_sft and add dataloader_idx to clear outputs Signed-off-by: Abhishree * Reference checkpoint with trainer.ckpt_path Signed-off-by: Abhishree * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add _prefetch to NLPModel and minor fixes Signed-off-by: Abhishree * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add limit_val_batches in JenkinsFile for NMT 1) Add trainer.limit_val_batches in Megatron NMT Training TP=2 2) Remove unused import in ModelPT Signed-off-by: Abhishree --------- Signed-off-by: Abhishree Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Include the scripts for preprocessing OAST and unit tests for chat sft datasets (#7112) * scripts for sft Signed-off-by: Yi Dong * fix style Signed-off-by: Yi Dong * adde special token only for huggingface model Signed-off-by: Yi Dong * change default name Signed-off-by: Yi Dong * print out error datapoint content Signed-off-by: Yi Dong * show error id Signed-off-by: Yi Dong * annotation script working Signed-off-by: Yi Dong * try to be compatible with huggingface tokenizer Signed-off-by: Yi Dong * added examples Signed-off-by: Yi Dong * added lang Signed-off-by: Yi Dong * added lang Signed-off-by: Yi Dong * text to value special case Signed-off-by: Yi Dong * configure the slider Signed-off-by: Yi Dong * annoatation handles lang Signed-off-by: Yi Dong * added the unit test for chat sft dataset Signed-off-by: Yi Dong * used the file in the test dir Signed-off-by: Yi Dong * fix json error Signed-off-by: Yi Dong * load local tokenizer Signed-off-by: Yi Dong * remove mask count check Signed-off-by: Yi Dong * added HF dataset backend Signed-off-by: Yi Dong * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Yi Dong Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * add paths to labeler. (#7087) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: dorotat * T5 metrics fix (#7037) * Fix race condition when executing with multi-node where some ranks does not wait for setup (#7016) Signed-off-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Signed-off-by: jubick1337 * Added bool types to neural_types export (#7032) Signed-off-by: tbartley94 Signed-off-by: jubick1337 * rnnt and char utils (#6971) * rnnt_ngram_merge Signed-off-by: Nikolay Karpov * char level bug Signed-off-by: Nikolay Karpov * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Nikolay Karpov Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Signed-off-by: jubick1337 * fix tab text gen (#7022) (#7031) Signed-off-by: Yi Dong Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> Signed-off-by: jubick1337 * Fixed kwargs for metric instance init Signed-off-by: jubick1337 * Fixed kwargs for metric instance init Signed-off-by: jubick1337 * removed kwagrs Signed-off-by: jubick1337 * Updated config desc Signed-off-by: jubick1337 * ASR Confidence update and tutorial (#6810) * small fixes and tests Signed-off-by: Aleksandr Laptev * various fixes for the tutorial Signed-off-by: Aleksandr Laptev * tutorial added Signed-off-by: Aleksandr Laptev * for for a little oops after rebasement Signed-off-by: Aleksandr Laptev * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix tests Signed-off-by: Aleksandr Laptev * unused import removed Signed-off-by: Aleksandr Laptev * fix review comments Signed-off-by: Aleksandr Laptev * deprecated parameters for greedy configs Signed-off-by: Aleksandr Laptev * move re-assigning to configs Signed-off-by: Aleksandr Laptev * fix comments 2 Signed-off-by: Aleksandr Laptev * fix config tests Signed-off-by: Aleksandr Laptev * fix ece test (my env was bugged apparently) Signed-off-by: Aleksandr Laptev * renamings for confidence ensemble Signed-off-by: Aleksandr Laptev * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fox comments 3 Signed-off-by: Aleksandr Laptev * return dropped tutorial Signed-off-by: Aleksandr Laptev * CI flips back and forth, increasing tolerance Signed-off-by: Aleksandr Laptev --------- Signed-off-by: Aleksandr Laptev Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * install_bs (#7019) (#7028) Signed-off-by: Nikolay Karpov Co-authored-by: Nikolay Karpov Signed-off-by: jubick1337 * fixes for spellmapper (#6994) (#7000) Signed-off-by: Alexandra Antonova Co-authored-by: bene-ges Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Signed-off-by: jubick1337 * added back the retro documents (#7033) Signed-off-by: Yi Dong Signed-off-by: jubick1337 * Remove pyyaml (#7052) (#7054) Signed-off-by: smajumdar Co-authored-by: Somshubra Majumdar Signed-off-by: jubick1337 * st standalone model (#6969) * st standalone model Signed-off-by: AlexGrinch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * style fix Signed-off-by: AlexGrinch * sacrebleu import fix, unused imports removed Signed-off-by: AlexGrinch * import guard for nlp inside asr transformer bpe model Signed-off-by: AlexGrinch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * codeql fixes Signed-off-by: AlexGrinch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * comments answered Signed-off-by: AlexGrinch * import ordering fix Signed-off-by: AlexGrinch * yttm for asr removed Signed-off-by: AlexGrinch * logging added Signed-off-by: AlexGrinch * added inference and translate method Signed-off-by: AlexGrinch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: AlexGrinch Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * remove pos emb from state dict for old models (#7068) * remove pos emb from state dict Signed-off-by: Evelina * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * move to nlp_model Signed-off-by: Evelina * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update comment Signed-off-by: Evelina * fix nmt test Signed-off-by: Evelina * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix nmt test Signed-off-by: Evelina --------- Signed-off-by: Evelina Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * Fix typo in ASR-TTS tutorial (#7049) Signed-off-by: Vladimir Bataev Signed-off-by: jubick1337 * Fixed tutorial's name (#7047) Signed-off-by: Vitaly Lavrukhin Co-authored-by: Vladimir Bataev Signed-off-by: jubick1337 * Fix documentation for Numba (#7065) (#7077) * Fix documentation for Numba * Update force float32 flag dynamically * Update force float32 flag dynamically * Fix nemo version --------- Signed-off-by: smajumdar Co-authored-by: Somshubra Majumdar Co-authored-by: Eric Harper Signed-off-by: jubick1337 * Update Frame-VAD doc and fix onnx export (#7076) * update fvad doc Signed-off-by: stevehuang52 * fix typo Signed-off-by: stevehuang52 * update fvad example Signed-off-by: stevehuang52 * update Signed-off-by: stevehuang52 * fix onnx export Signed-off-by: stevehuang52 * update test Signed-off-by: stevehuang52 * refactor Signed-off-by: stevehuang52 * update doc Signed-off-by: stevehuang52 * update Signed-off-by: stevehuang52 --------- Signed-off-by: stevehuang52 Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Signed-off-by: jubick1337 * memmap worker arg (#7062) * memmap worker arg Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update Signed-off-by: arendu * update Signed-off-by: arendu --------- Signed-off-by: arendu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * Fix caching bug in causal convolutions for cache-aware ASR models (#7034) (#7082) Co-authored-by: Vahid Noroozi Signed-off-by: jubick1337 * Fast Conformer global token fix (#7085) * old way Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * remove extra Signed-off-by: sam1373 * clean Signed-off-by: sam1373 * clean Signed-off-by: sam1373 * clean Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * fix Signed-off-by: sam1373 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: sam1373 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * Refined export_config (#7053) (#7066) * Refined export_config * Rolling back hierarchy change --------- Signed-off-by: Boris Fomitchev Co-authored-by: Boris Fomitchev Signed-off-by: jubick1337 * small Bugfix (#7081) * small Bugfix (#7079) * fix branch Signed-off-by: fayejf * fix typo Signed-off-by: fayejf * fix link Signed-off-by: fayejf --------- Signed-off-by: fayejf * Update tutorials/nlp/SpellMapper_English_ASR_Customization.ipynb Signed-off-by: Somshubra Majumdar * Update tutorials/nlp/SpellMapper_English_ASR_Customization.ipynb Signed-off-by: Somshubra Majumdar --------- Signed-off-by: fayejf Signed-off-by: Somshubra Majumdar Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Signed-off-by: jubick1337 * Added script to extract ASR CTC and RNNT models from ASR hybrid models (#7092) * Added script to extract ctc and rnnt models from hybrid models Signed-off-by: Daniel Egert * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated hybrid extraction script for review request 1 Signed-off-by: Daniel Egert * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated hybrid convert script to remove --cuda flag Signed-off-by: Daniel Egert --------- Signed-off-by: Daniel Egert Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Signed-off-by: jubick1337 * Adding docs and models for multiple lookahead cache-aware ASR (#7067) (#7094) Signed-off-by: jubick1337 * update TTS readme (#7088) * update TTS readme Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --------- Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: jubick1337 * Fix absolute path in path join call (#7099) Signed-off-by: Jan Beckmann Signed-off-by: jubick1337 * Disable distopt contiguous param buffer by default (#7095) Signed-off-by: Tim Moon Signed-off-by: jubick1337 * microphone demo (#7110) Signed-off-by: Linnea Pari Leaver Co-authored-by: Linnea Pari Leaver Signed-off-by: jubick1337 * [Fix] load_state_dict in nlp_model.py (#7086) * Fix load_state_dict in nlp_model.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * Fix plot function in vad_utils.py (#7113) Fix plot function in vad_utils.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: jubick1337 * Fixed small bug with NoisePerturbationWithNormalization (#7118) Signed-off-by: Daniel Egert Signed-off-by: jubick1337 * Fix import guard checks (#7124) Signed-off-by: smajumdar Signed-off-by: jubick1337 * Revert "Fix import guard checks (#7124)" (#7125) This reverts commit ae7624da7d773a6b9436ff61903dc4b99c7c27cb. Signed-off-by: jubick1337 * Fix import guard checks (#7126) * Fix import guard checks Signed-off-by: smajumdar * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: smajumdar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * Add updated fc ctc and rnnt xxl models (#7128) (#7130) Signed-off-by: jubick1337 * [TTS] Create EnCodec training recipe (#6852) * [TTS] Create EnCodec training recipe Signed-off-by: Ryan * [TTS] Update encodec recipe Signed-off-by: Ryan * [TTS] Rename EnCodec to AudioCodec Signed-off-by: Ryan * [TTS] Add EnCodec unit tests Signed-off-by: Ryan * [TTS] Add copyright header to distributed.py Signed-off-by: Ryan --------- Signed-off-by: Ryan Signed-off-by: jubick1337 * Fix rank where torch.distributed may not be initialized yet and would not wait for tokenizer file caching (#7061) Signed-off-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Co-authored-by: David Signed-off-by: jubick1337 * fix default attention size (#7141) (#7143) Signed-off-by: jubick1337 * fix evaluator.py for various exceptions by ast (#7150) Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: jubick1337 * [TTS][ZH] add Chinese TTS recipes based on IPA symbol sets. (#6893) * [TTS] add Chinese TTS recipe based on IPA. * add new pinyin and ipa dictionaries with 36 finals. * add yaml configs for 24-final pinyin and ipa. * add copyright header * add a directory level 24finals to discriminate from 36 finals. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * unify configs into a single one and add detailed comments providing supported candidates. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * choose 36-final IPA as default phoneme dict Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --------- Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: jubick1337 * [TTS] Add output audio format to preprocessing (#6889) * [TTS] Add output audio format to preprocessing Signed-off-by: Ryan * [TTS] Add format validation Signed-off-by: Ryan * [TTS] Fix data tutorial Signed-off-by: Ryan --------- Signed-off-by: Ryan Signed-off-by: jubick1337 * freeze (#7152) Signed-off-by: arendu Signed-off-by: jubick1337 * make sure any empty segments are removed (#7155) Signed-off-by: Elena Rastorgueva Signed-off-by: jubick1337 * Update RIR generation scripts (#6547) - fix: reduce room size if evaluation of params fails - added randomized mic placement - added diffuse noise generation - added an option to specify the format and subtype for saved audio Signed-off-by: Ante Jukić Signed-off-by: jubick1337 * A quickstart speech enhancement tutorial (#6492) A simple example of training a model for speech enhancement task Signed-off-by: Ante Jukić Signed-off-by: jubick1337 * NFA subtitle file config - specify colors and vertical alignment (#7160) * allow specifying colors of text in ASS subtitle file Signed-off-by: Elena Rastorgueva * specify vertical_alignment instead of marginv in ass_file_config Signed-off-by: Elena Rastorgueva * add documentation of CTMFileConfig and ASSFileConfig to NFA README Signed-off-by: Elena Rastorgueva --------- Signed-off-by: Elena Rastorgueva Signed-off-by: jubick1337 * Eagerly accumulate embedding grads into fp32 buffer (#6958) (#7153) Signed-off-by: Tim Moon Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Signed-off-by: jubick1337 * TE bug fix (#7027) (#7036) Signed-off-by: Dmytro Pykhtar Co-authored-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Signed-off-by: jubick1337 * [TTS] Remove nested TTS configs (#7154) * [TTS] Remove nested TTS configs Signed-off-by: Ryan * [TTS] Modify tutorial to support multiple sampling rates Signed-off-by: Ryan * [TTS] Clarify min_duration unit Signed-off-by: Ryan * [TTS] Default 22.05kHz highfreq to null Signed-off-by: Ryan --------- Signed-off-by: Ryan Signed-off-by: jubick1337 * Merge release r1.20.0 to main (#7167) * update package info Signed-off-by: ericharper * Add ASR with TTS Tutorial. Fix enhancer usage. (#6955) * Add ASR with TTS Tutorial * Fix enhancer usage Signed-off-by: Vladimir Bataev * install_bs (#7019) Signed-off-by: Nikolay Karpov * Fix typo and branch in tutorial (#7048) Signed-off-by: Vladimir Bataev * fix syntax error introduced in PR-7079 (#7102) * fix syntax error introduced in PR-7079 Signed-off-by: Alexandra Antonova * fixes for pr review Signed-off-by: Alexandra Antonova --------- Signed-off-by: Alexandra Antonova * fix links for TN (#7117) Signed-off-by: Evelina * update branch (#7135) Signed-off-by: ericharper * Fixed main and merging this to r1.20 (#7127) * Fixed main and merging this to r1.20 Signed-off-by: Taejin Park * Update vad_utils.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> --------- Signed-off-by: Taejin Park Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> * update branch Signed-off-by: ericharper * fix version Signed-off-by: ericharper * resolve conflict the other way Signed-off-by: ericharper * keep both Signed-off-by: ericharper * revert keep both Signed-off-by: ericharper --------- Signed-off-by: ericharper Signed-off-by: Vladimir Bataev Signed-off-by: Nikolay Karpov Signed-off-by: Alexandra Antonova Signed-off-by: Evelina Signed-off-by: Taejin Park Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: Vladimir Bataev Co-authored-by: Nikolay Karpov Co-authored-by: bene-ges Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Taejin Park Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: jubick1337 * Upgrade to pytorch lightning 2.0 (#6433) * Upgrade pytorch lightning version in requirements Signed-off-by: Abhishree * Initial fixes for PTL2.0 Signed-off-by: Abhishree * Add further fixes to support lightning 2.0 Signed-off-by: Abhishree * Add replacements for replace_sampler_ddp, resume_from_checkpoint_fit_path and few occurances of validation_epoch_end Signed-off-by: Abhishree * Replace all occurances of validation_epoch_end to on_validation_epoch_end Signed-off-by: Abhishree * Replace training_epoch_end, test_epoch_end with on_train_epoch_end and on_test_epoch_end respectively Signed-off-by: Abhishree * Change logger=None to logger=False in Trainer object Signed-off-by: Abhishree * Remove PTL2.0 deprecated Trainer args from TrainerConfig dataclass Signed-off-by: Abhishree * Modify trainer.precision check and other small edits Signed-off-by: Abhishree * Replace logger=None with logger=False in test_ptl_stateless_timer.py Trainer Signed-off-by: Abhishree * Add default values for args to fix Attribute Error Signed-off-by: Abhishree * Add the following modifications 1) Remove outputs arg from on_validation_epoch_end, on_test_epoch_end and make it an arg of the class 2) Replace resume_from_checkpoint with ckpt_path as needed 3) Explicitly add accelerator as 'CPU' in UTs being run on CPU Signed-off-by: Abhishree * Remove outputs arg from on_validation_epoch_end, on_test_epoch_end Signed-off-by: Abhishree * Remove outputs arg in on_validation_epoch_end in MultiBinaryAccuracy docstrings Signed-off-by: Abhishree * Add val, test outputs as instance vars in PunctuationCapitalizationModel and TokenClassificationModel Signed-off-by: Abhishree * Replace trainer.fit_loop.max_steps with trainer.fit_loop.epoch_loop.max_steps in test_optimizers_schedulers.py Signed-off-by: Abhishree * Revert an extra space that was mistakenly added Signed-off-by: Abhishree * Use self.validation_step_outputs and self.test_step_outputs in test_ema.py for uniformity Signed-off-by: Abhishree * Use self.validation_step_outputs and self.test_step_outputs in test_ptl_stateless_timer.py and check_for_ranks.py for uniformity Signed-off-by: Abhishree * Add self.validation_step_outputs.clear() and self.test_step_outputs.clear() wherever missing Signed-off-by: Abhishree * Remove outputs arg from on_train_epoch_end Signed-off-by: Abhishree * Remove outputs from on_validation_epoch_end in multi_binary_acc.py Signed-off-by: Abhishree * Remove output args from on_validation_epoch_end in the docstrings of some ASR files Signed-off-by: Abhishree * Remove output args from on_validation_epoch_end and clear memory from validation_step_outputs Signed-off-by: Abhishree * Add on_validation_epoch_end and remove outputs args for nlp models Signed-off-by: Abhishree * Append output of validation_step to validation_step_outputs in EncDecClassificationModel Signed-off-by: Abhishree * Add the following changes 1) Index self.validation_step_outputs and self.test_step.outputs with dataloader_idx wherever needed 2) Initialize self.validation_step_outputs and self.test_step.outputs as empty lists and add support for multi dataloaders if they exist 3) Remove self.pre_configure_ddp from NLPDDPStrategy class as its removed in PTL 2.0 Signed-off-by: Abhishree * Add default value dataloader_idx=0 for on_validation_batch_end() in megatron_base_model.py Signed-off-by: Abhishree * TypeCast precision to str in attention.py and utils_funcs.py to avoid TypeError Signed-off-by: Abhishree * Add if condition check for multiple dataloaders when appending to validation outputs Signed-off-by: Abhishree * Separate validation pass to be used with both validation_step and test_step Signed-off-by: Abhishree * Add if condition check for multiple dataloader while appending to test_step_outputs in punctuation_capitalization_model.py Signed-off-by: Abhishree * Add condition check for multiple dataloaders based on type of trainer.val/test_dataloaders or self._validation/test_dl instead of len Signed-off-by: Abhishree * Comment Megatron T5 IA3 PP=2 in CI pipeline due to dataloader_iter issue with PTL 2.0 Signed-off-by: Abhishree * Modify precision checks to account for 16-mixed and bf16-mixed Signed-off-by: Abhishree * Append output of validation/test_step to self.validation/test_step_outputs in CTCG2PModel Signed-off-by: Abhishree * Modify find_unused_parameters=True in g2p_heteronym model 1) Add find_unused_parameters=True for DDP strategy in g2p_heteronym_classification_train_and_evaluate.py 2) Remove args output in validation/test_step and add instance variables instead for heteronym_classification.py Signed-off-by: Abhishree * Remove outputs from on_test_epoch_end in DialogueGPTClassificationModel Signed-off-by: Abhishree * Add validation/test outputs in sgdqa_model and modify dialogue_config.yaml Signed-off-by: Abhishree * Add split arg self.test_step_outputs to TextClassificationModel Signed-off-by: Abhishree * Add test_step_outputs to dialogue and text classification models Signed-off-by: Abhishree * Change condition check for multiple dataloaders: 1) Replace ds_item as list in dialogue_config.yaml 2) Check for len of val/test_dataloaders or validation/test_dl along with type check of list in sgdqa_model.py while appending outputs of validation/test_step 3) Check for len of _validation/test_dl for creating self.validation/test_step_outputs in ModelPT and punctuation_cpitalization_model.py Signed-off-by: Abhishree * Add additional condition for multi dataloaders Check len(self.trainer.val/test_dataloaders) > 1 along with type(self.trainer.val/test_dataloaders) == list for multi dataloaders in validation/test_step Signed-off-by: Abhishree * Add val step outputs and default val for dataloader_idx 1) Append validation_step outout to self.validation_step_outputs in MultiLabelIntentSlotClassificationMode 2) Add default val for dataloader_idx for on_test_batch_start/end in TimingCallback 3) Add self.validation/test_step_outputs in BERTQAModel and remove outputs arg Signed-off-by: Abhishree * Add val/test_step_outputs to S2SQAModel and GPTQAModel Signed-off-by: Abhishree * Edit JenkinsFile for bert_pretrainig.py Edit Jenkinsfile for this test to disable validation as a workaround for trainer.val_dataloader None error Signed-off-by: Abhishree * Modify precision to support 16-mixed, bf16-mixed in megatron_gpt_pretraining.py Signed-off-by: Abhishree * Add ddp_find_unused_parameters_true and remove output args 1) Add ddp_find_unused_parameters_true fro trainer.strategy in self_alignment_pretraining.py as it has unused parameters 2) Remove output args and add self.validation/test_step_outputs to validation/test_step in mt_enc_dec_model.py 3) Comment tests in JenkinsFile that need to be fixed Signed-off-by: Abhishree * Precision fix in megatron_nmt_training.py for 16-mixed, bf16-mixed Signed-off-by: Abhishree * Precision fix for megatron_bert_pretraining.py and megatron_bert_model.py Signed-off-by: Abhishree * Precision fix and validation/test_step_outputs 1) Add fix to account for 16-mixed and bf16-mixed in megatron_retro_mutransfer_pretrain.py, megatron_retro_pretraining.py 2) Reset ckpt_path for test in enc_dec_nmt.py 3) Remove outputs args and add validation/test_step_outputs in megatron_retrieval_model.py 4) Comment Megatron Bert Pretraining and Resume Training with Pipeline Paralleism and add back NMT Training Post-LN Signed-off-by: Abhishree * Precision fix and skip few failing tests Signed-off-by: Abhishree * Add missing comment lines in JenkinsFile Signed-off-by: Abhishree * Comment jenkin tests and super().on_validation_epoch_end() in megatron_gpt_sft_model.py Signed-off-by: Abhishree * Minor edit JenkinsFile Signed-off-by: Abhishree * Minor edit in jenkins file Signed-off-by: Abhishree * Edit in Jenkins file Signed-off-by: Abhishree * Comment missed lines in Jenkins file Signed-off-by: Abhishree * Fix precision and validation/test outputs 1) Add precision fix to account for 16-mixed and bf16-mixed in megatron_t5_pretraining.py 2) Remove outputs args and add append loss to self.validation/test_step_outputs in megatron_lm_encoder_decoder_model.py 3) Add back resume_from_checkpoint in the megatron_t5_config.yaml 4) Comment out certain tests in Jenkins file Signed-off-by: Abhishree * Fix precision and validation/test/predict errors in megatron_t5_prompt_learning.py Signed-off-by: Abhishree * Precision fix and edit precision typo in all files 1) Account for 16-mixed and bf16-mixed in megatron_bart_pretraining.py and megatron_t5_seq2seq_finetune.py 2) Fix precision typo in all files Signed-off-by: Abhishree * Fix all CI TTS tests and comment few Jenkins tests Signed-off-by: Abhishree * Combine xx_epoch_end and on_xx_epoch_end Add on_inference_epoch_end to inference_epoch_end function and have a single on_validation/test_epoch_end in megatron_finetune_model.py and megatron_gpt_sft_model.py Signed-off-by: Abhishree * Add a missing comment in JenkinsFile Signed-off-by: Abhishree * Add try except StopIteration in validation_step for models with dataloader_iter Signed-off-by: Abhishree * Remove pyyaml from requirements Signed-off-by: Abhishree * Add try except for inference_step in megatron_finetune_model.py Signed-off-by: Abhishree * Remove limit_val_batches for mockGPTDataset test Signed-off-by: Abhishree * Add new self.validation_step_outputs for MegatronGPTSFTModel Signed-off-by: Abhishree * Minor edit Jenkinsfile Signed-off-by: Abhishree * Initialize self.validation/test_step_outputs in megatron_gpt_sft_model.py Initialize self.validation/test_step_outputs in setup of MegatronGPTSFTModel to take care of cases when datalaoders are not setup in ModelPT for example while restoring the model. Signed-off-by: Abhishree * Remove resume_from_checkpoint if trainer arg in conf yaml files Signed-off-by: Abhishree * Remove resume_from_checkpoint as trainer arg in GPT, T5 configs Signed-off-by: Abhishree * Remove resume_from_checkpoint in duplex_tn_config.yaml Signed-off-by: Abhishree * Fix typos, unused imports and refactor code to remove redundant funcs Signed-off-by: Abhishree * Remove commented code in megatron_nmt_model.py Signed-off-by: Abhishree * Fix overriden functions to match parent class functions Signed-off-by: Abhishree * Prefetch dataloader_iter to prevent hang for PP>1 Signed-off-by: Abhishree * Override setup() in NLPDDPStrategy to avoid hang during predict with PP>1 Signed-off-by: Abhishree * Uncomment tests in JenkinsFile Signed-off-by: Abhishree * Add '16' to precision checks and other minor fixes Signed-off-by: Abhishree * Clear validation/test_step_outputs with dataloader_idx for multi dataloaders Signed-off-by: Abhishree * Minor edits Signed-off-by: Abhishree * Modify precision checks to avoid indexing Signed-off-by: Abhishree * Remove self.validation_step_outputs_sft and add dataloader_idx to clear outputs Signed-off-by: Abhishree * Reference checkpoint with trainer.ckpt_path Signed-off-by: Abhishree * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add _prefetch to NLPModel and minor fixes Signed-off-by: Abhishree * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add limit_val_batches in JenkinsFile for NMT 1) Add trainer.limit_val_batches in Megatron NMT Training TP=2 2) Remove unused import in ModelPT Signed-off-by: Abhishree --------- Signed-off-by: Abhishree Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * Include the scripts for preprocessing OAST and unit tests for chat sft datasets (#7112) * scripts for sft Signed-off-by: Yi Dong * fix style Signed-off-by: Yi Dong * adde special token only for huggingface model Signed-off-by: Yi Dong * change default name Signed-off-by: Yi Dong * print out error datapoint content Signed-off-by: Yi Dong * show error id Signed-off-by: Yi Dong * annotation script working Signed-off-by: Yi Dong * try to be compatible with huggingface tokenizer Signed-off-by: Yi Dong * added examples Signed-off-by: Yi Dong * added lang Signed-off-by: Yi Dong * added lang Signed-off-by: Yi Dong * text to value special case Signed-off-by: Yi Dong * configure the slider Signed-off-by: Yi Dong * annoatation handles lang Signed-off-by: Yi Dong * added the unit test for chat sft dataset Signed-off-by: Yi Dong * used the file in the test dir Signed-off-by: Yi Dong * fix json error Signed-off-by: Yi Dong * load local tokenizer Signed-off-by: Yi Dong * remove mask count check Signed-off-by: Yi Dong * added HF dataset backend Signed-off-by: Yi Dong * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Yi Dong Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: jubick1337 * add paths to labeler. (#7087) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: jubick1337 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Signed-off-by: jubick1337 Signed-off-by: tbartley94 Signed-off-by: Nikolay Karpov Signed-off-by: Yi Dong Signed-off-by: Aleksandr Laptev Signed-off-by: Alexandra Antonova Signed-off-by: smajumdar Signed-off-by: AlexGrinch Signed-off-by: Evelina Signed-off-by: Vladimir Bataev Signed-off-by: Vitaly Lavrukhin Signed-off-by: stevehuang52 Signed-off-by: arendu Signed-off-by: sam1373 Signed-off-by: Boris Fomitchev Signed-off-by: fayejf Signed-off-by: Somshubra Majumdar Signed-off-by: Daniel Egert Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Jan Beckmann Signed-off-by: Tim Moon Signed-off-by: Linnea Pari Leaver Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: Ryan Signed-off-by: Elena Rastorgueva Signed-off-by: Ante Jukić Signed-off-by: Dmytro Pykhtar Signed-off-by: ericharper Signed-off-by: Taejin Park Signed-off-by: Abhishree Co-authored-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Co-authored-by: tbartley94 <90423858+tbartley94@users.noreply.github.com> Co-authored-by: Nikolay Karpov Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> Co-authored-by: Aleksandr Laptev Co-authored-by: bene-ges Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Aleksey Grinchuk (Oleksii Hrinchuk) Co-authored-by: Vladimir Bataev Co-authored-by: Vitaly Lavrukhin Co-authored-by: Eric Harper Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Co-authored-by: Adi Renduchintala Co-authored-by: Vahid Noroozi Co-authored-by: Samuel Kriman Co-authored-by: Boris Fomitchev Co-authored-by: trias702 <25867060+trias702@users.noreply.github.com> Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: Jan Beckmann Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Co-authored-by: lleaver <137942999+lleaver@users.noreply.github.com> Co-authored-by: Linnea Pari Leaver Co-authored-by: Ryan Langman Co-authored-by: David Co-authored-by: Elena Rastorgueva <80532067+erastorgueva-nv@users.noreply.github.com> Co-authored-by: anteju <108555623+anteju@users.noreply.github.com> Co-authored-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Co-authored-by: Taejin Park Co-authored-by: Abhishree Thittenamane <47577437+athitten@users.noreply.github.com> Signed-off-by: dorotat * fix a bug in filtering checkpoints (#6851) Signed-off-by: Xin Yao Signed-off-by: dorotat * Update README.rst (#7175) * Update README.rst Signed-off-by: fayejf <36722593+fayejf@users.noreply.github.com> * Update setup.py Signed-off-by: fayejf <36722593+fayejf@users.noreply.github.com> * Update pyproject.toml Signed-off-by: fayejf <36722593+fayejf@users.noreply.github.com> * Update setup.py Signed-off-by: fayejf <36722593+fayejf@users.noreply.github.com> --------- Signed-off-by: fayejf <36722593+fayejf@users.noreply.github.com> Signed-off-by: dorotat * Make NFA subtitles stay until end of video (#7189) * write extra subtitles line to make sure final subtitles are visible until end of video Signed-off-by: Elena Rastorgueva * add some comments for final subtitle line logic Signed-off-by: Elena Rastorgueva --------- Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * [TTS] Fix TTS recipes with PTL 2.0 (#7188) Signed-off-by: Ryan Signed-off-by: dorotat * minor fix for conformer subsampling docstring. (#7195) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: dorotat * Uncomment removal of exp_dir (#7198) The removal of result dir after the test 'Megatron GPT with KERPLE Pretraining and Resume Training TP=2' was commented out by mistake that was leading to the next test using the checkpoint from this test. Hence rolling it back. Signed-off-by: Abhishree Signed-off-by: dorotat * megatron gpt training fix (#7199) * megatron gpt training fix when sequence parallel=True, transformer engine=False, and bias=False * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: ANMOL GUPTA Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * replace ellipses in text with 3 periods (#7208) Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * Fix T5 using FA (#7196) * Fix T5 using FA Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove auto cast Signed-off-by: Cheng-Ping Hsieh * Fix tests Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix tests Signed-off-by: Cheng-Ping Hsieh --------- Signed-off-by: Cheng-Ping Hsieh Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * fix-causal-fa-infer (#7200) * fix-causal-fa-infer Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Cheng-Ping Hsieh Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix gpt trainer test (#6915) * Add trainer.test() for GPT Signed-off-by: hsiehjackson * Remove unused part Signed-off-by: hsiehjackson * Add trainer.test() for GPT Signed-off-by: hsiehjackson * Remove unused part Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix training part Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix config Signed-off-by: hsiehjackson * Fix references and add CI Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix config error Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix dataset Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add metadata Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix config Signed-off-by: hsiehjackson * Fix empty batch Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix config Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix config Signed-off-by: hsiehjackson * Fix max seq length Signed-off-by: hsiehjackson * Fix dataset Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix dataset Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add token f1 Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add FA in sft Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add inference config Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix bug Signed-off-by: hsiehjackson * Fix pad Signed-off-by: hsiehjackson * Fix num batch Signed-off-by: hsiehjackson * Add query_key Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove pdb Signed-off-by: hsiehjackson * Fix write json Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix dataset bug and refactor Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add logging for prediction Signed-off-by: hsiehjackson * Fix retrain Signed-off-by: hsiehjackson * Add query_key in config Signed-off-by: hsiehjackson * Fix bug Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix config Signed-off-by: hsiehjackson * Fix bug Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add inference config Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix bug Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix mask Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix mask Signed-off-by: hsiehjackson * Split PR Signed-off-by: Cheng-Ping Hsieh * Undo commit Signed-off-by: Cheng-Ping Hsieh * Add query_key to doc_string Signed-off-by: Cheng-Ping Hsieh * Adjust yzhang123 comments Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix error and follow comments Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove query key Signed-off-by: Cheng-Ping Hsieh * Remove logic and query Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove query from model Signed-off-by: Cheng-Ping Hsieh * Remove query_key Signed-off-by: Cheng-Ping Hsieh * Fix error Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix pdb Signed-off-by: Cheng-Ping Hsieh * Add default tokens_to_generate Signed-off-by: Cheng-Ping Hsieh * Revert prompt truncate re-prompt Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Revert Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * skip generation with metric loss Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Revert Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix bug Signed-off-by: Cheng-Ping Hsieh * support GPTSFTChatDataset Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add comment Signed-off-by: Cheng-Ping Hsieh --------- Signed-off-by: hsiehjackson Signed-off-by: Cheng-Ping Hsieh <37269846+hsiehjackson@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * [TTS] Add license to ported EnCodec code (#7197) Signed-off-by: Ryan Signed-off-by: dorotat * Fix discriminator update in AudioCodecModel (#7209) Signed-off-by: Ante Jukić Signed-off-by: dorotat * NFA tutorial notebook (#7210) * add NFA notebook tutorial Signed-off-by: Elena Rastorgueva * change single quotes to backticks around model_path Signed-off-by: Elena Rastorgueva --------- Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * Megatron hidden transformations (#6332) * [TTS] bugfix for missing configs. (#4725) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * docs typo fix Signed-off-by: Oleksii Kuchaiev * Fix pynini install in TTS tutorials (#4729) Signed-off-by: Jocelyn Huang Signed-off-by: Jocelyn Huang * Fix ASR notebooks (#4738) Signed-off-by: smajumdar Signed-off-by: smajumdar * Multilingual VAD model (#4734) * add ngc link Signed-off-by: fayejf * add tuned VAD config on ASR data Signed-off-by: fayejf * yaml note Signed-off-by: fayejf * update vad asr notebook with mVAD Signed-off-by: fayejf * update vad infer config comment Signed-off-by: fayejf * fix Signed-off-by: fayejf * mvad sd config for ch109 Signed-off-by: fayejf * update sd readme Signed-off-by: fayejf * add new mVAD model to doc Signed-off-by: fayejf * style fix Signed-off-by: fayejf * update sd tutorial with mVAD Signed-off-by: fayejf * typo fix Signed-off-by: fayejf Signed-off-by: fayejf * publish pretrained itn t5 model for English (#4748) Signed-off-by: Alexandra Antonova Signed-off-by: Alexandra Antonova Co-authored-by: Alexandra Antonova * Updated docs and doc paths (#4754) * Updated docs and doc paths Signed-off-by: Virginia Adams * Update Multitask_Prompt_and_PTuning.ipynb * Update README.rst * Changed branch name to use single quotes Signed-off-by: Virginia Adams Signed-off-by: Virginia Adams * fix bug relating to ddp strategy in joint intent slot classification tutorial (#4762) * [TTS] updated config with a German IPA phoneme tokenizer (#4756) * [TTS] added a German IPA phoneme tokenizer * [TTS][ASR] enabled customized arguments for trimming the leading and trailing silence. * [TTS] disabled spline interpolation for beta-binomial distribution. Let it generate align prior and save to disks. Use a new phoneme tokenizer. * [TTS] use consistent spline interpolation with fastpitch checkpoint when generating mel-spectrograms for hifigan finetune. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * Update r1.11 to new heteronyms list (#4745) * Update configs to new heteronyms list * Remove old heteronyms list, add alt 'merchandise' pron to CMUdict * Update remaining references to old heteronyms list Signed-off-by: Jocelyn Huang Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * [TTS] Add multi-speaker German FastPitch and HiFiGAN NGC checkpoints (#4763) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * [TTS] Add single male speaker German FastPitch and HiFiGAN NGC checkpoints (#4770) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * Update CMUdict with more recent 0.7b entries (#4768) Signed-off-by: Jocelyn Huang Signed-off-by: Jocelyn Huang Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * Install pynini in docker container (#4733) Signed-off-by: Vladimir Bataev Signed-off-by: Vladimir Bataev Co-authored-by: Nithin Rao Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: Eric Harper * Fix tutorial formatting (#4778) Signed-off-by: Jocelyn Huang * [TTS] deprecated old scripts for ljspeech. (#4780) * deprecated old scripts for ljspeech. * removed relevent function calls in TTS docs. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * update branch and typos (#4788) Signed-off-by: ericharper Signed-off-by: ericharper * Adding support for models trained with full context for cache-aware streaming. (#4687) * added support for models trained with full context. Signed-off-by: Vahid * fixed style. Signed-off-by: Vahid * dropped seq_range Signed-off-by: Vahid * fixed indexing in caching methods. Signed-off-by: Vahid * fixed code style. Signed-off-by: Vahid * fixed code style. Signed-off-by: Vahid * updated docs. Signed-off-by: Vahid * addressed comments. Signed-off-by: Vahid * fixed code style. Signed-off-by: Vahid * fixed code style. Signed-off-by: Vahid * fixed code style. Signed-off-by: Vahid * change frame-wise to cache-aware. Signed-off-by: Vahid * change frame-wise to cache-aware. Signed-off-by: Vahid * change frame-wise to cache-aware. Signed-off-by: Vahid * fixed code style. Signed-off-by: Vahid Signed-off-by: Vahid * Update megatron encoder decoder model to support py37 for colab (#4791) * [ASR] Add pretrained ASR models for Croatian (#4682) * [ASR] Add pretrained ASR models for Croatian Signed-off-by: Ante Jukić * Fix style for import Signed-off-by: Ante Jukić Signed-off-by: Ante Jukić Co-authored-by: Ante Jukić Co-authored-by: Nithin Rao Co-authored-by: Eric Harper Co-authored-by: Somshubra Majumdar * added/fixed export for Megatron models (#4712) * added/fixed export for Megatron models Signed-off-by: David Mosallanezhad * fixed style Signed-off-by: David Mosallanezhad * fixed FusedScaleMaskSoftmax in BioMegatron Signed-off-by: David Mosallanezhad * included comments Signed-off-by: David Mosallanezhad Signed-off-by: David Mosallanezhad Co-authored-by: David Mosallanezhad Co-authored-by: Eric Harper * update branch for qa notebook Signed-off-by: ericharper * Fix initializing weights from ptl ckpt with exclude (#4807) Signed-off-by: sam1373 Signed-off-by: sam1373 * Fix index error from addition of voiced_mask and p_voiced (#4811) Signed-off-by: Jocelyn Huang Signed-off-by: Jocelyn Huang * T5 prompt learning fixes (#4771) * RPE, hidden size and config fixes Signed-off-by: MaximumEntropy * Update to reflect new config names Signed-off-by: MaximumEntropy * Sentencepiece fixes Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Fix finetuning Signed-off-by: MaximumEntropy * Add encoder seq len to gpt Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Add finetune eval script Signed-off-by: MaximumEntropy * Fix name Signed-off-by: MaximumEntropy * Update Jenkinsfile Signed-off-by: MaximumEntropy * Update config Signed-off-by: MaximumEntropy * Fix CI test Signed-off-by: MaximumEntropy * Update check Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Backward compat Signed-off-by: MaximumEntropy * Update CI test Signed-off-by: MaximumEntropy * Split rank for Enc-Dec models Signed-off-by: MaximumEntropy * Address comments Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy Signed-off-by: MaximumEntropy Co-authored-by: Virginia Adams <78445382+vadam5@users.noreply.github.com> * G2P docs (#4841) * g2p docs added Signed-off-by: ekmb * fix references Signed-off-by: ekmb * address review feedback Signed-off-by: ekmb Signed-off-by: ekmb * Fix providing glue in seq2seq eval (#4843) * Fix providing glue in seq2seq eval Signed-off-by: MaximumEntropy * Fix Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy Signed-off-by: MaximumEntropy * Updated inference code and squad scripts (#4835) * Updated inference code and squad scripts Signed-off-by: Virginia Adams * Reverted GPT & T5 inference files back to use NLPDDPlugin Signed-off-by: Virginia Adams * Overwrite frozen LM to use fused adam Signed-off-by: Virginia Adams * Added padded vocab size Signed-off-by: Virginia Adams * Fixed val check interval value Signed-off-by: Virginia Adams * Python format fix Signed-off-by: Virginia Adams * Make t5 prompt learning preds write to file Signed-off-by: Virginia Adams * Added back dp=1 check Signed-off-by: Virginia Adams Signed-off-by: Virginia Adams Co-authored-by: Sandeep Subramanian * Update README.rst * Fix uppercasing mismatch for IPA heteronyms (#4860) Signed-off-by: Jocelyn Huang Signed-off-by: Jocelyn Huang * Set the number of workers to 0 for validation and test sets in all enc-dec models (#4790) * Set workers to 0 for validation and test Signed-off-by: MaximumEntropy * Revert pin memory Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy Signed-off-by: MaximumEntropy Co-authored-by: Sean Naren * Fix mha (#4866) * fix bug in mha forward function related to cache update return type Signed-off-by: Yang Zhang * fix lgtm Signed-off-by: Yang Zhang Signed-off-by: Yang Zhang Co-authored-by: Sean Naren * ipa bug fix (#4871) Signed-off-by: ekmb Signed-off-by: ekmb * Fix Megatron NMT consumed samples and ckpt_to_nemo split rank (#4884) * Fix nmt and ckpt_to_nemo Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy Signed-off-by: MaximumEntropy * added utf8 encoding (#4892) Signed-off-by: Virginia Adams Signed-off-by: Virginia Adams * 1. Applying the same patch to r1.11.0 (#4894) Signed-off-by: Micha Livne Signed-off-by: Micha Livne * Update tutorials.rst (#4897) * update readme with apex commit Signed-off-by: ericharper * Add support for Apex distributed Adam optimizer with GPT-3 (#4487) * Add support for Apex distributed Adam optimizer with GPT-3 Signed-off-by: Tim Moon * Fix bug in grad clipping with dist Adam Grad norm was computed over all params, not respecting model parallelism. Signed-off-by: Tim Moon * Fix bug with DDP initialization Signed-off-by: Tim Moon * Make distopt dependent on megatron_amp_o2 Signed-off-by: Tim Moon * Fix code formatting Signed-off-by: Tim Moon * Handle dist Adam in optimizer unit tests Signed-off-by: Tim Moon Signed-off-by: Tim Moon Co-authored-by: Eric Harper * update readme Signed-off-by: ericharper * update readme Signed-off-by: ericharper * latent model support * 1. Debugging. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. * update branch Signed-off-by: ericharper * fix replace_bos_with_pad not found (#6443) Signed-off-by: Abhinav Khattar * Support Swiglu in TP PP Conversion (#6437) * Support Swiglu in TP PP Conversion Signed-off-by: smajumdar * Guard activation Signed-off-by: smajumdar * Guard activation Signed-off-by: smajumdar --------- Signed-off-by: smajumdar * BERT pre-training mp fork to spawn (#6442) * change bert fork to spawn Signed-off-by: Abhinav Khattar * num_workers=0 fix Signed-off-by: Abhinav Khattar --------- Signed-off-by: Abhinav Khattar * Meagtron encoder decoder fix for empty validation outputs (#6459) * 1. Meagtron encoder decoder fix for empty validation outputs. Signed-off-by: Micha Livne * 1. Debugging. --------- Signed-off-by: Micha Livne Co-authored-by: Micha Livne * Added/updated new Conformer configs (#6426) * updated conf files. Signed-off-by: Vahid * added confs. Signed-off-by: Vahid * moved longconformer confs. Signed-off-by: Vahid * updated readme. Signed-off-by: Vahid * updated readme. Signed-off-by: Vahid * updated batch sizes and added fastconformer ctc streaming configs. Signed-off-by: Vahid * updated batch sizes. Signed-off-by: Vahid * added hybrid support. Signed-off-by: Vahid * added hybrid support. Signed-off-by: Vahid --------- Signed-off-by: Vahid * reduce workers on NMT CI (#6472) Signed-off-by: Abhinav Khattar * move to nvidia megatron repo (#6465) Signed-off-by: Abhinav Khattar * Megatron KERPLE positional embeddings (#6478) * [TTS] FastPitch adapter fine-tune and conditional layer normalization (#6416) [TTS] FastPitch adapter fine-tune and conditional layer normalization (#6416) --------- Signed-off-by: hsiehjackson Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [TTS] whitelist broken path fix. (#6412) * [TTS] whitelist broken path fix. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * [TTS] FastPitch speaker encoder (#6417) * Add initial codes Signed-off-by: hsiehjackson * Remove wemb Signed-off-by: hsiehjackson * Fix import Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Restore aligner loss Signed-off-by: hsiehjackson * Add ConditionalInput Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix error and support pre-trained config Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Follow comments Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Rename config Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Change copyright and random weight test Signed-off-by: hsiehjackson * Add initial codes Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Fix import error Signed-off-by: hsiehjackson * Add initial codes Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Fix dataset error Signed-off-by: hsiehjackson * Remove reference speaker embedding Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Remove SV encoder Signed-off-by: hsiehjackson * Follow comments Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Fix length type Signed-off-by: hsiehjackson * Fix append Signed-off-by: hsiehjackson * Move error msg Signed-off-by: hsiehjackson * Add look-up into speaker encoder Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Add valueerror msg Signed-off-by: hsiehjackson * Move lookup Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Remove unused Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: hsiehjackson * Fix error Signed-off-by: hsiehjackson * Rebase and Fix error Signed-off-by: hsiehjackson * Fix spk encoder Signed-off-by: hsiehjackson * Rename n_speakers Signed-off-by: hsiehjackson * Follow comments Signed-off-by: hsiehjackson * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix n_speakers None error Signed-off-by: hsiehjackson --------- Signed-off-by: hsiehjackson Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Sharded manifests for tarred datasets (#6395) * testing sharded manifests Signed-off-by: Dima Rekesh * compatibility Signed-off-by: Dima Rekesh * proper fixes Signed-off-by: Dima Rekesh * adding flag tot convert_to_tarred_audio_dataset Signed-off-by: Dima Rekesh * shard_manifests conf param Signed-off-by: Dima Rekesh * propagating the shard_manifests param Signed-off-by: Dima Rekesh * propagating the shard_manifests param Signed-off-by: Dima Rekesh * distributed checks Signed-off-by: Dima Rekesh * typo Signed-off-by: Dima Rekesh * typo Signed-off-by: Dima Rekesh * fixes Signed-off-by: Dima Rekesh * fixes Signed-off-by: Dima Rekesh * fixes Signed-off-by: Dima Rekesh * fixes Signed-off-by: Dima Rekesh * fixes Signed-off-by: Dima Rekesh * fixes Signed-off-by: Dima Rekesh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixes based on PR comments and tests Signed-off-by: Dima Rekesh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixes to convert_to_tarred_audio_dataset.py Signed-off-by: Dima Rekesh * reversing manifest shards flag Signed-off-by: Dima Rekesh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * tests Signed-off-by: Dima Rekesh * excluding manifests from webdataset url expansion Signed-off-by: Dima Rekesh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * expand manifest paths before attempting to cache from datastore Signed-off-by: Dima Rekesh * explicit use of UTF-8 for manifest i/o Signed-off-by: Dima Rekesh --------- Signed-off-by: Dima Rekesh Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Update wfst_text_normalization.rst (#6374) Add Hungarian (incoming in NeMo-text-processing) Signed-off-by: Jim O’Regan * Support Swiglu in TP PP Conversion (#6437) (#6451) * Support Swiglu in TP PP Conversion * Guard activation * Guard activation --------- Signed-off-by: smajumdar Co-authored-by: Somshubra Majumdar * Update NeMo_TTS_Primer.ipynb (#6436) * Update NeMo_TTS_Primer.ipynb Changed a mistake in line 782. Instead of frequency band (ie. pitch) we should write frequency bin. Note that frequency bins in FFT are not related to pitch. Signed-off-by: Mostafa Ghorbandoost * Update NeMo_TTS_Primer.ipynb Corrected the description of spectrogram and mel spectrogram calculations in lines 782 & 783 and added a fourth point to the description and added a reference for more mathematical details at the end of this point. Signed-off-by: Mostafa Ghorbandoost --------- Signed-off-by: Mostafa Ghorbandoost * add rampup batch size support for Megatron GPT (#6424) * added rampup batch size support Signed-off-by: Dmytro Pykhtar * added tests for rampup batch size Signed-off-by: Dmytro Pykhtar * fixed the typos Signed-off-by: Dmytro Pykhtar * added assertions Signed-off-by: Dmytro Pykhtar * changed assertion rules Signed-off-by: Dmytro Pykhtar * deleted unused imports Signed-off-by: Dmytro Pykhtar * changed tests for rampup batch size Signed-off-by: Dmytro Pykhtar * updated rampup batch size tests Signed-off-by: Dmytro Pykhtar * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed styling Signed-off-by: Dmytro Pykhtar * rampup batch size tests changes Signed-off-by: Dmytro Pykhtar --------- Signed-off-by: Dmytro Pykhtar Signed-off-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Co-authored-by: Dmytro Pykhtar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Harper * Meagtron encoder decoder fix for empty validation outputs (#6459) (#6461) * 1. Meagtron encoder decoder fix for empty validation outputs. * 1. Debugging. --------- Signed-off-by: Micha Livne Co-authored-by: Micha Livne Co-authored-by: Micha Livne * Code-Switching dataset creation - upgrading to aggregate tokenizer manifest format (#6448) * added functionality to create agg tokenizer compatible manifest for CS, flag to use this mode by default Signed-off-by: Kunal Dhawan * updated README with the new agg_tokenizer_manifest flag Signed-off-by: Kunal Dhawan * fixed typo in scripts/speech_recognition/code_switching/README.md Signed-off-by: Kunal Dhawan * changed agg_tokenizer_manifest to is_lid_manifest Signed-off-by: Kunal Dhawan --------- Signed-off-by: Kunal Dhawan Co-authored-by: Dima Rekesh * Added/updated new Conformer configs (#6426) (#6467) * Update script for ngram rnnt and hat beam search decoding (#6370) * add rnnt ngram beamsearch script Signed-off-by: andrusenkoau * add return encoding embedding option Signed-off-by: andrusenkoau * update script Signed-off-by: andrusenkoau * add rnnt and hat ngram decoding script Signed-off-by: andrusenkoau * add some parameters Signed-off-by: andrusenkoau * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add return_encoder_embeddings parameter to RNNTDecodingConfig Signed-off-by: andrusenkoau * replace return_encoder_embeddings parameter Signed-off-by: andrusenkoau * generalization of scipt behavior Signed-off-by: andrusenkoau * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove return_encoder_embeddings parameter Signed-off-by: andrusenkoau * remove return_encoder_embeddings parameter Signed-off-by: andrusenkoau * add manual encoder_embeddings calculation Signed-off-by: andrusenkoau * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix beam_width value to 8 Signed-off-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> * fix rescoring description Signed-off-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> --------- Signed-off-by: andrusenkoau Signed-off-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Somshubra Majumdar * BERT pre-training mp fork to spawn (#6442) (#6454) * change bert fork to spawn * num_workers=0 fix --------- Signed-off-by: Abhinav Khattar Co-authored-by: Abhinav Khattar * fix replace_bos_with_pad not found (#6443) (#6450) Signed-off-by: Abhinav Khattar Co-authored-by: Abhinav Khattar * reduce workers on NMT CI (#6472) (#6474) Signed-off-by: Abhinav Khattar Co-authored-by: Abhinav Khattar * 1. Added KERPLE positional embeddings to encoder-decoder. Signed-off-by: Micha Livne * 1. Added a missing file. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Fixing commits. Signed-off-by: Micha Livne * 1. Debugging. * 1. Debugging. * 1. Debugging. * 1. Debugging. --------- Signed-off-by: hsiehjackson Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Dima Rekesh Signed-off-by: Jim O’Regan Signed-off-by: smajumdar Signed-off-by: Mostafa Ghorbandoost Signed-off-by: Dmytro Pykhtar Signed-off-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Signed-off-by: Micha Livne Signed-off-by: Kunal Dhawan Signed-off-by: andrusenkoau Signed-off-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Signed-off-by: Abhinav Khattar Co-authored-by: Cheng-Ping Hsieh <37269846+hsiehjackson@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: Dima Rekesh Co-authored-by: Jim O’Regan Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Co-authored-by: Mostafa Ghorbandoost Co-authored-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Co-authored-by: Dmytro Pykhtar Co-authored-by: Eric Harper Co-authored-by: Micha Livne Co-authored-by: Kunal Dhawan Co-authored-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Co-authored-by: Abhinav Khattar * 1. Added external index sample. (#6462) Signed-off-by: Micha Livne * Fix cache aware hybrid bugs (#6466) * Update README to add core installation (#6488) * update README for megatron-core Signed-off-by: Abhinav Khattar * fix Signed-off-by: Abhinav Khattar --------- Signed-off-by: Abhinav Khattar * Fix typos (#6494) Signed-off-by: smajumdar * fix broken links r1.18.0 (#6501) * fix broken links Signed-off-by: Evelina * fix broken links Signed-off-by: Evelina --------- Signed-off-by: Evelina * 1. Fixed gaussian hidden transform. Signed-off-by: Micha Livne * 1. Finished updating hidden loss for MIM. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix custom forward_torch_softmax (#6512) Signed-off-by: Abhinav Khattar * [BugFix] Force _get_batch_preds() to keep logits in decoder timestamp… (#6500) * [BugFix] Force _get_batch_preds() to keep logits in decoder timestamps generator r1.18.0 Signed-off-by: Taejin Park * ignore keep_logits in FrameBatchASRLogits Signed-off-by: Taejin Park --------- Signed-off-by: Taejin Park * [TTS] fixed broken path. (#6514) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * 1. Added a hiddens module. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix typos (#6523) (#6539) * Fix typos Signed-off-by: smajumdar * Fix typos Signed-off-by: smajumdar --------- Signed-off-by: smajumdar (cherry picked from commit 7fa47bfcd359f1ea80ad056d8bb2d78f97c6206a) * added back the fast emit section to the configs. (#6540) * added back the fast emit section to the configs. Signed-off-by: Vahid * added back the fast emit section to the configs. Signed-off-by: Vahid --------- Signed-off-by: Vahid * Fix fp16 (#6543) Signed-off-by: MaximumEntropy * fix (#6529) Signed-off-by: Abhinav Khattar * pass .scale instead of scaler object to core (#6545) Signed-off-by: Abhinav Khattar Co-authored-by: Eric Harper * Change Megatron Enc Dec model to use persistent_workers (#6548) * persistent workers Signed-off-by: Abhinav Khattar * fix Signed-off-by: Abhinav Khattar --------- Signed-off-by: Abhinav Khattar Co-authored-by: Eric Harper * Add FastConformer Hybrid ASR models for EN, ES, IT, DE, PL, HR, UA, BY (#6549) * Added fastconfomer hybrid asr models for en, es, it, de, pl, hr, ua, by Signed-off-by: KunalDhawan * updated ASR docs with the fastconformer hybrid checkpoints Signed-off-by: KunalDhawan * added the fastconformer RNNT and CTC models Signed-off-by: KunalDhawan --------- Signed-off-by: KunalDhawan * Add scores for FastConformer models (#6557) Signed-off-by: smajumdar * Patch transcribe and support offline transcribe for hybrid model (#6550) Signed-off-by: fayejf * Not doing CastToFloat by default (#6524) * Not doing CastToFloat by default Signed-off-by: Boris Fomitchev * Added docustring Signed-off-by: Boris Fomitchev * Dummy commit Signed-off-by: Boris Fomitchev --------- Signed-off-by: Boris Fomitchev * temp rtd fix (#6568) Signed-off-by: Abhinav Khattar * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update manifest.py for speedup (#6565) * Update manifest.py Re-order the checks for faster processing audio filepaths that are already absolute paths Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> * Update manifest.py Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> --------- Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: Vahid Noroozi * Turn autocast off when precision is fp32 (#6554) * Turn autocast off when precision is fp32 Signed-off-by: Abhinav Khattar * address review Signed-off-by: Abhinav Khattar * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixes Signed-off-by: Abhinav Khattar * merge Signed-off-by: Abhinav Khattar --------- Signed-off-by: Abhinav Khattar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Harper * More streaming conformer export fixes (#6567) Signed-off-by: Greg Clark Co-authored-by: Vahid Noroozi * Fix batch size reconf for T5 FT for multi-validation (#6582) Signed-off-by: Abhinav Khattar * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Updated Megatron LM encoder/decoder to use cfg for hiddens. Signed-off-by: Micha Livne * 1. Added support to register externalhidden loss / transforms. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Make tensor split contiguous (#6580) Signed-off-by: Abhinav Khattar * Patches from main to r1.18.0 for Virtual Parallel (#6592) * Add interleaved pp support (#6498) * Add support for Virtual Pipeline Parallel conversion Signed-off-by: smajumdar * Add support for Virtual Pipeline Parallel conversion Signed-off-by: smajumdar * Switch to megatron core Signed-off-by: smajumdar * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: smajumdar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> (cherry picked from commit fa9ab5b41301c9d92c97bf37894f9c94de6b91b5) * Add patches for Virtual Parallel conversion (#6589) * Add patches for Virtual Parllel conversion Signed-off-by: smajumdar * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: smajumdar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> (cherry picked from commit 4d639ef36b10d8536db02bc47ea30f4acbc061bb) * Documentation for ASR-TTS models (#6594) * Add docs about hybrid ASR-TTS models Signed-off-by: Vladimir Bataev * Add docs about text-only datasets Signed-off-by: Vladimir Bataev * Add docs about ASR-TTS checkpoints Signed-off-by: Vladimir Bataev * Add docs about ASR-TTS configs and training Signed-off-by: Vladimir Bataev * Clean up Signed-off-by: Vladimir Bataev * ASR-TTS docs: add to api, fix imports Signed-off-by: Vladimir Bataev * Clean up Signed-off-by: Vladimir Bataev * Wrap optional import Signed-off-by: Vladimir Bataev * Revert general ASR import Signed-off-by: Vladimir Bataev --------- Signed-off-by: Vladimir Bataev * Update SDP docs (#6485) * add info about SDP e.g. processor classes in docs Signed-off-by: Elena Rastorgueva * add link to SDP docs in README Signed-off-by: Elena Rastorgueva * address code review comments and add SDP overview diagram Signed-off-by: Elena Rastorgueva * Fix spelling typo Signed-off-by: Elena Rastorgueva --------- Signed-off-by: Elena Rastorgueva * Create dummy iters to satisy len checks (#6600) Signed-off-by: Abhinav Khattar * 1. Debugging. Signed-off-by: Micha Livne * Restore GPT support for interleaved pipeline parallelism (#6528) * Restore logic for data-parallel communication with pipeline parallelism in GPT Signed-off-by: Tim Moon * Support dynamic attention masks in GPT Signed-off-by: Tim Moon * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Debug typos Signed-off-by: Tim Moon * Debug data iterator caching with interleaved pipeline parallelism Each model chunk accesses the data iterator multiple times, so we need to cache multiple samples. Signed-off-by: Tim Moon * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Megatron-LM commit Signed-off-by: Tim Moon * Distinguish between list of data iterators and data iterator that is a list Signed-off-by: Tim Moon * Create dummy iters to satisy len checks Signed-off-by: Abhinav Khattar * Kludge while waiting for Megatron-LM update Signed-off-by: Tim Moon * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * set transformers offline to avoid rate limiting Signed-off-by: ericharper --------- Signed-off-by: Tim Moon Signed-off-by: Eric Harper Signed-off-by: Abhinav Khattar Signed-off-by: ericharper Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Harper Co-authored-by: Abhinav Khattar * Patch transcribe_util for steaming mode and add wer calculation back to inference scripts (#6601) * fix write Signed-off-by: fayejf * decoding ctc Signed-off-by: fayejf * temp set rnnt decoding return_best_hypothesis to true Signed-off-by: fayejf * add wer cal back to transcribe_speech as requested Signed-off-by: fayejf * add wer cal back to speech_to_text_buffered_infer_rnnt as requested Signed-off-by: fayejf * add wer cal back to speech_to_text_buffered_infer_ctc as requested Signed-off-by: fayejf * style fix Signed-off-by: fayejf * reflect change in asr_evaluator Signed-off-by: fayejf * reflect som and vahid comment Signed-off-by: fayejf * remove return_best_hy=true in transcribe_speech Signed-off-by: fayejf * no text skip Signed-off-by: fayejf --------- Signed-off-by: fayejf * 1. Added example conf YAML. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Added support in tensor_parallel. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add hat image to docs (#6619) Signed-off-by: andrusenkoau * update core commit hash in readme (#6622) Signed-off-by: Abhinav Khattar * Patch decoding for PC models (#6630) * Patch decoding logic for PC models Signed-off-by: smajumdar * Patch decoding logic for PC models Signed-off-by: smajumdar --------- Signed-off-by: smajumdar * Fix wer.py where 'errors' variable was not set (#6633) Fix wer.py where 'errors' variable was not set when both reference and hypothesis are empty strings Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> * fix att_context_size bug for older models. (#6635) Signed-off-by: Vahid * Add megatron_core to requirements (#6639) * add megatron_core to requirements Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: ericharper Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Remove from jenkins (#6641) * add megatron_core to requirements Signed-off-by: ericharper * remove from jenkins Signed-off-by: ericharper --------- Signed-off-by: ericharper * remove dup (#6643) Signed-off-by: ericharper * 1. Fixed config to use names, and added better error messages. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Added support to pass extra data to hiddens for loss computation. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Working on passing extra data to hiddnes. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Fixed support in loading .nemo without hiddnes module. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Improved and fixed logging of validation and testing. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Fixed training logging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Fixed logging of hidden loss. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Fixed logging names. 2. Added logging to hiddens and tokens loss. Signed-off-by: Micha Livne * 1. Fixed conflicts. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne --------- Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Oleksii Kuchaiev Signed-off-by: Jocelyn Huang Signed-off-by: smajumdar Signed-off-by: fayejf Signed-off-by: Alexandra Antonova Signed-off-by: Virginia Adams Signed-off-by: Vladimir Bataev Signed-off-by: ericharper Signed-off-by: Vahid Signed-off-by: Ante Jukić Signed-off-by: David Mosallanezhad Signed-off-by: sam1373 Signed-off-by: MaximumEntropy Signed-off-by: ekmb Signed-off-by: Yang Zhang Signed-off-by: Micha Livne Signed-off-by: Tim Moon Signed-off-by: Abhinav Khattar Signed-off-by: smajumdar Signed-off-by: Micha Livne Signed-off-by: hsiehjackson Signed-off-by: Dima Rekesh Signed-off-by: Jim O’Regan Signed-off-by: Mostafa Ghorbandoost Signed-off-by: Dmytro Pykhtar Signed-off-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Signed-off-by: Kunal Dhawan Signed-off-by: andrusenkoau Signed-off-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Signed-off-by: Evelina Signed-off-by: Taejin Park Signed-off-by: KunalDhawan Signed-off-by: Boris Fomitchev Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: Greg Clark Signed-off-by: Elena Rastorgueva Signed-off-by: Eric Harper Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: Oleksii Kuchaiev Co-authored-by: Jocelyn Co-authored-by: Somshubra Majumdar Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Co-authored-by: bene-ges <61418381+bene-ges@users.noreply.github.com> Co-authored-by: Alexandra Antonova Co-authored-by: Virginia Adams <78445382+vadam5@users.noreply.github.com> Co-authored-by: Zhilin Wang Co-authored-by: Vladimir Bataev Co-authored-by: Nithin Rao Co-authored-by: Eric Harper Co-authored-by: Vahid Noroozi Co-authored-by: anteju <108555623+anteju@users.noreply.github.com> Co-authored-by: Ante Jukić Co-authored-by: David Co-authored-by: David Mosallanezhad Co-authored-by: Samuel Kriman Co-authored-by: Sandeep Subramanian Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Sean Naren Co-authored-by: Yang Zhang Co-authored-by: Sean Naren Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Co-authored-by: Neha Tadimeti Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Abhinav Khattar Co-authored-by: Cheng-Ping Hsieh <37269846+hsiehjackson@users.noreply.github.com> Co-authored-by: Dima Rekesh Co-authored-by: Jim O’Regan Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Mostafa Ghorbandoost Co-authored-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Co-authored-by: Dmytro Pykhtar Co-authored-by: Kunal Dhawan Co-authored-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Co-authored-by: Taejin Park Co-authored-by: Boris Fomitchev Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: Greg Clark Co-authored-by: Elena Rastorgueva <80532067+erastorgueva-nv@users.noreply.github.com> Signed-off-by: dorotat * Load ub_cfg from hydra config (#7003) * Pass tp config via hydra Signed-off-by: Jan Baczek * Remove self.ub_cfgs field - it isn't used anywhere else Signed-off-by: Jan Baczek * Allow tp_overlap tree substitution in hydra config Signed-off-by: Jan Baczek * Add warning in case of usage of the default tp config * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Change warning message Signed-off-by: Jan Baczek * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add compute capability resolver Signed-off-by: Jan Baczek * Bugfix Signed-off-by: Jan Baczek * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add guards to pynvml import Signed-off-by: Jan Baczek --------- Signed-off-by: Jan Baczek Signed-off-by: yaoyu-33 <54727607+yaoyu-33@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: yaoyu-33 <54727607+yaoyu-33@users.noreply.github.com> Signed-off-by: dorotat * NFA docs: update READMEs and links, add docs page (#7219) * move some of NFA readme to new NFA docs page and add various links Signed-off-by: Elena Rastorgueva * address code review comments Signed-off-by: Elena Rastorgueva --------- Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * make image centering in NFA readme actually work (#7220) Signed-off-by: Elena Rastorgueva Signed-off-by: dorotat * Fixes for lightning 2.0 upgrade (#7176) * Remove trainer._checkpoint_connector = _CheckpointConnector(trainer) Signed-off-by: Abhishree * Remove trainer._checkpoint_connector = _CheckpointConnector(trainer) for NMT Signed-off-by: Abhishree * Add resume_if_exists=False in JenkinsFile 1) Add resume_if_exists=False for Megatron GPT Pretraining and Resume Training PP=2 as it can resume from the checkpoint of the previous model test in CI leading to Error Signed-off-by: Abhishree * Remove optimizer_idx in optimizer_step and fix typo 1) Remove arg optimizer_idx in optimizer_step func as the arg is not used my parent func of lightning Signed-off-by: Abhishree * Remove resume_if_exists=False in JenkinsFile Signed-off-by: Abhishree * Make trainer.val_check_interval=1 for few tests in JenkinsFile Signed-off-by: Abhishree * Change val_check_interval in JenkinsFile during resume to less than len(dataloader) Signed-off-by: Abhishree * Change val_check_interval to 1 for Megatron T5 with ALiBi resume Signed-off-by: Abhishree * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Abhishree Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix `consumed_samples` which was off by one batch (#7212) Signed-off-by: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Co-authored-by: Sandeep Subramanian Signed-off-by: dorotat * Start using ModelParallelConfig from Megatron Core (#6885) * start adding gpt from megatron core path Signed-off-by: ericharper * set model parallel config Signed-off-by: ericharper * use model parallel config object Signed-off-by: ericharper * update args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * set vp size to none if it is 1 Signed-off-by: ericharper * set vp size to none if it is 1 Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add TransformerConfig Signed-off-by: ericharper * start updating to TransformerConfig Signed-off-by: ericharper * add todo Signed-off-by: ericharper * revert to model parallel config Signed-off-by: ericharper * add hidden_size to model_parallel_config Signed-off-by: ericharper * remove imports Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove import Signed-off-by: ericharper * small clean up Signed-off-by: ericharper * update hidden size in peft base model, add mcore commit to jenkins Signed-off-by: ericharper * update module args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add config obj to flash attention tests Signed-off-by: ericharper * remove args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove sequence parallel arg Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update args Signed-off-by: ericharper * add config to self Signed-off-by: ericharper * update args Signed-off-by: ericharper * update args Signed-off-by: ericharper * update args Signed-off-by: ericharper * add config to test Signed-off-by: ericharper * get hidden_size from config Signed-off-by: ericharper * add try except Signed-off-by: ericharper * use default Signed-off-by: ericharper * update config with hidden size Signed-off-by: ericharper * remove arg Signed-off-by: ericharper * comment out jenkins test Signed-off-by: ericharper * revert import Signed-off-by: ericharper * remove optimizer_idx Signed-off-by: eharper * prefetch num microbatches Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove import Signed-off-by: eharper * temporarily comment jenkins test Signed-off-by: eharper * update seq_length Signed-off-by: eharper * remove commented code Signed-off-by: eharper * update arg Signed-off-by: eharper * update mbs and gbs of test Signed-off-by: eharper * update batch size in test Signed-off-by: eharper * fix precision in test Signed-off-by: eharper * update precision Signed-off-by: eharper * move hidden_size out of conditional Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: ericharper Signed-off-by: eharper Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * deprecation warning (#7193) * dep warning Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * added logging import Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * import fix Signed-off-by: arendu * pretty deprecation warning Signed-off-by: arendu --------- Signed-off-by: arendu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix attention mask inference (#7213) * Fix attention mask inference Signed-off-by: Cheng-Ping Hsieh * Fix config Signed-off-by: Cheng-Ping Hsieh --------- Signed-off-by: Cheng-Ping Hsieh Signed-off-by: dorotat * Use GPTModel from mcore (#7093) * start adding gpt from megatron core path Signed-off-by: ericharper * set model parallel config Signed-off-by: ericharper * use model parallel config object Signed-off-by: ericharper * update args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * set vp size to none if it is 1 Signed-off-by: ericharper * set vp size to none if it is 1 Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add TransformerConfig Signed-off-by: ericharper * start updating to TransformerConfig Signed-off-by: ericharper * add todo Signed-off-by: ericharper * revert to model parallel config Signed-off-by: ericharper * add hidden_size to model_parallel_config Signed-off-by: ericharper * remove imports Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove import Signed-off-by: ericharper * small clean up Signed-off-by: ericharper * update hidden size in peft base model, add mcore commit to jenkins Signed-off-by: ericharper * update module args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add config obj to flash attention tests Signed-off-by: ericharper * remove args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove sequence parallel arg Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update args Signed-off-by: ericharper * add config to self Signed-off-by: ericharper * update args Signed-off-by: ericharper * update args Signed-off-by: ericharper * update args Signed-off-by: ericharper * add config to test Signed-off-by: ericharper * get hidden_size from config Signed-off-by: ericharper * add try except Signed-off-by: ericharper * use default Signed-off-by: ericharper * update config with hidden size Signed-off-by: ericharper * remove arg Signed-off-by: ericharper * comment out jenkins test Signed-off-by: ericharper * revert import Signed-off-by: ericharper * remove optimizer_idx Signed-off-by: eharper * prefetch num microbatches Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * start adding gpt from megatron core path Signed-off-by: ericharper * set model parallel config Signed-off-by: ericharper * use model parallel config object Signed-off-by: ericharper * update args Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * start updating to TransformerConfig Signed-off-by: ericharper * revert to model parallel config Signed-off-by: ericharper * add hidden_size to model_parallel_config Signed-off-by: ericharper * remove imports Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update module args Signed-off-by: ericharper * add config to self Signed-off-by: ericharper * build transformer config Signed-off-by: ericharper * add model to provider func Signed-off-by: ericharper * update forward and float16 wrapper Signed-off-by: ericharper * instantiate model parallel config after init model parallel Signed-off-by: ericharper * set virtual rank Signed-off-by: ericharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add GQA config to megatron gpt model (#7096) * Add GQA config in gpt config file Signed-off-by: jasonwan * Verify mcore is enabled when using GQA Signed-off-by: jasonwan --------- Signed-off-by: jasonwan * revert Signed-off-by: ericharper * remove import Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update for dist adam Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * use get_gpt_module_list Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update megatron core commit Signed-off-by: eharper * revert change Signed-off-by: eharper * remove import Signed-off-by: eharper * remove import Signed-off-by: eharper * remove import Signed-off-by: eharper --------- Signed-off-by: ericharper Signed-off-by: eharper Signed-off-by: jasonwan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jason Wang Signed-off-by: dorotat * add mcore installation (#7237) Signed-off-by: eharper Signed-off-by: dorotat * Add bf16-mixed and 16-mixed in module.py (#7227) Signed-off-by: Abhishree Signed-off-by: dorotat * Refactor LLM pretraining examples (#7159) * add builder class Signed-off-by: Maanu Grover * formatting Signed-off-by: Maanu Grover * use trainer builder for gpt pretraining example Signed-off-by: Maanu Grover * subclass trainer builder for bert Signed-off-by: Maanu Grover * use trainer builder for bert pretraining example Signed-off-by: Maanu Grover * subclass t5 builder and use in t5 pretraining Signed-off-by: Maanu Grover * move resume_from_checkpoint logic to exp_manager Signed-off-by: Maanu Grover * add docstring for resume_from_checkpoint Signed-off-by: Maanu Grover * set resume_from_checkpoint with interpolation Signed-off-by: Maanu Grover * remove refactored lines Signed-off-by: Maanu Grover * unused import Signed-off-by: Maanu Grover * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * another unused import Signed-off-by: Maanu Grover * bug fix Signed-off-by: Maanu Grover * another bug missed in rebase Signed-off-by: Maanu Grover * add copyright Signed-off-by: Maanu Grover * add type annotation Signed-off-by: Maanu Grover * docstrings for trainer builder Signed-off-by: Maanu Grover * move trainer builder file Signed-off-by: Maanu Grover * not needed for ptl 2.0 Signed-off-by: Maanu Grover * disable resume_from_checkpoint logic in exp_manager Signed-off-by: Maanu Grover --------- Signed-off-by: Maanu Grover Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Abhishree Thittenamane <47577437+athitten@users.noreply.github.com> Signed-off-by: dorotat * Add only trainable parameters to optimizer group in PEFT (#7230) * Add only parameters with requires_grad=True to the optimizer group Signed-off-by: Guyue Huang Signed-off-by: dorotat * Dummy class for ModelParallelConfig (#7254) * guard ModelParallelConfig Signed-off-by: eharper * guard TransformerConfig Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * revert Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * import Signed-off-by: eharper * revert Signed-off-by: eharper * revert Signed-off-by: eharper * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * import Signed-off-by: eharper --------- Signed-off-by: eharper Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * [TN][Docs] update language coverage matrix and refs (#7247) Signed-off-by: Mariana Graterol Fuenmayor Signed-off-by: dorotat * Checkpoint averaging for model parallel (#7252) * Checkpoint averaging for model parallel Signed-off-by: Igor Gitman * Add a check for dir name Signed-off-by: Igor Gitman --------- Signed-off-by: Igor Gitman Co-authored-by: Sandeep Subramanian Signed-off-by: dorotat * [ASR] Fix GPU memory leak in transcribe_speech.py (#7249) Signed-off-by: Ryan Signed-off-by: dorotat * Adding Multilingual, Code-Switched, and Hybrid ASR models (#7250) * Updated docs with Hybrid, Multilingual and Code-Swithing ASR models Signed-off-by: KunalDhawan * updated pretrained model list for multilingual models Signed-off-by: KunalDhawan --------- Signed-off-by: KunalDhawan Signed-off-by: dorotat * Adapter ipa Tutorial and config update (#7260) * Added config for speaker adapter config for IPA * Updated epochs, added IPA support * Updated epochs, added IPA support Signed-off-by: Siddharth Tyagi --------- Signed-off-by: Siddharth Tyagi Co-authored-by: Siddharth Tyagi Signed-off-by: dorotat * [TTS] Audio codec fixes (#7266) Signed-off-by: Ryan Signed-off-by: dorotat * Upgrade hydra and omegaconf (#7243) * Upgrade hydra and omegaconf Signed-off-by: Abhishree * Edit omegaconf version Signed-off-by: Abhishree Thittenamane <47577437+athitten@users.noreply.github.com> --------- Signed-off-by: Abhishree Signed-off-by: Abhishree Thittenamane <47577437+athitten@users.noreply.github.com> Signed-off-by: dorotat * Update numba support (#7271) Signed-off-by: smajumdar Signed-off-by: dorotat * tied weights for adapters (#6928) * wip Signed-off-by: arendu * wip Signed-off-by: arendu * tied lora weights Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * lora and adapter tying Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * layer selection wip Signed-off-by: arendu * added layer selection Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * make ln optional Signed-off-by: arendu * layer section Signed-off-by: arendu * small dim pos emb Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * adapter w/o layer norm and weight tying Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * eval works with all pos embeddings strategy Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update Signed-off-by: arendu * mlp transform of pos embeddings Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * zero init position bias Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * merge Signed-off-by: arendu * minor fix Signed-off-by: arendu --------- Signed-off-by: arendu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix skip generation (#7270) * Fix skip generation Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add metric condition Signed-off-by: Cheng-Ping Hsieh --------- Signed-off-by: Cheng-Ping Hsieh Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Adi Renduchintala Signed-off-by: dorotat * Hidden transforms model parallel config + CI with Perceiver (#7241) * 1. Fixed ModelParallelConfig compatibility. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Fixed Perceiver. Signed-off-by: Micha Livne * 1. Added CI forhidden transformations (MIM) + Perceiver for Megatron. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * 1. Removed debugging code. Signed-off-by: Micha Livne * 1. Fixed CI command argument (removed space). Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Fixing CI command line. Signed-off-by: Micha Livne * 1. Fixed CI command. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne --------- Signed-off-by: Micha Livne Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Fix restore sequence parallel (#7273) * Fix restore Signed-off-by: Cheng-Ping Hsieh * reset and restore transformer config sequence parallel Signed-off-by: Jason Wang * modify model parallel config as well Signed-off-by: Jason Wang --------- Signed-off-by: Cheng-Ping Hsieh Signed-off-by: Jason Wang Co-authored-by: Jason Wang Signed-off-by: dorotat * fix ptuning and lora model_parallel_config (#7287) * fix ptuning and lora model_parallel_config Signed-off-by: jasonwan * support deprecated models Signed-off-by: jasonwan * update megatron connit sha Signed-off-by: jasonwan --------- Signed-off-by: jasonwan Signed-off-by: dorotat * Fix adapters and ptuning for amp O2 (#7285) * Transform adapter modules to fp16/bf16 under amp_O2 * Under megatron_amp_O2, transform the adapter modules to low precision after instantiation Signed-off-by: Guyue Huang Conflicts: nemo/collections/nlp/modules/common/megatron/adapters/parallel_adapters.py * Fix ptuning under amp O2 * Fix the first_stage_of_pipeline detection for half models * Fix the freezing of InferenceTable for half models Signed-off-by: Guyue Huang * Fix MegatronGPTAdapterPTuningModel * When unfreezing adapters, we explicitly set inference embedding table in prompt encoder to be untrainable. Signed-off-by: Guyue Huang * Add comments for feature explanation Signed-off-by: Guyue Huang * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix ptuning and lora model_parallel_config Signed-off-by: jasonwan * Put the casting of adapters in their instantiaion Signed-off-by: Guyue Huang * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * small fix for state dict Signed-off-by: jasonwan * optional model_parallel_config Signed-off-by: jasonwan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Guyue Huang Signed-off-by: jasonwan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jasonwan Signed-off-by: dorotat * fix partial transcribe (#7284) * fix partial transcribe Signed-off-by: stevehuang52 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refactor Signed-off-by: stevehuang52 --------- Signed-off-by: stevehuang52 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * remove additional line (#7293) Signed-off-by: Jason Wang Signed-off-by: dorotat * loss mask aware final layer applicaiton (#7275) * loss mask for final output and softmax Signed-off-by: arendu * bs2 working Signed-off-by: arendu * Fix skip generation Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add metric condition Signed-off-by: Cheng-Ping Hsieh * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * encoder_input is none check Signed-off-by: arendu * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: arendu Signed-off-by: Cheng-Ping Hsieh Co-authored-by: Cheng-Ping Hsieh Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * Adding server option to peft eval (#7292) * Adding server to peft eval Signed-off-by: David Mosallanezhad * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated trainer.test for server Signed-off-by: David Mosallanezhad --------- Signed-off-by: David Mosallanezhad Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: dorotat * remove deprecated scripts from ci (#7239) Signed-off-by: arendu Signed-off-by: dorotat * add log_model to MLFlowParams (#7258) Signed-off-by: Alireza Morsali Signed-off-by: dorotat * [TTS] minor fix typos and input_types (#7272) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: dorotat * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: dorotat * improved docs Signed-off-by: dorotat * fixed unit test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: dorotat Signed-off-by: arendu Signed-off-by: sam1373 Signed-off-by: Boris Fomitchev Signed-off-by: fayejf Signed-off-by: Somshubra Majumdar Signed-off-by: Daniel Egert Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Jan Beckmann Signed-off-by: Tim Moon Signed-off-by: Linnea Pari Leaver Signed-off-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Signed-off-by: smajumdar Signed-off-by: Ryan Signed-off-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Signed-off-by: Elena Rastorgueva Signed-off-by: Ante Jukić Signed-off-by: Dmytro Pykhtar Signed-off-by: ericharper Signed-off-by: Vladimir Bataev Signed-off-by: Nikolay Karpov Signed-off-by: Alexandra Antonova Signed-off-by: Evelina Signed-off-by: Taejin Park Signed-off-by: Abhishree Signed-off-by: Yi Dong Signed-off-by: jubick1337 Signed-off-by: tbartley94 Signed-off-by: Aleksandr Laptev Signed-off-by: AlexGrinch Signed-off-by: Vitaly Lavrukhin Signed-off-by: stevehuang52 Signed-off-by: Xin Yao Signed-off-by: fayejf <36722593+fayejf@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh Signed-off-by: hsiehjackson Signed-off-by: Cheng-Ping Hsieh <37269846+hsiehjackson@users.noreply.github.com> Signed-off-by: Oleksii Kuchaiev Signed-off-by: Jocelyn Huang Signed-off-by: smajumdar Signed-off-by: Alexandra Antonova Signed-off-by: Virginia Adams Signed-off-by: Vahid Signed-off-by: David Mosallanezhad Signed-off-by: MaximumEntropy Signed-off-by: ekmb Signed-off-by: Yang Zhang Signed-off-by: Micha Livne Signed-off-by: Abhinav Khattar Signed-off-by: Micha Livne Signed-off-by: Dima Rekesh Signed-off-by: Jim O’Regan Signed-off-by: Mostafa Ghorbandoost Signed-off-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Signed-off-by: Kunal Dhawan Signed-off-by: andrusenkoau Signed-off-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Signed-off-by: KunalDhawan Signed-off-by: Greg Clark Signed-off-by: Eric Harper Signed-off-by: Jan Baczek Signed-off-by: yaoyu-33 <54727607+yaoyu-33@users.noreply.github.com> Signed-off-by: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Signed-off-by: eharper Signed-off-by: jasonwan Signed-off-by: Maanu Grover Signed-off-by: Guyue Huang Signed-off-by: Mariana Graterol Fuenmayor Signed-off-by: Igor Gitman Signed-off-by: Siddharth Tyagi Signed-off-by: Abhishree Thittenamane <47577437+athitten@users.noreply.github.com> Signed-off-by: Jason Wang Signed-off-by: arendu Signed-off-by: Alireza Morsali Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Adi Renduchintala Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Vahid Noroozi Co-authored-by: Samuel Kriman Co-authored-by: Boris Fomitchev Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Co-authored-by: Somshubra Majumdar Co-authored-by: trias702 <25867060+trias702@users.noreply.github.com> Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: Jan Beckmann Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Co-authored-by: lleaver <137942999+lleaver@users.noreply.github.com> Co-authored-by: Linnea Pari Leaver Co-authored-by: He Huang (Steve) <105218074+stevehuang52@users.noreply.github.com> Co-authored-by: Ryan Langman Co-authored-by: Kim Ngo <6362111+findkim@users.noreply.github.com> Co-authored-by: David Co-authored-by: Elena Rastorgueva <80532067+erastorgueva-nv@users.noreply.github.com> Co-authored-by: anteju <108555623+anteju@users.noreply.github.com> Co-authored-by: Dmytro Pykhtar <37850217+dimapihtar@users.noreply.github.com> Co-authored-by: Eric Harper Co-authored-by: Vladimir Bataev Co-authored-by: Nikolay Karpov Co-authored-by: bene-ges Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Taejin Park Co-authored-by: Abhishree Thittenamane <47577437+athitten@users.noreply.github.com> Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> Co-authored-by: Matvei Novikov Co-authored-by: tbartley94 <90423858+tbartley94@users.noreply.github.com> Co-authored-by: Aleksandr Laptev Co-authored-by: Aleksey Grinchuk (Oleksii Hrinchuk) Co-authored-by: Vitaly Lavrukhin Co-authored-by: Xin Yao Co-authored-by: anmolgupt <14880251+anmolgupt@users.noreply.github.com> Co-authored-by: ANMOL GUPTA Co-authored-by: Cheng-Ping Hsieh <37269846+hsiehjackson@users.noreply.github.com> Co-authored-by: Micha Livne Co-authored-by: Oleksii Kuchaiev Co-authored-by: Jocelyn Co-authored-by: bene-ges <61418381+bene-ges@users.noreply.github.com> Co-authored-by: Alexandra Antonova Co-authored-by: Virginia Adams <78445382+vadam5@users.noreply.github.com> Co-authored-by: Zhilin Wang Co-authored-by: Nithin Rao Co-authored-by: Ante Jukić Co-authored-by: David Mosallanezhad Co-authored-by: Sandeep Subramanian Co-authored-by: Sean Naren Co-authored-by: Yang Zhang Co-authored-by: Sean Naren Co-authored-by: Neha Tadimeti Co-authored-by: Abhinav Khattar Co-authored-by: Dima Rekesh Co-authored-by: Jim O’Regan Co-authored-by: Mostafa Ghorbandoost Co-authored-by: Dmytro Pykhtar Co-authored-by: Kunal Dhawan Co-authored-by: Andrei Andrusenko <52885736+andrusenkoau@users.noreply.github.com> Co-authored-by: Greg Clark Co-authored-by: jbaczek <45043825+jbaczek@users.noreply.github.com> Co-authored-by: yaoyu-33 <54727607+yaoyu-33@users.noreply.github.com> Co-authored-by: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> Co-authored-by: Jason Wang Co-authored-by: Maanu Grover <109391026+maanug-nv@users.noreply.github.com> Co-authored-by: guyueh1 <140554423+guyueh1@users.noreply.github.com> Co-authored-by: Mariana <47233618+mgrafu@users.noreply.github.com> Co-authored-by: Igor Gitman Co-authored-by: styagi130 Co-authored-by: Siddharth Tyagi Co-authored-by: Cheng-Ping Hsieh Co-authored-by: Alireza Morsali <32244795+AlirezaMorsali@users.noreply.github.com> --- .../language_modeling/text_memmap_dataset.py | 57 +++++++++++++++++++ tests/collections/nlp/test_mem_map_dataset.py | 16 ++++++ 2 files changed, 73 insertions(+) diff --git a/nemo/collections/nlp/data/language_modeling/text_memmap_dataset.py b/nemo/collections/nlp/data/language_modeling/text_memmap_dataset.py index 706fdf1d2393b..28e7a0256197e 100644 --- a/nemo/collections/nlp/data/language_modeling/text_memmap_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/text_memmap_dataset.py @@ -333,6 +333,63 @@ def _build_data_from_text(self, text): return super()._build_data_from_text(text) +class CSVFieldsMemmapDataset(TextMemMapDataset): + """ + Allow per-line lazy access to multiple csv files using numpy memmap. + Returns a dictionary with multiple fields. + """ + + def __init__( + self, + dataset_paths, + newline_int=10, + header_lines=1, + workers=None, + tokenizer=None, + sort_dataset_paths=True, + data_sep=',', + data_fields={"data": 0}, + index_mapping_dir: Optional[str] = None, + ): + """ + Args: + dataset_paths: list of csv file paths to read data from + newline_int: ASCII code to use to interpret newlines in file. + header_lines: number of header lines in csv files. + workers: number of workers to use for creating index files. + tokenizer: tokenizer to use to convert text to tokens. + sort_dataset_paths: whether to sort datasets by paths. + data_sep: data separator. + data_fields: dict of field names and their corresponding column indices + index_mapping_dir: directory to save the index mapping to. + If None, will write to the same folder as the dataset. + """ + super().__init__( + dataset_paths=dataset_paths, + newline_int=newline_int, + header_lines=header_lines, + workers=workers, + tokenizer=tokenizer, + sort_dataset_paths=sort_dataset_paths, + index_mapping_dir=index_mapping_dir, + ) + + self._data_fields = data_fields + self._data_sep = data_sep + + def _build_data_from_text(self, text: str): + """ + + """ + _build_data_from_text = super()._build_data_from_text + data = {} + text_fields = text.split(self._data_sep) + for field_name, field_idx in self._data_fields.items(): + data[field_name] = _build_data_from_text(text_fields[field_idx]) + + return data + + class JSONLMemMapDataset(TextMemMapDataset): """ Memory-mapped iteration over a JSONL file. diff --git a/tests/collections/nlp/test_mem_map_dataset.py b/tests/collections/nlp/test_mem_map_dataset.py index 1e21b6d270c9f..20932b6c4e0df 100644 --- a/tests/collections/nlp/test_mem_map_dataset.py +++ b/tests/collections/nlp/test_mem_map_dataset.py @@ -84,6 +84,22 @@ def test_csv_mem_map_dataset(csv_file): assert indexed_dataset[2].strip() == "Bob" +def test_csv_fields_mem_map_dataset(csv_file): + """Test for CSV memory-mapped datasets.""" + + indexed_dataset = text_memmap_dataset.CSVFieldsMemmapDataset( + dataset_paths=[csv_file], data_fields={"ID": 0, "Name": 1}, header_lines=1 + ) + assert isinstance(indexed_dataset[0], dict) + assert sorted(indexed_dataset[0].keys()) == ["ID", "Name"] + assert indexed_dataset[0]["ID"] == "1" and indexed_dataset[1]["ID"] == "2" and indexed_dataset[2]["ID"] == "3" + assert ( + indexed_dataset[0]["Name"].strip() == "John" + and indexed_dataset[1]["Name"].strip() == "Jane" + and indexed_dataset[2]["Name"].strip() == "Bob" + ) + + @pytest.mark.parametrize( "dataset_class", [text_memmap_dataset.JSONLMemMapDataset, text_memmap_dataset.CSVMemMapDataset], )