From 78adca8e3356ca65230f675f4ca1345386fef162 Mon Sep 17 00:00:00 2001 From: Juhan Bae Date: Sun, 14 Jul 2024 14:21:47 -0400 Subject: [PATCH] Higher preicison for grad computation --- examples/openwebtext/compute_scores.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/openwebtext/compute_scores.py b/examples/openwebtext/compute_scores.py index f627769..59bd670 100644 --- a/examples/openwebtext/compute_scores.py +++ b/examples/openwebtext/compute_scores.py @@ -96,6 +96,7 @@ def main(): # We can invest some time in getting more accurate SVD results. score_args.use_full_svd = True score_args.precondition_dtype = torch.float32 + score_args.per_sample_gradient_dtype = torch.float32 analyzer.compute_pairwise_scores( scores_name=args.scores_name, score_args=score_args,