V2 Performance Signal Detected by TorchBench CI on '2.6.0.dev20240925+cu124'

[xuzhao9]

TorchBench CI has detected a performance signal.

Base PyTorch version: 2.4.0.dev20240606+cu121

Base PyTorch commit: ffaea656b5d8ff6518669494cc8f664b94f8e8b1

Affected PyTorch version: 2.6.0.dev20240925+cu124

Affected PyTorch commit: 780f4debdbb51347da5b416416392e5849cbfec5

Affected Tests:

• test_train[Super_SloMo-cuda-eager]: +14.62530%
• test_eval[Super_SloMo-cuda-eager]: +11.06564%
• test_train[Super_SloMo-cuda-jit]: +14.76551%
• test_eval[Super_SloMo-cuda-jit]: +11.21703%
• test_eval[alexnet-cuda-eager]: +10.44549%
• test_eval[alexnet-cuda-jit]: +13.53622%
• test_train[dcgan-cuda-eager]: +63.63544%
• test_train[dcgan-cuda-jit]: +63.63278%
• test_eval[pyhpc_equation_of_state-cpu-jit]: +13.75146%
• test_eval[pytorch_stargan-cpu-eager]: -7.47339%
• test_eval[pytorch_stargan-cpu-jit]: -16.44898%
• test_eval[resnext50_32x4d-cpu-jit]: +20.97126%
• test_eval[shufflenet_v2_x1_0-cpu-eager]: +13.76384%
• test_eval[shufflenet_v2_x1_0-cuda-jit]: -27.57779%
• test_train[timm_nfnet-cpu-jit]: -9.20727%
• test_eval[timm_regnet-cuda-eager]: +170.32362%
• test_eval[timm_regnet-cuda-jit]: +355.17416%
• test_train[yolov3-cuda-eager]: -9.90389%

cc @xuzhao9

Result json:

{
  "start": "ffaea656b5d8ff6518669494cc8f664b94f8e8b1",
  "end": "780f4debdbb51347da5b416416392e5849cbfec5",
  "threshold": 7,
  "timeout": 120,
  "torchbench_branch": "v2.0",
  "result": [
    {
      "commit1": "2b9465d62ab",
      "commit1_time": "2024-06-13 12:09:08 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06711476687341929
      },
      "commit2": "7cc07a3eb13",
      "commit2_time": "2024-06-13 13:36:39 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09414750961586833
      }
    },
    {
      "commit1": "edb45dce85a",
      "commit1_time": "2024-06-13 13:58:47 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09023285945877432
      },
      "commit2": "cf7adc2fa1c",
      "commit2_time": "2024-06-13 14:34:37 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08251784127205611
      }
    },
    {
      "commit1": "790138fdc77",
      "commit1_time": "2024-06-13 20:52:46 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07368147987872362
      },
      "commit2": "8763d44bf16",
      "commit2_time": "2024-06-13 21:15:09 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08161171982064844
      }
    },
    {
      "commit1": "9ac08dab1f7",
      "commit1_time": "2024-06-14 04:32:38 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08594924621284009
      },
      "commit2": "f48f7615dca",
      "commit2_time": "2024-06-14 05:18:07 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07616496244445443
      }
    },
    {
      "commit1": "f48f7615dca",
      "commit1_time": "2024-06-14 05:18:07 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07616496244445443
      },
      "commit2": "18f35d9e12b",
      "commit2_time": "2024-06-14 06:08:17 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09225645046681166
      }
    },
    {
      "commit1": "18f35d9e12b",
      "commit1_time": "2024-06-14 06:08:17 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09225645046681166
      },
      "commit2": "674be9d3be5",
      "commit2_time": "2024-06-14 07:04:34 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08528265990316868
      }
    },
    {
      "commit1": "4669c6d3ae8",
      "commit1_time": "2024-06-14 07:13:10 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08583004362881183
      },
      "commit2": "e886122e981",
      "commit2_time": "2024-06-14 07:30:31 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09325523599982262
      }
    },
    {
      "commit1": "e886122e981",
      "commit1_time": "2024-06-14 07:30:31 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09325523599982262
      },
      "commit2": "dd3b79a08fe",
      "commit2_time": "2024-06-14 07:30:31 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10814351234585047
      }
    },
    {
      "commit1": "a87d82abd74",
      "commit1_time": "2024-06-17 16:29:29 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08268363336101174
      },
      "commit2": "316b7296771",
      "commit2_time": "2024-06-17 16:42:43 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06731120152398944
      }
    },
    {
      "commit1": "c172b58fe01",
      "commit1_time": "2024-06-17 18:49:15 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0662929986603558
      },
      "commit2": "213eba7d2e9",
      "commit2_time": "2024-06-17 18:53:56 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07361583039164543
      }
    },
    {
      "commit1": "213eba7d2e9",
      "commit1_time": "2024-06-17 18:53:56 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07361583039164543
      },
      "commit2": "b181b588574",
      "commit2_time": "2024-06-17 18:55:47 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08877140348777175
      }
    },
    {
      "commit1": "b181b588574",
      "commit1_time": "2024-06-17 18:55:47 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08877140348777175
      },
      "commit2": "1577328ea40",
      "commit2_time": "2024-06-17 19:24:09 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0812253455631435
      }
    },
    {
      "commit1": "1577328ea40",
      "commit1_time": "2024-06-17 19:24:09 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0812253455631435
      },
      "commit2": "0f89e66d174",
      "commit2_time": "2024-06-17 20:07:13 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06737935505807399
      }
    },
    {
      "commit1": "0f89e66d174",
      "commit1_time": "2024-06-17 20:07:13 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06737935505807399
      },
      "commit2": "a59766ee058",
      "commit2_time": "2024-06-17 20:50:22 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08082860205322503
      }
    },
    {
      "commit1": "163847b1bb5",
      "commit1_time": "2024-06-17 21:25:59 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07743226950988173
      },
      "commit2": "1fd7496ab2e",
      "commit2_time": "2024-06-17 21:58:46 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08966684350743889
      }
    },
    {
      "commit1": "1835e3beab7",
      "commit1_time": "2024-06-17 22:20:33 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09242729544639587
      },
      "commit2": "3b8c9b8ab11",
      "commit2_time": "2024-06-17 22:51:12 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06779224071651697
      }
    },
    {
      "commit1": "3b8c9b8ab11",
      "commit1_time": "2024-06-17 22:51:12 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06779224071651697
      },
      "commit2": "8415a4ba98f",
      "commit2_time": "2024-06-17 22:52:27 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08009095480665565
      }
    },
    {
      "commit1": "11ff5345d24",
      "commit1_time": "2024-06-17 23:53:26 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10658697485923767
      },
      "commit2": "beb29836cd1",
      "commit2_time": "2024-06-18 00:20:32 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07564477706328035
      }
    },
    {
      "commit1": "fbc7559ceb3",
      "commit1_time": "2024-06-18 00:55:50 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07316059973090887
      },
      "commit2": "9e8443b56f5",
      "commit2_time": "2024-06-18 01:26:45 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09477473217993974
      }
    },
    {
      "commit1": "9e8443b56f5",
      "commit1_time": "2024-06-18 01:26:45 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09477473217993974
      },
      "commit2": "e12fa93b8bb",
      "commit2_time": "2024-06-18 02:00:04 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06794789889827371
      }
    },
    {
      "commit1": "e12fa93b8bb",
      "commit1_time": "2024-06-18 02:00:04 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06794789889827371
      },
      "commit2": "43998711a79",
      "commit2_time": "2024-06-18 02:07:07 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07565820412710308
      }
    },
    {
      "commit1": "43998711a79",
      "commit1_time": "2024-06-18 02:07:07 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07565820412710308
      },
      "commit2": "22f1793c0ac",
      "commit2_time": "2024-06-18 02:17:56 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06210884088650346
      }
    },
    {
      "commit1": "48171806010",
      "commit1_time": "2024-06-18 14:56:35 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08997715767472983
      },
      "commit2": "108318ad103",
      "commit2_time": "2024-06-18 15:40:45 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07453400176018476
      }
    },
    {
      "commit1": "108318ad103",
      "commit1_time": "2024-06-18 15:40:45 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07453400176018476
      },
      "commit2": "ec616da5184",
      "commit2_time": "2024-06-18 16:16:38 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.11012181285768748
      }
    },
    {
      "commit1": "ec616da5184",
      "commit1_time": "2024-06-18 16:16:38 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.11012181285768748
      },
      "commit2": "9818283da18",
      "commit2_time": "2024-06-18 17:08:58 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08110316200181841
      }
    },
    {
      "commit1": "4e03263224a",
      "commit1_time": "2024-06-18 17:26:23 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0661167873069644
      },
      "commit2": "84c86e56bd8",
      "commit2_time": "2024-06-18 17:48:47 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07898384258151055
      }
    },
    {
      "commit1": "c0b87afcade",
      "commit1_time": "2024-06-12 04:09:23 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08490734407678246
      },
      "commit2": "77a0ca66e4e",
      "commit2_time": "2024-06-12 04:13:36 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07925356533378362
      }
    },
    {
      "commit1": "77a0ca66e4e",
      "commit1_time": "2024-06-12 04:13:36 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07925356533378362
      },
      "commit2": "089f9a116ac",
      "commit2_time": "2024-06-12 05:22:24 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09490136289969087
      }
    },
    {
      "commit1": "1edcb31d34e",
      "commit1_time": "2024-06-12 08:37:16 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09398238696157932
      },
      "commit2": "2386045e4f0",
      "commit2_time": "2024-06-12 09:39:58 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06574655557051301
      }
    },
    {
      "commit1": "bf8a05f483c",
      "commit1_time": "2024-06-13 21:35:33 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07666015978902578
      },
      "commit2": "6211e67e49f",
      "commit2_time": "2024-06-13 21:49:16 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09253177354112267
      }
    },
    {
      "commit1": "dd3b79a08fe",
      "commit1_time": "2024-06-14 07:30:31 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10814351234585047
      },
      "commit2": "03725a05127",
      "commit2_time": "2024-06-14 07:30:31 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09180594393983484
      }
    },
    {
      "commit1": "03725a05127",
      "commit1_time": "2024-06-14 07:30:31 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09180594393983484
      },
      "commit2": "0344f95c2ea",
      "commit2_time": "2024-06-14 07:49:09 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07943073650822044
      }
    },
    {
      "commit1": "95b5ea9cdef",
      "commit1_time": "2024-06-17 23:39:48 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07916179271414876
      },
      "commit2": "b70440f0a7f",
      "commit2_time": "2024-06-17 23:42:40 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06594249494373798
      }
    },
    {
      "commit1": "b70440f0a7f",
      "commit1_time": "2024-06-17 23:42:40 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06594249494373798
      },
      "commit2": "11ff5345d24",
      "commit2_time": "2024-06-17 23:53:26 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10658697485923767
      }
    },
    {
      "commit1": "4e97d37fd94",
      "commit1_time": "2024-06-18 02:18:09 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.062194927223026754
      },
      "commit2": "c017c97333d",
      "commit2_time": "2024-06-18 02:18:09 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08754603192210197
      }
    },
    {
      "commit1": "9818283da18",
      "commit1_time": "2024-06-18 17:08:58 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08110316200181841
      },
      "commit2": "26e374e3ca3",
      "commit2_time": "2024-06-18 17:15:05 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06901781614869833
      }
    }
  ]
}

Bisection workflow link: https://github.com/pytorch/benchmark/actions/runs/11074863032