Batch size 32 results (#51)

pytorch-labs · Oct 25, 2023 · 3bd7461 · 3bd7461
1 parent 94ce454
commit 3bd7461
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 2 deletions.
diff --git a/experiments/results_bs1.csv → experiments/results/results_bs1.csv b/experiments/results_bs1.csv → experiments/results/results_bs1.csv
diff --git a/experiments/results/results_bs32.csv b/experiments/results/results_bs32.csv
@@ -0,0 +1,9 @@
+technique,time,sam_commit_name,pytorch_version,sam_model_type,batch_size,memory(MiB),memory(%),img_s(avg),batch_ms(avg)/batch_size,mIoU,use_compile,use_half,compress,epilogue_fusion_first,use_compile_decoder,use_nested_tensor,use_rel_pos,pad_input_image_batch,num_workers,num_batches,num_images,profile_path,memory_path
+fp32,0.23617535432179768,default,2.2.0.dev20231024+cu121,ERROR
+bf16,0.21061046520868937,codesign,2.2.0.dev20231024+cu121,ERROR
+compile,2.914792573451996,codesign,2.2.0.dev20231024+cu121,vit_b,32,31077,76,50.12497270213896,19.950135553037967,0.5407576752390846,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,154,4928,None,None
+SDPA,2.4382545590400695,sdpa-decoder,2.2.0.dev20231024+cu121,vit_b,32,18128,44,65.21504434452841,15.333885149522262,0.5355346808697282,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,154,4928,None,None
+Triton,2.0670506795247396,local-fork,2.2.0.dev20231024+cu121,vit_b,32,6224,15,84.64604662944608,11.813900823716994,0.5339075529136259,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,154,4928,None,None
+NT,1.9937538544336955,local-fork,2.2.0.dev20231024+cu121,vit_b,32,6963,17,94.91964930359119,10.535226450337992,0.533777680508926,max-autotune,torch.bfloat16,None,False,False,True,True,True,32,154,4928,None,None
+int8,2.257409866650899,local-fork,2.2.0.dev20231024+cu121,vit_b,32,6879,16,92.73502322451758,10.783412406970928,0.5331365436735569,max-autotune,torch.bfloat16,dynamic_quant,False,False,True,True,True,32,154,4928,None,None
+sparse,2.0192723433176676,local-fork,2.2.0.dev20231024+cu121,vit_b,32,7397,18,95.40203078844067,10.481957163129534,0.4781413896120807,max-autotune,torch.bfloat16,sparse,False,False,True,True,True,32,154,4928,None,None
diff --git a/experiments/results/results_bs32_vit_h.csv b/experiments/results/results_bs32_vit_h.csv
@@ -0,0 +1,9 @@
+technique,time,sam_commit_name,pytorch_version,sam_model_type,batch_size,memory(MiB),memory(%),img_s(avg),batch_ms(avg)/batch_size,mIoU,use_compile,use_half,compress,epilogue_fusion_first,use_compile_decoder,use_nested_tensor,use_rel_pos,pad_input_image_batch,num_workers,num_batches,num_images,profile_path,memory_path
+fp32,0.3408697446187337,default,2.2.0.dev20231024+cu121,ERROR
+bf16,0.3232296347618103,codesign,2.2.0.dev20231024+cu121,ERROR
+compile,1.4304784536361694,codesign,2.2.0.dev20231024+cu121,ERROR
+SDPA,5.974866807460785,sdpa-decoder,2.2.0.dev20231024+cu121,vit_h,32,27276,67,21.754509457085913,45.96748099389014,0.581191777206921,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,154,4928,None,None
+Triton,5.73175394932429,local-fork,2.2.0.dev20231024+cu121,vit_h,32,14424,35,22.71537227352625,44.02305134859952,0.5820036887609843,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,154,4928,None,None
+NT,5.5431528210639955,local-fork,2.2.0.dev20231024+cu121,vit_h,32,14424,35,23.333015236230114,42.85772712509353,0.5807765231991617,max-autotune,torch.bfloat16,None,False,False,True,True,True,32,154,4928,None,None
+int8,6.006167344252268,local-fork,2.2.0.dev20231024+cu121,vit_h,32,11074,27,24.945000391298656,40.08819339801739,0.5817226962602728,max-autotune,torch.bfloat16,dynamic_quant,False,False,True,True,True,32,154,4928,None,None
+sparse,5.685418633619944,local-fork,2.2.0.dev20231024+cu121,vit_h,32,15108,37,24.31964924223424,41.11901409595043,0.5286787009095467,max-autotune,torch.bfloat16,sparse,False,False,True,True,True,32,154,4928,None,None
diff --git a/experiments/results_bs8.csv → experiments/results/results_bs8.csv b/experiments/results_bs8.csv → experiments/results/results_bs8.csv
diff --git a/experiments/run_experiments.py b/experiments/run_experiments.py
@@ -143,7 +143,8 @@ def run(batch_size,
         run_experiments=False,
         traces_dir=None,
         num_workers=32,
-        print_header=True):
+        print_header=True,
+        capture_output=True):
 
     assert model == "vit_b" or model == "vit_h"
 
@@ -152,7 +153,8 @@ def run(batch_size,
                              sam_path,
                              model,
                              batch_size=batch_size,
-                             num_workers=num_workers)
+                             num_workers=num_workers,
+                             capture_output=capture_output)
 
     print_header = True
     if run_traces: