From b041627b7cfc75868a2b9f87fabd070cfdf6a424 Mon Sep 17 00:00:00 2001
From: Grace Nansamba <nansamba1@ruby961.llnl.gov>
Date: Wed, 13 Nov 2024 16:19:45 -0800
Subject: [PATCH] problem size and gt change for rocm and cuda

---
 experiments/amg2023/cuda/ramble.yaml | 143 +++++++++++++++++++-------
 experiments/amg2023/rocm/ramble.yaml | 145 ++++++++++++++++++++-------
 2 files changed, 216 insertions(+), 72 deletions(-)

diff --git a/experiments/amg2023/cuda/ramble.yaml b/experiments/amg2023/cuda/ramble.yaml
index 33b6db5e4..f2bb9c370 100644
--- a/experiments/amg2023/cuda/ramble.yaml
+++ b/experiments/amg2023/cuda/ramble.yaml
@@ -30,35 +30,72 @@ ramble:
                 px: '2'
                 py: '2'
                 pz: '2'
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                nx: ['120', '240', '240']
+                ny: ['120', '120', '240']
+                nz: ['120', '120', '120']
                 env_name: 'amg2023'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_cuda_strong_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_cuda_strong_problem1_120_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: '10'
-                ny: '10'
-                nz: '10'
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['1', '2', '2', '4']
+                nx: ['60', '30','30', '30']
+                ny: ['60', '60', '30', '30']
+                nz: ['60', '60', '60', '30']
                 env_name: 'amg2023'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_cuda_weak_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_cuda_strong_problem1_160_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['80', '40','40', '40']
+                ny: ['80', '80', '40', '40']
+                nz: ['80', '80', '80', '40']
                 env_name: 'amg2023'
                 n_gpus: '{px}*{py}*{pz}'
+            'amg2023_cuda_strong_problem1_200_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['100', '50','50', '50']
+                ny: ['100', '100', '50', '50']
+                nz: ['100', '100', '100', '50']
+                env_name: 'amg2023'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_cuda_weak_60_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '60'
+                ny: '60'
+                nz: '60'
+                env_name: 'amg2023'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_cuda_weak_40_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '40'
+                ny: '40'
+                nz: '40'
+                env_name: 'amg2023'
+                n_gpus: '{px}*{py}*{pz}'
+
         problem2:
           experiments:
             'amg2023_cuda_throughput_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
@@ -68,33 +105,69 @@ ramble:
                 px: '2'
                 py: '2'
                 pz: '2'
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                nx: ['120', '240', '240', '240']
+                ny: ['120', '120', '240', '240']
+                nz: ['120', '120', '120', '240']
+                env_name: 'amg2023'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_cuda_strong_problem2_200_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['100', '50','50', '50']
+                ny: ['100', '100', '50', '50']
+                nz: ['100', '100', '100', '50']
+                env_name: 'amg2023'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_cuda_strong_problem2_256_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['128', '64','64', '64']
+                ny: ['128', '128', '64', '64']
+                nz: ['128', '128', '128', '64']
+                env_name: 'amg2023'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_cuda_strong_problem2_320_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['160', '80','80', '80']
+                ny: ['160', '160', '80', '80']
+                nz: ['160', '160', '160', '80']
                 env_name: 'amg2023'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_cuda_strong_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_cuda_weak_80_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: '10'
-                ny: '10'
-                nz: '10'
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '80'
+                ny: '80'
+                nz: '80'
                 env_name: 'amg2023'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_cuda_weak_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_cuda_weak_60_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '60'
+                ny: '60'
+                nz: '60'
                 env_name: 'amg2023'
                 n_gpus: '{px}*{py}*{pz}'
   software:
@@ -103,7 +176,7 @@ ramble:
         pkg_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
         compiler: default-compiler
       hypre:
-        pkg_spec: hypre@2.31.0 +mpi+cuda+mixedint~fortran{modifier_spack_variant} cuda_arch=={cuda_arch}
+        pkg_spec: hypre@3.1_comm_cali +mpi+cuda+mixedint~fortran{modifier_spack_variant} cuda_arch=={cuda_arch}
         compiler: default-compiler
       amg2023:
         pkg_spec: amg2023@develop +mpi+cuda{modifier_spack_variant} cuda_arch=={cuda_arch}
diff --git a/experiments/amg2023/rocm/ramble.yaml b/experiments/amg2023/rocm/ramble.yaml
index 3e1e665f0..a4a87778d 100644
--- a/experiments/amg2023/rocm/ramble.yaml
+++ b/experiments/amg2023/rocm/ramble.yaml
@@ -30,33 +30,69 @@ ramble:
                 px: '2'
                 py: '2'
                 pz: '2'
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                nx: ['120', '240', '240']
+                ny: ['120', '120', '240']
+                nz: ['120', '120', '120']
                 env_name: 'amg2023-gpu'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_rocm_strong_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_rocm_strong_problem1_120_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: '10'
-                ny: '10'
-                nz: '10'
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['60', '30','30', '30']
+                ny: ['60', '60', '30', '30']
+                nz: ['60', '60', '60', '30']
                 env_name: 'amg2023-gpu'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_rocm_weak_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_rocm_strong_problem1_160_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['80', '40','40', '40']
+                ny: ['80', '80', '40', '40']
+                nz: ['80', '80', '80', '40']
+                env_name: 'amg2023-gpu'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_rocm_strong_problem1_200_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['100', '50','50', '50']
+                ny: ['100', '100', '50', '50']
+                nz: ['100', '100', '100', '50']
+                env_name: 'amg2023-gpu'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_rocm_weak_60_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '60'
+                ny: '60'
+                nz: '60'
+                env_name: 'amg2023-gpu'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_rocm_weak_40_problem1_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '40'
+                ny: '40'
+                nz: '40'
                 env_name: 'amg2023-gpu'
                 n_gpus: '{px}*{py}*{pz}'
         problem2:
@@ -68,39 +104,75 @@ ramble:
                 px: '2'
                 py: '2'
                 pz: '2'
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                nx: ['120', '240', '240', '240']
+                ny: ['120', '120', '240', '240']
+                nz: ['120', '120', '120', '240']
+                env_name: 'amg2023-gpu'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_rocm_strong_problem2_200_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['100', '50','50', '50']
+                ny: ['100', '100', '50', '50']
+                nz: ['100', '100', '100', '50']
+                env_name: 'amg2023-gpu'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_rocm_strong_problem2_256_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['128', '64','64', '64']
+                ny: ['128', '128', '64', '64']
+                nz: ['128', '128', '128', '64']
+                env_name: 'amg2023-gpu'
+                n_gpus: '{px}*{py}*{pz}'
+            'amg2023_rocm_strong_problem2_320_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+              variants:
+                package_manager: spack
+              variables:
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: ['160', '80','80', '80']
+                ny: ['160', '160', '80', '80']
+                nz: ['160', '160', '160', '80']
                 env_name: 'amg2023-gpu'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_rocm_strong_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_rocm_weak_80_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: '10'
-                ny: '10'
-                nz: '10'
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '80'
+                ny: '80'
+                nz: '80'
                 env_name: 'amg2023-gpu'
                 n_gpus: '{px}*{py}*{pz}'
-            'amg2023_rocm_weak_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
+            'amg2023_rocm_weak_60_problem2_{px}_{py}_{pz}_{nx}_{ny}_{nz}':
               variants:
                 package_manager: spack
               variables:
-                px: ['2', '2', '4']
-                py: ['2', '2', '2']
-                pz: ['1', '2', '2']
-                nx: ['10', '20', '20']
-                ny: ['10', '10', '20']
-                nz: ['10', '10', '10']
+                px: ['2', '4', '4', '4']
+                py: ['2', '2', '4', '4']
+                pz: ['2', '2', '2', '4']
+                nx: '60'
+                ny: '60'
+                nz: '60'
                 env_name: 'amg2023-gpu'
                 n_gpus: '{px}*{py}*{pz}'
   software:
     packages:
       hypre:
-        pkg_spec: hypre@2.31.0 +mpi+rocm+mixedint~fortran{modifier_spack_variant} amdgpu_target={rocm_arch}
+        pkg_spec: hypre@3.1_comm_cali +mpi+rocm+mixedint~fortran{modifier_spack_variant} amdgpu_target={rocm_arch}
         compiler: compiler-rocm
       amg2023-gpu:
         pkg_spec: amg2023@develop +mpi+rocm{modifier_spack_variant} amdgpu_target={rocm_arch}
@@ -110,9 +182,8 @@ ramble:
         packages:
         - blas-rocm
         - lapack
-        - mpi-rocm
         - hypre
-        - mpi-rocm
+        - mpi-rocm-no-gtl
         - hypre
         - amg2023-gpu
         - '{modifier_package_name}'