Skip to content

Commit

Permalink
Working advection cuda and cuda_mpi DH
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZamanLantra
ZamanLantra committed Oct 17, 2024
1 parent 25bcd4f commit 71bc64d
Show file tree
Hide file tree
Showing 44 changed files with 3,817 additions and 1,756 deletions.
2 changes: 1 addition & 1 deletion app_fempic_cg/fempic_opp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ int main(int argc, char **argv)
}

//*************************************************************************************************
// std::string f = std::string("F_") + std::to_string(ts + 1);
// std::string f = std::string("F_") + std::to_string(OPP_main_loop_iter + 1);
// opp_print_map_to_txtfile(c2n_map , f.c_str(), "c2n_map.dat");
// opp_print_dat_to_txtfile(n_charge_den, f.c_str(), "n_charge_den.dat");
// opp_mpi_print_dat_to_txtfile(c_sd, "c_sd.dat");
106 changes: 53 additions & 53 deletions app_neso_advection_cg/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,22 @@ HIPCC = $(HIP_COMPILER)
SYCLCC = icpx

CPPFLAGS = -std=c++17 -Wall -fPIC $(CPPFLAGS_ADD)
NVCCFLAGS = -m64 -Xptxas -dlcm=ca -Xptxas=-v $(NVCCFLAGS_ADD)
NVCCFLAGS = -m64 -Xptxas -dlcm=ca $(NVCCFLAGS_ADD)
HIPCCFLAGS = $(HIPCCFLAGS_ADD) -std=c++17 -Wall -isystem -D__HIP_PLATFORM_AMD__
SYCLFLAGS = -w -fsycl -std=c++17 $(SYCLFLAGS_ADD)

DEBUG ?= 0
ifeq ($(DEBUG), 1)
CPPFLAGS += -O0 -g
NVCCFLAGS += -O0 -g -lineinfo
NVCCFLAGS += -O0 -G -g -lineinfo
HIPCCFLAGS += -O0 -g
SYCLFLAGS += -O0 -g
DEBUG_LOG = 1
else
CPPFLAGS += -O3 -g
NVCCFLAGS += -O3 -g
HIPCCFLAGS += -O3 -g
SYCLFLAGS += -O3 -g
CPPFLAGS += -O3 -g -DLOG_HOPS
NVCCFLAGS += -O3 -g -DLOG_HOPS
HIPCCFLAGS += -O3 -g -DLOG_HOPS
SYCLFLAGS += -O3 -g -DLOG_HOPS
endif

DEBUG_LOG ?= 0
Expand All @@ -45,8 +45,8 @@ ifeq ($(IEEE), 1)
HIPCCFLAGS += -fmad=false -ftz=false -prec-div=true -prec-sqrt=true
endif

CUDA_INC = -I$(CUDA_INSTALL_PATH)/include
CUDA_LIB = -L$(CUDA_INSTALL_PATH)/lib64 -lcudart -lcuda
CUDA_INC = -I$(CUDA_INSTALL_PATH)/include -I$(MPI_INSTALL_PATH)/include
CUDA_LIB = -L$(CUDA_INSTALL_PATH)/lib64 -lcudart -lcuda -I$(MPI_INSTALL_PATH)/lib -lmpi

HIP_INC = -I$(ROCM_INSTALL_DIR)/include -I$(ROCM_THRUST_DIR)/include -I$(ROCM_PRIM_DIR)/include
HIP_LIB = -L$(ROCM_INSTALL_DIR)/lib -lamdhip64
Expand All @@ -60,11 +60,11 @@ mklib:
@mkdir -p $(OBJ) $(BIN)

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_seq.o: advec_opp.cpp
$(CPP) $(CPPFLAGS) -c advec_opp.cpp -o $@ $(ALL_INC)
$(OBJ)/opp_kernels_seq.o: seq/opp_kernels.cpp
$(CPP) $(CPPFLAGS) -c $< -o $@ $(ALL_INC)
seq: mklib $(OBJ)/advec_opp_seq.o $(OBJ)/opp_kernels_seq.o
advec_opp+seq:
$(CPP) $(CPPFLAGS) -c advec_opp.cpp -o $(OBJ)/advec_opp_seq.o $(ALL_INC)
opp_kernels+seq:
$(CPP) $(CPPFLAGS) -c seq/opp_kernels.cpp -o $(OBJ)/opp_kernels_seq.o $(ALL_INC)
seq: mklib advec_opp+seq opp_kernels+seq
$(CPP) $(CPPFLAGS) -o $(BIN)/seq \
$(OBJ)/advec_opp_seq.o \
$(OBJ)/opp_kernels_seq.o \
Expand All @@ -82,88 +82,88 @@ mpi: mklib advec_opp+mpi opp_kernels+mpi
$(ALL_INC) $(ALL_LIBS) -lopp_mpi

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_omp.o: advec_opp.cpp
$(CPP) $(CPPFLAGS) -fopenmp -DUSE_OMP -c advec_opp.cpp -o $@ $(ALL_INC)
$(OBJ)/opp_kernels_omp.o: omp/opp_kernels.cpp
$(CPP) $(CPPFLAGS) -fopenmp -DUSE_OMP -c $< -o $@ $(ALL_INC)
omp: mklib $(OBJ)/advec_opp_omp.o $(OBJ)/opp_kernels_omp.o
advec_opp+omp:
$(CPP) $(CPPFLAGS) -fopenmp -DUSE_OMP -c advec_opp.cpp -o $(OBJ)/advec_opp_omp.o $(ALL_INC)
opp_kernels+omp:
$(CPP) $(CPPFLAGS) -fopenmp -DUSE_OMP -c omp/opp_kernels.cpp -o $(OBJ)/opp_kernels_omp.o $(ALL_INC)
omp: mklib advec_opp+omp opp_kernels+omp
$(CPP) $(CPPFLAGS) -fopenmp -o $(BIN)/omp \
$(OBJ)/advec_opp_omp.o \
$(OBJ)/opp_kernels_omp.o \
$(ALL_INC) $(ALL_LIBS) -lopp_omp

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_omp_mpi.o: advec_opp.cpp
$(MPICPP) $(CPPFLAGS) -fopenmp -DUSE_MPI -DUSE_OMP -c advec_opp.cpp -o $@ $(ALL_INC)
$(OBJ)/opp_kernels_omp_mpi.o: omp/opp_kernels.cpp
$(MPICPP) $(CPPFLAGS) -fopenmp -DUSE_MPI -DUSE_OMP -c $< -o $@ $(ALL_INC)
omp_mpi: mklib $(OBJ)/advec_opp_omp_mpi.o $(OBJ)/opp_kernels_omp_mpi.o
advec_opp+omp_mpi:
$(MPICPP) $(CPPFLAGS) -fopenmp -DUSE_MPI -DUSE_OMP -c advec_opp.cpp -o $(OBJ)/advec_opp_omp_mpi.o $(ALL_INC)
opp_kernels+omp_mpi:
$(MPICPP) $(CPPFLAGS) -fopenmp -DUSE_MPI -DUSE_OMP -c omp/opp_kernels.cpp -o $(OBJ)/opp_kernels_omp_mpi.o $(ALL_INC)
omp_mpi: mklib advec_opp+omp_mpi opp_kernels+omp_mpi
$(MPICPP) $(CPPFLAGS) -fopenmp -o $(BIN)/omp_mpi \
$(OBJ)/advec_opp_omp_mpi.o \
$(OBJ)/opp_kernels_omp_mpi.o \
$(ALL_INC) $(ALL_LIBS) -lopp_omp_mpi

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_cuda.o: advec_opp.cpp
$(CPP) $(CPPFLAGS) -DUSE_CUDA -c advec_opp.cpp -o $@ $(ALL_INC)
$(OBJ)/opp_kernels_cuda.o: cuda/opp_kernels.cu
$(NVCC) $(NVCCFLAGS) -DUSE_CUDA -c $< -o $@ $(ALL_INC) $(CUDA_INC)
cuda: mklib $(OBJ)/advec_opp_cuda.o $(OBJ)/opp_kernels_cuda.o
advec_opp+cuda:
$(CPP) $(CPPFLAGS) -DUSE_CUDA -c advec_opp.cpp -o $(OBJ)/advec_opp_cuda.o $(ALL_INC)
opp_kernels+cuda:
$(NVCC) $(NVCCFLAGS) -DUSE_CUDA -c cuda/opp_kernels.cu -o $(OBJ)/opp_kernels_cuda.o $(ALL_INC) $(CUDA_INC)
cuda: mklib advec_opp+cuda opp_kernels+cuda
$(CPP) $(CPPFLAGS) -o $(BIN)/cuda \
$(OBJ)/advec_opp_cuda.o \
$(OBJ)/opp_kernels_cuda.o \
$(ALL_INC) $(CUDA_INC) $(ALL_LIBS) $(CUDA_LIB) -lopp_cuda

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_cuda_mpi.o: advec_opp.cpp
$(MPICPP) $(CPPFLAGS) -DUSE_CUDA -DUSE_MPI -c advec_opp.cpp -o $@ $(ALL_INC) $(CUDA_INC)
$(OBJ)/opp_kernels_cuda_mpi.o: cuda/opp_kernels.cu
$(NVCC) $(NVCCFLAGS) -DUSE_CUDA -DUSE_MPI -c $< -o $@ $(ALL_INC) $(CUDA_INC)
cuda_mpi: mklib $(OBJ)/advec_opp_cuda_mpi.o $(OBJ)/opp_kernels_cuda_mpi.o
advec_opp+cuda_mpi:
$(MPICPP) $(CPPFLAGS) -DUSE_CUDA -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_cuda_mpi.o $(ALL_INC) $(CUDA_INC)
opp_kernels+cuda_mpi:
$(NVCC) $(NVCCFLAGS) -DUSE_CUDA -DUSE_MPI -c cuda/opp_kernels.cu -o $(OBJ)/opp_kernels_cuda_mpi.o $(ALL_INC) $(CUDA_INC)
cuda_mpi: mklib advec_opp+cuda_mpi opp_kernels+cuda_mpi
$(MPICPP) $(CPPFLAGS) -DUSE_MPI -o $(BIN)/cuda_mpi \
$(OBJ)/advec_opp_cuda_mpi.o \
$(OBJ)/opp_kernels_cuda_mpi.o \
$(ALL_INC) $(CUDA_INC) $(ALL_LIBS) $(CUDA_LIB) -lopp_cuda_mpi

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_sycl.o: advec_opp.cpp
$(SYCLCC) $(SYCLFLAGS) -DUSE_SYCL -c advec_opp.cpp -o $@ $(ALL_INC)
$(OBJ)/opp_kernels_sycl.o: sycl/opp_kernels.cpp
$(SYCLCC) $(SYCLFLAGS) -DUSE_SYCL -c $< -o $@ $(ALL_INC)
sycl: mklib $(OBJ)/advec_opp_sycl.o $(OBJ)/opp_kernels_sycl.o
advec_opp+sycl:
$(SYCLCC) $(SYCLFLAGS) -DUSE_SYCL -c advec_opp.cpp -o $(OBJ)/advec_opp_sycl.o $(ALL_INC)
opp_kernels+sycl:
$(SYCLCC) $(SYCLFLAGS) -DUSE_SYCL -c sycl/opp_kernels.cpp -o $(OBJ)/opp_kernels_sycl.o $(ALL_INC)
sycl: mklib advec_opp+sycl opp_kernels+sycl
$(SYCLCC) $(SYCLFLAGS) -o $(BIN)/sycl \
$(OBJ)/advec_opp_sycl.o \
$(OBJ)/opp_kernels_sycl.o \
$(ALL_INC) $(ALL_LIBS) -lopp_sycl

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_sycl_mpi.o: advec_opp.cpp
$(MPICPP) $(SYCLFLAGS) -DUSE_SYCL -DUSE_MPI -c advec_opp.cpp -o $@ $(ALL_INC)
$(OBJ)/opp_kernels_sycl_mpi.o: sycl/opp_kernels.cpp
$(MPICPP) $(SYCLFLAGS) -DUSE_SYCL -DUSE_MPI -c $< -o $@ $(ALL_INC)
sycl_mpi: mklib $(OBJ)/advec_opp_sycl_mpi.o $(OBJ)/opp_kernels_sycl_mpi.o
advec_opp+sycl_mpi:
$(MPICPP) $(SYCLFLAGS) -DUSE_SYCL -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_sycl_mpi.o $(ALL_INC)
opp_kernels+sycl_mpi:
$(MPICPP) $(SYCLFLAGS) -DUSE_SYCL -DUSE_MPI -c sycl/opp_kernels.cpp -o $(OBJ)/opp_kernels_sycl_mpi.o $(ALL_INC)
sycl_mpi: mklib advec_opp+sycl_mpi opp_kernels+sycl_mpi
$(MPICPP) $(SYCLFLAGS) -o $(BIN)/sycl_mpi \
$(OBJ)/advec_opp_sycl_mpi.o \
$(OBJ)/opp_kernels_sycl_mpi.o \
$(ALL_INC) $(ALL_LIBS) -lopp_sycl_mpi

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_hip.o: advec_opp.cpp
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -c advec_opp.cpp -o $@ $(ALL_INC) $(HIP_INC)
$(OBJ)/opp_kernels_hip.o: hip/opp_kernels.cpp
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -c $< -o $@ $(ALL_INC) $(HIP_INC)
hip: mklib $(OBJ)/advec_opp_hip.o $(OBJ)/opp_kernels_hip.o
advec_opp+hip:
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -c advec_opp.cpp -o $(OBJ)/advec_opp_hip.o $(ALL_INC) $(HIP_INC)
opp_kernels+hip:
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -c hip/opp_kernels.cpp -o $(OBJ)/opp_kernels_hip.o $(ALL_INC) $(HIP_INC)
hip: mklib advec_opp+hip opp_kernels+hip
$(HIPCC) $(HIPCCFLAGS) -o $(BIN)/hip \
$(OBJ)/advec_opp_hip.o \
$(OBJ)/opp_kernels_hip.o \
$(ALL_INC) $(HIP_INC) $(ALL_LIBS) -lopp_hip

# ------------------------------------------------------------------------------------------
$(OBJ)/advec_opp_hip_mpi.o: advec_opp.cpp
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -DUSE_MPI -c advec_opp.cpp -o $@ $(ALL_INC) $(HIP_INC)
$(OBJ)/opp_kernels_hip_mpi.o: hip/opp_kernels.cpp
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -DUSE_MPI -c $< -o $@ $(ALL_INC) $(HIP_INC)
hip_mpi: mklib $(OBJ)/advec_opp_hip_mpi.o $(OBJ)/opp_kernels_hip_mpi.o
advec_opp+hip_mpi:
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_hip_mpi.o $(ALL_INC) $(HIP_INC)
opp_kernels+hip_mpi:
$(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -DUSE_MPI -c hip/opp_kernels.cpp -o $(OBJ)/opp_kernels_hip_mpi.o $(ALL_INC) $(HIP_INC)
hip_mpi: mklib advec_opp+hip_mpi opp_kernels+hip_mpi
$(HIPCC) $(HIPCCFLAGS) -o $(BIN)/hip_mpi \
$(OBJ)/advec_opp_hip_mpi.o \
$(OBJ)/opp_kernels_hip_mpi.o \
Expand Down
4 changes: 1 addition & 3 deletions app_neso_advection_cg/advec.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,11 @@ int main(int argc, char **argv)
{
opp_par_loop(update_pos_kernel, "update_pos", part_set, OPP_ITERATE_ALL,
opp_arg_dat(p_vel, OPP_READ),
opp_arg_dat(p_pos, OPP_RW),
opp_arg_dat(p_mdir, OPP_WRITE)
opp_arg_dat(p_pos, OPP_RW)
);

opp_particle_move(move_kernel, "move", part_set, c2c_map, p2c_map,
opp_arg_dat(p_pos, OPP_READ),
opp_arg_dat(p_mdir, OPP_RW),
opp_arg_dat(c_pos_ll, p2c_map, OPP_READ)
);

Expand Down
13 changes: 5 additions & 8 deletions app_neso_advection_cg/advec_opp.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

// Auto-generated at 2024-10-10 12:19:56.330538 by opp-translator
// Auto-generated at 2024-10-17 18:33:47.579352 by opp-translator
/*
BSD 3-Clause License
Expand Down Expand Up @@ -37,10 +37,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "opp_lib.h"

void opp_par_loop_all__update_pos_kernel(opp_set,opp_iterate_type,opp_arg,opp_arg,opp_arg);
void opp_particle_move__move_kernel(opp_set,opp_map,opp_map,opp_arg,opp_arg,opp_arg);
void opp_par_loop_all__update_pos_kernel(opp_set,opp_iterate_type,opp_arg,opp_arg);
void opp_particle_move__move_kernel(opp_set,opp_map,opp_map,opp_arg,opp_arg);
void opp_par_loop_all__verify_kernel(opp_set,opp_iterate_type,opp_arg,opp_arg,opp_arg);
void opp_init_direct_hop_cg(double,int,const opp_dat,const opp::BoundingBox&,opp_map,opp_map,opp_arg,opp_arg,opp_arg);
void opp_init_direct_hop_cg(double,int,const opp_dat,const opp::BoundingBox&,opp_map,opp_map,opp_arg,opp_arg);

#include "advec_misc.h"
// #include "kenels.h" // codegen commented...
Expand Down Expand Up @@ -102,7 +102,6 @@ int main(int argc, char **argv)
opp::BoundingBox bounding_box = opp::BoundingBox(c_pos_ll, DIM, expansion);
opp_init_direct_hop_cg(grid_spacing, DIM, c_idx, bounding_box, c2c_map, p2c_map,
opp_arg_dat(p_pos, OPP_READ),
opp_arg_dat(p_mdir, OPP_RW),
opp_arg_dat(c_pos_ll, p2c_map, OPP_READ));

opp_printf("Setup Completed", "Cells[%d] Particles[%d] max_iter[%d]", cell_set->size, part_set->size, max_iter);
Expand All @@ -114,13 +113,11 @@ int main(int argc, char **argv)
{
opp_par_loop_all__update_pos_kernel(part_set, OPP_ITERATE_ALL,
opp_arg_dat(p_vel, OPP_READ),
opp_arg_dat(p_pos, OPP_RW),
opp_arg_dat(p_mdir, OPP_WRITE)
opp_arg_dat(p_pos, OPP_RW)
);

opp_particle_move__move_kernel(part_set, c2c_map, p2c_map,
opp_arg_dat(p_pos, OPP_READ),
opp_arg_dat(p_mdir, OPP_RW),
opp_arg_dat(c_pos_ll, p2c_map, OPP_READ)
);

Expand Down
Loading

0 comments on commit 71bc64d

Please sign in to comment.