From cd48e76927a5acb63f1dc3116a22c47696d6c61f Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Tue, 11 Jun 2024 20:10:30 +0900 Subject: [PATCH 1/3] mapper_mpi revised --- src/py2dmat/algorithm/mapper_mpi.py | 143 +++++++++++++++------------- 1 file changed, 75 insertions(+), 68 deletions(-) diff --git a/src/py2dmat/algorithm/mapper_mpi.py b/src/py2dmat/algorithm/mapper_mpi.py index c26fd306..b498f27e 100644 --- a/src/py2dmat/algorithm/mapper_mpi.py +++ b/src/py2dmat/algorithm/mapper_mpi.py @@ -48,83 +48,90 @@ def _run(self) -> None: label_list = self.label_list dimension = self.dimension run = self.runner - print("Make ColorMap") - with open("ColorMap.txt", "w") as file_CM: - fx_list = [] + + fx_list = [] + self.timer["run"]["submit"] = 0.0 + + iterations = len(self.mesh_list) + for iteration_count, mesh in enumerate(self.mesh_list): + print("Iteration : {}/{}".format(iteration_count + 1, iterations)) + # print("mesh before:", mesh) + + # update information + args = (int(mesh[0]), 0) + x = np.array(mesh[1:]) + time_sta = time.perf_counter() - file_CM.write("#") - for label in label_list: - file_CM.write(f"{label} ") - file_CM.write("fval\n") + fx = run.submit(x, args) time_end = time.perf_counter() + self.timer["run"]["submit"] += time_end - time_sta + + fx_list.append(fx) + # print("mesh after:", mesh) + + self.fx_list = fx_list + + if iterations > 0: + fx_order = np.argsort(fx_list) + # minimum_point = [] + # print("mesh_list[fx_order[0]]:") + # print(self.mesh_list[fx_order[0]]) + # for index in range(1, dimension + 1): + # minimum_point.append(self.mesh_list[fx_order[0]][index]) + opt_index = fx_order[0] + self.opt_fx = fx_list[opt_index] + self.opt_mesh = np.array(self.mesh_list[opt_index]) + + print("minimum_point: {}".format(self.opt_mesh)) + print("minimum_value: {}".format(self.opt_fx)) + + self._output_results() - self.timer["run"]["file_CM"] = time_end - time_sta - self.timer["run"]["submit"] = 0.0 - - iterations = len(self.mesh_list) - for iteration_count, mesh in enumerate(self.mesh_list): - print("Iteration : {}/{}".format(iteration_count + 1, iterations)) - # print("mesh before:", mesh) - - time_sta = time.perf_counter() - for value in mesh[1:]: - file_CM.write("{:8f} ".format(value)) - time_end = time.perf_counter() - self.timer["run"]["file_CM"] += time_end - time_sta - - # update information - args = (int(mesh[0]), 0) - x = np.array(mesh[1:]) - - time_sta = time.perf_counter() - fx = run.submit(x, args) - time_end = time.perf_counter() - self.timer["run"]["submit"] += time_end - time_sta - - fx_list.append(fx) - time_sta = time.perf_counter() - file_CM.write("{:8f}\n".format(fx)) - time_end = time.perf_counter() - self.timer["run"]["file_CM"] += time_end - time_sta - - # print("mesh after:", mesh) - - if iterations > 0: - fx_order = np.argsort(fx_list) - minimum_point = [] - print("mesh_list[fx_order[0]]:") - print(self.mesh_list[fx_order[0]]) - for index in range(1, dimension + 1): - minimum_point.append(self.mesh_list[fx_order[0]][index]) - - time_sta = time.perf_counter() - file_CM.write("#Minimum point :") - for value in minimum_point: - file_CM.write(" {:8f}".format(value)) - file_CM.write("\n") - file_CM.write("#R-factor : {:8f}\n".format(fx_list[fx_order[0]])) - file_CM.write("#see Log{}\n".format(round(self.mesh_list[fx_order[0]][0]))) - time_end = time.perf_counter() - self.timer["run"]["file_CM"] += time_end - time_sta + print("complete main process : rank {:08d}/{:08d}".format(self.mpirank, self.mpisize)) + + def _output_results(self): + print("Make ColorMap") + + time_sta = time.perf_counter() + + with open("ColorMap.txt", "w") as fp: + fp.write("#" + " ".join(self.label_list) + " fval\n") + + for x, fx in zip(self.mesh_list, self.fx_list): + fp.write(" ".join( + map(lambda v: "{:8f}".format(v), (*x[1:], fx)) + ) + "\n") + + if len(self.mesh_list) > 0: + fp.write("#Minimum point : " + " ".join( + map(lambda v: "{:8f}".format(v), self.opt_mesh[1:]) + ) + "\n") + fp.write("#R-factor : {:8f}\n".format(self.opt_fx)) + fp.write("#see Log{:d}\n".format(round(self.opt_mesh[0]))) else: - file_CM.write("# No mesh point\n") + fp.write("# No mesh point\n") - print( - "complete main process : rank {:08d}/{:08d}".format( - self.mpirank, self.mpisize - ) - ) + time_end = time.perf_counter() + self.timer["run"]["file_CM"] = time_end - time_sta def _prepare(self) -> None: # do nothing pass def _post(self) -> None: + if self.mpisize > 1: + fx_lists = self.mpicomm.allgather(self.fx_list) + results = [v for vs in fx_lists for v in vs] + + mesh_lists = self.mpicomm.allgather(self.mesh_list) + coords = [v for vs in mesh_lists for v in vs] + else: + results = self.fx_list + coords = self.mesh_list + if self.mpirank == 0: - with open("ColorMap.txt", "w") as file_output: - for i in range(self.mpisize): - with open(os.path.join(str(i), "ColorMap.txt"), "r") as file_input: - for line in file_input: - line = line.lstrip() - if not line.startswith("#"): - file_output.write(line) + with open("ColorMap.txt", "w") as fp: + for x, fx in zip(coords, results): + fp.write(" ".join( + map(lambda v: "{:8f}".format(v), (*x[1:], fx)) + ) + "\n") From fbf8f76ef3d28794c8fdf00d2d32c564a7483740 Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 10 Aug 2024 08:40:21 +0900 Subject: [PATCH 2/3] modify mapper to reduce gathered data --- src/py2dmat/algorithm/mapper_mpi.py | 34 ++++++++++++----------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/py2dmat/algorithm/mapper_mpi.py b/src/py2dmat/algorithm/mapper_mpi.py index 789e7f1b..b1cdee65 100644 --- a/src/py2dmat/algorithm/mapper_mpi.py +++ b/src/py2dmat/algorithm/mapper_mpi.py @@ -66,24 +66,22 @@ def _run(self) -> None: time_end = time.perf_counter() self.timer["run"]["submit"] += time_end - time_sta - fx_list.append(fx) + fx_list.append([mesh[0], fx]) # print("mesh after:", mesh) self.fx_list = fx_list if iterations > 0: - fx_order = np.argsort(fx_list) - # minimum_point = [] - # print("mesh_list[fx_order[0]]:") - # print(self.mesh_list[fx_order[0]]) - # for index in range(1, dimension + 1): - # minimum_point.append(self.mesh_list[fx_order[0]][index]) - opt_index = fx_order[0] - self.opt_fx = fx_list[opt_index] - self.opt_mesh = np.array(self.mesh_list[opt_index]) - - print("minimum_point: {}".format(self.opt_mesh)) - print("minimum_value: {}".format(self.opt_fx)) + opt_index = np.argsort(fx_list, axis=0)[0][1] + opt_id, opt_fx = fx_list[opt_index] + opt_mesh = self.mesh_list[opt_index] + + # assert opt_id == opt_mesh[0] + + self.opt_fx = opt_fx + self.opt_mesh = opt_mesh + + print(f"[{self.mpirank}] minimum_value: {opt_fx:12.8e} at {opt_mesh[1:]} (mesh {opt_mesh[0]})") self._output_results() @@ -91,13 +89,12 @@ def _run(self) -> None: def _output_results(self): print("Make ColorMap") - time_sta = time.perf_counter() with open("ColorMap.txt", "w") as fp: fp.write("#" + " ".join(self.label_list) + " fval\n") - for x, fx in zip(self.mesh_list, self.fx_list): + for x, (idx, fx) in zip(self.mesh_list, self.fx_list): fp.write(" ".join( map(lambda v: "{:8f}".format(v), (*x[1:], fx)) ) + "\n") @@ -122,16 +119,13 @@ def _post(self) -> Dict: if self.mpisize > 1: fx_lists = self.mpicomm.allgather(self.fx_list) results = [v for vs in fx_lists for v in vs] - - mesh_lists = self.mpicomm.allgather(self.mesh_list) - coords = [v for vs in mesh_lists for v in vs] else: results = self.fx_list - coords = self.mesh_list if self.mpirank == 0: with open("ColorMap.txt", "w") as fp: - for x, fx in zip(coords, results): + for x, (idx, fx) in zip(self.domain.grid, results): + assert x[0] == idx fp.write(" ".join( map(lambda v: "{:8f}".format(v), (*x[1:], fx)) ) + "\n") From 2b216c3eb3575dcd76f7d5d548f75325ad24de1c Mon Sep 17 00:00:00 2001 From: "T.Aoyama" Date: Sat, 10 Aug 2024 08:42:09 +0900 Subject: [PATCH 3/3] modify mapper to introduce colormap parameter for output file name --- src/py2dmat/algorithm/mapper_mpi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/py2dmat/algorithm/mapper_mpi.py b/src/py2dmat/algorithm/mapper_mpi.py index b1cdee65..d553511d 100644 --- a/src/py2dmat/algorithm/mapper_mpi.py +++ b/src/py2dmat/algorithm/mapper_mpi.py @@ -42,6 +42,8 @@ def __init__(self, info: py2dmat.Info, self.domain.do_split() self.mesh_list = self.domain.grid_local + self.colormap_file = info.algorithm.get("colormap", "ColorMap.txt") + def _run(self) -> None: # Make ColorMap @@ -91,7 +93,7 @@ def _output_results(self): print("Make ColorMap") time_sta = time.perf_counter() - with open("ColorMap.txt", "w") as fp: + with open(self.colormap_file, "w") as fp: fp.write("#" + " ".join(self.label_list) + " fval\n") for x, (idx, fx) in zip(self.mesh_list, self.fx_list): @@ -123,7 +125,7 @@ def _post(self) -> Dict: results = self.fx_list if self.mpirank == 0: - with open("ColorMap.txt", "w") as fp: + with open(self.colormap_file, "w") as fp: for x, (idx, fx) in zip(self.domain.grid, results): assert x[0] == idx fp.write(" ".join(