diff --git a/.gitignore b/.gitignore index 0233da08..d443496e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ doc/sphinx/output.log doc/sphinx/07_miniem/*.png doc/sphinx/08_sparta/*.png doc/sphinx/*/*.png +doc/sphinx/*/*/*.png doc/sphinx/09_Microbenchmarks/*/*.png doc/sphinx/07_miniem/PanzerMiniEM_BlockPrec.exe diff --git a/doc/sphinx/03_vibe/scaling/plots/parthenon-pct.png b/doc/sphinx/03_vibe/scaling/plots/parthenon-pct.png index baaf46c3..ee1098a1 100644 Binary files a/doc/sphinx/03_vibe/scaling/plots/parthenon-pct.png and b/doc/sphinx/03_vibe/scaling/plots/parthenon-pct.png differ diff --git a/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-area.png b/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-area.png index b590c56d..625dc554 100644 Binary files a/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-area.png and b/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-area.png differ diff --git a/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-line.png b/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-line.png index 3be9099a..1efabc02 100644 Binary files a/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-line.png and b/doc/sphinx/03_vibe/scaling/plots/parthenon-totaltime-line.png differ diff --git a/doc/sphinx/03_vibe/scaling/weak-august.csv b/doc/sphinx/03_vibe/scaling/weak-august.csv new file mode 100644 index 00000000..df538131 --- /dev/null +++ b/doc/sphinx/03_vibe/scaling/weak-august.csv @@ -0,0 +1,12 @@ +Nodes,aggregate-FOM,per-node-FOM,per-node-Ideal +4096,7.33E+09,1.79E+06,9.60E+06 +2048,8.85E+09,4.32E+06,9.60E+06 +1024,4.28E+09,4.18E+06,9.60E+06 +512,3.29E+09,6.43E+06,9.60E+06 +256,1.73E+09,6.76E+06,9.60E+06 +128,9.26E+08,7.23E+06,9.60E+06 +64,5.18E+08,8.09E+06,9.60E+06 +32,2.65E+08,8.28E+06,9.60E+06 +16,1.32E+08,8.25E+06,9.60E+06 +8,7.70E+07,9.63E+06,9.60E+06 +4,3.84E+07,9.60E+06,9.60E+06 \ No newline at end of file diff --git a/doc/sphinx/03_vibe/scaling/weak.gp b/doc/sphinx/03_vibe/scaling/weak.gp index 4be2bb80..3ac9f7d7 100644 --- a/doc/sphinx/03_vibe/scaling/weak.gp +++ b/doc/sphinx/03_vibe/scaling/weak.gp @@ -20,6 +20,8 @@ set key autotitle columnheader set style line 1 linetype 6 dashtype 1 linecolor rgb "#FF0000" linewidth 2 pointtype 6 pointsize 3 set style line 2 linetype 1 dashtype 2 linecolor rgb "#FF0000" linewidth 2 - set output "weak.png" plot "weak.csv" using 1:3 with linespoints linestyle 1, "" using 1:4 with line linestyle 2 + +set output "weak-august.png" +plot "weak-august.csv" using 1:3 with linespoints linestyle 1, "" using 1:4 with line linestyle 2 diff --git a/doc/sphinx/03_vibe/vibe.rst b/doc/sphinx/03_vibe/vibe.rst index 15612c24..0a45562f 100644 --- a/doc/sphinx/03_vibe/vibe.rst +++ b/doc/sphinx/03_vibe/vibe.rst @@ -207,18 +207,24 @@ Multi-node scaling on Crossroads The results of the scaling runs performed on Crossroads are presented below. Parthenon was built with intel oneapi 2023.1.0 and cray-mpich 8.1.25. -These runs used between 2 and 4096 nodes with 8 ranks per node and 14 threads (using Kokkos OpenMP) per rank. -NXs=(208 256 320 400 512 640 800 1024 1280 1616 2048 2576) -NODES=(2 4 8 16 32 64 128 256 512 1024 2048 4096) +These runs used between 4 and 4096 nodes with 8 ranks per node and 14 threads per rank (using Kokkos OpenMP) with the following problem sizes. + +.. code-block:: bash + +NXs=(256 320 400 512 640 800 1024 1280 1616 2048 2576) +NODES=(4 8 16 32 64 128 256 512 1024 2048 4096) + +.. + Output files can be found in ``./docs/sphinx/03_vibe/scaling/output/`` -.. figure:: ./scaling/weak.png +.. figure:: ./scaling/weak-august.png :align: center :scale: 50% :alt: VIBE Weak scaling per node. .. csv-table:: Multi Node Scaling Parthenon - :file: ./scaling/weak.csv + :file: ./scaling/weak-august.csv :align: center :widths: 10, 10, 10, 10 :header-rows: 1 diff --git a/utils/read_lanl_amg.py b/utils/read_lanl_amg.py new file mode 100644 index 00000000..6b9dc3c4 --- /dev/null +++ b/utils/read_lanl_amg.py @@ -0,0 +1,107 @@ +# %% +import sys +from glob import glob + +import matplotlib.pyplot as plt +from cycler import cycler + +default_cycler = (cycler(color=['r', 'g', 'b', 'y']) * + cycler(linestyle=['-', '--', ':', '-.'])) + +plt.rc('axes', prop_cycle=default_cycler) + +# sys.path.append("/usr/gapps/spot/dev/hatchet-venv/x86_64/lib/python3.9/site-packages/") +# sys.path.append("/usr/gapps/spot/dev/hatchet/x86_64/") +# sys.path.append("/usr/gapps/spot/dev/thicket-playground-dev/") + +import thicket as th + +# %% +tk = th.Thicket.from_caliperreader(glob("AMG/*.cali")) + +# %% +problem_sizes = list(sorted(tk.metadata["Problem"].unique())) +ranks = list(sorted(tk.metadata["mpi.world.size"].unique())) + +gb = tk.groupby("mpi.world.size") +thickets = list(gb.values()) +ctk = th.Thicket.concat_thickets( + thickets=thickets, + headers=list(gb.keys()), + axis="columns", + metadata_key="Problem", +) + +# %% +for p in problem_sizes: + for r in ranks: + ctk.dataframe.loc[(slice(None), p), (r, "perc")] = (ctk.dataframe.loc[(slice(None), p), (r, "Avg time/rank (exc)")] / ctk.dataframe.loc[(slice(None), p), (r, "Avg time/rank (exc)")].sum()) * 100 + +# %% +for p in problem_sizes: + ax = ctk.dataframe.loc[(slice(None), p), [(r, "perc") for r in ranks]].T.reset_index(1, drop=True).plot( + kind="area", + title=f"AMG2023 Problem {p} (Weak Scaling)", + xlabel="Number of MPI ranks", + ylabel="% of Runtime", + figsize=(10,5) + ) + # Custom legend + handles, labels = ax.get_legend_handles_labels() + legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left') + labels = ctk.dataframe.loc[(slice(None), 1), ("name", "")].tolist() + labels.reverse() + for i, label in enumerate(legend.get_texts()): + label.set_text(labels[i]) + # Set custom x labels + xlabels = [item.get_text() for item in ax.get_xticklabels()] + ax.set_xticklabels([xlabels[0]] + ranks + [xlabels[-1]]) + plt.savefig(f"prob{p}-pct.png", bbox_inches="tight") + #plt.show() + +# %% +for p in problem_sizes: + ax = ctk.dataframe.loc[(slice(None), p), [(r, "Avg time/rank") for r in ranks]].T.reset_index(1, drop=True).plot( + kind="area", + title=f"AMG2023 Problem {p} (Weak Scaling)", + xlabel="Number of MPI ranks", + ylabel="Runtime (sec)", + figsize=(10,5) + ) + ax.set_prop_cycle(default_cycler) + # Custom legend + handles, labels = ax.get_legend_handles_labels() + legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left') + labels = ctk.dataframe.loc[(slice(None), 1), ("name", "")].tolist() + labels.reverse() + for i, label in enumerate(legend.get_texts()): + label.set_text(labels[i]) + # Set custom x labels + xlabels = [item.get_text() for item in ax.get_xticklabels()] + ax.set_xticklabels([xlabels[0]] + ranks + [xlabels[-1]]) + plt.savefig(f"prob{p}-totaltime.png", bbox_inches="tight") + #plt.show() + +# %% +for p in problem_sizes: + + ax = ctk.dataframe.loc[(slice(None), p), [(r, "Avg time/rank") for r in ranks]].T.reset_index(1, drop=True).plot( + kind="line", + title=f"AMG2023 Problem {p} (Weak Scaling)", + xlabel="Number of MPI ranks", + ylabel="Runtime (sec)", + figsize=(10,5) + ) + # Custom legend + handles, labels = ax.get_legend_handles_labels() + legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left') + labels = ctk.dataframe.loc[(slice(None), 1), ("name", "")].tolist() + labels.reverse() + for i, label in enumerate(legend.get_texts()): + label.set_text(labels[i]) + # Set custom x labels + xlabels = [item.get_text() for item in ax.get_xticklabels()] + ax.set_xticklabels([xlabels[0]] + ranks + [xlabels[-1]]) + ax.set_prop_cycle(default_cycler) + plt.savefig(f"prob{p}-time.png", bbox_inches="tight") + #plt.show() diff --git a/utils/read_lanl_vibe.py b/utils/read_lanl_vibe.py new file mode 100755 index 00000000..94f189e4 --- /dev/null +++ b/utils/read_lanl_vibe.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +## TAKES ONE ARGUMENT: PATH TO DIR WITH CALIPER FILES RELATIVE OR ABSOLUTE +# WILL CREATE A DIRECTORY ADJACENT TO ARGUMENT DIR WITH THE NAME OF THAT DIR + _plots +## REQUIRES THICKET: +# pip install llnl-thicket +# IF THEY'RE IN A NON_STANDARD PLACE, MAKE SURE TO ADD THEM TO PYTHONPATH ON CL. +# IF RUNNING IN IPYTHON, JUPYTER, ETC, import sys AND APPEND THEIR LOC TO sys.path BEFORE LAUNCHING. + + +import sys +import os +import os.path as op +from glob import glob +# from matplotlib.style import context +import matplotlib.pyplot as plt +import thicket as th + +# from cycler import cycler + +# default_cycler = (cycler(color=['r', 'g', 'b', 'y']) * +# cycler(linestyle=['-', '--', ':', '-.'])) + +# sys.path.append("/usr/gapps/spot/dev/hatchet-venv/x86_64/lib/python3.9/site-packages/") +# sys.path.append("/usr/gapps/spot/dev/hatchet/x86_64/") +# sys.path.append("/usr/gapps/spot/dev/thicket-playground-dev/") + +### CONFIG +plotx=12 +ploty=7 +cat_to_plot=10 #Number of categories to plot. + +### FILTER DF +def filter_df(df, category, ncat=10, criteria=None): + dfclean = df.reset_index('profile', drop=True).T + totalmeans = dfclean.loc[(slice(None), category),:].mean().sort_values(ascending=False) + + # If given a criteria (a minimum value cutoff), use that instead + if criteria: + mean_filt = list(totalmeans[totalmeans > criteria].index) + else: + mean_filt = list(totalmeans.head(ncat).index) + + cols = list(dfclean.columns) + for code_section in cols: + if code_section not in mean_filt: + dfclean.drop(code_section, axis=1, inplace=True) + + return dfclean + +def get_cali_files(rawpath): + cali_path = op.realpath(rawpath) + califiles = glob(cali_path + "/*.cali") + + if not califiles: + print(f"\nERROR:\n Path {cali_path} has no caliper files.\n") + sys.exit(1) + + cali_name = op.basename(cali_path) + cali_dirname = op.dirname(cali_path) + return cali_name, cali_dirname, califiles + + +if __name__ == "__main__": + + # %% + if len(sys.argv) < 2: + error="\nERROR:\n TAKES ONE ARGUMENT: PATH TO DIR WITH CALIPER FILES RELATIVE OR ABSOLUTE\n" + error+=" WILL CREATE A DIRECTORY ADJACENT TO ARGUMENT DIR WITH THE NAME OF THAT DIR + _plots\n" + print(error) + sys.exit(1) + + cali_name, cali_dirname, cali_files = get_cali_files(sys.argv[1]) + + plt_outdir = op.join(cali_dirname, cali_name+"_plots") + os.makedirs(plt_outdir, exist_ok=True) + + tk = th.Thicket.from_caliperreader(cali_files) + + # %% + #problem_sizes = list(sorted(tk.metadata["Problem"].unique())) + ranks = list(sorted(tk.metadata["mpi.world.size"].unique())) + + gb = tk.groupby("mpi.world.size") + thickets = list(gb.values()) + ranks = list(gb.keys()) + problem_sizes = [0] + ctk = th.Thicket.concat_thickets( + thickets=thickets, + headers=ranks, + axis="columns" + ) + rank_str=[str(r) for r in ranks] + # print(ranks) + + df_filtered = filter_df(ctk.dataframe, 'Avg time/rank', cat_to_plot) + + # %% + for r in ranks: + df_filtered.loc[(r, "perc"),:] = (df_filtered.loc[(r, "Avg time/rank (exc)"),:] / df_filtered.loc[(r, "Avg time/rank (exc)"),:].sum()) * 100 + + ''' + PLOT PCT + ''' + fig_path = op.join(plt_outdir, "parthenon-pct.png") + plt.style.use('fivethirtyeight') + ax = df_filtered.loc[[(r, "perc") for r in ranks],:].reset_index(1, drop=True).plot( + kind="area", + title=f"Parthenon-VIBE (Weak Scaling)", + xlabel="Number of Nodes", + ylabel="% of Runtime (Exclusive)", + figsize=(10,5), + colormap="tab20" + ) + # Custom legend + handles, labels = ax.get_legend_handles_labels() + legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left') + #print (ctk.dataframe[(slice(None), 1), ("name", "")].tolist()) + labels = df_filtered.loc[("name", "")].tolist() + + labels.reverse() + print(labels) + for i, label in enumerate(legend.get_texts()): + label.set_text(labels[i]) + + f=ax.get_figure() + f.set_size_inches(plotx,ploty) + plt.tight_layout() + plt.savefig(fig_path, bbox_inches="tight") + #plt.show() + + ''' + PLOT TOTAL TIME LINES + ''' + for p in problem_sizes: + fig_path = op.join(plt_outdir, f"parthenon-totaltime-line_{p}.png") + plt.style.use('fivethirtyeight') + ax = df_filtered.loc[[(r, "Avg time/rank (exc)") for r in ranks],:].reset_index(1, drop=True).plot( + kind="line", + title=f"Parthenon-VIBE (Weak Scaling)", + xlabel="Number of Nodes", + ylabel="Runtime (sec) (Exclusive)", + figsize=(10,5), + colormap="tab20" + ) + # Custom legend + handles, labels = ax.get_legend_handles_labels() + legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left') + labels = df_filtered.loc[("name", "")].tolist() + labels.reverse() + for i, label in enumerate(legend.get_texts()): + label.set_text(labels[i]) + f=ax.get_figure() + f.set_size_inches(plotx,ploty) + plt.tight_layout() + plt.savefig(fig_path, bbox_inches="tight") + #plt.show() + + ''' + PLOT TOTAL TIME AREA + ''' + for p in problem_sizes: + fig_path = op.join(plt_outdir, f"parthenon-totaltime-area_{p}.png") + plt.style.use('fivethirtyeight') + ax = df_filtered.loc[[(r, "Avg time/rank (exc)") for r in ranks],:].reset_index(1, drop=True).plot( + kind="area", + title=f"Parthenon-VIBE (Weak Scaling)", + xlabel="Number of Nodes", + ylabel="Runtime (sec) (Exclusive)", + figsize=(10,5), + colormap="tab20" + ) + + # Custom legend + handles, labels = ax.get_legend_handles_labels() + legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left') + labels = df_filtered.loc[("name", "")].tolist() + labels.reverse() + for i, label in enumerate(legend.get_texts()): + label.set_text(labels[i]) + + f=ax.get_figure() + f.set_size_inches(plotx,ploty) + plt.tight_layout() + plt.savefig(fig_path, bbox_inches="tight") + + #ctk.dataframe.to_csv("out.csv", index=True) + #print (plt.style.available) + #print(plt.colormaps)