Merge pull request #110 from lanl/parthenon-rslt-update

Parthenon rslt update

.gitignore

@@ -3,6 +3,7 @@ doc/sphinx/output.log
 doc/sphinx/07_miniem/*.png
 doc/sphinx/08_sparta/*.png
 doc/sphinx/*/*.png
+doc/sphinx/*/*/*.png
 doc/sphinx/09_Microbenchmarks/*/*.png
 
 doc/sphinx/07_miniem/PanzerMiniEM_BlockPrec.exe

doc/sphinx/03_vibe/scaling/weak-august.csv

@@ -0,0 +1,12 @@
+Nodes,aggregate-FOM,per-node-FOM,per-node-Ideal
+4096,7.33E+09,1.79E+06,9.60E+06
+2048,8.85E+09,4.32E+06,9.60E+06
+1024,4.28E+09,4.18E+06,9.60E+06
+512,3.29E+09,6.43E+06,9.60E+06
+256,1.73E+09,6.76E+06,9.60E+06
+128,9.26E+08,7.23E+06,9.60E+06
+64,5.18E+08,8.09E+06,9.60E+06
+32,2.65E+08,8.28E+06,9.60E+06
+16,1.32E+08,8.25E+06,9.60E+06
+8,7.70E+07,9.63E+06,9.60E+06
+4,3.84E+07,9.60E+06,9.60E+06

doc/sphinx/03_vibe/scaling/weak.gp

@@ -20,6 +20,8 @@ set key autotitle columnheader
 set style line 1 linetype 6 dashtype 1 linecolor rgb "#FF0000" linewidth 2 pointtype 6 pointsize 3
 set style line 2 linetype 1 dashtype 2 linecolor rgb "#FF0000" linewidth 2
 
-
 set output "weak.png"
 plot "weak.csv" using 1:3 with linespoints linestyle 1, "" using 1:4 with line linestyle 2
+
+set output "weak-august.png"
+plot "weak-august.csv" using 1:3 with linespoints linestyle 1, "" using 1:4 with line linestyle 2

doc/sphinx/03_vibe/vibe.rst

@@ -207,18 +207,24 @@ Multi-node scaling on Crossroads
 
 The results of the scaling runs performed on Crossroads are presented below.
 Parthenon was built with intel oneapi 2023.1.0 and cray-mpich 8.1.25.
-These runs used between 2 and 4096 nodes with 8 ranks per node and 14 threads (using Kokkos OpenMP) per rank.
-NXs=(208 256 320 400 512 640 800 1024 1280 1616 2048 2576)  
-NODES=(2 4 8 16 32 64 128 256 512 1024 2048 4096)
+These runs used between 4 and 4096 nodes with 8 ranks per node and 14 threads per rank (using Kokkos OpenMP) with the following problem sizes. 
+
+.. code-block:: bash
+
+NXs=(256 320 400 512 640 800 1024 1280 1616 2048 2576)
+NODES=(4 8 16 32 64 128 256 512 1024 2048 4096)
+
+..
+
 Output files can be found in ``./docs/sphinx/03_vibe/scaling/output/``
 
-.. figure:: ./scaling/weak.png
+.. figure:: ./scaling/weak-august.png
    :align: center
    :scale: 50%
    :alt: VIBE Weak scaling per node.
 
 .. csv-table:: Multi Node Scaling Parthenon
-   :file: ./scaling/weak.csv
+   :file: ./scaling/weak-august.csv
    :align: center
    :widths: 10, 10, 10, 10
    :header-rows: 1

utils/read_lanl_amg.py

@@ -0,0 +1,107 @@
+# %%
+import sys
+from glob import glob
+
+import matplotlib.pyplot as plt
+from cycler import cycler
+
+default_cycler = (cycler(color=['r', 'g', 'b', 'y']) *
+                  cycler(linestyle=['-', '--', ':', '-.']))
+
+plt.rc('axes', prop_cycle=default_cycler)
+
+# sys.path.append("/usr/gapps/spot/dev/hatchet-venv/x86_64/lib/python3.9/site-packages/")
+# sys.path.append("/usr/gapps/spot/dev/hatchet/x86_64/")
+# sys.path.append("/usr/gapps/spot/dev/thicket-playground-dev/")
+
+import thicket as th
+
+# %%
+tk = th.Thicket.from_caliperreader(glob("AMG/*.cali"))
+
+# %%
+problem_sizes = list(sorted(tk.metadata["Problem"].unique()))
+ranks = list(sorted(tk.metadata["mpi.world.size"].unique()))
+
+gb = tk.groupby("mpi.world.size")
+thickets = list(gb.values())
+ctk = th.Thicket.concat_thickets(
+    thickets=thickets,
+    headers=list(gb.keys()),
+    axis="columns",
+    metadata_key="Problem",
+)
+
+# %%
+for p in problem_sizes:
+    for r in ranks:
+        ctk.dataframe.loc[(slice(None), p), (r, "perc")] = (ctk.dataframe.loc[(slice(None), p), (r, "Avg time/rank (exc)")] / ctk.dataframe.loc[(slice(None), p), (r, "Avg time/rank (exc)")].sum()) * 100
+
+# %%
+for p in problem_sizes:
+    ax = ctk.dataframe.loc[(slice(None), p), [(r, "perc") for r in ranks]].T.reset_index(1, drop=True).plot(
+        kind="area",
+        title=f"AMG2023 Problem {p} (Weak Scaling)",
+        xlabel="Number of MPI ranks",
+        ylabel="% of Runtime",
+        figsize=(10,5)
+    )
+    # Custom legend
+    handles, labels = ax.get_legend_handles_labels()
+    legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left')
+    labels = ctk.dataframe.loc[(slice(None), 1), ("name", "")].tolist()
+    labels.reverse()
+    for i, label in enumerate(legend.get_texts()):
+        label.set_text(labels[i])
+    # Set custom x labels
+    xlabels = [item.get_text() for item in ax.get_xticklabels()]
+    ax.set_xticklabels([xlabels[0]] + ranks + [xlabels[-1]])
+    plt.savefig(f"prob{p}-pct.png", bbox_inches="tight")
+    #plt.show()
+
+# %%
+for p in problem_sizes:
+    ax = ctk.dataframe.loc[(slice(None), p), [(r, "Avg time/rank") for r in ranks]].T.reset_index(1, drop=True).plot(
+        kind="area",
+        title=f"AMG2023 Problem {p} (Weak Scaling)",
+        xlabel="Number of MPI ranks",
+        ylabel="Runtime (sec)",
+        figsize=(10,5)
+    )
+    ax.set_prop_cycle(default_cycler)
+    # Custom legend
+    handles, labels = ax.get_legend_handles_labels()
+    legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left')
+    labels = ctk.dataframe.loc[(slice(None), 1), ("name", "")].tolist()
+    labels.reverse()
+    for i, label in enumerate(legend.get_texts()):
+        label.set_text(labels[i])
+    # Set custom x labels
+    xlabels = [item.get_text() for item in ax.get_xticklabels()]
+    ax.set_xticklabels([xlabels[0]] + ranks + [xlabels[-1]])
+    plt.savefig(f"prob{p}-totaltime.png", bbox_inches="tight")
+    #plt.show()
+
+# %%
+for p in problem_sizes:
+
+    ax = ctk.dataframe.loc[(slice(None), p), [(r, "Avg time/rank") for r in ranks]].T.reset_index(1, drop=True).plot(
+        kind="line",
+        title=f"AMG2023 Problem {p} (Weak Scaling)",
+        xlabel="Number of MPI ranks",
+        ylabel="Runtime (sec)",
+        figsize=(10,5)
+    )
+    # Custom legend
+    handles, labels = ax.get_legend_handles_labels()
+    legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left')
+    labels = ctk.dataframe.loc[(slice(None), 1), ("name", "")].tolist()
+    labels.reverse()
+    for i, label in enumerate(legend.get_texts()):
+        label.set_text(labels[i])
+    # Set custom x labels
+    xlabels = [item.get_text() for item in ax.get_xticklabels()]
+    ax.set_xticklabels([xlabels[0]] + ranks + [xlabels[-1]])
+    ax.set_prop_cycle(default_cycler)
+    plt.savefig(f"prob{p}-time.png", bbox_inches="tight")
+    #plt.show()

utils/read_lanl_vibe.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+
+## TAKES ONE ARGUMENT: PATH TO DIR WITH CALIPER FILES RELATIVE OR ABSOLUTE
+# WILL CREATE A DIRECTORY ADJACENT TO ARGUMENT DIR WITH THE NAME OF THAT DIR + _plots 
+## REQUIRES THICKET:
+# pip install llnl-thicket
+# IF THEY'RE IN A NON_STANDARD PLACE, MAKE SURE TO ADD THEM TO PYTHONPATH ON CL.
+# IF RUNNING IN IPYTHON, JUPYTER, ETC, import sys AND APPEND THEIR LOC TO sys.path BEFORE LAUNCHING.
+
+
+import sys
+import os
+import os.path as op
+from glob import glob
+# from matplotlib.style import context
+import matplotlib.pyplot as plt
+import thicket as th
+
+# from cycler import cycler
+
+# default_cycler = (cycler(color=['r', 'g', 'b', 'y']) *
+#                   cycler(linestyle=['-', '--', ':', '-.']))
+
+# sys.path.append("/usr/gapps/spot/dev/hatchet-venv/x86_64/lib/python3.9/site-packages/")
+# sys.path.append("/usr/gapps/spot/dev/hatchet/x86_64/")
+# sys.path.append("/usr/gapps/spot/dev/thicket-playground-dev/")
+
+### CONFIG
+plotx=12
+ploty=7
+cat_to_plot=10 #Number of categories to plot.
+
+### FILTER DF
+def filter_df(df, category, ncat=10, criteria=None):
+    dfclean    = df.reset_index('profile', drop=True).T
+    totalmeans = dfclean.loc[(slice(None), category),:].mean().sort_values(ascending=False)
+
+    # If given a criteria (a minimum value cutoff), use that instead
+    if criteria:
+        mean_filt = list(totalmeans[totalmeans > criteria].index)
+    else:
+        mean_filt = list(totalmeans.head(ncat).index)
+
+    cols = list(dfclean.columns)
+    for code_section in cols:
+        if code_section not in mean_filt:
+            dfclean.drop(code_section, axis=1, inplace=True)
+
+    return dfclean
+
+def get_cali_files(rawpath):
+    cali_path = op.realpath(rawpath)
+    califiles = glob(cali_path + "/*.cali")
+
+    if not califiles:
+        print(f"\nERROR:\n    Path {cali_path} has no caliper files.\n")
+        sys.exit(1)
+
+    cali_name = op.basename(cali_path)
+    cali_dirname = op.dirname(cali_path)
+    return cali_name, cali_dirname, califiles
+
+
+if __name__ == "__main__":
+
+    # %%
+    if len(sys.argv) < 2:
+        error="\nERROR:\n  TAKES ONE ARGUMENT: PATH TO DIR WITH CALIPER FILES RELATIVE OR ABSOLUTE\n"
+        error+="  WILL CREATE A DIRECTORY ADJACENT TO ARGUMENT DIR WITH THE NAME OF THAT DIR + _plots\n"
+        print(error)
+        sys.exit(1)
+
+    cali_name, cali_dirname, cali_files = get_cali_files(sys.argv[1])
+
+    plt_outdir = op.join(cali_dirname, cali_name+"_plots")
+    os.makedirs(plt_outdir, exist_ok=True)
+
+    tk = th.Thicket.from_caliperreader(cali_files)
+
+    # %%
+    #problem_sizes = list(sorted(tk.metadata["Problem"].unique()))
+    ranks = list(sorted(tk.metadata["mpi.world.size"].unique()))
+
+    gb = tk.groupby("mpi.world.size")
+    thickets = list(gb.values())
+    ranks = list(gb.keys())
+    problem_sizes = [0]
+    ctk = th.Thicket.concat_thickets(
+        thickets=thickets,
+        headers=ranks,
+        axis="columns"
+    )
+    rank_str=[str(r) for r in ranks]
+    # print(ranks)
+
+    df_filtered = filter_df(ctk.dataframe, 'Avg time/rank', cat_to_plot)
+
+    # %%
+    for r in ranks:
+        df_filtered.loc[(r, "perc"),:] = (df_filtered.loc[(r, "Avg time/rank (exc)"),:] / df_filtered.loc[(r, "Avg time/rank (exc)"),:].sum()) * 100
+
+    '''
+    PLOT PCT
+    '''
+    fig_path = op.join(plt_outdir, "parthenon-pct.png")
+    plt.style.use('fivethirtyeight')
+    ax = df_filtered.loc[[(r, "perc") for r in ranks],:].reset_index(1, drop=True).plot(
+        kind="area",
+        title=f"Parthenon-VIBE (Weak Scaling)",
+        xlabel="Number of Nodes",
+        ylabel="% of Runtime (Exclusive)",
+        figsize=(10,5), 
+        colormap="tab20"
+    )
+    # Custom legend
+    handles, labels = ax.get_legend_handles_labels()
+    legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left')
+    #print (ctk.dataframe[(slice(None), 1), ("name", "")].tolist())
+    labels = df_filtered.loc[("name", "")].tolist()
+
+    labels.reverse()
+    print(labels)
+    for i, label in enumerate(legend.get_texts()):
+        label.set_text(labels[i])
+
+    f=ax.get_figure()
+    f.set_size_inches(plotx,ploty)
+    plt.tight_layout()
+    plt.savefig(fig_path, bbox_inches="tight")
+    #plt.show()
+
+    '''
+    PLOT TOTAL TIME LINES
+    '''
+    for p in problem_sizes:
+        fig_path = op.join(plt_outdir, f"parthenon-totaltime-line_{p}.png")
+        plt.style.use('fivethirtyeight')
+        ax = df_filtered.loc[[(r, "Avg time/rank (exc)") for r in ranks],:].reset_index(1, drop=True).plot(
+            kind="line",
+            title=f"Parthenon-VIBE (Weak Scaling)",
+            xlabel="Number of Nodes",
+            ylabel="Runtime (sec) (Exclusive)",
+            figsize=(10,5),
+            colormap="tab20"
+        )
+        # Custom legend
+        handles, labels = ax.get_legend_handles_labels()
+        legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left')
+        labels = df_filtered.loc[("name", "")].tolist()
+        labels.reverse()
+        for i, label in enumerate(legend.get_texts()):
+            label.set_text(labels[i])
+        f=ax.get_figure()
+        f.set_size_inches(plotx,ploty)
+        plt.tight_layout()
+        plt.savefig(fig_path, bbox_inches="tight")
+        #plt.show()
+
+    '''
+    PLOT TOTAL TIME AREA
+    '''
+    for p in problem_sizes:
+        fig_path = op.join(plt_outdir, f"parthenon-totaltime-area_{p}.png")
+        plt.style.use('fivethirtyeight')
+        ax = df_filtered.loc[[(r, "Avg time/rank (exc)") for r in ranks],:].reset_index(1, drop=True).plot(
+            kind="area",
+            title=f"Parthenon-VIBE (Weak Scaling)",
+            xlabel="Number of Nodes",
+            ylabel="Runtime (sec) (Exclusive)",
+            figsize=(10,5),
+            colormap="tab20"
+        )
+
+        # Custom legend
+        handles, labels = ax.get_legend_handles_labels()
+        legend = ax.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1, 1))#, title='Line', loc='upper left')
+        labels = df_filtered.loc[("name", "")].tolist()
+        labels.reverse()
+        for i, label in enumerate(legend.get_texts()):
+            label.set_text(labels[i])
+
+        f=ax.get_figure()
+        f.set_size_inches(plotx,ploty)
+        plt.tight_layout()
+        plt.savefig(fig_path, bbox_inches="tight")
+
+    #ctk.dataframe.to_csv("out.csv", index=True)
+        #print (plt.style.available) 
+        #print(plt.colormaps)