Skip to content

Commit

Permalink
updated pydarshan to support DFS/DAOS heatmaps
Browse files Browse the repository at this point in the history
  • Loading branch information
@shanedsnyder
shanedsnyder committed Nov 11, 2024
1 parent b14c589 commit 47a1a97
Show file tree
Hide file tree
Showing 7 changed files with 22 additions and 20 deletions.
Binary file removed ...les/example_logs/snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan
View file
Binary file not shown.
Binary file added ...es/example_logs/snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan
View file
Binary file not shown.
4 changes: 3 additions & 1 deletion darshan-util/pydarshan/darshan/report.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -608,7 +608,9 @@ def read_all_heatmap_records(self):
_nrecs_heatmap = {
16592106915301738621: "heatmap:POSIX",
3989511027826779520: "heatmap:STDIO",
3668870418325792824: "heatmap:MPIIO"
3668870418325792824: "heatmap:MPIIO",
1597927878319380788: "heatmap:DFS",
4131494093108637317: "heatmap:DAOS"
}

def heatmap_rec_to_module_name(rec, nrecs=None):
Expand Down
8 changes: 4 additions & 4 deletions darshan-util/pydarshan/darshan/tests/test_plot_exp_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@
"log_path, mod, func, expected_xticklabels",
[
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
"DFS",
plot_access_histogram,
["0-100", "101-1K", "1K-10K", "10K-100K", "100K-1M",
"1M-4M", "4M-10M", "10M-100M", "100M-1G", "1G+"]
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
"DAOS",
plot_access_histogram,
["0-100", "101-1K", "1K-10K", "10K-100K", "100K-1M",
Expand Down Expand Up @@ -79,13 +79,13 @@
"1M-4M", "4M-10M", "10M-100M", "100M-1G", "1G+"],
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
"DFS",
plot_opcounts,
['Read', 'Readx', 'Write', 'Writex', 'Open', 'GlobalOpen', 'Lookup', 'Get Size', 'Punch', 'Remove', 'Stat'],
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
"DAOS",
plot_opcounts,
['ObjFetch', 'ObjUpdate', 'ObjOpen', 'ArrRead', 'ArrWrite', 'ArrOpen', 'KVGet', 'KVPut', 'KVOpen'],
Expand Down
20 changes: 10 additions & 10 deletions darshan-util/pydarshan/darshan/tests/test_plot_io_cost.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@
),
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
pd.DataFrame(
np.array([
[0.0, 0.0, 0.0, 0.0],
[0.0, 4.515051841e-06, 0.0, 0.0],
[0.001456562, 0.002266062, 0.007923812, 0.0],
[0.001492562, 0.002273217, 0.007910812, 0.0],
[0.0, 0.0, 1.490116119e-08, 0.0],
[0.0, 4.872679710e-06, 0.0, 0.0],
[0.001452714, 0.001865715, 0.007635623, 0.0],
[0.001490027, 0.001865178, 0.007622942, 0.0],
]),
["POSIX", "STDIO", "DFS", "DAOS"],
["Read", "Write", "Meta", "Wait"],
Expand Down Expand Up @@ -101,8 +101,8 @@ def test_get_io_cost_df(logname, expected_df):
[0.0, 1111.0],
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
[0.0, 0.80276728],
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
[0.0, 0.75089884],
),
],
)
Expand Down Expand Up @@ -134,9 +134,9 @@ def test_plot_io_cost_ylims(logname, expected_ylims):
['0', '156', '312', '468', '624', '780'],
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
[0.0 , 0.16055346, 0.32110691, 0.48166037, 0.64221382, 0.80276728],
['0.0000' , '0.1606', '0.3211', '0.4817', '0.6422', '0.8028'],
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
[0.0 , 0.15017977, 0.30035954, 0.4505393 , 0.60071907, 0.75089884],
['0.0000', '0.1502', '0.3004', '0.4505', '0.6007', '0.7509'],
),
],
)
Expand Down
2 changes: 1 addition & 1 deletion darshan-util/pydarshan/darshan/tests/test_report.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def test_dfs_daos_posix_match():
# backend should produce matching counters where
# comparable data fields exist
posix_ior_report = darshan.DarshanReport(get_log_path("snyder_ior-POSIX_id1057716-202103_11-8-64415-6936117869459351096_1.darshan"))
dfs_ior_report = darshan.DarshanReport(get_log_path("snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan"))
dfs_ior_report = darshan.DarshanReport(get_log_path("snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan"))
posix_ior_report.mod_read_all_records("POSIX")
dfs_ior_report.mod_read_all_records("DFS")
dfs_ior_report.mod_read_all_records("DAOS")
Expand Down
8 changes: 4 additions & 4 deletions darshan-util/pydarshan/darshan/tests/test_summary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ def test_metadata_table(self, log_path, expected_df):
2,
),
(
"snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan",
"snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan",
pd.DataFrame(
index=[
"Log Filename", "Runtime Library Version", "Log Format Version",
Expand All @@ -503,9 +503,9 @@ def test_metadata_table(self, log_path, expected_df):
"DFS (ver=1)", "DAOS (ver=1)"
],
data=[
["snyder_ior-DFS_id1057716-201712_11-8-64400-1922568413188514066_1.darshan"], ["3.4.7"], ["3.41"],
["0.07 KiB"], ["0.07 KiB"], ["0.05 KiB"],
["0.17 KiB"], ["0.39 KiB"],
["snyder_ior-DFS_id1069383-23981_11-11-71996-14940917661149127511_1.darshan"], ["3.4.7"], ["3.41"],
["0.08 KiB"], ["0.07 KiB"], ["1.09 KiB"],
["0.17 KiB"], ["0.40 KiB"],
],
),
0,
Expand Down

0 comments on commit 47a1a97

Please sign in to comment.