From 4c769ff6de6fbb3586f666ac20aeb0b2ffacf4a0 Mon Sep 17 00:00:00 2001 From: Fabian Ruffy <5960321+fruffy@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:02:34 +0100 Subject: [PATCH] Benchmarking fixes for P4Testgen. (#4205) asdasd --- .../modules/testgen/benchmarks/plots.py | 67 ++++++++++--------- .../testgen/benchmarks/test_coverage.py | 31 ++++++--- .../bmv2/backend/metadata/metadata.cpp | 2 +- .../testgen/targets/pna/backend/ptf/ptf.cpp | 37 ++-------- tools/testutils.py | 2 + 5 files changed, 65 insertions(+), 74 deletions(-) diff --git a/backends/p4tools/modules/testgen/benchmarks/plots.py b/backends/p4tools/modules/testgen/benchmarks/plots.py index 07b81a8228b..dd569ada0bb 100755 --- a/backends/p4tools/modules/testgen/benchmarks/plots.py +++ b/backends/p4tools/modules/testgen/benchmarks/plots.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import os import re import sys from operator import attrgetter @@ -83,6 +84,8 @@ def get_strategy_data(input_dir): folders = input_dir.glob("*/") program_data = {} for folder in folders: + if not os.path.isdir(folder): + continue folder = Path(folder) program_name = folder.stem # Make sure we escape hyphens @@ -105,38 +108,40 @@ def get_strategy_data(input_dir): def plot_strategies(args, extra_args): - program_data = get_strategy_data(args.input_dir) - program_data = program_data["tna_simple_switch"] - pruned_data = [] - name_map = { - "depth_first": "DFS", - "random_backtrack": "Random", - "greedy_statement_search": "Coverage-Optimized", - } - for strategy, candidate_data in program_data.items(): - candidate_data = candidate_data.drop("Seed", axis=1) - candidate_data["Strategy"] = name_map[strategy] - candidate_data.Time = pd.to_numeric(candidate_data.Time) - candidate_data = candidate_data.sort_values(by=["Time"]) - bins = np.arange( - 0, - candidate_data.Time.max() + candidate_data.Time.max() / 1000, - candidate_data.Time.max() / 1000, + program_datas = get_strategy_data(args.input_dir) + for program_name, program_data in program_datas.items(): + pruned_data = [] + name_map = { + "depth_first": "DFS", + "random_backtrack": "Random", + "greedy_statement_search": "Coverage-Optimized", + } + for strategy, candidate_data in program_data.items(): + candidate_data = candidate_data.drop("Seed", axis=1) + candidate_data["Strategy"] = name_map[strategy] + candidate_data.Time = pd.to_numeric(candidate_data.Time) + candidate_data = candidate_data.sort_values(by=["Time"]) + bins = np.arange( + 0, + candidate_data.Time.max() + candidate_data.Time.max() / 1000, + candidate_data.Time.max() / 1000, + ) + candidate_data["Minutes"] = pd.cut( + candidate_data.Time, bins.astype(np.int64), include_lowest=True + ).map(attrgetter("right")) + candidate_data.Minutes = pd.to_timedelta(candidate_data.Minutes, unit="nanoseconds") + candidate_data.Minutes = candidate_data.Minutes / pd.Timedelta(minutes=1) + candidate_data = candidate_data[candidate_data.Minutes <= 60] + pruned_data.append(candidate_data) + concat_data = pd.concat(pruned_data) + ax = sns.lineplot( + x="Minutes", y="Coverage", hue="Strategy", data=concat_data, errorbar=None ) - candidate_data["Minutes"] = pd.cut( - candidate_data.Time, bins.astype(np.int64), include_lowest=True - ).map(attrgetter("right")) - candidate_data.Minutes = pd.to_timedelta(candidate_data.Minutes, unit="nanoseconds") - candidate_data.Minutes = candidate_data.Minutes / pd.Timedelta(minutes=1) - candidate_data = candidate_data[candidate_data.Minutes <= 60] - pruned_data.append(candidate_data) - concat_data = pd.concat(pruned_data) - ax = sns.lineplot(x="Minutes", y="Coverage", hue="Strategy", data=concat_data, errorbar=None) - sns.move_legend(ax, "lower center", bbox_to_anchor=(0.5, 0.98), ncol=3, title=None) - outdir = Path(args.out_dir).joinpath("strategy_coverage") - plt.savefig(outdir.with_suffix(".png"), bbox_inches="tight") - plt.savefig(outdir.with_suffix(".pdf"), bbox_inches="tight") - plt.gcf().clear() + sns.move_legend(ax, "lower center", bbox_to_anchor=(0.5, 0.98), ncol=3, title=None) + outdir = Path(args.out_dir).joinpath(f"{program_name}_strategy_coverage") + plt.savefig(outdir.with_suffix(".png"), bbox_inches="tight") + plt.savefig(outdir.with_suffix(".pdf"), bbox_inches="tight") + plt.gcf().clear() def main(args, extra_args): diff --git a/backends/p4tools/modules/testgen/benchmarks/test_coverage.py b/backends/p4tools/modules/testgen/benchmarks/test_coverage.py index 53c93fd3447..769df31cc3a 100755 --- a/backends/p4tools/modules/testgen/benchmarks/test_coverage.py +++ b/backends/p4tools/modules/testgen/benchmarks/test_coverage.py @@ -247,15 +247,26 @@ def run_strategies_for_max_tests(options, test_args): ) perf_file = test_args.test_dir.joinpath(test_args.p4_program.stem + "_perf").with_suffix(".csv") - perf = pd.read_csv(perf_file, index_col=0) - summarized_data = [ - float(final_cov) * 100, - num_tests, - time_needed, - perf["Percentage"]["z3"], - perf["Percentage"]["step"], - perf["Percentage"]["backend"], - ] + if perf_file.exists(): + perf = pd.read_csv(perf_file, index_col=0) + summarized_data = [ + float(final_cov) * 100, + num_tests, + time_needed, + perf["Percentage"]["z3"], + perf["Percentage"]["step"], + perf["Percentage"]["backend"], + ] + else: + # In some cases, we do not have performance data. Nullify it. + summarized_data = [ + float(final_cov) * 100, + num_tests, + time_needed, + None, + None, + None, + ] return summarized_data, nodes_cov, timestamps @@ -313,7 +324,7 @@ def main(args, extra_args): if options.test_mode == "DPDK": options.target = "dpdk" options.arch = "pna" - options.test_backend = "METADATA" + options.test_backend = "PTF" # 7189 is an example of a good seed, which gets cov 1 with less than 100 tests # in random access stack. diff --git a/backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.cpp b/backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.cpp index e307e8c20d8..69e618b03fa 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/backend/metadata/metadata.cpp @@ -83,7 +83,7 @@ std::string Metadata::getTestCaseTemplate() { # Seed used to generate this test. seed: {{ default(seed, "none") }} # Test timestamp. -data: {{timestamp}} +date: {{timestamp}} # Percentage of nodes covered at the time of this test. node_coverage: {{coverage}} diff --git a/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp b/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp index fe91dcb775e..2fa7b5b3619 100644 --- a/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp +++ b/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp @@ -296,12 +296,13 @@ std::string PTF::getTestCaseTemplate() { static std::string TEST_CASE( R"""( class Test{{test_id}}(AbstractTest): - # Date generated: {{timestamp}} + ''' + Date generated: {{timestamp}} + Current node coverage: {{coverage}} ## if length(selected_branches) > 0 - # {{selected_branches}} + Selected branches: {{selected_branches}} ## endif - ''' - # Current statement coverage: {{coverage}} + Trace: ## for trace_item in trace {{trace_item}} ##endfor @@ -343,11 +344,6 @@ class Test{{test_id}}(AbstractTest): ) ## endfor ## endfor -## endif -## if exists("clone_specs") -## for clone_pkt in clone_specs.clone_pkts - self.insert_pre_clone_session({{clone_pkt.session_id}}, [{{clone_pkt.clone_port}}]) -## endfor ## endif @@ -367,20 +363,9 @@ class Test{{test_id}}(AbstractTest): ## for ignore_mask in verify.ignore_masks exp_pkt.set_do_not_care({{ignore_mask.0}}, {{ignore_mask.1}}) ## endfor -## if exists("clone_specs") -## for clone_pkt in clone_specs.clone_pkts -## if clone_pkt.cloned - ptfutils.verify_packet(self, exp_pkt, {{clone_pkt.clone_port}}) -##endif -##endfor -## if not clone_specs.has_clone - ptfutils.verify_packet(self, exp_pkt, eg_port) -##endif -## else ptfutils.verify_packet(self, exp_pkt, eg_port) bt.testutils.log.info("Verifying no other packets ...") ptfutils.verify_no_other_packets(self, self.device_id, timeout=self.packet_wait_time) -## endif ## else ptfutils.verify_no_other_packets(self, self.device_id, timeout=self.packet_wait_time) ## endif @@ -409,18 +394,6 @@ void PTF::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_ coverageStr << std::setprecision(2) << currentCoverage; dataJson["coverage"] = coverageStr.str(); - // The following few lins are commented temporarily, they are copied from BMv2 - // Check whether this test has a clone configuration. - // These are special because they require additional instrumentation and produce two output - // packets. - // auto cloneSpecs = testSpec->getTestObjectCategory("clone_specs"); - - // if (!cloneSpecs.empty()) { - // dataJson["clone_specs"] = getClone(cloneSpecs); - // } - // auto meterValues = testSpec->getTestObjectCategory("meter_values"); - // dataJson["meter_values"] = getMeter(meterValues); - LOG5("PTF backend: emitting testcase:" << std::setw(4) << dataJson); inja::render_to(ptfFileStream, testCase, dataJson); diff --git a/tools/testutils.py b/tools/testutils.py index f410103238a..f57469a0864 100644 --- a/tools/testutils.py +++ b/tools/testutils.py @@ -246,6 +246,8 @@ def exec_process(args: str, **extra_args) -> ProcessResult: except subprocess.TimeoutExpired as exception: if errpipe: out = errpipe.out + else: + out = str(exception.stderr) returncode = FAILURE cmd = exception.cmd # Rejoin the list for better readability.