From bf763f9e037f7ccbb50e7f69a11715181a658b67 Mon Sep 17 00:00:00 2001 From: James Curtis Date: Tue, 6 Aug 2024 12:17:38 +0000 Subject: [PATCH] Update test to write results to S3 bucket Only manual test being used now, due to udev rules being too slow. Change the manual test to save results to s3 bucket, so that the results can more easily be processed Signed-off-by: James Curtis --- tests/conftest.py | 5 +- tests/host_tools/hotplug.sh | 2 +- .../performance/test_vcpu_hotplug.py | 249 +++++++++--------- 3 files changed, 131 insertions(+), 125 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a6985272a552..4b8598aaa84e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -362,7 +362,10 @@ def rootfs_fxt(request, record_property): guest_kernel_fxt, params=kernel_params("vmlinux-5.10*") ) guest_kernel_linux_acpi_only = pytest.fixture( - guest_kernel_fxt, params=kernel_params("vmlinux-5.10.221") + guest_kernel_fxt, params=kernel_params("vmlinux-5.10.219") +) +guest_kernel_linux_6_5 = pytest.fixture( + guest_kernel_fxt, params=kernel_params("vmlinux-6.5.*", select=kernels_unfiltered) ) # Use the unfiltered selector, since we don't officially support 6.1 yet. # TODO: switch to default selector once we add full 6.1 support. diff --git a/tests/host_tools/hotplug.sh b/tests/host_tools/hotplug.sh index 69254f4d9fb4..13a8bd994eec 100644 --- a/tests/host_tools/hotplug.sh +++ b/tests/host_tools/hotplug.sh @@ -9,7 +9,7 @@ done readarray -t offline_cpus < <(lscpu -p=cpu --offline | sed '/^#/d') for cpu_idx in ${offline_cpus[@]}; do - echo 1 >/sys/devices/system/cpu/cpu$cpu_idx/online + echo 1 | tee cpu*/online done /home/hotplug_time.o diff --git a/tests/integration_tests/performance/test_vcpu_hotplug.py b/tests/integration_tests/performance/test_vcpu_hotplug.py index 03e422da377d..e25148a8d541 100644 --- a/tests/integration_tests/performance/test_vcpu_hotplug.py +++ b/tests/integration_tests/performance/test_vcpu_hotplug.py @@ -12,133 +12,133 @@ from host_tools.cargo_build import gcc_compile - -@pytest.mark.parametrize( - "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] -) -def test_custom_udev_rule_latency( - microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count -): - """Test the latency for hotplugging and booting CPUs in the guest""" - api_durations = [] - onlining_durations = [] - print(f"Vcpu count: {vcpu_count}") - for i in range(5): - uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw) - uvm_hotplug.jailer.extra_args.update({"no-seccomp": None}) - uvm_hotplug.help.enable_console() - uvm_hotplug.spawn() - uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128) - uvm_hotplug.add_net_iface() - uvm_hotplug.start() - uvm_hotplug.ssh.run("rm /usr/lib/udev/rules.d/40-vm-hotadd.rules") - uvm_hotplug.ssh.scp_put( - Path("./host_tools/1-cpu-hotplug.rules"), - Path("/usr/lib/udev/rules.d/1-cpu-hotplug.rules"), - ) - - time.sleep(0.25) - - uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count}) - time.sleep(0.25) - _, stdout, _ = uvm_hotplug.ssh.run("dmesg") - - # Extract API call duration - api_duration = ( - float( - re.findall( - r"Total previous API call duration: (\d+) us\.", - uvm_hotplug.log_data, - )[-1] - ) - / 1000 - ) - - # Extract onlining timings - start = float( - re.findall(r"\[\s+(\d+\.\d+)\] CPU1 has been hot-added\n", stdout)[0] - ) - end = float(re.findall(r"\[\s+(\d+\.\d+)\] \w+", stdout)[-1]) - elapsed_time = (end - start) * 1000 - print(f"Api call duration: {api_duration} ms") - print(f"Onlining duration: {elapsed_time} ms") - api_durations.append(api_duration) - onlining_durations.append(elapsed_time) - uvm_hotplug.kill() - time.sleep(1) - - avg_api_duration = sum(api_durations) / 5 - avg_onlining_duration = sum(onlining_durations) / 5 - print(f"Averages for {vcpu_count} hotplugged vcpus:") - print(f"\tAverage API call duration: {avg_api_duration} ms") - print(f"\tAverage onliing duration: {avg_onlining_duration} ms") - - -@pytest.mark.parametrize( - "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] -) -def test_default_udev_rule_latency( - microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count -): - """Test the latency for hotplugging and booting CPUs in the guest""" - api_durations = [] - onlining_durations = [] - print(f"Vcpu count: {vcpu_count}") - for i in range(5): - uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw) - uvm_hotplug.jailer.extra_args.update({"no-seccomp": None}) - uvm_hotplug.help.enable_console() - uvm_hotplug.spawn() - uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128) - uvm_hotplug.add_net_iface() - uvm_hotplug.start() - - time.sleep(0.25) - - _, stdout, _ = uvm_hotplug.ssh.run("ls /usr/lib/udev/rules.d") - default_rule = re.search(r"40-vm-hotadd\.rules", stdout) - assert default_rule is not None - - uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count}) - time.sleep(0.25) - _, stdout, _ = uvm_hotplug.ssh.run("dmesg") - - # Extract API call duration - api_duration = ( - float( - re.findall( - r"Total previous API call duration: (\d+) us\.", - uvm_hotplug.log_data, - )[-1] - ) - / 1000 - ) - - # Extract onlining timings - start = float( - re.findall(r"\[\s+(\d+\.\d+)\] CPU1 has been hot-added\n", stdout)[0] - ) - end = float(re.findall(r"\[\s+(\d+\.\d+)\] \w+", stdout)[-1]) - elapsed_time = (end - start) * 1000 - print(f"Api call duration: {api_duration} ms") - print(f"Onlining duration: {elapsed_time} ms") - api_durations.append(api_duration) - onlining_durations.append(elapsed_time) - uvm_hotplug.kill() - time.sleep(1) - - avg_api_duration = sum(api_durations) / 5 - avg_onlining_duration = sum(onlining_durations) / 5 - print(f"Averages for {vcpu_count} hotplugged vcpus:") - print(f"\tAverage API call duration: {avg_api_duration} ms") - print(f"\tAverage onliing duration: {avg_onlining_duration} ms") +# @pytest.mark.parametrize( +# "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] +# ) +# def test_custom_udev_rule_latency( +# microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count +# ): +# """Test the latency for hotplugging and booting CPUs in the guest""" +# api_durations = [] +# onlining_durations = [] +# print(f"Vcpu count: {vcpu_count}") +# for i in range(5): +# uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw) +# uvm_hotplug.jailer.extra_args.update({"no-seccomp": None}) +# uvm_hotplug.help.enable_console() +# uvm_hotplug.spawn() +# uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128) +# uvm_hotplug.add_net_iface() +# uvm_hotplug.start() +# uvm_hotplug.ssh.run("rm /usr/lib/udev/rules.d/40-vm-hotadd.rules") +# uvm_hotplug.ssh.scp_put( +# Path("./host_tools/1-cpu-hotplug.rules"), +# Path("/usr/lib/udev/rules.d/1-cpu-hotplug.rules"), +# ) +# +# time.sleep(0.25) +# +# uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count}) +# time.sleep(0.25) +# _, stdout, _ = uvm_hotplug.ssh.run("dmesg") +# +# # Extract API call duration +# api_duration = ( +# float( +# re.findall( +# r"Total previous API call duration: (\d+) us\.", +# uvm_hotplug.log_data, +# )[-1] +# ) +# / 1000 +# ) +# +# # Extract onlining timings +# start = float( +# re.findall(r"\[\s+(\d+\.\d+)\] CPU1 has been hot-added\n", stdout)[0] +# ) +# end = float(re.findall(r"\[\s+(\d+\.\d+)\] \w+", stdout)[-1]) +# elapsed_time = (end - start) * 1000 +# print(f"Api call duration: {api_duration} ms") +# print(f"Onlining duration: {elapsed_time} ms") +# api_durations.append(api_duration) +# onlining_durations.append(elapsed_time) +# uvm_hotplug.kill() +# time.sleep(1) +# +# avg_api_duration = sum(api_durations) / 5 +# avg_onlining_duration = sum(onlining_durations) / 5 +# print(f"Averages for {vcpu_count} hotplugged vcpus:") +# print(f"\tAverage API call duration: {avg_api_duration} ms") +# print(f"\tAverage onliing duration: {avg_onlining_duration} ms") +# +# +# @pytest.mark.parametrize( +# "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] +# ) +# def test_default_udev_rule_latency( +# microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count +# ): +# """Test the latency for hotplugging and booting CPUs in the guest""" +# api_durations = [] +# onlining_durations = [] +# print(f"Vcpu count: {vcpu_count}") +# for i in range(5): +# uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw) +# uvm_hotplug.jailer.extra_args.update({"no-seccomp": None}) +# uvm_hotplug.help.enable_console() +# uvm_hotplug.spawn() +# uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128) +# uvm_hotplug.add_net_iface() +# uvm_hotplug.start() +# +# time.sleep(0.25) +# +# _, stdout, _ = uvm_hotplug.ssh.run("ls /usr/lib/udev/rules.d") +# default_rule = re.search(r"40-vm-hotadd\.rules", stdout) +# assert default_rule is not None +# +# uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count}) +# time.sleep(0.25) +# _, stdout, _ = uvm_hotplug.ssh.run("dmesg") +# +# # Extract API call duration +# api_duration = ( +# float( +# re.findall( +# r"Total previous API call duration: (\d+) us\.", +# uvm_hotplug.log_data, +# )[-1] +# ) +# / 1000 +# ) +# +# # Extract onlining timings +# start = float( +# re.findall(r"\[\s+(\d+\.\d+)\] CPU1 has been hot-added\n", stdout)[0] +# ) +# end = float(re.findall(r"\[\s+(\d+\.\d+)\] \w+", stdout)[-1]) +# elapsed_time = (end - start) * 1000 +# print(f"Api call duration: {api_duration} ms") +# print(f"Onlining duration: {elapsed_time} ms") +# api_durations.append(api_duration) +# onlining_durations.append(elapsed_time) +# uvm_hotplug.kill() +# time.sleep(1) +# +# avg_api_duration = sum(api_durations) / 5 +# avg_onlining_duration = sum(onlining_durations) / 5 +# print(f"Averages for {vcpu_count} hotplugged vcpus:") +# print(f"\tAverage API call duration: {avg_api_duration} ms") +# print(f"\tAverage onliing duration: {avg_onlining_duration} ms") +# @pytest.mark.parametrize( "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] ) def test_manual_latency( - microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count + microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count, results_dir ): """Test the latency for hotplugging and booting CPUs in the guest""" gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o")) @@ -189,8 +189,11 @@ def test_manual_latency( # Extract onlining timings data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp}) - df = pandas.DataFrame.from_dict(data).to_csv( - f"../test_results/manual-hotplug_{vcpu_count}.csv", + output_file = results_dir / f"hotplug-{vcpu_count}.csv" + + csv_data = pandas.DataFrame.from_dict(data).to_csv( index=False, float_format="%.3f", ) + + output_file.write_text(csv_data)