Monitor PkgManager operations

Summary: Adding monitoring on failures on * processAll * loadRequiredKmods * unloadKmods * processRpm Some functions have a couple place where it exits or throws on failures. So using SCOPE_* macro. Reviewed By: rationalis Differential Revision: D66711104 fbshipit-source-id: e0e71e7379f3c757d18cac8251e44217877a5ee0
facebook · Dec 13, 2024 · d2eb5a5 · d2eb5a5
1 parent 14fcc2b
commit d2eb5a5
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 0 deletions.
diff --git a/fboss/platform/platform_manager/PkgManager.cpp b/fboss/platform/platform_manager/PkgManager.cpp
@@ -141,6 +141,13 @@ PkgManager::PkgManager(
       platformFsUtils_(platformFsUtils) {}
 
 void PkgManager::processAll() const {
+  SCOPE_SUCCESS {
+    fb303::fbData->setCounter(kProcessAllFailure, 0);
+  };
+  SCOPE_FAIL {
+    fb303::fbData->setCounter(kProcessAllFailure, 1);
+  };
+
   if (FLAGS_local_rpm_path.size()) {
     fb303::fbData->setExportedValue(
         kBspKmodsRpmName, "local_rpm: " + FLAGS_local_rpm_path);
@@ -186,6 +193,13 @@ void PkgManager::processAll() const {
 }
 
 void PkgManager::processRpms() const {
+  SCOPE_SUCCESS {
+    fb303::fbData->setCounter(kProcessRpmFailure, 0);
+  };
+  SCOPE_FAIL {
+    fb303::fbData->setCounter(kProcessRpmFailure, 1);
+  };
+
   int exitStatus{0};
   if (auto installedRpms =
           systemInterface_->getInstalledRpms(getKmodsRpmBaseWithKernelName());
@@ -242,6 +256,13 @@ void PkgManager::processLocalRpms() const {
 }
 
 void PkgManager::unloadBspKmods() const {
+  SCOPE_SUCCESS {
+    fb303::fbData->setCounter(kUnloadKmodsFailure, 0);
+  };
+  SCOPE_FAIL {
+    fb303::fbData->setCounter(kUnloadKmodsFailure, 1);
+  };
+
   std::string keyword{};
   re2::RE2::FullMatch(
       *platformConfig_.bspKmodsRpmName(), kBspRpmNameRe, &keyword);
@@ -305,10 +326,12 @@ void PkgManager::loadRequiredKmods() const {
   for (const auto& requiredKmod : *platformConfig_.requiredKmodsToLoad()) {
     XLOG(INFO) << fmt::format("Loading {}", requiredKmod);
     if (!systemInterface_->loadKmod(requiredKmod)) {
+      fb303::fbData->setCounter(kLoadKmodsFailure, 1);
       throw std::runtime_error(
           fmt::format("Failed to load ({})", requiredKmod));
     }
   }
+  fb303::fbData->setCounter(kLoadKmodsFailure, 0);
 }
 
 std::string PkgManager::getKmodsRpmName() const {

diff --git a/fboss/platform/platform_manager/PkgManager.h b/fboss/platform/platform_manager/PkgManager.h
@@ -31,6 +31,16 @@ class SystemInterface {
 
 class PkgManager {
  public:
+  // ODS Counters
+  constexpr static auto kProcessAllFailure =
+      "package_manager.process_all_failure";
+  constexpr static auto kLoadKmodsFailure =
+      "package_manager.load_kmods_failure";
+  constexpr static auto kUnloadKmodsFailure =
+      "package_manager.unload_kmods_failure";
+  constexpr static auto kProcessRpmFailure =
+      "package_manager.process_rpm_failure";
+
   explicit PkgManager(
       const PlatformConfig& config,
       const std::shared_ptr<package_manager::SystemInterface>& systemInterface =

diff --git a/fboss/platform/platform_manager/tests/BUCK b/fboss/platform/platform_manager/tests/BUCK
@@ -102,6 +102,7 @@ cpp_unittest(
     ],
     deps = [
         "fbsource//third-party/googletest:gmock",
+        "//fb303:service_data",
         "//fboss/platform/platform_manager:pkg_manager",
         "//thrift/lib/cpp2/protocol:protocol",
     ],

diff --git a/fboss/platform/platform_manager/tests/PkgManagerTest.cpp b/fboss/platform/platform_manager/tests/PkgManagerTest.cpp
@@ -1,5 +1,6 @@
 // (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
+#include <fb303/ServiceData.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <thrift/lib/cpp2/protocol/Serializer.h>
@@ -177,6 +178,8 @@ TEST_F(PkgManagerTest, processRpms) {
       .WillOnce(Return(0));
   EXPECT_CALL(*mockSystemInterface_, depmod()).WillOnce(Return(0));
   EXPECT_NO_THROW(pkgManager_.processRpms());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 0);
   // Installed rpms
   EXPECT_CALL(
       *mockSystemInterface_,
@@ -199,6 +202,8 @@ TEST_F(PkgManagerTest, processRpms) {
       .WillOnce(Return(0));
   EXPECT_CALL(*mockSystemInterface_, depmod()).WillOnce(Return(0));
   EXPECT_NO_THROW(pkgManager_.processRpms());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 0);
   // Remove installed rpms failed.
   EXPECT_CALL(
       *mockSystemInterface_,
@@ -213,6 +218,8 @@ TEST_F(PkgManagerTest, processRpms) {
           "fboss_bsp_kmods-6.4.3-0_fbk1_755_ga25447393a1d-2.4.0-1"}))
       .WillOnce(Return(1));
   EXPECT_THROW(pkgManager_.processRpms(), std::runtime_error);
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 1);
   // depmod failed
   EXPECT_CALL(
       *mockSystemInterface_,
@@ -234,6 +241,8 @@ TEST_F(PkgManagerTest, processRpms) {
       .WillOnce(Return(0));
   EXPECT_CALL(*mockSystemInterface_, depmod()).WillOnce(Return(1));
   EXPECT_NO_THROW(pkgManager_.processRpms());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 0);
   // Rpm install failed
   EXPECT_CALL(
       *mockSystemInterface_,
@@ -255,6 +264,8 @@ TEST_F(PkgManagerTest, processRpms) {
       .Times(3)
       .WillRepeatedly(Return(1));
   EXPECT_THROW(pkgManager_.processRpms(), std::runtime_error);
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 1);
 }
 
 TEST_F(PkgManagerTest, unloadBspKmods) {
@@ -271,6 +282,8 @@ TEST_F(PkgManagerTest, unloadBspKmods) {
       .WillOnce(Return(std::vector<std::string>{}));
   EXPECT_CALL(*mockSystemInterface_, lsmod()).Times(0);
   EXPECT_NO_THROW(pkgManager_.unloadBspKmods());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 0);
   // No kmods.json when it should exist
   EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_))
       .WillOnce(Return(std::nullopt));
@@ -282,6 +295,8 @@ TEST_F(PkgManagerTest, unloadBspKmods) {
       .WillOnce(Return(std::vector<std::string>{
           "fboss_bsp_kmods-6.4.3-0_fbk1_755_ga25447393a1d-2.4.0-1"}));
   EXPECT_THROW(pkgManager_.unloadBspKmods(), std::runtime_error);
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 1);
   // kmods.json exist and all kmods are loaded
   EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_))
       .WillOnce(Return(jsonBspKmodsFile_));
@@ -294,19 +309,25 @@ TEST_F(PkgManagerTest, unloadBspKmods) {
           bspKmodsFile_.bspKmods()->size())
       .WillRepeatedly(Return(true));
   EXPECT_NO_THROW(pkgManager_.unloadBspKmods());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 0);
   // kmods.json exists but unload fails
   EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_))
       .WillOnce(Return(jsonBspKmodsFile_));
   EXPECT_CALL(*mockSystemInterface_, lsmod())
       .WillOnce(Return(jsonBspKmodsFile_));
   EXPECT_CALL(*mockSystemInterface_, unloadKmod(_)).WillOnce(Return(false));
   EXPECT_THROW(pkgManager_.unloadBspKmods(), std::runtime_error);
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 1);
   // kmods.json exist and all kmods aren't loaded
   EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_))
       .WillOnce(Return(jsonBspKmodsFile_));
   EXPECT_CALL(*mockSystemInterface_, lsmod()).WillOnce(Return(""));
   EXPECT_CALL(*mockSystemInterface_, unloadKmod(_)).Times(0);
   EXPECT_NO_THROW(pkgManager_.unloadBspKmods());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 0);
 }
 
 TEST_F(PkgManagerTest, loadRequiredKmods) {
@@ -315,8 +336,12 @@ TEST_F(PkgManagerTest, loadRequiredKmods) {
       .Times(platformConfig_.requiredKmodsToLoad()->size())
       .WillRepeatedly(Return(true));
   EXPECT_NO_THROW(pkgManager_.loadRequiredKmods());
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kLoadKmodsFailure), 0);
   // Load kmods fail
   EXPECT_CALL(*mockSystemInterface_, loadKmod(_)).WillOnce(Return(false));
   EXPECT_THROW(pkgManager_.loadRequiredKmods(), std::runtime_error);
+  EXPECT_EQ(
+      facebook::fb303::fbData->getCounter(PkgManager::kLoadKmodsFailure), 1);
 }
 }; // namespace facebook::fboss::platform::platform_manager