fix load and run file order bug and add feature importantce

BilyZ98 · May 23, 2024 · cc12b68 · cc12b68
1 parent 7e2a62f
commit cc12b68
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 4 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,8 @@
 make_config.mk
 rocksdb.pc
 
+tools/zipfian_ycsb_abf*
+
 LOG
 0.8
 *.a

diff --git a/db/builder.cc b/db/builder.cc
@@ -207,7 +207,7 @@ Status BuildTable(
         uint64_t now_seq = versions->LastSequence();
         uint64_t end_seq;
         // if(now_seq < 25000000) {
-        //   end_seq = now_seq + 25000000;
+        //   end_seq = now_seq + 50000000;
         // } else {
         //   end_seq = now_seq + tboptions.db_impl->GetDefaultLifetimeThreshold();
         // }

diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc
@@ -3110,6 +3110,21 @@ void DBImpl::BackgroundCallDataCollection() {
       uint64_t training_data_size = training_data_->GetTrainingDataSize();
       ROCKS_LOG_INFO(immutable_db_options_.info_log, "model len: %lu, out len: %lu, training data size: %lu", model_len, out_len, training_data_size);
       uint64_t end_time = env_->NowMicros();
+
+      double feature_importance_split[50];
+      res = LGBM_BoosterFeatureImportance(*new_model, 0, C_API_FEATURE_IMPORTANCE_SPLIT, &feature_importance_split[0]);  
+      if(res != 0) {
+        assert(false);
+      }
+      double feauture_importance_gain[50];
+      res = LGBM_BoosterFeatureImportance(*new_model, 0, C_API_FEATURE_IMPORTANCE_GAIN, &feauture_importance_gain[0]);
+      if(res != 0) {
+        assert(false);
+      }
+      for(int i=0; i < 50; i++) {
+        ROCKS_LOG_INFO(immutable_db_options_.info_log, "feature %d, split: %lf, gain: %lf", i, feature_importance_split[i], feauture_importance_gain[i]);
+      }
+
       uint64_t duration_sec = (end_time - start_time) / 1000000;
       assert(s.ok());
       // uint64_t num_class = std::stoi(training_params_["num_class"]);

diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc
@@ -6760,12 +6760,14 @@ class Benchmark {
       std::cout << "FLAGS_ycsb_a_run_path is empty" << std::endl;
       return;
     }
-    std::vector<std::string> files = {
-      FLAGS_ycsb_a_run_path
-    };
+    std::vector<std::string> files; 
+    // = {
+    //   FLAGS_ycsb_a_run_path
+    // };
     if(!FLAGS_ycsb_a_load_path.empty()){
       files.push_back(FLAGS_ycsb_a_load_path);
     }
+    files.push_back(FLAGS_ycsb_a_run_path);
     std::vector<std::vector<std::string>> datas;
     read_ycsb_a(files, &datas);
     int64_t val_size = FLAGS_value_size; //1000;
@@ -6811,6 +6813,7 @@ class Benchmark {
           fprintf(stderr, "Get returned an error: %s\n", s.ToString().c_str());
           abort();
         }
+        assert(s.ok());
 
         if (thread->shared->read_rate_limiter && (gets + seek) % 100 == 0) {
           thread->shared->read_rate_limiter->Request(100, Env::IO_HIGH,