Merge pull request #12 from ninja-cloudbuild/for_ci_test

Add Cloudbuild  ci test and  support for custom command filtering in project root

.github/workflows/cloudbuild-ci.yml

@@ -0,0 +1,119 @@
+name: Cloudbuild TEST
+
+on:
+  pull_request:
+  push:
+  release:
+    types: published
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    services:
+      redis:
+        image: redis:6.2
+        ports:
+          - 6379:6379
+    steps:
+      # 测试redis服务
+      - name: Test Redis connection
+        run: |
+          docker exec $(docker ps -q --filter ancestor=redis:6.2) redis-cli -h localhost ping
+      # 1. 检出代码
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # 2. 配置 apt 使用更快的镜像源（可选，根据地域选择）
+      - name: Configure apt to use faster mirrors
+        run: |
+          sudo sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/g' /etc/apt/sources.list
+          sudo apt-get update
+
+      # 4. 安装客户端依赖
+      - name: Install client dependencies
+        # if: steps.apt-cache.outputs.cache-hit != 'true' （仅在缓存未命中时）
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git cmake g++ gcc googletest libgmock-dev libssl-dev pkg-config uuid-dev grpc++ libprotobuf-dev protobuf-compiler-grpc ninja-build libyaml-cpp-dev zip unzip
+
+      # # 5. 缓存 CMake 和构建产物（可选）
+      # - name: Cache CMake and build
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: |
+      #       ~/.cache
+      #       build
+      #     key: ${{ runner.os }}-cmake-${{ hashFiles('**/CMakeLists.txt') }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-cmake-
+
+      # 6. 运行构建脚本
+      - name: Build ninja2
+        run: |
+          chmod +x build.sh
+          ./build.sh build
+          ./build.sh package
+
+      # 7. 安装构建好的 ninja2
+      - name: Install ninja2
+        run: |
+          sudo ./build.sh install
+
+      # 8. 验证 ninja2 安装
+      - name: Verify ninja2 installation
+        run: |
+          ninja --version
+
+      # 9. 运行 Ninja 单元测试套件
+      - name: Run Ninja unit-test suite
+        run: |
+          cd build/bin
+          ctest --output-on-failure
+
+      # 10. 克隆 re2 项目
+      - name: Clone re2 repository
+        run: |
+          wget https://github.com/google/re2/archive/refs/tags/2021-11-01.tar.gz
+          tar -zxvf 2021-11-01.tar.gz
+          mv re2-2021-11-01 re2
+      # 11. 安装服务端需要的依赖
+      - name: Install server dependencies
+        run: |
+          wget https://github.com/bazelbuild/bazel/releases/download/6.0.0/bazel-6.0.0-installer-linux-x86_64.sh
+          chmod +x bazel-6.0.0-installer-linux-x86_64.sh 
+          ./bazel-6.0.0-installer-linux-x86_64.sh --user
+          export PATH=$PATH:~/bin
+     # 12. 安装服务端
+      - name: build server 
+        run: |
+          git clone https://gitee.com/cloudbuild888/buildbuddy.git
+          cd buildbuddy
+          bazel build //enterprise/server:server 
+     # 13. 安装executor 
+      - name: build executor
+        run: |
+          cd buildbuddy
+          bazel build //enterprise/server/cmd/executor:executor 
+      - name: run server 
+        run: |
+          cd buildbuddy
+          nohup bazel run //enterprise/server:server &
+      - name: run executor
+        run: |
+          cd buildbuddy
+          nohup bazel run //enterprise/server/cmd/executor:executor &
+      - name: cloudbuild re2
+        run: |
+          cd re2
+          mkdir -p build
+          cd build
+          cmake -G Ninja ..
+          ninja -c grpc://127.0.0.1:1985
+     # 测试 cloudbuild 编译出来的 re2 是否正常
+      - name: Test re2(cloudbuild)
+        run: |
+          cd re2/build
+          ./re2_test
+          # ctest --output-on-failure
+
+ 

src/build.cc

@@ -814,9 +814,29 @@ size_t CloudCommandRunner::CanRunMore() const {
   return capacity;
 }
 
+
 bool CloudCommandRunner::StartCommand(Edge* edge) {
   string command = edge->EvaluateCommand();
   auto spawn = RemoteExecutor::RemoteSpawn::CreateRemoteSpawn(edge);
+  string cmd_rule =spawn->edge->rule().name();
+  if(config_.rbe_config.local_only_rules.find(cmd_rule) != config_.rbe_config.local_only_rules.end()){
+    SubprocessSet subprocset;
+    Subprocess* subproc = subprocset.Add(spawn->command,edge->use_console());
+    if (!subproc) {
+      return false;
+    }   
+    return true;
+  }
+  for(auto &cmd:config_.rbe_config.fuzzy_rules){
+    if(cmd.find(cmd_rule)!=std::string::npos){
+    SubprocessSet subprocset;
+    Subprocess* subproc = subprocset.Add(spawn->command,edge->use_console());
+    if (!subproc) {
+      return false;
+    }   
+    return true;
+    }
+  }
   RemoteProcess* remoteproc = remote_procs_.Add(spawn);
   if (!remoteproc)
     return false;

src/build.h

@@ -168,6 +168,9 @@ struct ProjectConfig {
   std::string project_root;                               // project root directory. eg: ~/proj
   std::map<std::string, std::string> rbe_properties;      // remote build execution properties
   std::string grpc_url;
+  std::set<std::string>  local_only_rules;
+  std::set<std::string>  remote_no_cache_rules;
+  std::set<std::string>  fuzzy_rules;
 };
 /// Options (e.g. verbosity, parallelism) passed to a build.
 struct BuildConfig {

src/ninja.cc

@@ -1596,6 +1596,11 @@ NORETURN void real_main(int argc, char** argv) {
   } 
   std::string project_root = "../";
   if (config.cloud_run || config.share_run) {
+     string err;
+    if (!GetCurrentDirectory(&config.rbe_config.cwd, &err)) {
+      Error(err.c_str());
+      exit(1);
+    }
     if (!config.rbe_config.project_root.empty()) {
       project_root = config.rbe_config.project_root;
     }

src/rbe_config.cc

@@ -10,6 +10,7 @@
 #include <yaml-cpp/yaml.h>
 #include <json/json.h>
 #include "rbe_config.h"
+#include <filesystem>
 
 bool load_config_file(BuildConfig &config) {
   std::string config_file = "/etc/ninja2.conf";
@@ -65,3 +66,35 @@ void load_devcontainer_config(const std::string& project_root, BuildConfig &conf
         config.rbe_config.rbe_properties["workload-isolation-type"] = "docker";  
     }  
 } 
+
+const std::string COMMANDFILE ="/command_cloudbuild.yml";
+
+void load_command_file(const std::string& project_root, BuildConfig &config){
+   std::string commandFilePath=project_root+COMMANDFILE;
+    std::ifstream file(commandFilePath);  
+    YAML::Node  command_set;
+    if (file.is_open()){
+      command_set = YAML::LoadFile(commandFilePath);
+        } else {
+            std::cout << "YAML file not found,no filter command.\n";
+            return;
+        }
+     if (command_set) {
+        if (command_set["commands"]["local_only"]) {
+            for (const auto& cmd : command_set["commands"]["local_only"]) {
+                config.rbe_config.local_only_rules.insert(cmd.as<std::string>());
+            }
+        }
+        if (command_set["commands"]["remote_no_cache"]) {
+            for (const auto& cmd : command_set["commands"]["remote_no_cache"]) {
+                 config.rbe_config.remote_no_cache_rules.insert(cmd.as<std::string>());
+            }
+        }
+         if (command_set["commands"]["fuzzy_rule"]) {
+            for (const auto& cmd : command_set["commands"]["fuzzy_rule"]) {
+                 config.rbe_config.fuzzy_rules.insert(cmd.as<std::string>());
+            }
+        }
+
+    }
+}

src/rbe_config.h

@@ -12,3 +12,5 @@
 bool load_config_file(BuildConfig &config);
 // Loads the devcontainer configuration from the specified project root  
 void load_devcontainer_config(const std::string& project_root, BuildConfig &config);
+
+void load_command_file(const std::string& project_root, BuildConfig &config);

src/remote_executor/execution_context.cc

@@ -80,6 +80,19 @@ FileNode File::ToFileNode(const std::string& name) const {
   return result;
 }
 
+OutputFile File::ToOutputFile(const std::string& path) const {
+  OutputFile result;
+  result.set_path(path);
+  *result.mutable_digest() = digest;
+  result.set_is_executable(executable);
+  if (mtime_set) {
+    auto node_properties = result.mutable_node_properties();
+    node_properties->mutable_mtime()->CopyFrom(MakeTimestamp(mtime));
+  }
+  return result;
+}
+
+
 void NestedDirectory::Add(const File& file, const char* relative_path) {
   const char *slash = strchr(relative_path, '/');
   if (slash) {
@@ -243,45 +256,66 @@ Action BuildAction(RemoteSpawn* spawn, const std::string& cwd,
   return action;
 }
 
-void BuildActionOutputs(RemoteSpawn* spawn, const std::string& cwd,
+bool BuildActionOutputs(RemoteSpawn* spawn, const std::string& cwd,
     DigestStringMap* blobs, DigestStringMap* digest_files,
     ActionResult *result) {
+  std::set<std::string> products;
   std::set<std::string> deps;
   for (auto i : spawn->outputs)
+    products.insert(i);
+   for (auto i : spawn->inputs)
     deps.insert(i);
-
-  NestedDirectory nested_dir;
-  auto cmd_work_dir = CommonAncestorPath(deps, deps, cwd);
+  //NestedDirectory nested_dir;
+  auto cmd_work_dir = CommonAncestorPath(deps, products, cwd);
   // BuildMerkleTree(deps, cmd_work_dir, &nested_dir, digest_files);
-  for (auto& dep : deps) {
-    std::string merklePath(dep);
-    if (merklePath[0] != '/' && !cwd.empty())
-      merklePath = cwd + "/" + merklePath;
-    merklePath = StaticFileUtils::NormalizePath(merklePath.c_str());
+  //file exist?
+  // if(!StaticFileUtils::WaitForPipeClose(spawn->local_pipe_fd_))
+  //     Fatal("local output produce failed!");
+  for (auto& product : products) {
+    if(product.find(".o.d")!=std::string::npos){
+      continue;
+    }
+    std::string merklePath(product);
     if (merklePath[0] == '/' &&
         !StaticFileUtils::HasPathPrefix(merklePath,
-            RemoteSpawn::config->rbe_config.project_root)) {
+                                        RemoteSpawn::config->rbe_config.project_root)) {
       continue;
     }
-    File file(dep.c_str());
-    nested_dir.Add(file, merklePath.c_str());
-    (*digest_files)[file.digest] = dep;
-    OutputFile output;
-    output.mutable_digest()->CopyFrom(file.digest);
-    output.set_path(merklePath.c_str());
-    *result->add_output_files() = output;
-  }
-  if (!cmd_work_dir.empty()) {
-    cmd_work_dir = StaticFileUtils::NormalizePath(cmd_work_dir.c_str());
-    nested_dir.AddDirectory(cmd_work_dir.c_str());
+    char currentPath[PATH_MAX];
+    getcwd(currentPath, sizeof(currentPath));
+    strcat(currentPath,"/");
+    strcat(currentPath,product.c_str());
+    if(StaticFileUtils::IsSymlink(currentPath)){
+      // return false;
+       char target[PATH_MAX];
+       ssize_t len=readlink(currentPath,target,sizeof(target)-1);
+       std::string real_target;
+       if(len==-1){
+          Error("Error read symbol link %s",product.c_str());
+       }else{
+         target[len] = '\0';
+         real_target=target;
+       }
+       OutputSymlink outsymlink;
+       outsymlink.set_path(product);
+       outsymlink.set_target(real_target);
+       result->add_output_symlinks()->CopyFrom(outsymlink);
+    }else{
+      File file(product.c_str());
+     (*digest_files)[file.digest] = product;
+
+      auto  outputF=file.ToOutputFile(std::string(merklePath.c_str()));
+      result->add_output_files()->CopyFrom(outputF);
+    }
   }
-  const auto dir_digest = nested_dir.ToDigest(blobs);
-  const auto cmd_proto = GenerateCommandProto(spawn->arguments, deps, cmd_work_dir, 
-                                              spawn->config->rbe_config.rbe_properties);
+
+  const auto cmd_proto = GenerateCommandProto(spawn->arguments, products,
+                                                 cmd_work_dir,spawn->config->rbe_config.rbe_properties);
   const auto cmd_digest = MakeDigest(cmd_proto);
   (*blobs)[cmd_digest] = cmd_proto.SerializeAsString();
 
-  return ;
+  result->set_exit_code(0);
+  return true;
 }
 
 constexpr auto kMetadataToolName = "Ninja_Remote";
@@ -341,6 +375,9 @@ void ExecutionContext::Execute(int fd, RemoteExecutor::RemoteSpawn* spawn,
   bool cached = false;
   ActionResult result;
   cached = re_client.FetchFromActionCache(action_digest, products, &result);
+   Warning("Execute locally CMD: %s,is it cached? %d", spawn->command.c_str(),cached);
+  if(cached) 
+    exit_code = result.exit_code();
   // Info("action cached is %d", cached);
 
   // 本地和远程共享一套 cache
@@ -353,8 +390,7 @@ void ExecutionContext::Execute(int fd, RemoteExecutor::RemoteSpawn* spawn,
     SubprocessSet subprocset;
     Subprocess* subproc = subprocset.Add(spawn->command);
     if (!subproc) {
-      Warning("Error while `Execute locally and Update to ActionCache`"); 
-      return;
+      Fatal("Error while `Execute locally and Update to ActionCache`"); 
     }    
     //wait for local run finished
     subproc = NULL;
@@ -369,7 +405,12 @@ void ExecutionContext::Execute(int fd, RemoteExecutor::RemoteSpawn* spawn,
     delete subproc;
 
     DigestStringMap outblobs, outputs_digest_files;
-    BuildActionOutputs(spawn, cwd, &outblobs, &outputs_digest_files, &result);
+     bool ret = BuildActionOutputs(spawn, cwd, &outblobs, &outputs_digest_files, &result);
+      if(!ret){
+        exit_code = 0;
+        close(fd);
+        return;
+      }
     outblobs[action_digest] = action.SerializeAsString();
     try {
       // 上传文件至 CAS cache
@@ -399,6 +440,10 @@ void ExecutionContext::Execute(int fd, RemoteExecutor::RemoteSpawn* spawn,
 
   const int _exit_code = result.exit_code();
   exit_code = _exit_code;
+    if ( exit_code != 0) {
+        close(fd);
+        return;
+      }
   if (_exit_code == 0 && result.output_files_size() == 0 && products.size() != 0)
     Fatal("Action produced none of the of the expected output_files");
 

src/remote_executor/execution_context.h

@@ -37,6 +37,7 @@ struct File {
        const std::vector<std::string>& capture_properties = {});
 
   FileNode ToFileNode(const std::string &name) const;
+  OutputFile ToOutputFile(const std::string &name) const;
 
 private:
   void Init(int fd, const FileDigestFunction& digest_func,