Skip to content

Commit

Permalink
重构了配置管理相关代码
Browse files Browse the repository at this point in the history
  • Loading branch information
@mengning
mengning committed Dec 15, 2024
1 parent 585a520 commit fd27b15
Show file tree
Hide file tree
Showing 12 changed files with 156 additions and 316 deletions.
18 changes: 17 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,27 @@ CloudBuild refer to https://gitee.com/cloudbuild888/cloudbuild

## QuickStart

### Install the latest prebuild release version
Install the latest prebuild release version

```
wget -c https://raw.githubusercontent.com/ninja-cloudbuild/ninja2/refs/heads/main/install.sh && chmod +x install.sh && sudo ./install.sh
```

Enable CloudBuild as default in /etc/ninja2.conf

```
cloudbuild: true
grpc_url: "grpc://CLOUDBUILD_SERVICE_IP:PORT"
```

Enable CloudBuild as once

```
$ ninja --cloudbuild grpc://CLOUDBUILD_SERVICE_IP:PORT [-r PROJECT_ROOT_DIR]
```
- `--cloudbuild` can be used instead of `-c`
- When the working directory is not the root directory, you should use `-r`, which can replace `--project-root-dir`,

### Build & install from source code

```
Expand Down
7 changes: 5 additions & 2 deletions build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,12 @@ EOL
function build {
# Create build directory and build
mkdir -p build
(cd build && cmake -G Ninja .. && ninja)
if [ ! -f "build/build.ninja" ]; then
(cd build && cmake -G Ninja ..)
fi
(cd build && ninja)

success "Build succeed. Build output is located in $(pwd)"
success "Build successfully. Build output is located in build/bin/"
}

# Main script
Expand Down
2 changes: 1 addition & 1 deletion install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ function install {
# Backup existing /etc/ninja2.conf if it exists
if [ -f "$config_file" ]; then
sudo mv "$config_file" "$backup_config_file"
success "Backed up original ninja to $backup_config_file"
success "Backed up original $config_file to $backup_config_file"
fi

# Install new ninja binary
Expand Down
5 changes: 2 additions & 3 deletions src/build.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
#endif

using namespace std;
#include "rbe_config.h"
#include "share_build/share_thread.h"

#include <grpcpp/grpcpp.h>
Expand Down Expand Up @@ -969,12 +968,12 @@ bool Builder::Build(string* err) {
if (config_.dry_run)
command_runner_.reset(new DryRunCommandRunner);
#ifdef CLOUD_BUILD_SUPPORT
else if (config_.rbe_config_ptr->cloud_build) {
else if (config_.cloud_run) {
RemoteExecutor::RemoteSpawn::config = &config_;
command_runner_.reset(new CloudCommandRunner(config_));
}
#endif
else if (config_.rbe_config_ptr->share_build) {
else if (config_.share_run) {
command_runner_.reset(new ShareCommandRunner(config_));
} else {
command_runner_.reset(new RealCommandRunner(config_));
Expand Down
21 changes: 12 additions & 9 deletions src/build.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,16 @@ struct Builder;
struct DiskInterface;
struct Edge;
struct Node;
struct RBEConfig;
struct State;
struct Status;


/// Plan stores the state of a build plan: what we intend to build,
/// which steps we're ready to execute.
struct Plan {
Plan(Builder* builder = NULL);

/// Add a target to our plan (including all its dependencies).
/// Returns false if we don't need to build this target; mayz
/// Returns false if we don't need to build this target; may
/// fill in |err| with an error message if there's a problem.
bool AddTarget(const Node* target, std::string* err);

Expand Down Expand Up @@ -150,7 +148,6 @@ struct CommandRunner {
virtual size_t CanRunMore() const = 0;
virtual bool StartCommand(Edge* edge) = 0;


/// The result of waiting for a command.
struct Result {
Result() : edge(NULL) {}
Expand All @@ -165,10 +162,16 @@ struct CommandRunner {
virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
virtual void Abort() {}
};

struct ProjectConfig {
// project config
std::string cwd; // current working directory(i.e. ninjaDir). eg: ~/proj/build
std::string project_root; // project root directory. eg: ~/proj
std::map<std::string, std::string> rbe_properties; // remote build execution properties
std::string grpc_url;
};
/// Options (e.g. verbosity, parallelism) passed to a build.
struct BuildConfig {
BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
BuildConfig() : verbosity(NORMAL), dry_run(false), cloud_run(false), share_run(false), parallelism(1),
failures_allowed(1), max_load_average(-0.0f) {}

enum Verbosity {
Expand All @@ -179,9 +182,9 @@ struct BuildConfig {
};
Verbosity verbosity;
bool dry_run;

RBEConfig* rbe_config_ptr;

bool cloud_run;
ProjectConfig rbe_config;
bool share_run;
int parallelism;
int failures_allowed;
/// The maximum load average we must not exceed. A negative value
Expand Down
149 changes: 42 additions & 107 deletions src/ninja.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,6 @@

#include <algorithm>
#include <cstdlib>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <net/if.h>
#include <arpa/inet.h>
#include <ifaddrs.h>
#include <cstdlib>
#include <fstream>


#ifdef _WIN32
#include "getopt.h"
Expand Down Expand Up @@ -63,23 +54,10 @@

#ifndef _WIN32
#include "thread_pool.h"
#endif

#include <deque>
#include <unordered_map>
#include <unordered_set>
#include "thread_pool.h"
#include "share_build/ninjaRegisterService.h"
#include "share_build/ninjaUnregisterService.h"
#include "rbe_config.h"
#endif

using namespace std;
using ninjaRegister::RegisterRequest;
using ninjaRegister::RegisterResponse;
using ninjaRegister::RegisterService;
using ninjaUnregister::UnregisterRequest;
using ninjaUnregister::UnregisterResponse;
using ninjaUnregister::UnregisterService;

#ifdef _WIN32
// Defined in msvc_helper_main-win32.cc.
Expand Down Expand Up @@ -251,11 +229,11 @@ void Usage(const BuildConfig& config) {
" -v, --verbose show all command lines while building\n"
" --quiet don't show progress status, just command output\n"
"\n"
" remote options: Make sure a remote compilation server is running at the\n"
" specified `server_addr` before using these options.\n"
" -c server_addr remote cloud build mode (remote execution api)\n"
" -s server_addr remote share build mode (p2p)\n"
" -r project_root_dir only valid in remote build mode\n"
"cloudbuild/sharebuild options: Make sure a cloudbuild or sharebuild is running\n"
" -c , --cloudbuild grpc://CLOUDBUILD_SERVICE_IP:PORT\n"
" Enable cloudbuild mode (remote execution api)\n"
" -s , --sharebuild Enable sharebuild mode (p2p)\n"
" -r , --project-root-dir only valid in cloudbuild or sharebuild mode\n"
"\n"
" -C DIR change to DIR before doing anything else\n"
" -f FILE specify input build file [default=build.ninja]\n"
Expand Down Expand Up @@ -1469,12 +1447,21 @@ int ReadFlags(int* argc, char*** argv,
Options* options, BuildConfig* config) {
DeferGuessParallelism deferGuessParallelism(config);

enum { OPT_VERSION = 1, OPT_QUIET = 2 };
enum {
OPT_VERSION = 1,
OPT_QUIET = 2,
OPT_SHAREBUILD = 3,
OPT_CLOUDBUILD = 4,
OPT_PROJECT_ROOT_DIR = 5
};
const option kLongOptions[] = {
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, OPT_VERSION },
{ "verbose", no_argument, NULL, 'v' },
{ "quiet", no_argument, NULL, OPT_QUIET },
{ "sharebuild", required_argument, NULL, OPT_SHAREBUILD },
{ "cloudbuild", required_argument, NULL, OPT_CLOUDBUILD },
{ "project-root-dir", required_argument, NULL, OPT_PROJECT_ROOT_DIR },
{ NULL, 0, NULL, 0 }
};

Expand All @@ -1483,21 +1470,24 @@ int ReadFlags(int* argc, char*** argv,
(opt = getopt_long(*argc, *argv, "s:c:r:d:f:j:k:l:nt:vw:C:h",
kLongOptions, NULL)) != -1) {
switch (opt) {
case 's':
config->rbe_config_ptr->share_build = true;
config->rbe_config_ptr->master_addr = optarg;
break;
case 'c':
config->rbe_config_ptr->cloud_build = true;
config->rbe_config_ptr->grpc_url = optarg;
if (config->rbe_config_ptr->grpc_url.compare(0, 7, "grpc://") != 0)
Fatal("invalid grpc url");
break;
case 'r': {
std::string project_root = optarg;
config->rbe_config_ptr->init_proj_config(project_root);
break;
}
case 's':
case OPT_SHAREBUILD: // -s or --sharebuild
config->share_run = true;
break;
case 'c':
case OPT_CLOUDBUILD: // -c or --cloudbuild
config->cloud_run = true;
config->rbe_config.grpc_url = optarg;
if (config->rbe_config.grpc_url.compare(0, 7, "grpc://") != 0) {
Fatal("invalid grpc url");
}
break;
case 'r':
case OPT_PROJECT_ROOT_DIR: // -r or --project-root-dir
{
config->rbe_config.project_root = optarg;
break;
}
case 'd':
if (!DebugEnable(optarg))
return 1;
Expand Down Expand Up @@ -1572,33 +1562,13 @@ int ReadFlags(int* argc, char*** argv,
*argc -= optind;

return -1;
}

bool reg(const std::string& ninja_host, const std::string& ninja_dir,
const std::string& scheduler_addr, const std::string& root_dir, const std::string& container_image) {
RegisterClient registerClient(
grpc::CreateChannel(scheduler_addr, grpc::InsecureChannelCredentials()));
bool registerSuccess =
registerClient.Register(ninja_host, ninja_dir, root_dir, container_image);
cout << "注册结果 registerSuccess is " << registerSuccess << endl;
return registerSuccess;
}

bool unReg(const std::string& ninja_host, const std::string& ninja_dir,
const std::string& scheduler_addr, const std::string& root_dir) {
UnregisterClient unregisterClient(
grpc::CreateChannel(scheduler_addr, grpc::InsecureChannelCredentials()));
bool unregisterSuccess =
unregisterClient.Unregister(ninja_host, ninja_dir, root_dir);
cout << "注册结果 unregisterSuccess is " << unregisterSuccess << endl;
return unregisterSuccess;
}
}

NORETURN void real_main(int argc, char** argv) {
// Use exit() instead of return in this function to avoid potentially
// expensive cleanup when destructing NinjaMain.
BuildConfig config;
config.rbe_config_ptr = &g_rbe_config;
load_config_file(config);
Options options = {};
options.input_file = "build.ninja";

Expand All @@ -1622,38 +1592,14 @@ NORETURN void real_main(int argc, char** argv) {
if (chdir(options.working_dir) < 0) {
Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno));
}
config.rbe_config_ptr->cwd = options.working_dir;
} else if (config.rbe_config_ptr->cloud_build || config.rbe_config_ptr->share_build) { // TODO: 这里的 else if 其实可以删除
string err;
if (!GetCurrentDirectory(&(config.rbe_config_ptr->cwd), &err)) {
Error(err.c_str());
exit(1);
}
if (config.rbe_config_ptr->project_root.empty())
config.rbe_config_ptr->project_root = config.rbe_config_ptr->cwd;
}

if (g_rbe_config.share_build) {
Warning("启动 p2p share 分布式编译模式");
// 注册ninja
bool registerSuccess =
reg(g_rbe_config.self_ipv4_address, g_rbe_config.cwd,
g_rbe_config.master_addr, g_rbe_config.project_root,
g_rbe_config.rbe_properties["container-image"]);
if (!registerSuccess) {
//注册失败
Warning("注册失败");
config.rbe_config_ptr->share_build = false; //shut down share build
}
std::string project_root = ".";
if (config.cloud_run || config.share_run) {
if (!config.rbe_config.project_root.empty()) {
project_root = config.rbe_config.project_root;
}
Warning("sleep(3)");
sleep(3);
} else if (g_rbe_config.cloud_build) {
Info("启动 remote api 分布式编译模式");
} else {
Info("启动本地编译模式");
load_devcontainer_config(project_root, config);
}



#ifndef _WIN32
SetThreadPoolThreadCount(GetProcessorCount());
Expand Down Expand Up @@ -1712,17 +1658,6 @@ NORETURN void real_main(int argc, char** argv) {
int result = ninja.RunBuild(argc, argv, status);
if (g_metrics)
ninja.DumpMetrics();

if (g_rbe_config.share_build) {
Info("注销");
// 注销ninja
bool unregisterSuccess = unReg(g_rbe_config.self_ipv4_address, g_rbe_config.cwd, g_rbe_config.master_addr, g_rbe_config.project_root);

if (!unregisterSuccess) {
//注销失败
Fatal("注销失败");
}
}
exit(result);
}

Expand Down
Loading

0 comments on commit fd27b15

Please sign in to comment.