diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57f09b9..bf9f091 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Build JVM Docker Image uses: docker/build-push-action@v1 with: - dockerfile: 'agent/Dockerfile' + dockerfile: 'agent/docker/jvm/Dockerfile' username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} repository: verizondigital/kubectl-flame @@ -23,11 +23,19 @@ jobs: - name: Build JVM Alpine Docker Image uses: docker/build-push-action@v1 with: - dockerfile: 'agent/Dockerfile.alpine' + dockerfile: 'agent/docker/jvm/Dockerfile.alpine' username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} repository: verizondigital/kubectl-flame tags: ${{ steps.vars.outputs.tag }}-jvm-alpine + - name: Build BPF Docker Image + uses: docker/build-push-action@v1 + with: + dockerfile: 'agent/docker/bpf/Dockerfile' + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + repository: verizondigital/kubectl-flame + tags: ${{ steps.vars.outputs.tag }}-bpf - name: Setup Go uses: actions/setup-go@v1 with: diff --git a/README.md b/README.md index 8036b17..8a8855e 100644 --- a/README.md +++ b/README.md @@ -15,25 +15,33 @@ Running `kubectlf-flame` does **not** require any modification to existing pods. - [License](#license) ## Requirements -* Currently, only Java applications are supported. (Golang support coming soon!) +* Supported languages: Go, Java (any JVM based language) * Kubernetes cluster that use Docker as the container runtime (tested on GKE, EKS and AKS) ## Usage ### Profiling Kubernetes Pod -In order to profile pod `mypod` for 1 minute and save the flamegraph as `/tmp/flamegraph.svg` run: +In order to profile a Java application in pod `mypod` for 1 minute and save the flamegraph as `/tmp/flamegraph.svg` run: ```shell -kubectl flame mypod -t 1m -f /tmp/flamegraph.svg +kubectl flame mypod -t 1m --lang java -f /tmp/flamegraph.svg ``` ### Profiling Alpine based container -Profiling alpine based containers require using `--alpine` flag: +Profiling Java application in alpine based containers require using `--alpine` flag: ```shell -kubectl flame mypod -t 1m -f /tmp/flamegraph.svg --alpine +kubectl flame mypod -t 1m -f /tmp/flamegraph.svg --lang java --alpine ``` +*NOTICE*: this is only required for Java apps, the `--alpine` flag is unnecessary for Go profiling. + ### Profiling sidecar container Pods that contains more than one container require specifying the target container as an argument: ```shell -kubectl flame mypod -t 1m -f /tmp/flamegraph.svg mycontainer +kubectl flame mypod -t 1m --lang go -f /tmp/flamegraph.svg mycontainer +``` +### Profiling Golang multi-process container +Profiling Go application in pods that contains more than one process require specifying the target process name via `--pgrep` flag: +```shell +kubectl flame mypod -t 1m --lang go -f /tmp/flamegraph.svg --pgrep go-app ``` +Java profiling assumes that the process name is `java`. Use `--pgrep` flag if your process name is different. ## Installing @@ -54,7 +62,7 @@ See the release page for the full list of pre-built assets. `kubectl-flame` launch a Kubernetes Job on the same node as the target pod. Under the hood `kubectl-flame` use [async-profiler](https://github.com/jvm-profiling-tools/async-profiler) in order to generate flame graphs for Java applications. Interaction with the target JVM is done via a shared `/tmp` folder. -Other languages support (such as the upcoming Golang support) will be based on [ebpf profiling](https://en.wikipedia.org/wiki/Berkeley_Packet_Filter). +Golang support is based on [ebpf profiling](https://en.wikipedia.org/wiki/Berkeley_Packet_Filter). ## Contribute Please refer to [the contributing.md file](Contributing.md) for information about how to get involved. We welcome issues, questions, and pull requests. diff --git a/agent/details/profiling_job.go b/agent/details/profiling_job.go index 7408cf3..8442d7a 100644 --- a/agent/details/profiling_job.go +++ b/agent/details/profiling_job.go @@ -2,12 +2,17 @@ //: Licensed under the terms of the Apache 2.0 License. See LICENSE file in the project root for terms. package details -import "time" +import ( + "github.com/VerizonMedia/kubectl-flame/api" + "time" +) type ProfilingJob struct { - Duration time.Duration - ID string - ContainerID string - ContainerName string - PodUID string + Duration time.Duration + ID string + ContainerID string + ContainerName string + PodUID string + Language api.ProgrammingLanguage + TargetProcessName string } diff --git a/agent/docker/bpf/Dockerfile b/agent/docker/bpf/Dockerfile new file mode 100644 index 0000000..69ee428 --- /dev/null +++ b/agent/docker/bpf/Dockerfile @@ -0,0 +1,29 @@ +ARG KERNEL_VERSION=4.9.125 + +FROM linuxkit/kernel:$KERNEL_VERSION AS ksrc + +FROM golang:1.14-buster as agentbuild +WORKDIR /go/src/github.com/VerizonMedia/kubectl-flame +ADD . /go/src/github.com/VerizonMedia/kubectl-flame +RUN go get -d -v ./... +RUN cd agent && go build -o /go/bin/agent + +FROM alpine as builder +COPY --from=ksrc /kernel-dev.tar / +RUN tar xf /kernel-dev.tar -C / +RUN mv /usr/src/*/ /usr/src/kernel-source/ +RUN apk add git +RUN git clone https://github.com/brendangregg/FlameGraph +RUN git clone https://gist.github.com/edeNFed/83a9438156288661e2283c28fee18b8b bcc-profiler + +FROM alpine +COPY --from=builder /usr/src /usr/src +RUN apk add bcc-tools perl +RUN ln -s $(which python3) /usr/bin/python +RUN mkdir -p /app/FlameGraph +COPY --from=builder /FlameGraph /app/FlameGraph +COPY --from=agentbuild /go/bin/agent /app +COPY --from=builder /bcc-profiler /app/bcc-profiler/ +RUN chmod +x /app/bcc-profiler/profile + +CMD [ "/app/agent" ] \ No newline at end of file diff --git a/agent/Dockerfile b/agent/docker/jvm/Dockerfile similarity index 100% rename from agent/Dockerfile rename to agent/docker/jvm/Dockerfile diff --git a/agent/Dockerfile.alpine b/agent/docker/jvm/Dockerfile.alpine similarity index 100% rename from agent/Dockerfile.alpine rename to agent/docker/jvm/Dockerfile.alpine diff --git a/agent/main.go b/agent/main.go index 93412a1..05a5df0 100644 --- a/agent/main.go +++ b/agent/main.go @@ -17,45 +17,33 @@ import ( func main() { args, err := validateArgs() - if err != nil { - api.PublishError(err) - os.Exit(1) - } + handleError(err) err = api.PublishEvent(api.Progress, &api.ProgressData{Time: time.Now(), Stage: api.Started}) - if err != nil { - api.PublishError(err) - os.Exit(1) - } + handleError(err) - err = profiler.SetUp(args) - if err != nil { - api.PublishError(err) - os.Exit(1) - } + p, err := profiler.ForLanguage(args.Language) + handleError(err) + + err = p.SetUp(args) + handleError(err) done := handleSignals() - err = profiler.Invoke(args) - if err != nil { - api.PublishError(err) - os.Exit(1) - } + err = p.Invoke(args) + handleError(err) err = api.PublishEvent(api.Progress, &api.ProgressData{Time: time.Now(), Stage: api.Ended}) - if err != nil { - api.PublishError(err) - os.Exit(1) - } + handleError(err) <-done } func validateArgs() (*details.ProfilingJob, error) { - if len(os.Args) != 6 { - return nil, errors.New("expected 6 arguments") + if len(os.Args) != 7 && len(os.Args) != 8 { + return nil, errors.New("expected 6 or 7 arguments") } - duration, err := time.ParseDuration((os.Args[5])) + duration, err := time.ParseDuration(os.Args[5]) if err != nil { return nil, err } @@ -66,6 +54,10 @@ func validateArgs() (*details.ProfilingJob, error) { currentJob.ContainerName = os.Args[3] currentJob.ContainerID = strings.Replace(os.Args[4], "docker://", "", 1) currentJob.Duration = duration + currentJob.Language = api.ProgrammingLanguage(os.Args[6]) + if len(os.Args) == 8 { + currentJob.TargetProcessName = os.Args[7] + } return currentJob, nil } @@ -84,3 +76,10 @@ func handleSignals() chan bool { return done } + +func handleError(err error) { + if err != nil { + api.PublishError(err) + os.Exit(1) + } +} diff --git a/agent/profiler/bpf.go b/agent/profiler/bpf.go new file mode 100644 index 0000000..5637d6f --- /dev/null +++ b/agent/profiler/bpf.go @@ -0,0 +1,105 @@ +package profiler + +import ( + "fmt" + "github.com/VerizonMedia/kubectl-flame/agent/details" + "github.com/VerizonMedia/kubectl-flame/agent/utils" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" +) + +const ( + kernelSourcesDir = "/usr/src/kernel-source/" + profilerLocation = "/app/bcc-profiler/profile" + rawProfilerOutputFile = "/tmp/raw_profile.txt" + flameGraphScriptLocation = "/app/FlameGraph/flamegraph.pl" + flameGraphOutputLocation = "/tmp/flamegraph.svg" +) + +type BpfProfiler struct{} + +func (b *BpfProfiler) SetUp(job *details.ProfilingJob) error { + exitCode, kernelVersion, err := utils.ExecuteCommand(exec.Command("uname", "-r")) + if err != nil { + return fmt.Errorf("failed to get kernel version, exit code: %d, error: %s", exitCode, err) + } + + expectedSourcesLocation, err := os.Readlink(fmt.Sprintf("/lib/modules/%s/build", + strings.TrimSuffix(kernelVersion, "\n"))) + if err != nil { + return fmt.Errorf("failed to read source link, error: %s", err) + } + + return b.moveSources(expectedSourcesLocation) +} + +func (b *BpfProfiler) Invoke(job *details.ProfilingJob) error { + err := b.runProfiler(job) + if err != nil { + return fmt.Errorf("profiling failed: %s", err) + } + + err = b.generateFlameGraph() + if err != nil { + return fmt.Errorf("flamegraph generation failed: %s", err) + } + + return utils.PublishFlameGraph(flameGraphOutputLocation) +} + +func (b *BpfProfiler) runProfiler(job *details.ProfilingJob) error { + pid, err := utils.FindProcessId(job) + if err != nil { + return err + } + + f, err := os.Create(rawProfilerOutputFile) + if err != nil { + return err + } + defer f.Close() + + duration := strconv.Itoa(int(job.Duration.Seconds())) + profileCmd := exec.Command(profilerLocation, "-df", "-p", pid, duration) + profileCmd.Stdout = f + + return profileCmd.Run() +} + +func (b *BpfProfiler) generateFlameGraph() error { + inputFile, err := os.Open(rawProfilerOutputFile) + if err != nil { + return err + } + defer inputFile.Close() + + outputFile, err := os.Create(flameGraphOutputLocation) + if err != nil { + return err + } + defer outputFile.Close() + + flameGraphCmd := exec.Command(flameGraphScriptLocation) + flameGraphCmd.Stdin = inputFile + flameGraphCmd.Stdout = outputFile + + return flameGraphCmd.Run() +} + +func (b *BpfProfiler) moveSources(target string) error { + parent, _ := filepath.Split(target) + err := os.MkdirAll(parent, os.ModePerm) + if err != nil { + return err + } + + _, _, err = utils.ExecuteCommand(exec.Command("mv", kernelSourcesDir, target)) + if err != nil { + return fmt.Errorf("failed moving source files, error: %s, tried to move to: %s", err, target) + } + + return nil +} diff --git a/agent/profiler/invoke.go b/agent/profiler/invoke.go deleted file mode 100644 index 83d3bcf..0000000 --- a/agent/profiler/invoke.go +++ /dev/null @@ -1,117 +0,0 @@ -//: Copyright Verizon Media -//: Licensed under the terms of the Apache 2.0 License. See LICENSE file in the project root for terms. -package profiler - -import ( - "bufio" - "bytes" - "encoding/base64" - "errors" - "io" - "io/ioutil" - "os" - "os/exec" - "path" - "strconv" - "strings" - - "github.com/VerizonMedia/kubectl-flame/agent/details" - "github.com/VerizonMedia/kubectl-flame/api" - "github.com/fntlnz/mountinfo" -) - -const ( - profilerDir = "/tmp/async-profiler" - fileName = profilerDir + "/flamegraph.svg" - profilerSh = profilerDir + "/profiler.sh" -) - -func Invoke(job *details.ProfilingJob) error { - pid, err := findJavaProcessId(job) - if err != nil { - return err - } - - duration := strconv.Itoa(int(job.Duration.Seconds())) - cmd := exec.Command(profilerSh, "-d", duration, "-f", fileName, "-e", "wall", pid) - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - err = cmd.Run() - if err != nil { - return err - } - - return publishFlameGraph() -} - -func findJavaProcessId(job *details.ProfilingJob) (string, error) { - proc, err := os.Open("/proc") - if err != nil { - return "", err - } - - defer proc.Close() - - for { - dirs, err := proc.Readdir(15) - if err == io.EOF { - break - } - if err != nil { - return "", err - } - - for _, di := range dirs { - if !di.IsDir() { - continue - } - - dname := di.Name() - if dname[0] < '0' || dname[0] > '9' { - continue - } - - mi, err := mountinfo.GetMountInfo(path.Join("/proc", dname, "mountinfo")) - if err != nil { - continue - } - - for _, m := range mi { - root := m.Root - if strings.Contains(root, job.PodUID) && - strings.Contains(root, job.ContainerName) { - - exeName, err := os.Readlink(path.Join("/proc", dname, "exe")) - if err != nil { - continue - } - - if strings.Contains(exeName, "java") { - return dname, nil - } - } - } - } - } - return "", errors.New("Could not find any process") -} - -func publishFlameGraph() error { - file, err := os.Open(fileName) - if err != nil { - return err - } - - reader := bufio.NewReader(file) - content, err := ioutil.ReadAll(reader) - if err != nil { - return err - } - - encoded := base64.StdEncoding.EncodeToString(content) - fgData := api.FlameGraphData{EncodedFile: encoded} - - return api.PublishEvent(api.FlameGraph, fgData) -} diff --git a/agent/profiler/jvm.go b/agent/profiler/jvm.go new file mode 100644 index 0000000..836aa1c --- /dev/null +++ b/agent/profiler/jvm.go @@ -0,0 +1,63 @@ +package profiler + +import ( + "bytes" + "github.com/VerizonMedia/kubectl-flame/agent/details" + "github.com/VerizonMedia/kubectl-flame/agent/utils" + "os" + "os/exec" + "path" + "strconv" +) + +const ( + profilerDir = "/tmp/async-profiler" + fileName = profilerDir + "/flamegraph.svg" + profilerSh = profilerDir + "/profiler.sh" +) + +type JvmProfiler struct{} + +func (j *JvmProfiler) SetUp(job *details.ProfilingJob) error { + targetFs, err := utils.GetTargetFileSystemLocation(job.ContainerID) + if err != nil { + return err + } + + err = os.RemoveAll("/tmp") + if err != nil { + return err + } + + err = os.Symlink(path.Join(targetFs, "tmp"), "/tmp") + if err != nil { + return err + } + + return j.copyProfilerToTempDir() +} + +func (j *JvmProfiler) Invoke(job *details.ProfilingJob) error { + pid, err := utils.FindProcessId(job) + if err != nil { + return err + } + + duration := strconv.Itoa(int(job.Duration.Seconds())) + cmd := exec.Command(profilerSh, "-d", duration, "-f", fileName, "-e", "wall", pid) + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { + return err + } + + return utils.PublishFlameGraph(fileName) +} + +func (j *JvmProfiler) copyProfilerToTempDir() error { + cmd := exec.Command("cp", "-r", "/app/async-profiler", "/tmp") + return cmd.Run() +} diff --git a/agent/profiler/root.go b/agent/profiler/root.go new file mode 100644 index 0000000..c2edc05 --- /dev/null +++ b/agent/profiler/root.go @@ -0,0 +1,28 @@ +package profiler + +import ( + "fmt" + "github.com/VerizonMedia/kubectl-flame/agent/details" + "github.com/VerizonMedia/kubectl-flame/api" +) + +type FlameGraphProfiler interface { + SetUp(job *details.ProfilingJob) error + Invoke(job *details.ProfilingJob) error +} + +var ( + jvm = JvmProfiler{} + bpf = BpfProfiler{} +) + +func ForLanguage(lang api.ProgrammingLanguage) (FlameGraphProfiler, error) { + switch lang { + case api.Java: + return &jvm, nil + case api.Go: + return &bpf, nil + default: + return nil, fmt.Errorf("could not find profiler for language %s", lang) + } +} diff --git a/agent/profiler/setup.go b/agent/profiler/setup.go deleted file mode 100644 index afd0ce3..0000000 --- a/agent/profiler/setup.go +++ /dev/null @@ -1,52 +0,0 @@ -//: Copyright Verizon Media -//: Licensed under the terms of the Apache 2.0 License. See LICENSE file in the project root for terms. -package profiler - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path" - - "github.com/VerizonMedia/kubectl-flame/agent/details" -) - -const ( - mountIdLocation = "/var/lib/docker/image/overlay2/layerdb/mounts/%s/mount-id" - targetFileSystemLocation = "/var/lib/docker/overlay2/%s/merged" -) - -func SetUp(job *details.ProfilingJob) error { - targetFs, err := getTargetFileSystemLocation(job.ContainerID) - if err != nil { - return err - } - - err = os.RemoveAll("/tmp") - if err != nil { - return err - } - - err = os.Symlink(path.Join(targetFs, "tmp"), "/tmp") - if err != nil { - return err - } - - return copyProfilerToTempDir() -} - -func copyProfilerToTempDir() error { - cmd := exec.Command("cp", "-r", "/app/async-profiler", "/tmp") - return cmd.Run() -} - -func getTargetFileSystemLocation(containerId string) (string, error) { - fileName := fmt.Sprintf(mountIdLocation, containerId) - mountId, err := ioutil.ReadFile(fileName) - if err != nil { - return "", err - } - - return fmt.Sprintf(targetFileSystemLocation, string(mountId)), nil -} diff --git a/agent/utils/exec.go b/agent/utils/exec.go new file mode 100644 index 0000000..d94d690 --- /dev/null +++ b/agent/utils/exec.go @@ -0,0 +1,16 @@ +package utils + +import "os/exec" + +func ExecuteCommand(cmd *exec.Cmd) (int, string, error) { + exitCode := 0 + output, err := cmd.CombinedOutput() + + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + exitCode = exitError.ExitCode() + } + } + + return exitCode, string(output), err +} diff --git a/agent/utils/filesystem.go b/agent/utils/filesystem.go new file mode 100644 index 0000000..62804ae --- /dev/null +++ b/agent/utils/filesystem.go @@ -0,0 +1,21 @@ +package utils + +import ( + "fmt" + "io/ioutil" +) + +const ( + mountIdLocation = "/var/lib/docker/image/overlay2/layerdb/mounts/%s/mount-id" + targetFileSystemLocation = "/var/lib/docker/overlay2/%s/merged" +) + +func GetTargetFileSystemLocation(containerId string) (string, error) { + fileName := fmt.Sprintf(mountIdLocation, containerId) + mountId, err := ioutil.ReadFile(fileName) + if err != nil { + return "", err + } + + return fmt.Sprintf(targetFileSystemLocation, string(mountId)), nil +} diff --git a/agent/utils/flamegraph.go b/agent/utils/flamegraph.go new file mode 100644 index 0000000..62ef31d --- /dev/null +++ b/agent/utils/flamegraph.go @@ -0,0 +1,27 @@ +package utils + +import ( + "bufio" + "encoding/base64" + "github.com/VerizonMedia/kubectl-flame/api" + "io/ioutil" + "os" +) + +func PublishFlameGraph(flameFile string) error { + file, err := os.Open(flameFile) + if err != nil { + return err + } + + reader := bufio.NewReader(file) + content, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + + encoded := base64.StdEncoding.EncodeToString(content) + fgData := api.FlameGraphData{EncodedFile: encoded} + + return api.PublishEvent(api.FlameGraph, fgData) +} diff --git a/agent/utils/process.go b/agent/utils/process.go new file mode 100644 index 0000000..d4d2bc8 --- /dev/null +++ b/agent/utils/process.go @@ -0,0 +1,99 @@ +package utils + +import ( + "errors" + "github.com/VerizonMedia/kubectl-flame/agent/details" + "github.com/VerizonMedia/kubectl-flame/api" + "github.com/fntlnz/mountinfo" + "io" + "os" + "path" + "strings" +) + +var ( + defaultProcessNames = map[api.ProgrammingLanguage]string{ + api.Java: "java", + } +) + +func getProcessName(job *details.ProfilingJob) string { + if job.TargetProcessName != "" { + return job.TargetProcessName + } + + if val, ok := defaultProcessNames[job.Language]; ok { + return val + } + + return "" +} + +func FindProcessId(job *details.ProfilingJob) (string, error) { + name := getProcessName(job) + foundProc := "" + proc, err := os.Open("/proc") + if err != nil { + return "", err + } + + defer proc.Close() + + for { + dirs, err := proc.Readdir(15) + if err == io.EOF { + break + } + if err != nil { + return "", err + } + + for _, di := range dirs { + if !di.IsDir() { + continue + } + + dname := di.Name() + if dname[0] < '0' || dname[0] > '9' { + continue + } + + mi, err := mountinfo.GetMountInfo(path.Join("/proc", dname, "mountinfo")) + if err != nil { + continue + } + + for _, m := range mi { + root := m.Root + if strings.Contains(root, job.PodUID) && + strings.Contains(root, job.ContainerName) { + + exeName, err := os.Readlink(path.Join("/proc", dname, "exe")) + if err != nil { + continue + } + + if name != "" { + // search by process name + if strings.Contains(exeName, name) { + return dname, nil + } + } else { + if foundProc != "" { + return "", errors.New("found more than one process on container," + + " specify process name using --pgrep flag") + } else { + foundProc = dname + } + } + } + } + } + } + + if foundProc != "" { + return foundProc, nil + } + + return "", errors.New("could not find any process") +} diff --git a/api/langs.go b/api/langs.go new file mode 100644 index 0000000..c770d76 --- /dev/null +++ b/api/langs.go @@ -0,0 +1,30 @@ +package api + +type ProgrammingLanguage string + +const ( + Java ProgrammingLanguage = "java" + Go ProgrammingLanguage = "go" +) + +var ( + supportedLangs = []ProgrammingLanguage{Java, Go} +) + +func AvailableLanguages() []ProgrammingLanguage { + return supportedLangs +} + +func IsSupportedLanguage(lang string) bool { + return containsLang(ProgrammingLanguage(lang), AvailableLanguages()) +} + +func containsLang(l ProgrammingLanguage, langs []ProgrammingLanguage) bool { + for _, current := range langs { + if l == current { + return true + } + } + + return false +} diff --git a/cli/cmd/data/target.go b/cli/cmd/data/target.go index faf2882..f7fb332 100644 --- a/cli/cmd/data/target.go +++ b/cli/cmd/data/target.go @@ -2,7 +2,10 @@ //: Licensed under the terms of the Apache 2.0 License. See LICENSE file in the project root for terms. package data -import "time" +import ( + "github.com/VerizonMedia/kubectl-flame/api" + "time" +) type TargetDetails struct { Namespace string @@ -15,4 +18,6 @@ type TargetDetails struct { Alpine bool DryRun bool Image string + Language api.ProgrammingLanguage + Pgrep string } diff --git a/cli/cmd/kubernetes/job/bpf.go b/cli/cmd/kubernetes/job/bpf.go new file mode 100644 index 0000000..431a8f5 --- /dev/null +++ b/cli/cmd/kubernetes/job/bpf.go @@ -0,0 +1,103 @@ +package job + +import ( + "fmt" + "github.com/VerizonMedia/kubectl-flame/cli/cmd/data" + "github.com/VerizonMedia/kubectl-flame/cli/cmd/version" + batchv1 "k8s.io/api/batch/v1" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" +) + +type bpfCreator struct{} + +func (b *bpfCreator) create(targetPod *v1.Pod, targetDetails *data.TargetDetails) (string, *batchv1.Job) { + id := string(uuid.NewUUID()) + var imageName string + if targetDetails.Image != "" { + imageName = targetDetails.Image + } else { + imageName = fmt.Sprintf("%s:%s-bpf", baseImageName, version.GetCurrent()) + } + + commonMeta := metav1.ObjectMeta{ + Name: fmt.Sprintf("kubectl-flame-%s", id), + Namespace: targetDetails.Namespace, + Labels: map[string]string{ + "kubectl-flame/id": id, + }, + } + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + }, + ObjectMeta: commonMeta, + Spec: batchv1.JobSpec{ + Parallelism: int32Ptr(1), + Completions: int32Ptr(1), + TTLSecondsAfterFinished: int32Ptr(5), + Template: v1.PodTemplateSpec{ + ObjectMeta: commonMeta, + Spec: v1.PodSpec{ + HostPID: true, + Volumes: []apiv1.Volume{ + { + Name: "sys", + VolumeSource: apiv1.VolumeSource{ + HostPath: &apiv1.HostPathVolumeSource{ + Path: "/sys", + }, + }, + }, + { + Name: "modules", + VolumeSource: apiv1.VolumeSource{ + HostPath: &apiv1.HostPathVolumeSource{ + Path: "/lib/modules", + }, + }, + }, + }, + InitContainers: nil, + Containers: []apiv1.Container{ + { + ImagePullPolicy: v1.PullAlways, + Name: "kubectl-flame", + Image: imageName, + Command: []string{"/app/agent"}, + Args: []string{id, + string(targetPod.UID), + targetDetails.ContainerName, + targetDetails.ContainerId, + targetDetails.Duration.String(), + string(targetDetails.Language), + targetDetails.Pgrep, + }, + VolumeMounts: []apiv1.VolumeMount{ + { + Name: "sys", + MountPath: "/sys", + }, + { + Name: "modules", + MountPath: "/lib/modules", + }, + }, + SecurityContext: &v1.SecurityContext{ + Privileged: boolPtr(true), + }, + }, + }, + RestartPolicy: "Never", + NodeName: targetPod.Spec.NodeName, + }, + }, + }, + } + + return id, job +} diff --git a/cli/cmd/kubernetes/job/jvm.go b/cli/cmd/kubernetes/job/jvm.go new file mode 100644 index 0000000..344ed58 --- /dev/null +++ b/cli/cmd/kubernetes/job/jvm.go @@ -0,0 +1,96 @@ +package job + +import ( + "fmt" + "github.com/VerizonMedia/kubectl-flame/cli/cmd/data" + "github.com/VerizonMedia/kubectl-flame/cli/cmd/version" + batchv1 "k8s.io/api/batch/v1" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" +) + +type jvmCreator struct{} + +func (c *jvmCreator) create(targetPod *v1.Pod, targetDetails *data.TargetDetails) (string, *batchv1.Job) { + id := string(uuid.NewUUID()) + imageName := c.getAgentImage(targetDetails) + args := []string{id, string(targetPod.UID), + targetDetails.ContainerName, targetDetails.ContainerId, + targetDetails.Duration.String(), string(targetDetails.Language)} + + if targetDetails.Pgrep != "" { + args = append(args, targetDetails.Pgrep) + } + + commonMeta := metav1.ObjectMeta{ + Name: fmt.Sprintf("kubectl-flame-%s", id), + Namespace: targetDetails.Namespace, + Labels: map[string]string{ + "kubectl-flame/id": id, + }, + } + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + }, + ObjectMeta: commonMeta, + Spec: batchv1.JobSpec{ + Parallelism: int32Ptr(1), + Completions: int32Ptr(1), + TTLSecondsAfterFinished: int32Ptr(5), + Template: v1.PodTemplateSpec{ + ObjectMeta: commonMeta, + Spec: v1.PodSpec{ + HostPID: true, + Volumes: []apiv1.Volume{ + { + Name: "target-filesystem", + VolumeSource: apiv1.VolumeSource{ + HostPath: &apiv1.HostPathVolumeSource{ + Path: "/var/lib/docker", + }, + }, + }, + }, + InitContainers: nil, + Containers: []apiv1.Container{ + { + ImagePullPolicy: v1.PullAlways, + Name: "kubectl-flame", + Image: imageName, + Command: []string{"/app/agent"}, + Args: args, + VolumeMounts: []apiv1.VolumeMount{ + { + Name: "target-filesystem", + MountPath: "/var/lib/docker", + }, + }, + }, + }, + RestartPolicy: "Never", + NodeName: targetPod.Spec.NodeName, + }, + }, + }, + } + + return id, job +} + +func (c *jvmCreator) getAgentImage(targetDetails *data.TargetDetails) string { + if targetDetails.Image != "" { + return targetDetails.Image + } + + tag := fmt.Sprintf("%s-jvm", version.GetCurrent()) + if targetDetails.Alpine { + tag = fmt.Sprintf("%s-alpine", tag) + } + + return fmt.Sprintf("%s:%s", baseImageName, tag) +} diff --git a/cli/cmd/kubernetes/job/root.go b/cli/cmd/kubernetes/job/root.go new file mode 100644 index 0000000..bac783b --- /dev/null +++ b/cli/cmd/kubernetes/job/root.go @@ -0,0 +1,31 @@ +package job + +import ( + "github.com/VerizonMedia/kubectl-flame/api" + "github.com/VerizonMedia/kubectl-flame/cli/cmd/data" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" +) + +const baseImageName = "verizondigital/kubectl-flame" + +var ( + jvm = jvmCreator{} + bpf = bpfCreator{} +) + +type creator interface { + create(targetPod *v1.Pod, targetDetails *data.TargetDetails) (string, *batchv1.Job) +} + +func Create(targetPod *v1.Pod, targetDetails *data.TargetDetails) (string, *batchv1.Job) { + switch targetDetails.Language { + case api.Java: + return jvm.create(targetPod, targetDetails) + case api.Go: + return bpf.create(targetPod, targetDetails) + } + + // Should not happen + panic("got language without job creator") +} diff --git a/cli/cmd/kubernetes/utils.go b/cli/cmd/kubernetes/job/utils.go similarity index 92% rename from cli/cmd/kubernetes/utils.go rename to cli/cmd/kubernetes/job/utils.go index 2732b0a..1096a7d 100644 --- a/cli/cmd/kubernetes/utils.go +++ b/cli/cmd/kubernetes/job/utils.go @@ -1,6 +1,6 @@ //: Copyright Verizon Media //: Licensed under the terms of the Apache 2.0 License. See LICENSE file in the project root for terms. -package kubernetes +package job func int32Ptr(i int32) *int32 { return &i } func boolPtr(b bool) *bool { return &b } diff --git a/cli/cmd/kubernetes/launch.go b/cli/cmd/kubernetes/launch.go index fd6f6ee..b4eb98a 100644 --- a/cli/cmd/kubernetes/launch.go +++ b/cli/cmd/kubernetes/launch.go @@ -4,94 +4,28 @@ package kubernetes import ( "context" - "fmt" - "github.com/VerizonMedia/kubectl-flame/cli/cmd/version" + "github.com/VerizonMedia/kubectl-flame/cli/cmd/kubernetes/job" "os" "github.com/VerizonMedia/kubectl-flame/cli/cmd/data" batchv1 "k8s.io/api/batch/v1" - apiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/serializer/json" - "k8s.io/apimachinery/pkg/util/uuid" ) -const imageName = "verizondigital/kubectl-flame" - func LaunchFlameJob(targetPod *v1.Pod, targetDetails *data.TargetDetails, ctx context.Context) (string, *batchv1.Job, error) { - id := string(uuid.NewUUID()) - imageName := getAgentImage(targetDetails) - - commonMeta := metav1.ObjectMeta{ - Name: fmt.Sprintf("kubectl-flame-%s", id), - Namespace: targetDetails.Namespace, - Labels: map[string]string{ - "kubectl-flame/id": id, - }, - } - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - Kind: "Job", - APIVersion: "batch/v1", - }, - ObjectMeta: commonMeta, - Spec: batchv1.JobSpec{ - Parallelism: int32Ptr(1), - Completions: int32Ptr(1), - TTLSecondsAfterFinished: int32Ptr(5), - Template: v1.PodTemplateSpec{ - ObjectMeta: commonMeta, - Spec: v1.PodSpec{ - HostPID: true, - Volumes: []apiv1.Volume{ - { - Name: "target-filesystem", - VolumeSource: apiv1.VolumeSource{ - HostPath: &apiv1.HostPathVolumeSource{ - Path: "/var/lib/docker", - }, - }, - }, - }, - InitContainers: nil, - Containers: []apiv1.Container{ - { - ImagePullPolicy: v1.PullAlways, - Name: "kubectl-flame", - Image: imageName, - Command: []string{"/app/agent"}, - Args: []string{id, - string(targetPod.UID), - targetDetails.ContainerName, - targetDetails.ContainerId, - targetDetails.Duration.String(), - }, - VolumeMounts: []apiv1.VolumeMount{ - { - Name: "target-filesystem", - MountPath: "/var/lib/docker", - }, - }, - }, - }, - RestartPolicy: "Never", - NodeName: targetPod.Spec.NodeName, - }, - }, - }, - } + id, flameJob := job.Create(targetPod, targetDetails) if targetDetails.DryRun { - err := printJob(job) + err := printJob(flameJob) return "", nil, err } createJob, err := clientSet. BatchV1(). Jobs(targetDetails.Namespace). - Create(ctx, job, metav1.CreateOptions{}) + Create(ctx, flameJob, metav1.CreateOptions{}) if err != nil { return "", nil, err @@ -108,19 +42,6 @@ func printJob(job *batchv1.Job) error { return encoder.Encode(job, os.Stdout) } -func getAgentImage(targetDetails *data.TargetDetails) string { - if targetDetails.Image != "" { - return targetDetails.Image - } - - tag := fmt.Sprintf("%s-jvm", version.GetCurrent()) - if targetDetails.Alpine { - tag = fmt.Sprintf("%s-alpine", tag) - } - - return fmt.Sprintf("%s:%s", imageName, tag) -} - func DeleteProfilingJob(job *batchv1.Job, targetDetails *data.TargetDetails, ctx context.Context) error { deleteStrategy := metav1.DeletePropagationForeground return clientSet. diff --git a/cli/cmd/logic.go b/cli/cmd/logic.go index e3cae4b..49c810b 100644 --- a/cli/cmd/logic.go +++ b/cli/cmd/logic.go @@ -33,7 +33,7 @@ func Flame(target *data.TargetDetails, configFlags *genericclioptions.ConfigFlag os.Exit(1) } - containerName, err := validatePod(pod, target.ContainerName) + containerName, err := validatePod(pod, target) if err != nil { p.PrintError() fmt.Println(err.Error()) @@ -84,7 +84,7 @@ func Flame(target *data.TargetDetails, configFlags *genericclioptions.ConfigFlag <-done } -func validatePod(pod *v1.Pod, specificContainer string) (string, error) { +func validatePod(pod *v1.Pod, targetDetails *data.TargetDetails) (string, error) { if pod == nil { return "", errors.New(fmt.Sprintf("Could not find pod %s in Namespace %s", targetDetails.PodName, targetDetails.Namespace)) @@ -93,7 +93,7 @@ func validatePod(pod *v1.Pod, specificContainer string) (string, error) { if len(pod.Spec.Containers) != 1 { var containerNames []string for _, container := range pod.Spec.Containers { - if container.Name == specificContainer { + if container.Name == targetDetails.ContainerName { return container.Name, nil // Found given container } diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 93b3589..490f542 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -4,10 +4,12 @@ package cmd import ( "fmt" + "github.com/VerizonMedia/kubectl-flame/api" "github.com/VerizonMedia/kubectl-flame/cli/cmd/data" "github.com/VerizonMedia/kubectl-flame/cli/cmd/version" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" + "os" "time" ) @@ -29,9 +31,6 @@ These commands help you identify application performance issues. ` ) -var targetDetails data.TargetDetails -var showVersion bool - type FlameOptions struct { configFlags *genericclioptions.ConfigFlags genericclioptions.IOStreams @@ -45,6 +44,10 @@ func NewFlameOptions(streams genericclioptions.IOStreams) *FlameOptions { } func NewFlameCommand(streams genericclioptions.IOStreams) *cobra.Command { + var targetDetails data.TargetDetails + var showVersion bool + var chosenLang string + options := NewFlameOptions(streams) cmd := &cobra.Command{ Use: "flame [pod-name]", @@ -66,6 +69,11 @@ func NewFlameCommand(streams genericclioptions.IOStreams) *cobra.Command { return } + if err := validateFlags(chosenLang, &targetDetails); err != nil { + fmt.Fprintln(streams.Out, err) + os.Exit(1) + } + targetDetails.PodName = args[0] if len(args) > 1 { targetDetails.ContainerName = args[1] @@ -81,7 +89,23 @@ func NewFlameCommand(streams genericclioptions.IOStreams) *cobra.Command { cmd.Flags().BoolVar(&targetDetails.Alpine, "alpine", false, "Target image is based on Alpine") cmd.Flags().BoolVar(&targetDetails.DryRun, "dry-run", false, "Simulate profiling") cmd.Flags().StringVar(&targetDetails.Image, "image", "", "Manually choose agent docker image") + cmd.Flags().StringVarP(&targetDetails.Pgrep, "pgrep", "p", "", "name of the target process") + cmd.Flags().StringVarP(&chosenLang, "lang", "l", "", fmt.Sprintf("Programming language of "+ + "the target application, choose one of %v", api.AvailableLanguages())) options.configFlags.AddFlags(cmd.Flags()) return cmd } + +func validateFlags(langString string, details *data.TargetDetails) error { + if langString == "" { + return fmt.Errorf("use -l flag to select one of the supported languages %s", api.AvailableLanguages()) + } + + if !api.IsSupportedLanguage(langString) { + return fmt.Errorf("unsupported language, choose one of %s", api.AvailableLanguages()) + } + + details.Language = api.ProgrammingLanguage(langString) + return nil +}