Skip to content

Commit

Permalink
Merge pull request #11 from VerizonMedia/add-golang-support
Browse files Browse the repository at this point in the history
Golang Support via BPF
  • Loading branch information
edeNFed authored Sep 8, 2020
2 parents f014fd2 + c7c75ab commit 66922d9
Show file tree
Hide file tree
Showing 25 changed files with 749 additions and 300 deletions.
12 changes: 10 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,27 @@ jobs:
- name: Build JVM Docker Image
uses: docker/build-push-action@v1
with:
dockerfile: 'agent/Dockerfile'
dockerfile: 'agent/docker/jvm/Dockerfile'
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
repository: verizondigital/kubectl-flame
tags: ${{ steps.vars.outputs.tag }}-jvm
- name: Build JVM Alpine Docker Image
uses: docker/build-push-action@v1
with:
dockerfile: 'agent/Dockerfile.alpine'
dockerfile: 'agent/docker/jvm/Dockerfile.alpine'
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
repository: verizondigital/kubectl-flame
tags: ${{ steps.vars.outputs.tag }}-jvm-alpine
- name: Build BPF Docker Image
uses: docker/build-push-action@v1
with:
dockerfile: 'agent/docker/bpf/Dockerfile'
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
repository: verizondigital/kubectl-flame
tags: ${{ steps.vars.outputs.tag }}-bpf
- name: Setup Go
uses: actions/setup-go@v1
with:
Expand Down
22 changes: 15 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,33 @@ Running `kubectlf-flame` does **not** require any modification to existing pods.
- [License](#license)

## Requirements
* Currently, only Java applications are supported. (Golang support coming soon!)
* Supported languages: Go, Java (any JVM based language)
* Kubernetes cluster that use Docker as the container runtime (tested on GKE, EKS and AKS)

## Usage
### Profiling Kubernetes Pod
In order to profile pod `mypod` for 1 minute and save the flamegraph as `/tmp/flamegraph.svg` run:
In order to profile a Java application in pod `mypod` for 1 minute and save the flamegraph as `/tmp/flamegraph.svg` run:
```shell
kubectl flame mypod -t 1m -f /tmp/flamegraph.svg
kubectl flame mypod -t 1m --lang java -f /tmp/flamegraph.svg
```
### Profiling Alpine based container
Profiling alpine based containers require using `--alpine` flag:
Profiling Java application in alpine based containers require using `--alpine` flag:
```shell
kubectl flame mypod -t 1m -f /tmp/flamegraph.svg --alpine
kubectl flame mypod -t 1m -f /tmp/flamegraph.svg --lang java --alpine
```
*NOTICE*: this is only required for Java apps, the `--alpine` flag is unnecessary for Go profiling.

### Profiling sidecar container
Pods that contains more than one container require specifying the target container as an argument:
```shell
kubectl flame mypod -t 1m -f /tmp/flamegraph.svg mycontainer
kubectl flame mypod -t 1m --lang go -f /tmp/flamegraph.svg mycontainer
```
### Profiling Golang multi-process container
Profiling Go application in pods that contains more than one process require specifying the target process name via `--pgrep` flag:
```shell
kubectl flame mypod -t 1m --lang go -f /tmp/flamegraph.svg --pgrep go-app
```
Java profiling assumes that the process name is `java`. Use `--pgrep` flag if your process name is different.

## Installing

Expand All @@ -54,7 +62,7 @@ See the release page for the full list of pre-built assets.
`kubectl-flame` launch a Kubernetes Job on the same node as the target pod.
Under the hood `kubectl-flame` use [async-profiler](https://github.com/jvm-profiling-tools/async-profiler) in order to generate flame graphs for Java applications.
Interaction with the target JVM is done via a shared `/tmp` folder.
Other languages support (such as the upcoming Golang support) will be based on [ebpf profiling](https://en.wikipedia.org/wiki/Berkeley_Packet_Filter).
Golang support is based on [ebpf profiling](https://en.wikipedia.org/wiki/Berkeley_Packet_Filter).

## Contribute
Please refer to [the contributing.md file](Contributing.md) for information about how to get involved. We welcome issues, questions, and pull requests.
Expand Down
17 changes: 11 additions & 6 deletions agent/details/profiling_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@
//: Licensed under the terms of the Apache 2.0 License. See LICENSE file in the project root for terms.
package details

import "time"
import (
"github.com/VerizonMedia/kubectl-flame/api"
"time"
)

type ProfilingJob struct {
Duration time.Duration
ID string
ContainerID string
ContainerName string
PodUID string
Duration time.Duration
ID string
ContainerID string
ContainerName string
PodUID string
Language api.ProgrammingLanguage
TargetProcessName string
}
29 changes: 29 additions & 0 deletions agent/docker/bpf/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
ARG KERNEL_VERSION=4.9.125

FROM linuxkit/kernel:$KERNEL_VERSION AS ksrc

FROM golang:1.14-buster as agentbuild
WORKDIR /go/src/github.com/VerizonMedia/kubectl-flame
ADD . /go/src/github.com/VerizonMedia/kubectl-flame
RUN go get -d -v ./...
RUN cd agent && go build -o /go/bin/agent

FROM alpine as builder
COPY --from=ksrc /kernel-dev.tar /
RUN tar xf /kernel-dev.tar -C /
RUN mv /usr/src/*/ /usr/src/kernel-source/
RUN apk add git
RUN git clone https://github.com/brendangregg/FlameGraph
RUN git clone https://gist.github.com/edeNFed/83a9438156288661e2283c28fee18b8b bcc-profiler

FROM alpine
COPY --from=builder /usr/src /usr/src
RUN apk add bcc-tools perl
RUN ln -s $(which python3) /usr/bin/python
RUN mkdir -p /app/FlameGraph
COPY --from=builder /FlameGraph /app/FlameGraph
COPY --from=agentbuild /go/bin/agent /app
COPY --from=builder /bcc-profiler /app/bcc-profiler/
RUN chmod +x /app/bcc-profiler/profile

CMD [ "/app/agent" ]
File renamed without changes.
File renamed without changes.
49 changes: 24 additions & 25 deletions agent/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,45 +17,33 @@ import (

func main() {
args, err := validateArgs()
if err != nil {
api.PublishError(err)
os.Exit(1)
}
handleError(err)

err = api.PublishEvent(api.Progress, &api.ProgressData{Time: time.Now(), Stage: api.Started})
if err != nil {
api.PublishError(err)
os.Exit(1)
}
handleError(err)

err = profiler.SetUp(args)
if err != nil {
api.PublishError(err)
os.Exit(1)
}
p, err := profiler.ForLanguage(args.Language)
handleError(err)

err = p.SetUp(args)
handleError(err)

done := handleSignals()
err = profiler.Invoke(args)
if err != nil {
api.PublishError(err)
os.Exit(1)
}
err = p.Invoke(args)
handleError(err)

err = api.PublishEvent(api.Progress, &api.ProgressData{Time: time.Now(), Stage: api.Ended})
if err != nil {
api.PublishError(err)
os.Exit(1)
}
handleError(err)

<-done
}

func validateArgs() (*details.ProfilingJob, error) {
if len(os.Args) != 6 {
return nil, errors.New("expected 6 arguments")
if len(os.Args) != 7 && len(os.Args) != 8 {
return nil, errors.New("expected 6 or 7 arguments")
}

duration, err := time.ParseDuration((os.Args[5]))
duration, err := time.ParseDuration(os.Args[5])
if err != nil {
return nil, err
}
Expand All @@ -66,6 +54,10 @@ func validateArgs() (*details.ProfilingJob, error) {
currentJob.ContainerName = os.Args[3]
currentJob.ContainerID = strings.Replace(os.Args[4], "docker://", "", 1)
currentJob.Duration = duration
currentJob.Language = api.ProgrammingLanguage(os.Args[6])
if len(os.Args) == 8 {
currentJob.TargetProcessName = os.Args[7]
}

return currentJob, nil
}
Expand All @@ -84,3 +76,10 @@ func handleSignals() chan bool {

return done
}

func handleError(err error) {
if err != nil {
api.PublishError(err)
os.Exit(1)
}
}
105 changes: 105 additions & 0 deletions agent/profiler/bpf.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package profiler

import (
"fmt"
"github.com/VerizonMedia/kubectl-flame/agent/details"
"github.com/VerizonMedia/kubectl-flame/agent/utils"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
)

const (
kernelSourcesDir = "/usr/src/kernel-source/"
profilerLocation = "/app/bcc-profiler/profile"
rawProfilerOutputFile = "/tmp/raw_profile.txt"
flameGraphScriptLocation = "/app/FlameGraph/flamegraph.pl"
flameGraphOutputLocation = "/tmp/flamegraph.svg"
)

type BpfProfiler struct{}

func (b *BpfProfiler) SetUp(job *details.ProfilingJob) error {
exitCode, kernelVersion, err := utils.ExecuteCommand(exec.Command("uname", "-r"))
if err != nil {
return fmt.Errorf("failed to get kernel version, exit code: %d, error: %s", exitCode, err)
}

expectedSourcesLocation, err := os.Readlink(fmt.Sprintf("/lib/modules/%s/build",
strings.TrimSuffix(kernelVersion, "\n")))
if err != nil {
return fmt.Errorf("failed to read source link, error: %s", err)
}

return b.moveSources(expectedSourcesLocation)
}

func (b *BpfProfiler) Invoke(job *details.ProfilingJob) error {
err := b.runProfiler(job)
if err != nil {
return fmt.Errorf("profiling failed: %s", err)
}

err = b.generateFlameGraph()
if err != nil {
return fmt.Errorf("flamegraph generation failed: %s", err)
}

return utils.PublishFlameGraph(flameGraphOutputLocation)
}

func (b *BpfProfiler) runProfiler(job *details.ProfilingJob) error {
pid, err := utils.FindProcessId(job)
if err != nil {
return err
}

f, err := os.Create(rawProfilerOutputFile)
if err != nil {
return err
}
defer f.Close()

duration := strconv.Itoa(int(job.Duration.Seconds()))
profileCmd := exec.Command(profilerLocation, "-df", "-p", pid, duration)
profileCmd.Stdout = f

return profileCmd.Run()
}

func (b *BpfProfiler) generateFlameGraph() error {
inputFile, err := os.Open(rawProfilerOutputFile)
if err != nil {
return err
}
defer inputFile.Close()

outputFile, err := os.Create(flameGraphOutputLocation)
if err != nil {
return err
}
defer outputFile.Close()

flameGraphCmd := exec.Command(flameGraphScriptLocation)
flameGraphCmd.Stdin = inputFile
flameGraphCmd.Stdout = outputFile

return flameGraphCmd.Run()
}

func (b *BpfProfiler) moveSources(target string) error {
parent, _ := filepath.Split(target)
err := os.MkdirAll(parent, os.ModePerm)
if err != nil {
return err
}

_, _, err = utils.ExecuteCommand(exec.Command("mv", kernelSourcesDir, target))
if err != nil {
return fmt.Errorf("failed moving source files, error: %s, tried to move to: %s", err, target)
}

return nil
}
Loading

0 comments on commit 66922d9

Please sign in to comment.