From 4092a549b2820c72df71296728a3e48ab565aacf Mon Sep 17 00:00:00 2001 From: kunlongli <16629885+cnlkl@users.noreply.github.com> Date: Tue, 28 Mar 2023 17:44:28 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E6=90=9C=E7=B4=A2=E5=B7=A5=E5=85=B7=20#30?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deploy-bkrepo-filepath-search.yml | 12 ++ filepath-search/Dockerfile | 7 + filepath-search/README.md | 8 + filepath-search/cmd/main.go | 10 ++ filepath-search/go.mod | 17 ++ filepath-search/go.sum | 37 +++++ filepath-search/pkg/executor.go | 145 ++++++++++++++++++ 7 files changed, 236 insertions(+) create mode 100644 .github/workflows/deploy-bkrepo-filepath-search.yml create mode 100644 filepath-search/Dockerfile create mode 100644 filepath-search/README.md create mode 100644 filepath-search/cmd/main.go create mode 100644 filepath-search/go.mod create mode 100644 filepath-search/go.sum create mode 100644 filepath-search/pkg/executor.go diff --git a/.github/workflows/deploy-bkrepo-filepath-search.yml b/.github/workflows/deploy-bkrepo-filepath-search.yml new file mode 100644 index 0000000..5a9da10 --- /dev/null +++ b/.github/workflows/deploy-bkrepo-filepath-search.yml @@ -0,0 +1,12 @@ +name: Create and publish bkrepo-filepath-search image + +on: + push: + tags: + - "filepath-search/v*" + +jobs: + build-and-push-image: + uses: ./.github/workflows/deploy-golang-analysis-tool.yml + with: + tool_name: filepath-search diff --git a/filepath-search/Dockerfile b/filepath-search/Dockerfile new file mode 100644 index 0000000..6def9b1 --- /dev/null +++ b/filepath-search/Dockerfile @@ -0,0 +1,7 @@ +FROM golang:1.18-alpine + +COPY bkrepo-filepath-search /bkrepo-filepath-search +RUN chmod +x /bkrepo-filepath-search + +CMD [] +ENTRYPOINT [] diff --git a/filepath-search/README.md b/filepath-search/README.md new file mode 100644 index 0000000..79a0533 --- /dev/null +++ b/filepath-search/README.md @@ -0,0 +1,8 @@ +# FilepathSearch +在镜像Tar包中搜索匹配指定正则表达式的路径 + +## 使用方式 +在制品库Admin中添加standalone类型的分析工具，并设置下列参数 + +1. 镜像地址`ghcr.io/TencentBlueKing/ci-repoanalysis/bkrepo-filepath-search:latest` +2. 增加string类型的参数regex，值为用于路径匹配的正则表达式 diff --git a/filepath-search/cmd/main.go b/filepath-search/cmd/main.go new file mode 100644 index 0000000..9f1e646 --- /dev/null +++ b/filepath-search/cmd/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang/framework" + "github.com/TencentBlueKing/ci-repoAnalysis/filepath-search/pkg" +) + +func main() { + framework.Analyze(pkg.FilepathSearch{}) +} diff --git a/filepath-search/go.mod b/filepath-search/go.mod new file mode 100644 index 0000000..2dcfca0 --- /dev/null +++ b/filepath-search/go.mod @@ -0,0 +1,17 @@ +module github.com/TencentBlueKing/ci-repoAnalysis/filepath-search + +go 1.18 + +require ( + github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang v0.0.14 + github.com/google/go-containerregistry v0.14.0 +) + +require ( + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/vbatts/tar-split v0.11.2 // indirect + golang.org/x/sync v0.1.0 // indirect +) diff --git a/filepath-search/go.sum b/filepath-search/go.sum new file mode 100644 index 0000000..45c2d0c --- /dev/null +++ b/filepath-search/go.sum @@ -0,0 +1,37 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang v0.0.14 h1:EANOcN18mkPhyPvUeTQHtDkOcYnptJajQeMg+1b/1oo= +github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang v0.0.14/go.mod h1:8fgb+y0YWqULX/oVg4dsWAq6ftsVZfFrXQjCKXwAE1Q= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw= +github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/filepath-search/pkg/executor.go b/filepath-search/pkg/executor.go new file mode 100644 index 0000000..10a5b4a --- /dev/null +++ b/filepath-search/pkg/executor.go @@ -0,0 +1,145 @@ +package pkg + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "errors" + "fmt" + "github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang/object" + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "io" + "os" + "path" + "regexp" + "strings" +) + +// FilepathSearch 文件路径匹配工具 +type FilepathSearch struct{} + +// Execute 执行扫描，在镜像tar包中搜索匹配指定正则表达式的路径 +func (e FilepathSearch) Execute(config *object.ToolConfig, file *os.File) (*object.ToolOutput, error) { + regStr := config.GetStringArg("regex") + if len(regStr) == 0 { + return nil, errors.New("regex config not found") + } + + securityResults, err := scan(file.Name(), regStr) + if err != nil { + return nil, err + } + + return object.NewOutput( + object.StatusSuccess, + &object.Result{ + SecurityResults: securityResults, + }, + ), nil +} + +func scan(filepath string, regex string) ([]object.SecurityResult, error) { + reg, _ := regexp.Compile(regex) + img, err := tarball.Image(fileOpener(filepath), nil) + if err != nil { + return nil, err + } + + layers, err := img.Layers() + if err != nil { + return nil, err + } + + searchedLayers := make(map[string]struct{}) + securityResults := make([]object.SecurityResult, 0) + + for i := range layers { + l := layers[i] + diffId, err := l.DiffID() + if err != nil { + return nil, err + } + + // 已经搜索过的layer直接跳过 + if _, ok := searchedLayers[diffId.Hex]; ok { + continue + } + + if err := walk(l, func(filePath string, info os.FileInfo, reader io.Reader) error { + if reg.MatchString(filePath) { + vulId := "fp-" + filePath + securityResults = append(securityResults, object.SecurityResult{ + VulId: vulId, + CveId: vulId, + VulName: vulId, + Path: filePath, + PkgName: filePath, + PkgVersions: []string{}, + References: []string{}, + Des: fmt.Sprintf("File path [%s] matches the regex [%s]", filePath, reg), + Severity: "CRITICAL", + }) + } + return nil + }); err != nil { + return nil, err + } + } + + return securityResults, nil +} + +type WalkFunc func(filePath string, info os.FileInfo, reader io.Reader) error + +func walk(layer v1.Layer, processFunc WalkFunc) error { + rc, err := layer.Uncompressed() + if err != nil { + return err + } + defer rc.Close() + tr := tar.NewReader(rc) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + filePath := strings.TrimLeft(path.Clean(hdr.Name), "/") + + if err := processFunc(filePath, hdr.FileInfo(), tr); err != nil { + return err + } + } + + return nil +} + +func fileOpener(file string) func() (io.ReadCloser, error) { + return func() (io.ReadCloser, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + br := bufio.NewReader(f) + var r io.Reader = br + if isGzip(br) { + var err error + if r, err = gzip.NewReader(br); err != nil { + return nil, err + } + } + return io.NopCloser(r), nil + } +} + +func isGzip(f *bufio.Reader) bool { + buf, err := f.Peek(3) + if err != nil { + return false + } + return buf[0] == 0x1F && buf[1] == 0x8B && buf[2] == 0x8 +}