feat: 增加文件路径搜索工具 #30

.github/workflows/deploy-bkrepo-filepath-search.yml

@@ -0,0 +1,12 @@
+name: Create and publish bkrepo-filepath-search image
+
+on:
+  push:
+    tags:
+      - "filepath-search/v*"
+
+jobs:
+  build-and-push-image:
+    uses: ./.github/workflows/deploy-golang-analysis-tool.yml
+    with:
+      tool_name: filepath-search

filepath-search/Dockerfile

@@ -0,0 +1,7 @@
+FROM golang:1.18-alpine
+
+COPY bkrepo-filepath-search /bkrepo-filepath-search
+RUN chmod +x /bkrepo-filepath-search
+
+CMD []
+ENTRYPOINT []

filepath-search/README.md

@@ -0,0 +1,8 @@
+# FilepathSearch
+在镜像Tar包中搜索匹配指定正则表达式的路径
+
+## 使用方式
+在制品库Admin中添加standalone类型的分析工具，并设置下列参数
+
+1. 镜像地址`ghcr.io/TencentBlueKing/ci-repoanalysis/bkrepo-filepath-search:latest`
+2. 增加string类型的参数regex，值为用于路径匹配的正则表达式

filepath-search/cmd/main.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang/framework"
+	"github.com/TencentBlueKing/ci-repoAnalysis/filepath-search/pkg"
+)
+
+func main() {
+	framework.Analyze(pkg.FilepathSearch{})
+}

filepath-search/go.mod

@@ -0,0 +1,17 @@
+module github.com/TencentBlueKing/ci-repoAnalysis/filepath-search
+
+go 1.18
+
+require (
+	github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang v0.0.14
+	github.com/google/go-containerregistry v0.14.0
+)
+
+require (
+	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	github.com/vbatts/tar-split v0.11.2 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+)

filepath-search/go.sum

@@ -0,0 +1,37 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang v0.0.14 h1:EANOcN18mkPhyPvUeTQHtDkOcYnptJajQeMg+1b/1oo=
+github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang v0.0.14/go.mod h1:8fgb+y0YWqULX/oVg4dsWAq6ftsVZfFrXQjCKXwAE1Q=
+github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
+github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM=
+github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
+github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY=
+github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
+github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
+github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
+github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
+github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
+github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

filepath-search/pkg/executor.go

@@ -0,0 +1,145 @@
+package pkg
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"github.com/TencentBlueKing/ci-repoAnalysis/analysis-tool-sdk-golang/object"
+	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"io"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+)
+
+// FilepathSearch 文件路径匹配工具
+type FilepathSearch struct{}
+
+// Execute 执行扫描，在镜像tar包中搜索匹配指定正则表达式的路径
+func (e FilepathSearch) Execute(config *object.ToolConfig, file *os.File) (*object.ToolOutput, error) {
+	regStr := config.GetStringArg("regex")
+	if len(regStr) == 0 {
+		return nil, errors.New("regex config not found")
+	}
+
+	securityResults, err := scan(file.Name(), regStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return object.NewOutput(
+		object.StatusSuccess,
+		&object.Result{
+			SecurityResults: securityResults,
+		},
+	), nil
+}
+
+func scan(filepath string, regex string) ([]object.SecurityResult, error) {
+	reg, _ := regexp.Compile(regex)
+	img, err := tarball.Image(fileOpener(filepath), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	layers, err := img.Layers()
+	if err != nil {
+		return nil, err
+	}
+
+	searchedLayers := make(map[string]struct{})
+	securityResults := make([]object.SecurityResult, 0)
+
+	for i := range layers {
+		l := layers[i]
+		diffId, err := l.DiffID()
+		if err != nil {
+			return nil, err
+		}
+
+		// 已经搜索过的layer直接跳过
+		if _, ok := searchedLayers[diffId.Hex]; ok {
+			continue
+		}
+
+		if err := walk(l, func(filePath string, info os.FileInfo, reader io.Reader) error {
+			if reg.MatchString(filePath) {
+				vulId := "fp-" + filePath
+				securityResults = append(securityResults, object.SecurityResult{
+					VulId:       vulId,
+					CveId:       vulId,
+					VulName:     vulId,
+					Path:        filePath,
+					PkgName:     filePath,
+					PkgVersions: []string{},
+					References:  []string{},
+					Des:         fmt.Sprintf("File path [%s] matches the regex [%s]", filePath, reg),
+					Severity:    "CRITICAL",
+				})
+			}
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+	}
+
+	return securityResults, nil
+}
+
+type WalkFunc func(filePath string, info os.FileInfo, reader io.Reader) error
+
+func walk(layer v1.Layer, processFunc WalkFunc) error {
+	rc, err := layer.Uncompressed()
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+	tr := tar.NewReader(rc)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		filePath := strings.TrimLeft(path.Clean(hdr.Name), "/")
+
+		if err := processFunc(filePath, hdr.FileInfo(), tr); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func fileOpener(file string) func() (io.ReadCloser, error) {
+	return func() (io.ReadCloser, error) {
+		f, err := os.Open(file)
+		if err != nil {
+			return nil, err
+		}
+		br := bufio.NewReader(f)
+		var r io.Reader = br
+		if isGzip(br) {
+			var err error
+			if r, err = gzip.NewReader(br); err != nil {
+				return nil, err
+			}
+		}
+		return io.NopCloser(r), nil
+	}
+}
+
+func isGzip(f *bufio.Reader) bool {
+	buf, err := f.Peek(3)
+	if err != nil {
+		return false
+	}
+	return buf[0] == 0x1F && buf[1] == 0x8B && buf[2] == 0x8
+}