diff --git a/.github/workflows/ci-test-py.yml b/.github/workflows/ci-test-py.yml index 4d6ba59..081eb5f 100644 --- a/.github/workflows/ci-test-py.yml +++ b/.github/workflows/ci-test-py.yml @@ -44,7 +44,7 @@ jobs: - name: Lint with Ruff run: | pip install ruff - ruff --output-format=github . + ruff check --output-format=github . working-directory: ai-engine py-lint-ai-sentryflow: diff --git a/clients/log-client/README.md b/clients/log-client/README.md new file mode 100644 index 0000000..ad85f72 --- /dev/null +++ b/clients/log-client/README.md @@ -0,0 +1,33 @@ +# Log Client +Log client collects AccessLogs and Metrics from SentryFlow and prints them to the terminal or saves them to a log file. + +## Log Client Deployment +Log client can be deployed using kubectl command. The deployment can be accomplished with the following +commands: +```bash +$ cd SentryFlow/deployments +$ kubectl apply -f log-client.yaml +``` + +## Log client options +These are the default env value in the log-client.yaml file. +```bash +env: +- name: LOG_CFG + value: "stdout" +- name: METRIC_CFG + value: "stdout" +- name: METRIC_FILTER + value: "api" +``` + +If you want to change the default env value, you can refer to the following options. +```bash +env: +- name: LOG_CFG + value: {"stdout"|"file"|"none"} +- name: METRIC_CFG + value: {"stdout"|"file"|"none"} +- name: METRIC_FILTER + value: {"all"|"api"|"envoy"} +``` diff --git a/clients/mongo-client/README.md b/clients/mongo-client/README.md new file mode 100644 index 0000000..49fa621 --- /dev/null +++ b/clients/mongo-client/README.md @@ -0,0 +1,29 @@ +# Mongo Client +Mongo client collects AccessLogs and Metrics from SentryFlow and stores them to database. + +## Mongo Client Deployment +Mongo client can be deployed using kubectl command. The deployment can be accomplished with the following +commands: +```bash +$ cd SentryFlow/deployments +$ kubectl apply -f mongo-client.yaml +``` + +## Mongo client options +These are the default env value. +```bash +- LOG_CFG: mongodb +- METRIC_CFG: mongodb +- METRIC_FILTER: envoy +``` + +If you want to change the default env value, you can refer to the following options. +```bash +env: +- name: LOG_CFG + value: {"mongodb"|"none"} +- name: METRIC_CFG + value: {"mongodb"|"none"} +- name: METRIC_FILTER + value: {"all"|"api"|"envoy"} +``` diff --git a/clients/neo4j-client/Dockerfile b/clients/neo4j-client/Dockerfile new file mode 100644 index 0000000..2e75b9e --- /dev/null +++ b/clients/neo4j-client/Dockerfile @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 + +### Builder + +FROM golang:1.21-alpine3.17 as builder + +RUN apk --no-cache update +RUN apk add --no-cache git clang llvm make gcc protobuf musl-dev + +RUN mkdir /app +RUN mkdir /protobuf + +WORKDIR /protobuf + +COPY /protobuf . + +WORKDIR /app + +COPY /clients/neo4j-client . + +RUN go build -o neo4j-client + +### Make executable image + +FROM alpine:3.18 as client + +RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories + +RUN apk --no-cache update +RUN apk add bash + +COPY --from=builder /app/neo4j-client / + +CMD ["/neo4j-client"] diff --git a/clients/neo4j-client/Makefile b/clients/neo4j-client/Makefile new file mode 100644 index 0000000..afae23b --- /dev/null +++ b/clients/neo4j-client/Makefile @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 + +CLIENT_NAME = sentryflow-neo4j-client +IMAGE_NAME = 5gsec/$(CLIENT_NAME) +TAG = v0.1 + +.PHONY: gofmt +gofmt: + cd $(CURDIR); gofmt -w -s -d $(shell find . -type f -name '*.go' -print) + +.PHONY: golint +golint: +ifeq (, $(shell which golint)) + @{ \ + set -e ;\ + GOLINT_TEMP_DIR=$$(mktemp -d) ;\ + cd $$GOLINT_TEMP_DIR ;\ + go mod init tmp ;\ + go get golang.org/x/lint/golint ;\ + go install golang.org/x/lint/golint ;\ + rm -rf $$GOLINT_TEMP_DIR ;\ + } +endif + cd $(CURDIR); golint ./... + +.PHONY: gosec +gosec: +ifeq (, $(shell which gosec)) + @{ \ + set -e ;\ + GOSEC_TEMP_DIR=$$(mktemp -d) ;\ + cd $$GOSEC_TEMP_DIR ;\ + go mod init tmp ;\ + go get github.com/securego/gosec/v2/cmd/gosec ;\ + go install github.com/securego/gosec/v2/cmd/gosec ;\ + rm -rf $$GOSEC_TEMP_DIR ;\ + } +endif + cd $(CURDIR); gosec -exclude=G402 ./... + +.PHONY: build gofmt golint gosec +build: + go mod tidy + go build -o $(CLIENT_NAME) + +.PHONY: clean +clean: + rm -f $(CLIENT_NAME) + +.PHONY: build-image +build-image: + docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../../ + +.PHONY: clean-image +clean-image: + docker rmi $(IMAGE_NAME):$(TAG) + +.PHONY: run-image +run-image: + docker run -it --rm $(IMAGE_NAME):$(TAG) diff --git a/clients/neo4j-client/README.md b/clients/neo4j-client/README.md new file mode 100644 index 0000000..053ed35 --- /dev/null +++ b/clients/neo4j-client/README.md @@ -0,0 +1,59 @@ +# Neo4j Client +The Neo4j client collects AccessLogs from SentryFlow, stores them, and visualizes them. + +## Neo4j Client Deployment +Neo4j client can be deployed using kubectl command. The deployment can be accomplished with the following +commands: +```bash +$ cd SentryFlow/deployments +$ kubectl apply -f neo4j-client.yaml +``` + +## Neo4j settings +### Step 1. Create Neo4j account +Go to https://neo4j.com/ and create an account + +### Step 2. Create Neo4j Instance +Remember the Username and Password you created when creating the instance. + +### Step 3. Modify env value in neo4j-client.yaml file. +Put the Connection URI specified in the instance into NEO4J_URI, and the information created in Step 2 into NEO4J_USERNAME and NEO4J_PASSWORD, respectively. + +```bash +env: +- name: NEO4J_URI + value: "" +- name: NEO4J_USERNAME + value: "" +- name: NEO4J_PASSWORD + value: "" +``` + +## Neo4j client options +These are the default env value in the neo4j-client.yaml file. +```bash +env: +- name: NODE_LEVEL + value: "simple" +- name: EDGE_LEVEL + value: "simple" +``` + +If you want to change the default env value, you can refer to the following options. +```bash +env: +- name: NODE_LEVEL + value: {"simple"|"detail"} +- name: EDGE_LEVEL + value: {"simple"|"detail"} +``` + +## Example with robot-shop +### Example 1 (NODE_LEVEL: simple, EDGE_LEVEL: simple) +![Neo4j example1](/docs/neo4j_01.png) + +### Example 2 (NODE_LEVEL: simple, EDGE_LEVEL: detail) +![Neo4j example2](/docs/neo4j_02.png) + +### Example 3 (NODE_LEVEL: detail, EDGE_LEVEL: detail) +![Neo4j example3](/docs/neo4j_03.png) diff --git a/clients/neo4j-client/client/client.go b/clients/neo4j-client/client/client.go new file mode 100644 index 0000000..e41fc72 --- /dev/null +++ b/clients/neo4j-client/client/client.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + pb "SentryFlow/protobuf" + "context" + "io" + "log" + "neo4j-client/neo4jdb" +) + +// Feeder Structure +type Feeder struct { + Running bool + + client pb.SentryFlowClient + logStream pb.SentryFlow_GetAPILogClient + + DbHandler neo4jdb.Neo4jHandler + + Done chan struct{} +} + +// NewClient Function +func NewClient(client pb.SentryFlowClient, clientInfo *pb.ClientInfo, nodeLevel string, edgeLevel string, neo4jHost string, neo4jId string, neo4jPassword string) *Feeder { + fd := &Feeder{} + + fd.Running = true + fd.client = client + fd.Done = make(chan struct{}) + + // Contact the server and print out its response + logStream, err := client.GetAPILog(context.Background(), clientInfo) + if err != nil { + log.Fatalf("[Client] Could not get API log: %v", err) + } + + fd.logStream = logStream + + // Initialize DB + dbHandler, err := neo4jdb.NewNeo4jHandler(nodeLevel, edgeLevel, neo4jHost, neo4jId, neo4jPassword) + if err != nil { + log.Fatalf("[MongoDB] Unable to intialize DB: %v", err) + return nil + } + fd.DbHandler = *dbHandler + + return fd +} + +// APILogRoutine Function +func (fd *Feeder) APILogRoutine() { + for fd.Running { + select { + default: + data, err := fd.logStream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to receive log: %v", err) + } + log.Printf("[Client] Inserting log") + err = fd.DbHandler.CreateOrUpdateRelationship(data) + if err != nil { + log.Printf("[Client] Failed to insert log: %v", err) + } + case <-fd.Done: + return + } + } +} diff --git a/clients/neo4j-client/config/config.go b/clients/neo4j-client/config/config.go new file mode 100644 index 0000000..225ac3a --- /dev/null +++ b/clients/neo4j-client/config/config.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "errors" + "fmt" + "os" + "strconv" +) + +// Config structure +type Config struct { + Hostname string + + ServerAddr string + ServerPort int + + NodeLevel string + EdgeLevel string + + Neo4jURI string + Neo4jUsername string + Neo4jPassword string +} + +// Cfg is for global reference +var Cfg Config + +// LoadEnvVars loads environment variables and stores them as global variable +func LoadEnvVars() (Config, error) { + var err error + + Cfg.Hostname, err = os.Hostname() + if err != nil { + msg := fmt.Sprintf("[Config] Could not find hostname: %v", err) + return Cfg, errors.New(msg) + } + + // load listen address and check if valid + Cfg.ServerAddr = os.Getenv("SERVER_ADDR") + + // load listen port and check if valid + Cfg.ServerPort, err = strconv.Atoi(os.Getenv("SERVER_PORT")) + if err != nil { + msg := fmt.Sprintf("invalid server port %s: %v", os.Getenv("SERVER_PORT"), err) + return Cfg, errors.New(msg) + } + + Cfg.NodeLevel = os.Getenv("NODE_LEVEL") + Cfg.EdgeLevel = os.Getenv("EDGE_LEVEL") + + Cfg.Neo4jURI = os.Getenv("NEO4J_URI") + Cfg.Neo4jUsername = os.Getenv("NEO4J_USERNAME") + Cfg.Neo4jPassword = os.Getenv("NEO4J_PASSWORD") + + return Cfg, nil +} diff --git a/clients/neo4j-client/go.mod b/clients/neo4j-client/go.mod new file mode 100644 index 0000000..6f7ce9c --- /dev/null +++ b/clients/neo4j-client/go.mod @@ -0,0 +1,19 @@ +module neo4j-client + +go 1.21 + +replace SentryFlow/protobuf => ../../protobuf + +require ( + SentryFlow/protobuf v0.0.0-00010101000000-000000000000 + github.com/neo4j/neo4j-go-driver/v4 v4.4.7 + google.golang.org/grpc v1.64.0 +) + +require ( + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/protobuf v1.34.1 // indirect +) diff --git a/clients/neo4j-client/go.sum b/clients/neo4j-client/go.sum new file mode 100644 index 0000000..03037b3 --- /dev/null +++ b/clients/neo4j-client/go.sum @@ -0,0 +1,102 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/neo4j/neo4j-go-driver/v4 v4.4.7 h1:6D0DPI7VOVF6zB8eubY1lav7RI7dZ2mytnr3fj369Ow= +github.com/neo4j/neo4j-go-driver/v4 v4.4.7/go.mod h1:NexOfrm4c317FVjekrhVV8pHBXgtMG5P6GeweJWCyo4= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/clients/neo4j-client/main.go b/clients/neo4j-client/main.go new file mode 100644 index 0000000..2fcfbba --- /dev/null +++ b/clients/neo4j-client/main.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + "SentryFlow/protobuf" + "neo4j-client/client" + "neo4j-client/config" + + "google.golang.org/grpc" +) + +func main() { + // Load environment variables + cfg, err := config.LoadEnvVars() + if err != nil { + log.Fatalf("Could not load environment variables: %v", err) + } + + // Get arguments + nodeLevelPtr := flag.String("nodeLevel", "simple", "NodeLevel for storing API logs, {simple|detail}") + edgeLevelPtr := flag.String("edgeLevel", "simple", "EdgeLevel for storing API logs, {simple|detail}") + neo4jURIPtr := flag.String("neo4jHost", "", "Neo4j Host") + neo4jUsernamePtr := flag.String("neo4jId", "", "Neo4j Id") + neo4jPasswordPtr := flag.String("neo4jPassword", "", "Neo4j Password") + flag.Parse() + + if cfg.NodeLevel != "" { + *nodeLevelPtr = cfg.NodeLevel + } + if cfg.EdgeLevel != "" { + *edgeLevelPtr = cfg.EdgeLevel + } + if cfg.Neo4jURI != "" { + *neo4jURIPtr = cfg.Neo4jURI + } + if cfg.Neo4jUsername != "" { + *neo4jUsernamePtr = cfg.Neo4jUsername + } + if cfg.Neo4jPassword != "" { + *neo4jPasswordPtr = cfg.Neo4jPassword + } + + if *nodeLevelPtr == "" && *edgeLevelPtr == "" { + flag.PrintDefaults() + return + } + + // == // + + // Construct a string "ServerAddr:ServerPort" + addr := fmt.Sprintf("%s:%d", cfg.ServerAddr, cfg.ServerPort) + + // Connect to the gRPC server of SentryFlow + conn, err := grpc.Dial(addr, grpc.WithInsecure()) + if err != nil { + log.Fatalf("[gRPC] Failed to connect: %v", err) + return + } + defer conn.Close() + + // Connected to the gRPC server + log.Printf("[gRPC] Started to collect Logs from %s", addr) + + // Define clientInfo + clientInfo := &protobuf.ClientInfo{ + HostName: cfg.Hostname, + } + + // Create a gRPC client for the SentryFlow service + sfClient := protobuf.NewSentryFlowClient(conn) + + // Create a log client with the gRPC client + logClient := client.NewClient(sfClient, clientInfo, *nodeLevelPtr, *edgeLevelPtr, *neo4jURIPtr, *neo4jUsernamePtr, *neo4jPasswordPtr) + if logClient == nil { + log.Fatalf("[gRPC] Failed to create gRPC client") + return + } + + go logClient.APILogRoutine() + fmt.Printf("[APILog] Started to watch API logs\n") + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + <-signalChan + + logClient.DbHandler.Close() + close(logClient.Done) +} diff --git a/clients/neo4j-client/neo4jdb/neo4jHandler.go b/clients/neo4j-client/neo4jdb/neo4jHandler.go new file mode 100644 index 0000000..1c87d0d --- /dev/null +++ b/clients/neo4j-client/neo4jdb/neo4jHandler.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +package neo4jdb + +import ( + "log" + + pb "SentryFlow/protobuf" + + "github.com/neo4j/neo4j-go-driver/v4/neo4j" +) + +type Neo4jHandler struct { + Driver neo4j.Driver + Session neo4j.Session + + NodeLevel string + EdgeLevel string +} + +// NewHandler creates a new Neo4j handler +func NewNeo4jHandler(nodeLevel string, edgeLevel string, dbURI string, dbUsername string, dbPassword string) (*Neo4jHandler, error) { + var dbHandler Neo4jHandler + + // Create a new driver for Neo4j + driver, err := neo4j.NewDriver(dbURI, neo4j.BasicAuth(dbUsername, dbPassword, "")) + if err != nil { + log.Fatalf("Error connecting Neo4j %s: %v", dbURI, err) + return &dbHandler, err + } + + // Create session for Neo4j + session := driver.NewSession(neo4j.SessionConfig{}) + + dbHandler.Driver = driver + dbHandler.Session = session + + dbHandler.NodeLevel = nodeLevel + dbHandler.EdgeLevel = edgeLevel + + return &dbHandler, nil +} + +func (h *Neo4jHandler) Close() { + h.Session.Close() + h.Driver.Close() +} + +func (h *Neo4jHandler) CreateOrUpdateRelationship(APILog *pb.APILog) error { + var query string + var srcName string + var dstName string + + if h.NodeLevel == "simple" { + srcName = APILog.SrcLabel["app"] + dstName = APILog.DstLabel["app"] + } else { + srcName = APILog.SrcName + dstName = APILog.DstName + } + + if h.EdgeLevel == "simple" { + query = ` + MERGE (src:Pod {name: $srcNamem, namespace: $srcNamespace}) + ON CREATE SET src.name = $srcName + MERGE (dst:Pod {namespace: $dstNamespace, label: $dstLabel}) + ON CREATE SET dst.name = $dstName + MERGE (src)-[r:CALLS {method: $method}]->(dst) + ON CREATE SET r.weight = 1 + ON MATCH SET r.weight = r.weight + 1 + ` + } else { + query = ` + MERGE (src:Pod {name: $srcNamem, namespace: $srcNamespace}) + ON CREATE SET src.name = $srcName + MERGE (dst:Pod {namespace: $dstNamespace, label: $dstLabel}) + ON CREATE SET dst.name = $dstName + MERGE (src)-[r:CALLS {method: $method, path: $path}]->(dst) + ON CREATE SET r.weight = 1 + ON MATCH SET r.weight = r.weight + 1 + ` + } + + log.Printf("[HI] %v, %v", srcName, dstName) + + params := map[string]interface{}{ + "srcNamespace": APILog.SrcNamespace, + "srcName": srcName, + "dstNamespace": APILog.DstNamespace, + "dstName": dstName, + "method": APILog.Method, + "path": APILog.Path, + } + + _, err := h.Session.WriteTransaction(func(tx neo4j.Transaction) (interface{}, error) { + return tx.Run(query, params) + }) + + return err +} diff --git a/deployments/neo4j-client.yaml b/deployments/neo4j-client.yaml new file mode 100644 index 0000000..0a10baa --- /dev/null +++ b/deployments/neo4j-client.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: neo4j-client + namespace: sentryflow +spec: + replicas: 1 + selector: + matchLabels: + app: neo4j-client + template: + metadata: + labels: + app: neo4j-client + spec: + containers: + - name: neo4j-client + image: 5gsec/sentryflow-neo4j-client:v0.1 + ports: + - containerPort: 8080 + protocol: TCP + name: grpc + env: + - name: SERVER_ADDR + value: "sentryflow.sentryflow.svc.cluster.local" + - name: SERVER_PORT + value: "8080" + - name: NEO4J_URI + value: "" + - name: NEO4J_USERNAME + value: "" + - name: NEO4J_PASSWORD + value: "" + - name: NODE_LEVEL + value: "simple" + - name: EDGE_LEVEL + value: "simple" diff --git a/docs/neo4j_01.png b/docs/neo4j_01.png new file mode 100644 index 0000000..b22a084 Binary files /dev/null and b/docs/neo4j_01.png differ diff --git a/docs/neo4j_02.png b/docs/neo4j_02.png new file mode 100644 index 0000000..5ce16ff Binary files /dev/null and b/docs/neo4j_02.png differ diff --git a/docs/neo4j_03.png b/docs/neo4j_03.png new file mode 100644 index 0000000..6ca44ff Binary files /dev/null and b/docs/neo4j_03.png differ diff --git a/examples/README.md b/examples/README.md index f162656..d81fe1f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,5 @@ # Examples - [Single HTTP Requests](httpbin/README.md) +- [Bookinfo Demo Microservice](bookinfo/README.md) - [RobotShop Demo Microservice](robotshop/README.md)