neo4j image patch

.github/workflows/ci-test-py.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Lint with Ruff
         run: |
           pip install ruff
-          ruff --output-format=github .
+          ruff check --output-format=github .
         working-directory: ai-engine
 
   py-lint-ai-sentryflow:

clients/log-client/README.md

@@ -0,0 +1,33 @@
+# Log Client
+Log client collects AccessLogs and Metrics from SentryFlow and prints them to the terminal or saves them to a log file.
+
+## Log Client Deployment
+Log client can be deployed using kubectl command. The deployment can be accomplished with the following
+commands:
+```bash
+$ cd SentryFlow/deployments
+$ kubectl apply -f log-client.yaml
+```
+
+## Log client options
+These are the default env value in the log-client.yaml file.
+```bash
+env:
+- name: LOG_CFG
+  value: "stdout"
+- name: METRIC_CFG
+  value: "stdout"
+- name: METRIC_FILTER
+  value: "api"
+```
+
+If you want to change the default env value, you can refer to the following options.
+```bash
+env:
+- name: LOG_CFG
+  value: {"stdout"|"file"|"none"}
+- name: METRIC_CFG
+  value: {"stdout"|"file"|"none"}
+- name: METRIC_FILTER
+  value: {"all"|"api"|"envoy"}
+```

clients/mongo-client/README.md

@@ -0,0 +1,29 @@
+# Mongo Client
+Mongo client collects AccessLogs and Metrics from SentryFlow and stores them to database.
+
+## Mongo Client Deployment
+Mongo client can be deployed using kubectl command. The deployment can be accomplished with the following
+commands:
+```bash
+$ cd SentryFlow/deployments
+$ kubectl apply -f mongo-client.yaml
+```
+
+## Mongo client options
+These are the default env value.
+```bash
+- LOG_CFG: mongodb
+- METRIC_CFG: mongodb
+- METRIC_FILTER: envoy
+```
+
+If you want to change the default env value, you can refer to the following options.
+```bash
+env:
+- name: LOG_CFG
+  value: {"mongodb"|"none"}
+- name: METRIC_CFG
+  value: {"mongodb"|"none"}
+- name: METRIC_FILTER
+  value: {"all"|"api"|"envoy"}
+```

clients/neo4j-client/Dockerfile

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+
+### Builder
+
+FROM golang:1.21-alpine3.17 as builder
+
+RUN apk --no-cache update
+RUN apk add --no-cache git clang llvm make gcc protobuf musl-dev
+
+RUN mkdir /app
+RUN mkdir /protobuf
+
+WORKDIR /protobuf
+
+COPY /protobuf .
+
+WORKDIR /app
+
+COPY /clients/neo4j-client .
+
+RUN go build -o neo4j-client
+
+### Make executable image
+
+FROM alpine:3.18 as client
+
+RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories
+
+RUN apk --no-cache update
+RUN apk add bash
+
+COPY --from=builder /app/neo4j-client /
+
+CMD ["/neo4j-client"]

clients/neo4j-client/Makefile

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+
+CLIENT_NAME = sentryflow-neo4j-client
+IMAGE_NAME = 5gsec/$(CLIENT_NAME)
+TAG = v0.1
+
+.PHONY: gofmt
+gofmt:
+	cd $(CURDIR); gofmt -w -s -d $(shell find . -type f -name '*.go' -print)
+
+.PHONY: golint
+golint:
+ifeq (, $(shell which golint))
+	@{ \
+	set -e ;\
+	GOLINT_TEMP_DIR=$$(mktemp -d) ;\
+	cd $$GOLINT_TEMP_DIR ;\
+	go mod init tmp ;\
+	go get golang.org/x/lint/golint ;\
+	go install golang.org/x/lint/golint ;\
+	rm -rf $$GOLINT_TEMP_DIR ;\
+	}
+endif
+	cd $(CURDIR); golint ./...
+
+.PHONY: gosec
+gosec:
+ifeq (, $(shell which gosec))
+	@{ \
+	set -e ;\
+	GOSEC_TEMP_DIR=$$(mktemp -d) ;\
+	cd $$GOSEC_TEMP_DIR ;\
+	go mod init tmp ;\
+	go get github.com/securego/gosec/v2/cmd/gosec ;\
+	go install github.com/securego/gosec/v2/cmd/gosec ;\
+	rm -rf $$GOSEC_TEMP_DIR ;\
+	}
+endif
+	cd $(CURDIR); gosec -exclude=G402 ./...
+
+.PHONY: build gofmt golint gosec
+build:
+	go mod tidy
+	go build -o $(CLIENT_NAME)
+
+.PHONY: clean
+clean:
+	rm -f $(CLIENT_NAME)
+
+.PHONY: build-image
+build-image:
+	docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../../
+
+.PHONY: clean-image
+clean-image:
+	docker rmi $(IMAGE_NAME):$(TAG)
+
+.PHONY: run-image
+run-image:
+	docker run -it --rm $(IMAGE_NAME):$(TAG)

clients/neo4j-client/README.md

@@ -0,0 +1,59 @@
+# Neo4j Client
+The Neo4j client collects AccessLogs from SentryFlow, stores them, and visualizes them.
+
+## Neo4j Client Deployment
+Neo4j client can be deployed using kubectl command. The deployment can be accomplished with the following
+commands:
+```bash
+$ cd SentryFlow/deployments
+$ kubectl apply -f neo4j-client.yaml
+```
+
+## Neo4j settings
+### Step 1. Create Neo4j account
+Go to https://neo4j.com/ and create an account
+
+### Step 2. Create Neo4j Instance
+Remember the Username and Password you created when creating the instance.
+
+### Step 3. Modify env value in neo4j-client.yaml file.
+Put the Connection URI specified in the instance into NEO4J_URI, and the information created in Step 2 into NEO4J_USERNAME and NEO4J_PASSWORD, respectively.
+
+```bash
+env:
+- name: NEO4J_URI
+  value: ""
+- name: NEO4J_USERNAME
+  value: ""
+- name: NEO4J_PASSWORD
+  value: ""
+```
+
+## Neo4j client options
+These are the default env value in the neo4j-client.yaml file.
+```bash
+env:
+- name: NODE_LEVEL
+  value: "simple"
+- name: EDGE_LEVEL
+  value: "simple"
+```
+
+If you want to change the default env value, you can refer to the following options.
+```bash
+env:
+- name: NODE_LEVEL
+  value: {"simple"|"detail"}
+- name: EDGE_LEVEL
+  value: {"simple"|"detail"}
+```
+
+## Example with robot-shop
+### Example 1 (NODE_LEVEL: simple, EDGE_LEVEL: simple)
+![Neo4j example1](/docs/neo4j_01.png)
+
+### Example 2 (NODE_LEVEL: simple, EDGE_LEVEL: detail)
+![Neo4j example2](/docs/neo4j_02.png)
+
+### Example 3 (NODE_LEVEL: detail, EDGE_LEVEL: detail)
+![Neo4j example3](/docs/neo4j_03.png)

clients/neo4j-client/client/client.go

@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package client
+
+import (
+	pb "SentryFlow/protobuf"
+	"context"
+	"io"
+	"log"
+	"neo4j-client/neo4jdb"
+)
+
+// Feeder Structure
+type Feeder struct {
+	Running bool
+
+	client    pb.SentryFlowClient
+	logStream pb.SentryFlow_GetAPILogClient
+
+	DbHandler neo4jdb.Neo4jHandler
+
+	Done chan struct{}
+}
+
+// NewClient Function
+func NewClient(client pb.SentryFlowClient, clientInfo *pb.ClientInfo, nodeLevel string, edgeLevel string, neo4jHost string, neo4jId string, neo4jPassword string) *Feeder {
+	fd := &Feeder{}
+
+	fd.Running = true
+	fd.client = client
+	fd.Done = make(chan struct{})
+
+	// Contact the server and print out its response
+	logStream, err := client.GetAPILog(context.Background(), clientInfo)
+	if err != nil {
+		log.Fatalf("[Client] Could not get API log: %v", err)
+	}
+
+	fd.logStream = logStream
+
+	// Initialize DB
+	dbHandler, err := neo4jdb.NewNeo4jHandler(nodeLevel, edgeLevel, neo4jHost, neo4jId, neo4jPassword)
+	if err != nil {
+		log.Fatalf("[MongoDB] Unable to intialize DB: %v", err)
+		return nil
+	}
+	fd.DbHandler = *dbHandler
+
+	return fd
+}
+
+// APILogRoutine Function
+func (fd *Feeder) APILogRoutine() {
+	for fd.Running {
+		select {
+		default:
+			data, err := fd.logStream.Recv()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				log.Fatalf("failed to receive log: %v", err)
+			}
+			log.Printf("[Client] Inserting log")
+			err = fd.DbHandler.CreateOrUpdateRelationship(data)
+			if err != nil {
+				log.Printf("[Client] Failed to insert log: %v", err)
+			}
+		case <-fd.Done:
+			return
+		}
+	}
+}

clients/neo4j-client/config/config.go

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+)
+
+// Config structure
+type Config struct {
+	Hostname string
+
+	ServerAddr string
+	ServerPort int
+
+	NodeLevel string
+	EdgeLevel string
+
+	Neo4jURI      string
+	Neo4jUsername string
+	Neo4jPassword string
+}
+
+// Cfg is for global reference
+var Cfg Config
+
+// LoadEnvVars loads environment variables and stores them as global variable
+func LoadEnvVars() (Config, error) {
+	var err error
+
+	Cfg.Hostname, err = os.Hostname()
+	if err != nil {
+		msg := fmt.Sprintf("[Config] Could not find hostname: %v", err)
+		return Cfg, errors.New(msg)
+	}
+
+	// load listen address and check if valid
+	Cfg.ServerAddr = os.Getenv("SERVER_ADDR")
+
+	// load listen port and check if valid
+	Cfg.ServerPort, err = strconv.Atoi(os.Getenv("SERVER_PORT"))
+	if err != nil {
+		msg := fmt.Sprintf("invalid server port %s: %v", os.Getenv("SERVER_PORT"), err)
+		return Cfg, errors.New(msg)
+	}
+
+	Cfg.NodeLevel = os.Getenv("NODE_LEVEL")
+	Cfg.EdgeLevel = os.Getenv("EDGE_LEVEL")
+
+	Cfg.Neo4jURI = os.Getenv("NEO4J_URI")
+	Cfg.Neo4jUsername = os.Getenv("NEO4J_USERNAME")
+	Cfg.Neo4jPassword = os.Getenv("NEO4J_PASSWORD")
+
+	return Cfg, nil
+}

clients/neo4j-client/go.mod

@@ -0,0 +1,19 @@
+module neo4j-client
+
+go 1.21
+
+replace SentryFlow/protobuf => ../../protobuf
+
+require (
+	SentryFlow/protobuf v0.0.0-00010101000000-000000000000
+	github.com/neo4j/neo4j-go-driver/v4 v4.4.7
+	google.golang.org/grpc v1.64.0
+)
+
+require (
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+)