feat: use rust via FFI for memory efficient diff generation (#879)

* feat: use rust via FFI for memory efficient diff generation

* chore: add rust setup in test

* chore: add toggle via properties

* chore: use build tags for rust

* chore: add toggle via properties

* chore: add make build-prod

* chore: add properties to toggle rust invokation

.github/workflows/test.yml

@@ -27,3 +27,27 @@ jobs:
             cache-
       - name: Test
         run: make test
+
+  test-prod:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Install Go
+        uses: buildjet/setup-go@v5
+        with:
+          go-version: 1.22.x
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Checkout code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - uses: buildjet/cache@v4
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            .bin
+          key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
+          restore-keys: |
+            cache-
+      - name: Test
+        run: make test-prod

.gitignore

@@ -17,3 +17,7 @@ config-db.properties
 output-*/
 configs/
 *.pprof
+
+
+# Rust
+external/diffgen/target

Makefile

@@ -60,10 +60,18 @@ resources: fmt manifests
 test: manifests generate fmt vet envtest  ## Run tests.
 	$(MAKE) gotest
 
+test-prod: manifests generate fmt vet envtest  ## Run tests.
+	$(MAKE) gotest-prod
+
 .PHONY: gotest
 gotest:
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
 
+.PHONY: gotest-prod
+gotest-prod:
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -tags rustdiffgen ./... -coverprofile cover.out
+
+
 .PHONY: env
 env: envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
@@ -108,7 +116,11 @@ lint:
 
 .PHONY: build
 build:
-	go build -o ./.bin/$(NAME) -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+	go build -o ./.bin/$(NAME) -ldflags "-X \"main.version=$(VERSION_TAG)\"" .
+
+.PHONY: build-prod
+build-prod:
+	go build -o ./.bin/$(NAME) -ldflags "-X \"main.version=$(VERSION_TAG)\"" -tags rustdiffgen .
 
 .PHONY: install
 install:
@@ -177,9 +189,6 @@ helm-schema:
 	test -s $(LOCALBIN)/helm-schema  || \
 	GOBIN=$(LOCALBIN) go install github.com/dadav/helm-schema/cmd/helm-schema@latest
 
-
-
-
 .PHONY: controller-gen
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten.
 $(CONTROLLER_GEN): $(LOCALBIN)
@@ -192,3 +201,13 @@ CONTROLLER_RUNTIME_VERSION = v0.0.0-20240320141353-395cfc7486e6
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(CONTROLLER_RUNTIME_VERSION)
+
+.PHONY: rust-diffgen
+rust-diffgen:
+	cd external/diffgen && cargo build --release
+
+.PHONY: rust-generate-header
+rust-generate-header:
+	cargo install cbindgen
+	cd external/diffgen
+	cbindgen . -o libdiffgen.h --lang c

build/Dockerfile

@@ -1,4 +1,11 @@
-FROM golang:1.22.5@sha256:86a3c48a61915a8c62c0e1d7594730399caa3feb73655dfe96c7bc17710e96cf as builder
+FROM rust:bookworm@sha256:29fe4376919e25b7587a1063d7b521d9db735fc137d3cf30ae41eb326d209471 AS rust-builder
+
+WORKDIR /app
+COPY Makefile /app
+COPY external/diffgen /app/external/diffgen
+RUN make rust-diffgen
+
+FROM golang:1.22.5@sha256:86a3c48a61915a8c62c0e1d7594730399caa3feb73655dfe96c7bc17710e96cf AS builder
 WORKDIR /app
 
 ARG VERSION
@@ -8,7 +15,10 @@ COPY go.sum /app/go.sum
 RUN go mod download
 
 COPY ./ ./
-RUN make build
+
+COPY --from=rust-builder /app/external/diffgen/target ./external/diffgen/target
+
+RUN make build-prod
 
 FROM flanksource/base-image:v0.0.7@sha256:c3cda640ca7033a89e52c7f27776edfc95f825ece4b49de3b9c5af981d34a44e
 WORKDIR /app

db/diff.go

@@ -3,14 +3,20 @@ package db
 import (
 	"encoding/json"
 	"fmt"
+	"sync"
 
+	"github.com/flanksource/commons/properties"
 	dutyContext "github.com/flanksource/duty/context"
 	"github.com/hexops/gotextdiff"
 	"github.com/hexops/gotextdiff/myers"
 	"github.com/ohler55/ojg"
 	"github.com/ohler55/ojg/oj"
 )
 
+// We expose this function to replace it with a rust function called via FFI
+// when the build tag rustdiffgen is provided
+var DiffFunc func(string, string) string = TextDiff
+
 // NormalizeJSON returns an indented json string.
 // The keys are sorted lexicographically.
 func NormalizeJSONOj(object any) (string, error) {
@@ -90,16 +96,26 @@ func generateDiff(newConf, prevConfig string) (string, error) {
 		return "", fmt.Errorf("failed to normalize json for new config: %w", err)
 	}
 
-	// Compare again. They might be equal after normalization.
-	if newConf == prevConfig {
+	if before == after {
 		return "", nil
 	}
 
+	// If we compile the code with rustdiffgen tag, we still might
+	// want to disable rust invokation
+	var once sync.Once
+	once.Do(func() {
+		if properties.On(false, "diff.rust-gen") {
+			DiffFunc = TextDiff
+		}
+	})
+
+	return DiffFunc(before, after), nil
+}
+
+func TextDiff(before, after string) string {
 	edits := myers.ComputeEdits("", before, after)
 	if len(edits) == 0 {
-		return "", nil
+		return ""
 	}
-
-	diff := fmt.Sprint(gotextdiff.ToUnified("before", "after", before, edits))
-	return diff, nil
+	return fmt.Sprint(gotextdiff.ToUnified("before", "after", before, edits))
 }

external/diffgen/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "diffgen"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+libc = "0.2.158"
+similar = { git = "https://github.com/mitsuhiko/similar", version = "2.6.0" }
+
+
+[lib]
+crate-type = ["cdylib", "staticlib"]

external/diffgen/libdiffgen.h

@@ -0,0 +1,6 @@
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+char *diff(const char *before, const char *after);

external/diffgen/src/lib.rs

@@ -0,0 +1,41 @@
+extern crate similar;
+
+use similar::TextDiff;
+use std::ffi::{CStr, CString};
+
+#[no_mangle]
+pub extern "C" fn diff(
+    before: *const libc::c_char,
+    after: *const libc::c_char,
+) -> *mut libc::c_char {
+    let before_cstr = unsafe { CStr::from_ptr(before) };
+    let before_str = before_cstr.to_str().unwrap();
+
+    let after_cstr = unsafe { CStr::from_ptr(after) };
+    let after_str = after_cstr.to_str().unwrap();
+
+    let diff = TextDiff::from_lines(before_str, after_str);
+
+    CString::new(diff.unified_diff().to_string())
+        .unwrap()
+        .into_raw()
+}
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+    use std::ffi::CString;
+
+    #[test]
+    fn test_diff() {
+        let diff_result = diff(
+            CString::new("hello\nworld\n").unwrap().into_raw(),
+            CString::new("bye\nworld\n").unwrap().into_raw(),
+        );
+
+        assert_eq!(
+            unsafe { CStr::from_ptr(diff_result) }.to_str().unwrap(),
+            "@@ -1,2 +1,2 @@\n-hello\n+bye\n world\n"
+        )
+    }
+}

rustdiffgen.go

@@ -0,0 +1,37 @@
+//go:build rustdiffgen
+
+package main
+
+/*
+#cgo LDFLAGS: ${SRCDIR}/external/diffgen/target/release/libdiffgen.a -ldl
+#include "./external/diffgen/libdiffgen.h"
+#include <stdlib.h>
+*/
+import "C"
+
+import (
+	"unsafe"
+
+	"github.com/flanksource/config-db/db"
+)
+
+func init() {
+	db.DiffFunc = func(before, after string) string {
+		beforeCString := C.CString(before)
+		defer C.free(unsafe.Pointer(beforeCString))
+
+		afterCString := C.CString(after)
+		defer C.free(unsafe.Pointer(afterCString))
+
+		diffChar := C.diff(beforeCString, afterCString)
+		defer C.free(unsafe.Pointer(diffChar))
+		if diffChar == nil {
+			return ""
+		}
+
+		// prefix is required for UI
+		prefix := "--- before\n+++ after\n"
+		diff := C.GoString(diffChar)
+		return prefix + diff
+	}
+}