confgenerator: add NFS path detection

confgenerator/logging_receivers.go

@@ -18,8 +18,10 @@
 	"context"
 	"fmt"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
@@ -56,6 +58,7 @@
 		ExcludePaths:            r.ExcludePaths,
 		WildcardRefreshInterval: r.WildcardRefreshInterval,
 		RecordLogFilePath:       r.RecordLogFilePath,
+		syscallStatfs:           syscall.Statfs,
 	}
 }
 
@@ -67,13 +70,18 @@
 	return r.mixin().Pipelines(ctx)
 }
 
+type statfsFunc func(path string, buf *syscall.Statfs_t) error
+
 type LoggingReceiverFilesMixin struct {
 	IncludePaths            []string        `yaml:"include_paths,omitempty"`
 	ExcludePaths            []string        `yaml:"exclude_paths,omitempty"`
 	WildcardRefreshInterval *time.Duration  `yaml:"wildcard_refresh_interval,omitempty" validate:"omitempty,min=1s,multipleof_time=1s"`
 	MultilineRules          []MultilineRule `yaml:"-"`
 	BufferInMemory          bool            `yaml:"-"`
 	RecordLogFilePath       *bool           `yaml:"record_log_file_path,omitempty"`
+
+	// For mocking the statfs syscall
+	syscallStatfs statfsFunc
 }
 
 func (r LoggingReceiverFilesMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
@@ -115,6 +123,10 @@
 		// as a hint to set "how much data can be up in memory", once the limit is reached it continues writing to disk.
 		"Mem_Buf_Limit": "10M",
 	}
+	// If trying to read
+	if platform.FromContext(ctx).Type == platform.Linux && r.detectNFSPaths() {
+		config["Inotify_Watcher"] = "false"
+	}
 	if len(r.ExcludePaths) > 0 {
 		// TODO: Escaping?
 		config["Exclude_Path"] = strings.Join(r.ExcludePaths, ",")
@@ -169,6 +181,42 @@
 	return c
 }
 
+// See https://man7.org/linux/man-pages/man2/statfs.2.html for details.
+const NFS_SUPER_MAGIC = 0x6969
+
+func (r LoggingReceiverFilesMixin) detectNFSPaths() bool {
+	for _, path := range r.IncludePaths {
+		statPath, err := filepath.Abs(path)
+		if err != nil {
+			// This happens if os.Getwd fails in trying to get
+			// absolute path. In this scenario there is no way
+			// we can even try to detect an NFS path.
+			break
+		}
+
+		for strings.Contains(statPath, "*") {
+			// The statfs syscall doesn't work with wildcards in the path.
+			// Trim pieces away from the path until there are no wildcards.
+			filename := filepath.Base(statPath)
+			statPath = strings.TrimSuffix(statPath, filename)
+			// The clean is necessary because there will be a path separator on the
+			// end after the trim.
+			statPath = filepath.Clean(statPath)
+		}
+
+		var sfs syscall.Statfs_t
+		err = r.syscallStatfs(statPath, &sfs)
+		if err != nil {
+			// TODO print the error here?
+			continue
+		}
+		if sfs.Type == NFS_SUPER_MAGIC {
+			return true
+		}
+	}
+	return false
+}
+
 func (r LoggingReceiverFilesMixin) Pipelines(ctx context.Context) ([]otel.ReceiverPipeline, error) {
 	operators := []map[string]any{}
 	receiver_config := map[string]any{

confgenerator/logging_receivers_test.go

@@ -0,0 +1,94 @@
+package confgenerator
+
+import (
+	"runtime"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+// See https://man7.org/linux/man-pages/man2/statfs.2.html
+const EXT4_SUPER_MAGIC = 0xef53
+
+func TestDetectNFSPaths(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("This unit test only runs on Linux")
+	}
+
+	testCases := []struct {
+		name            string
+		includePaths    []string
+		nfsPaths        []string
+		shouldDetectNFS bool
+	}{
+		{
+			name: "direct path",
+			includePaths: []string{
+				"/a/b/x.log",
+			},
+			nfsPaths: []string{
+				"/a/b",
+			},
+			shouldDetectNFS: true,
+		},
+		{
+			name: "wildcard file",
+			includePaths: []string{
+				"/a/b/*.log",
+			},
+			nfsPaths: []string{
+				"/a/b",
+			},
+			shouldDetectNFS: true,
+		},
+		{
+			name: "wildcard in folder",
+			includePaths: []string{
+				"/a/b/c*/*.log",
+			},
+			nfsPaths: []string{
+				"/a/b",
+			},
+			shouldDetectNFS: true,
+		},
+		// If the NFS folder is part of a wildcard, we will not be able to
+		// detect it.
+		{
+			name: "wildcard in NFS folder",
+			includePaths: []string{
+				"/a/*/*.log",
+			},
+			nfsPaths: []string{
+				"/a/b",
+			},
+			shouldDetectNFS: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := LoggingReceiverFiles{
+				IncludePaths: tc.includePaths,
+			}
+			m := r.mixin()
+			m.syscallStatfs = pretendStatfs(tc.nfsPaths)
+			nfsFound := m.detectNFSPaths()
+			if nfsFound != tc.shouldDetectNFS {
+				t.Fatalf("expected detectNFSPaths to return %t but returned %t", tc.shouldDetectNFS, nfsFound)
+			}
+		})
+	}
+}
+
+func pretendStatfs(paths []string) statfsFunc {
+	return func(path string, buf *syscall.Statfs_t) error {
+		for _, testPath := range paths {
+			if strings.HasPrefix(path, testPath) {
+				buf.Type = NFS_SUPER_MAGIC
+				return nil
+			}
+		}
+		buf.Type = EXT4_SUPER_MAGIC
+		return nil
+	}
+}

integration_test/ops_agent_test/main_test.go

@@ -2809,7 +2809,7 @@ func TestPrometheusMetricsWithJSONExporter(t *testing.T) {
             module: [default]
           static_configs:
             - targets:
-              - http://localhost:8000/data.json 
+              - http://localhost:8000/data.json
           relabel_configs:
             - source_labels: [__address__]
               target_label: __param_target
@@ -2818,7 +2818,7 @@ func TestPrometheusMetricsWithJSONExporter(t *testing.T) {
               target_label: instance
               replacement: '$1'
             - target_label: __address__
-              replacement: localhost:7979 
+              replacement: localhost:7979
   service:
     pipelines:
       prom_pipeline:

integration_test/third_party_apps_test/applications/nfs/README.md

@@ -0,0 +1,12 @@
+# NFS
+
+This third party app test differs from the rest. The third party app test framework is used because it is somewhat similar (there is need to install and exercise a third party application before the test can work) but it does not directly map to a third party application receiver. A `files` receiver reads a log file from the shared NFS directory, and the expectations are:
+* The logs are read and sent to Cloud Logging
+* The tailed log file getting deleted from the directory is registered by the Ops Agent, and the `.nfsxxxxx` files are deleted as well
+
+## Shared values
+
+There is no official way to share values among these scripts so certain values are assumed across scripts:
+* NFS Share directory: `/mnt/nfssrv`
+* NFS Mount directory: `/var/nfsmnt`
+* Name of log file in NFS share directory: `example.log`

integration_test/third_party_apps_test/applications/nfs/centos_rhel/install

@@ -0,0 +1,16 @@
+function setup_nfs_share {
+    NFS_SERVER_SERVICE_NAME=$1
+    NFS_SERVE_DIR=/mnt/nfssrv
+    NFS_MOUNT_DIR=/var/nfsmnt
+    sudo mkdir -p $NFS_SERVE_DIR
+    sudo chown nobody:nogroup $NFS_SERVE_DIR
+    sudo chmod 777 $NFS_SERVE_DIR
+    echo "$NFS_SERVE_DIR localhost(rw,sync,no_subtree_check)"
+    sudo exportfs -a
+    sudo systemctl restart $NFS_SERVER_SERVICE_NAME
+    sudo mkdir -p $NFS_MOUNT_DIR
+    sudo mount -t nfs localhost:$NFS_SERVE_DIR $NFS_MOUNT_DIR
+}
+
+sudo yum -y install nfs-utils
+setup_nfs_share nfs-server

integration_test/third_party_apps_test/applications/nfs/debian_ubuntu/install

@@ -0,0 +1,16 @@
+function setup_nfs_share {
+    NFS_SERVER_SERVICE_NAME=$1
+    NFS_SERVE_DIR=/mnt/nfssrv
+    NFS_MOUNT_DIR=/var/nfsmnt
+    sudo mkdir -p $NFS_SERVE_DIR
+    sudo chown nobody:nogroup $NFS_SERVE_DIR
+    sudo chmod 777 $NFS_SERVE_DIR
+    echo "$NFS_SERVE_DIR localhost(rw,sync,no_subtree_check)"
+    sudo exportfs -a
+    sudo systemctl restart $NFS_SERVER_SERVICE_NAME
+    sudo mkdir -p $NFS_MOUNT_DIR
+    sudo mount -t nfs localhost:$NFS_SERVE_DIR $NFS_MOUNT_DIR
+}
+
+sudo apt install nfs-kernel-server
+setup_nfs_share nfs-kernel-server

integration_test/third_party_apps_test/applications/nfs/enable

@@ -0,0 +1,23 @@
+# Configures Ops Agent to collect telemetry from the app and restart Ops Agent.
+
+set -e
+
+# Create a back up of the existing file so existing configurations are not lost.
+sudo cp /etc/google-cloud-ops-agent/config.yaml /etc/google-cloud-ops-agent/config.yaml.bak
+
+# Configure the Ops Agent.
+sudo tee /etc/google-cloud-ops-agent/config.yaml > /dev/null << EOF
+logging:
+  receivers:
+    nfs_file:
+      type: files
+      include_paths: ["/var/nfsmnt/example.log"]
+  service:
+    pipelines:
+      files:
+        receivers:
+        - nfs_file
+EOF
+
+sudo service google-cloud-ops-agent restart
+sleep 60

integration_test/third_party_apps_test/applications/nfs/exercise

@@ -0,0 +1,10 @@
+NFS_SHARE_DIRECTORY=/mnt/nfssrv
+NFS_LOG_FILE=$NFS_SHARE_DIRECTORY/example.log
+
+sudo tee $NFS_LOG_FILE > /dev/null << EOF
+Example log line 1
+Example log line 2
+Example log line 3
+Example log line 4
+Example log line 5
+EOF

integration_test/third_party_apps_test/applications/nfs/sles/install

@@ -0,0 +1,16 @@
+function setup_nfs_share {
+    NFS_SERVER_SERVICE_NAME=$1
+    NFS_SERVE_DIR=/mnt/nfssrv
+    NFS_MOUNT_DIR=/var/nfsmnt
+    sudo mkdir -p $NFS_SERVE_DIR
+    sudo chown nobody:nogroup $NFS_SERVE_DIR
+    sudo chmod 777 $NFS_SERVE_DIR
+    echo "$NFS_SERVE_DIR localhost(rw,sync,no_subtree_check)"
+    sudo exportfs -a
+    sudo systemctl restart $NFS_SERVER_SERVICE_NAME
+    sudo mkdir -p $NFS_MOUNT_DIR
+    sudo mount -t nfs localhost:$NFS_SERVE_DIR $NFS_MOUNT_DIR
+}
+
+sudo zypper install nfs-kernel-server
+setup_nfs_share nfs-kernel-server