feat(bpf): implement stack bypass (#458)

Co-authored-by: Sumire (菫) <[email protected]>
daeuniverse · Mar 1, 2024 · 6f1db5e · 6f1db5e
1 parent bb3a07c
commit 6f1db5e
Show file tree

Hide file tree

Showing 16 changed files with 476 additions and 508 deletions.
diff --git a/.github/workflows/kernel-test.yml b/.github/workflows/kernel-test.yml
@@ -40,7 +40,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kernel: [ '5.10-v0.3', '5.15-v0.3', '6.3-main', 'bpf-next-20231030.012704' ]
+        kernel: [ '5.10-20240201.165956', '5.15-20240201.165956', '6.1-20240201.165956', 'bpf-next-20240204.012837' ]
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -52,7 +52,7 @@ jobs:
           path: dae
 
       - name: Provision LVH VMs
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           test-name: dae-test
           image-version: ${{ matrix.kernel }}
@@ -66,7 +66,7 @@ jobs:
             apt install -y unzip
 
       - name: Setup network
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -77,7 +77,7 @@ jobs:
             docker run -td --name dae --privileged --network dae -v /host:/host -v /sys:/sys ubuntu:22.04 bash
 
       - name: Setup v2ray server
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -125,7 +125,7 @@ jobs:
             echo '{"v":"2","ps":"test","add":"v2ray","port":"23333","id":"b004539e-0d7b-7996-c378-fb040e42de70","aid":"0","net":"tcp","tls":"","type":"none","path":"","host":"v2ray"}' > vmess.json
 
       - name: Setup dae server
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -165,11 +165,11 @@ jobs:
 
             chmod 600 ./conf.dae
             nohup docker exec dae /host/dae/dae run -c /host/conf.dae &> dae.log &
-            sleep 10s
+            sleep 5s
             cat dae.log
 
       - name: Check WAN IPv4 TCP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -180,7 +180,7 @@ jobs:
             cat /host/v2ray.access.log | grep -q 'accepted tcp:1.1.1.1:443'
 
       - name: Check WAN IPv4 UDP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -191,7 +191,7 @@ jobs:
             cat /host/v2ray.access.log | grep -q 'accepted udp:1.1.1.1:53'
 
       - name: Check WAN IPv6 TCP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -202,7 +202,41 @@ jobs:
             cat /host/dae.log | grep -F -- '-> [2606:4700:4700::1111]:443'
 
       - name: Check WAN IPv6 UDP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+
+            docker exec dae dig @2606:4700:4700::1111 one.one.one.one
+            cat /host/dae.log | grep -F -- '-> [2606:4700:4700::1111]:53'
+
+      - name: Setup WAN UDP port conflict
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+
+            docker restart -t0 dae v2ray
+            nohup docker exec v2ray v2ray -c /host/v2ray.json &> v2ray.log &
+            nohup docker exec dae /host/dae/dae run -c /host/conf.dae &> dae.log &
+            sleep 5s
+            nohup docker exec dae nc -lu 53 &> nc.log &
+
+      - name: Check WAN IPv4 UDP with port conflict
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+
+            docker exec dae dig @1.1.1.1 one.one.one.one
+            cat /host/dae.log | grep -F -- '-> 1.1.1.1:53'
+            cat /host/v2ray.access.log | grep -q 'accepted udp:1.1.1.1:53'
+
+      - name: Check WAN IPv6 UDP with port conflict
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -212,7 +246,7 @@ jobs:
             cat /host/dae.log | grep -F -- '-> [2606:4700:4700::1111]:53'
 
       - name: Setup LAN
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -261,7 +295,7 @@ jobs:
                 log_level: trace
 
                 lan_interface: dae-veth-peer
-                wan_interface: auto
+                wan_interface: dae-veth-peer,eth0
                 allow_insecure: false
                 auto_config_kernel_parameter: true
             }
@@ -284,11 +318,11 @@ jobs:
 
             chmod 600 ./conf.dae
             nohup docker exec dae /host/dae/dae run -c /host/conf.dae &> dae.log &
-            sleep 10s
+            sleep 5s
             cat dae.log
 
       - name: Check LAN IPv4 TCP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -299,7 +333,7 @@ jobs:
             cat /host/v2ray.access.log | grep -q 'accepted tcp:1.0.0.1:80'
 
       - name: Check LAN IPv4 UDP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -310,7 +344,7 @@ jobs:
             cat /host/v2ray.access.log | grep -q 'accepted udp:8.8.4.4:53'
 
       - name: Check LAN IPv6 TCP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |
@@ -321,7 +355,47 @@ jobs:
             cat /host/dae.log | grep -F -- '-> [2606:4700:4700::1001]:80'
 
       - name: Check LAN IPv6 UDP
-        uses: cilium/little-vm-helper@908ab1ff8a596a03cd5221a1f8602dc44c3f906d # v0.0.12
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+
+            docker exec dae ip net e dae dig @2001:4860:4860::8844 one.one.one.one
+            cat /host/dae.log | grep -F -- '-> [2001:4860:4860::8844]:53'
+
+      - name: Setup LAN UDP port conflict
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+
+            docker restart -t0 dae v2ray
+
+            docker exec dae rm -f /var/run/netns/dae
+            docker exec dae bash /host/lan.bash
+            docker exec dae sysctl net.ipv4.conf.dae-veth-peer.send_redirects=0
+            docker exec dae sysctl net.ipv6.conf.dae-veth-peer.forwarding=1
+
+            nohup docker exec v2ray v2ray -c /host/v2ray.json &> v2ray.log &
+            nohup docker exec dae /host/dae/dae run -c /host/conf.dae &> dae.log &
+            sleep 5s
+            nohup docker exec dae nc -lu 53 &> nc.log &
+
+      - name: Check LAN IPv4 UDP with port conflict
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+
+            docker exec dae ip net e dae dig @8.8.4.4 one.one.one.one
+            cat /host/dae.log | grep -F -- '-> 8.8.4.4:53'
+            cat /host/v2ray.access.log | grep -q 'accepted udp:8.8.4.4:53'
+
+      - name: Check LAN IPv6 UDP with port conflict
+        uses: cilium/little-vm-helper@9d758b756305e83718a51b792a5aeabd022a39ec # v0.0.16
         with:
           provision: 'false'
           cmd: |

diff --git a/cmd/run.go b/cmd/run.go
@@ -135,9 +135,12 @@ func Run(log *logrus.Logger, conf *config.Config, externGeoDataDirs []string) (e
 				_ = os.WriteFile(PidFilePath, []byte(strconv.Itoa(os.Getpid())), 0644)
 			}
 		}()
-		if listener, err = c.ListenAndServe(readyChan, conf.Global.TproxyPort); err != nil {
-			log.Errorln("ListenAndServe:", err)
-		}
+		control.GetDaeNetns().With(func() error {
+			if listener, err = c.ListenAndServe(readyChan, conf.Global.TproxyPort); err != nil {
+				log.Errorln("ListenAndServe:", err)
+			}
+			return err
+		})
 		sigs <- nil
 	}()
 	reloading := false

diff --git a/config/config.go b/config/config.go
@@ -35,10 +35,11 @@ type Global struct {
 	DialMode                  string        `mapstructure:"dial_mode" default:"domain"`
 	DisableWaitingNetwork     bool          `mapstructure:"disable_waiting_network" default:"false"`
 	AutoConfigKernelParameter bool          `mapstructure:"auto_config_kernel_parameter" default:"false"`
-	AutoConfigFirewallRule    bool          `mapstructure:"auto_config_firewall_rule" default:"false"`
-	SniffingTimeout           time.Duration `mapstructure:"sniffing_timeout" default:"100ms"`
-	TlsImplementation         string        `mapstructure:"tls_implementation" default:"tls"`
-	UtlsImitate               string        `mapstructure:"utls_imitate" default:"chrome_auto"`
+	// DEPRECATED: not used as of https://github.com/daeuniverse/dae/pull/458
+	AutoConfigFirewallRule bool          `mapstructure:"auto_config_firewall_rule" default:"false"`
+	SniffingTimeout        time.Duration `mapstructure:"sniffing_timeout" default:"100ms"`
+	TlsImplementation      string        `mapstructure:"tls_implementation" default:"tls"`
+	UtlsImitate            string        `mapstructure:"utls_imitate" default:"chrome_auto"`
 }
 
 type Utls struct {

diff --git a/control/anyfrom_pool.go b/control/anyfrom_pool.go
@@ -192,7 +192,12 @@ func (p *AnyfromPool) GetOrCreate(lAddr string, ttl time.Duration) (conn *Anyfro
 			},
 			KeepAlive: 0,
 		}
-		pc, err := d.ListenPacket(context.Background(), "udp", lAddr)
+		var err error
+		var pc net.PacketConn
+		GetDaeNetns().With(func() error {
+			pc, err = d.ListenPacket(context.Background(), "udp", lAddr)
+			return nil
+		})
 		if err != nil {
 			return nil, true, err
 		}

diff --git a/control/bpf_utils.go b/control/bpf_utils.go
@@ -217,13 +217,24 @@ func fullLoadBpfObjects(
 	opts *loadBpfOptions,
 ) (err error) {
 retryLoadBpf:
+	netnsID, err := GetDaeNetns().NetnsID()
+	if err != nil {
+		return fmt.Errorf("failed to get netns id: %w", err)
+	}
 	constants := map[string]interface{}{
 		"PARAM": struct {
 			tproxyPort      uint32
 			controlPlanePid uint32
+			dae0Ifindex     uint32
+			dae0NetnsId     uint32
+			dae0peerMac     [6]byte
+			padding         [2]byte
 		}{
 			tproxyPort:      uint32(opts.BigEndianTproxyPort),
 			controlPlanePid: uint32(os.Getpid()),
+			dae0Ifindex:     uint32(GetDaeNetns().Dae0().Attrs().Index),
+			dae0NetnsId:     uint32(netnsID),
+			dae0peerMac:     [6]byte(GetDaeNetns().Dae0Peer().Attrs().HardwareAddr),
 		},
 	}
 	if err = loadBpfObjectsWithConstants(bpf, opts.CollectionOptions, constants); err != nil {

diff --git a/control/control_plane.go b/control/control_plane.go
@@ -128,6 +128,15 @@ func NewControlPlane(
 	if err = rlimit.RemoveMemlock(); err != nil {
 		return nil, fmt.Errorf("rlimit.RemoveMemlock:%v", err)
 	}
+
+	InitDaeNetns(log)
+	if err = InitSysctlManager(log); err != nil {
+		return nil, err
+	}
+
+	if err = GetDaeNetns().Setup(); err != nil {
+		return nil, fmt.Errorf("failed to setup dae netns: %w", err)
+	}
 	pinPath := filepath.Join(consts.BpfPinRoot, consts.AppName)
 	if err = os.MkdirAll(pinPath, 0755); err != nil && !os.IsExist(err) {
 		if os.IsNotExist(err) {
@@ -194,20 +203,6 @@ func NewControlPlane(
 		}
 	}()
 
-	if len(global.LanInterface) > 0 || len(global.WanInterface) > 0 {
-		if err = core.setupRoutingPolicy(); err != nil {
-			return nil, err
-		}
-		if global.AutoConfigFirewallRule {
-			if ok := core.addAcceptInputMark(); ok {
-				core.deferFuncs = append(core.deferFuncs, func() error {
-					core.delAcceptInputMark()
-					return nil
-				})
-			}
-		}
-	}
-
 	/// Bind to links. Binding should be advance of dialerGroups to avoid un-routable old connection.
 	// Bind to LAN
 	if len(global.LanInterface) > 0 {
@@ -232,6 +227,10 @@ func NewControlPlane(
 			}
 		}
 	}
+	// Bind to dae0 and dae0peer
+	if err = core.bindDaens(); err != nil {
+		return nil, fmt.Errorf("bindDaens: %w", err)
+	}
 
 	/// DialerGroups (outbounds).
 	if global.AllowInsecure {
@@ -471,11 +470,6 @@ func NewControlPlane(
 	}
 	go dnsUpstream.InitUpstreams()
 
-	InitDaeNetns(log)
-	if err = InitSysctlManager(log); err != nil {
-		return nil, err
-	}
-
 	close(plane.ready)
 	return plane, nil
 }