Improve: use lwip stack instead of kernel

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "kaidl"]
 	path = kaidl
 	url = https://github.com/Kr328/kaidl.git
+[submodule "core/src/main/golang/tun2socket"]
+	path = core/src/main/golang/tun2socket
+	url = https://github.com/Kr328/tun2socket-lwip.git

buildSrc/src/main/java/GolangBuildTask.kt

@@ -9,6 +9,7 @@ import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.FileNotFoundException
 import java.io.IOException
+import java.util.*
 
 abstract class GolangBuildTask : DefaultTask() {
     abstract val debug: Property<Boolean>
@@ -23,7 +24,10 @@ abstract class GolangBuildTask : DefaultTask() {
     abstract val minSdkVersion: Property<Int>
         @Input get
 
-    abstract val cCompilerBasePath: DirectoryProperty
+    abstract val ndkDirectory: DirectoryProperty
+        @InputDirectory get
+
+    abstract val cmakeDirectory: DirectoryProperty
         @InputDirectory get
 
     abstract val inputDirectory: DirectoryProperty
@@ -36,14 +40,12 @@ abstract class GolangBuildTask : DefaultTask() {
     fun build() {
         val src = inputDirectory.get().asFile
 
-        val cmd = if (debug.get()) {
-            """
-                go build --buildmode=c-shared -trimpath -o "%s" -tags "without_gvisor,without_system,debug${if (premium.get()) ",premium" else ""}"
-            """.trimIndent().trim()
+        val generateCmd = """go run make/make.go bridge native build android %s"""
+
+        val buildCmd = if (debug.get()) {
+            """go build --buildmode=c-shared -trimpath -o "%s" -tags "without_gvisor,without_system,debug${if (premium.get()) ",premium" else ""}"""
         } else {
-            """
-                go build --buildmode=c-shared -trimpath -o "%s" -tags "without_gvisor,without_system${if (premium.get()) ",premium" else ""}" -ldflags "-s -w"
-            """.trimIndent().trim()
+            """go build --buildmode=c-shared -trimpath -o "%s" -tags "without_gvisor,without_system${if (premium.get()) ",premium" else ""}" -ldflags "-s -w""""
         }
 
         "go mod tidy".exec(pwd = src)
@@ -51,10 +53,23 @@ abstract class GolangBuildTask : DefaultTask() {
         nativeAbis.get().parallelStream().forEach {
             val out = outputDirectory.get().file("$it/libclash.so")
 
-            cmd.format(out).exec(pwd = src, env = generateGolangBuildEnvironment(it))
+            generateCmd.format(it.toGoArch()).exec(pwd = src.resolve("tun2socket/bridge"), env = generateGolangGenerateEnvironment(it))
+            buildCmd.format(out).exec(pwd = src, env = generateGolangBuildEnvironment(it))
         }
     }
 
+    private fun generateGolangGenerateEnvironment(abi: String): Map<String, String> {
+        val path = cmakeDirectory.get().asFile.absolutePath + File.pathSeparator + System.getenv("PATH")
+
+        return mapOf(
+            "PATH" to path,
+            "CMAKE_SYSTEM_NAME" to "Android",
+            "CMAKE_ANDROID_NDK" to ndkDirectory.get().asFile.absolutePath,
+            "CMAKE_ANDROID_ARCH_ABI" to abi,
+            "CMAKE_SYSTEM_VERSION" to minSdkVersion.get().toString()
+        )
+    }
+
     private fun generateGolangBuildEnvironment(abi: String): Map<String, String> {
         val (goArch, goArm) = when (abi) {
             "arm64-v8a" -> "arm64" to ""
@@ -81,15 +96,27 @@ abstract class GolangBuildTask : DefaultTask() {
         }
 
         return mapOf(
-            "CC" to cCompilerBasePath.get().asFile.resolve(compiler).absolutePath,
+            "CC" to compilerBasePath.resolve(compiler).absolutePath,
             "GOOS" to "android",
             "GOARCH" to goArch,
             "GOARM" to goArm,
             "CGO_ENABLED" to "1",
             "CFLAGS" to "-O3 -Werror",
+            "CMAKE_ARGS" to "-DCMAKE_TOOLCHAIN_FILE=${ndkDirectory.get().asFile.absolutePath}/build/cmake/android.toolchain.cmake -DANDROID_ABI=$abi -DANDROID_PLATFORM=android-${minSdkVersion.get()} -DCMAKE_BUILD_TYPE=Release",
+            "PATH" to cmakeDirectory.get().asFile.absolutePath + File.pathSeparator + System.getenv("PATH")
         )
     }
 
+    private fun String.toGoArch(): String {
+        return when (this) {
+            "arm64-v8a" -> "arm64"
+            "armeabi-v7a" -> "arm"
+            "x86" -> "386"
+            "x86_64" -> "amd64"
+            else -> throw UnsupportedOperationException("unsupported abi: $this")
+        }
+    }
+
     private fun String.exec(
         pwd: File,
         env: Map<String, String> = System.getenv()
@@ -118,4 +145,23 @@ abstract class GolangBuildTask : DefaultTask() {
 
         return outputStream.toString("utf-8")
     }
+
+    private val compilerBasePath: File
+        get() {
+            val host = when {
+                Os.isFamily(Os.FAMILY_WINDOWS) ->
+                    "windows"
+                Os.isFamily(Os.FAMILY_MAC) ->
+                    "darwin"
+                Os.isFamily(Os.FAMILY_UNIX) ->
+                    "linux"
+                else ->
+                    throw GradleScriptException(
+                        "Unsupported host",
+                        FileNotFoundException("Unsupported host")
+                    )
+            }
+
+            return ndkDirectory.get().asFile.resolve("toolchains/llvm/prebuilt/$host-x86_64/bin")
+        }
 }

buildSrc/src/main/java/LibraryGolangPlugin.kt

@@ -11,6 +11,11 @@ class LibraryGolangPlugin : Plugin<Project> {
     override fun apply(target: Project) {
         target.extensions.getByType(LibraryExtension::class.java).apply {
             target.afterEvaluate {
+                val properties = Properties().apply {
+                    target.rootProject.file("local.properties").inputStream().use(this::load)
+                }
+                val cmakeDirectory = target.rootProject.file(properties.getProperty("cmake.dir")!!)
+
                 libraryVariants.forEach { variant ->
                     val abis = defaultConfig.externalNativeBuild.cmake.abiFilters +
                             defaultConfig.externalNativeBuild.ndkBuild.abiFilters
@@ -26,7 +31,8 @@ class LibraryGolangPlugin : Plugin<Project> {
                         it.debug.set(variant.name == "debug")
                         it.nativeAbis.set(abis)
                         it.minSdkVersion.set(defaultConfig.minSdk!!)
-                        it.cCompilerBasePath.set(compilerBasePath)
+                        it.ndkDirectory.set(ndkDirectory)
+                        it.cmakeDirectory.set(cmakeDirectory)
                         it.inputDirectory.set(target.golangSource)
                         it.outputDirectory.set(golangBuildDir)
                     }
@@ -47,23 +53,4 @@ class LibraryGolangPlugin : Plugin<Project> {
             }
         }
     }
-
-    private val LibraryExtension.compilerBasePath: File
-        get() {
-            val host = when {
-                Os.isFamily(Os.FAMILY_WINDOWS) ->
-                    "windows"
-                Os.isFamily(Os.FAMILY_MAC) ->
-                    "darwin"
-                Os.isFamily(Os.FAMILY_UNIX) ->
-                    "linux"
-                else ->
-                    throw GradleScriptException(
-                        "Unsupported host",
-                        FileNotFoundException("Unsupported host")
-                    )
-            }
-
-            return ndkDirectory.resolve("toolchains/llvm/prebuilt/$host-x86_64/bin")
-        }
 }

core/src/main/cpp/main.c

@@ -97,17 +97,14 @@ Java_com_github_kr328_clash_core_bridge_Bridge_nativeNotifyInstalledAppChanged(J
 
 JNIEXPORT void JNICALL
 Java_com_github_kr328_clash_core_bridge_Bridge_nativeStartTun(JNIEnv *env, jobject thiz, jint fd,
-                                                              jint mtu, jstring gateway,
-                                                              jstring mirror, jstring dns,
+                                                              jint mtu, jstring dns,
                                                               jobject cb) {
     TRACE_METHOD();
 
-    scoped_string _gateway = get_string(gateway);
-    scoped_string _mirror = get_string(mirror);
     scoped_string _dns = get_string(dns);
     jobject _interface = new_global(cb);
 
-    startTun(fd, mtu, _gateway, _mirror, _dns, _interface);
+    startTun(fd, mtu, _dns, _interface);
 }
 
 JNIEXPORT void JNICALL

core/src/main/golang/go.mod

@@ -5,8 +5,8 @@ go 1.16
 require (
 	cfa/blob v0.0.0 // local generated
 	github.com/Dreamacro/clash v0.0.0 // local
+	github.com/kr328/tun2socket v0.0.0 // local
 	github.com/dlclark/regexp2 v1.4.0
-	github.com/kr328/tun2socket v0.0.0-20210412191540-3d56c47e2d99
 	github.com/miekg/dns v1.1.42
 	github.com/oschwald/geoip2-golang v1.5.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
@@ -15,4 +15,6 @@ require (
 
 replace github.com/Dreamacro/clash => ./clash
 
+replace github.com/kr328/tun2socket => ./tun2socket
+
 replace cfa/blob => ../../../build/intermediates/golang_blob

core/src/main/golang/tun.go

@@ -56,17 +56,14 @@ func (t *remoteTun) stop() {
 }
 
 //export startTun
-func startTun(fd, mtu C.int, gateway, mirror, dns C.c_string, callback unsafe.Pointer) C.int {
+func startTun(fd, mtu C.int, dns C.c_string, callback unsafe.Pointer) C.int {
 	f := int(fd)
 	m := int(mtu)
-
-	g := C.GoString(gateway)
-	mr := C.GoString(mirror)
 	d := C.GoString(dns)
 
 	remote := &remoteTun{callback: callback, closed: false, limit: semaphore.NewWeighted(4)}
 
-	if tun.Start(f, m, g, mr, d, remote.stop) != nil {
+	if tun.Start(f, m, d) != nil {
 		return 1
 	}
 

core/src/main/golang/tun/dns.go

@@ -7,24 +7,22 @@ import (
 	"time"
 
 	"github.com/Dreamacro/clash/component/resolver"
+	"github.com/kr328/tun2socket/bridge"
 
 	D "github.com/miekg/dns"
-
-	"github.com/kr328/tun2socket/binding"
-	"github.com/kr328/tun2socket/redirect"
 )
 
 const defaultDnsReadTimeout = time.Second * 30
 
-func shouldHijackDns(dnsAddr binding.Address, targetAddr binding.Address) bool {
-	if targetAddr.Port != 53 {
+func shouldHijackDns(dns net.IP, target net.IP, targetPort int) bool {
+	if targetPort != 53 {
 		return false
 	}
 
-	return dnsAddr.IP.Equal(net.IPv4zero) || dnsAddr.IP.Equal(targetAddr.IP)
+	return net.IPv4zero.Equal(dns) || target.Equal(dns)
 }
 
-func hijackUDPDns(pkt []byte, ep *binding.Endpoint, sender redirect.UDPSender) {
+func hijackUDPDns(pkt []byte, lAddr, rAddr net.Addr, udp bridge.UDP) {
 	go func() {
 		answer, err := relayDnsPacket(pkt)
 
@@ -32,10 +30,9 @@ func hijackUDPDns(pkt []byte, ep *binding.Endpoint, sender redirect.UDPSender) {
 			return
 		}
 
-		_ = sender(answer, &binding.Endpoint{
-			Source: ep.Target,
-			Target: ep.Source,
-		})
+		_, _ = udp.WriteTo(answer, lAddr, rAddr)
+
+		recycleUDP(pkt)
 	}()
 }
 

core/src/main/golang/tun/tcp.go

@@ -4,37 +4,24 @@ import (
 	"net"
 	"strconv"
 
-	"github.com/kr328/tun2socket/binding"
-
 	C "github.com/Dreamacro/clash/constant"
-	"github.com/Dreamacro/clash/context"
+	CTX "github.com/Dreamacro/clash/context"
 	"github.com/Dreamacro/clash/tunnel"
 )
 
-func handleTCP(conn net.Conn, endpoint *binding.Endpoint) {
-	src := &net.TCPAddr{
-		IP:   endpoint.Source.IP,
-		Port: int(endpoint.Source.Port),
-		Zone: "",
-	}
-	dst := &net.TCPAddr{
-		IP:   endpoint.Target.IP,
-		Port: int(endpoint.Target.Port),
-		Zone: "",
-	}
-
+func handleTCP(conn net.Conn, source *net.TCPAddr, target *net.TCPAddr) {
 	metadata := &C.Metadata{
 		NetWork:    C.TCP,
 		Type:       C.SOCKS,
-		SrcIP:      src.IP,
-		DstIP:      dst.IP,
-		SrcPort:    strconv.Itoa(src.Port),
-		DstPort:    strconv.Itoa(dst.Port),
+		SrcIP:      source.IP,
+		DstIP:      target.IP,
+		SrcPort:    strconv.Itoa(source.Port),
+		DstPort:    strconv.Itoa(target.Port),
 		AddrType:   C.AtypIPv4,
 		Host:       "",
-		RawSrcAddr: src,
-		RawDstAddr: dst,
+		RawSrcAddr: source,
+		RawDstAddr: target,
 	}
 
-	tunnel.Add(context.NewConnContext(conn, metadata))
+	tunnel.Add(CTX.NewConnContext(conn, metadata))
 }

core/src/main/golang/tun/tun.go

@@ -3,67 +3,126 @@ package tun
 import (
 	"net"
 	"os"
-	"strconv"
 	"sync"
 
-	"github.com/kr328/tun2socket/binding"
-	"github.com/kr328/tun2socket/redirect"
-
 	"github.com/kr328/tun2socket"
 )
 
+type context struct {
+	stack tun2socket.Stack
+	device *os.File
+}
+
 var lock sync.Mutex
-var tun *tun2socket.Tun2Socket
+var tun *context
 
-func Start(fd, mtu int, gateway, mirror, dns string, onStop func()) error {
+func Start(fd, mtu int, dns string) error {
 	lock.Lock()
 	defer lock.Unlock()
 
 	stopLocked()
 
-	dnsHost, dnsPort, err := net.SplitHostPort(dns)
-	if err != nil {
-		return err
-	}
+	dnsIP := net.ParseIP(dns)
+
+	device := os.NewFile(uintptr(fd), "/dev/tun")
 
-	dnsP, err := strconv.Atoi(dnsPort)
+	stack, err := tun2socket.NewStack(mtu)
 	if err != nil {
+		_ = device.Close()
+
 		return err
 	}
 
-	dnsAddr := binding.Address{
-		IP:   net.ParseIP(dnsHost),
-		Port: uint16(dnsP),
-	}
+	go func() {
+		// device -> lwip
 
-	t := tun2socket.NewTun2Socket(os.NewFile(uintptr(fd), "/dev/tun"), mtu, net.ParseIP(gateway), net.ParseIP(mirror))
+		defer device.Close()
+		defer stack.Close()
 
-	t.SetAllocator(allocUDP)
-	t.SetClosedHandler(onStop)
-	t.SetLogger(&logger{})
+		buf := make([]byte, mtu)
 
-	t.SetTCPHandler(func(conn net.Conn, endpoint *binding.Endpoint) {
-		if shouldHijackDns(dnsAddr, endpoint.Target) {
-			hijackTCPDns(conn)
+		for {
+			n, err := device.Read(buf)
+			if err != nil {
+				return
+			}
 
-			return
+			_, _ = stack.Link().Write(buf[:n])
 		}
+	}()
+
+	go func() {
+		// lwip -> device
+
+		defer device.Close()
+		defer stack.Close()
 
-		handleTCP(conn, endpoint)
-	})
-	t.SetUDPHandler(func(payload []byte, endpoint *binding.Endpoint, sender redirect.UDPSender) {
-		if shouldHijackDns(dnsAddr, endpoint.Target) {
-			hijackUDPDns(payload, endpoint, sender)
+		buf := make([]byte, mtu)
 
-			return
+		for {
+			n, err := stack.Link().Read(buf)
+			if err != nil {
+				return
+			}
+
+			_, _ = device.Write(buf[:n])
 		}
+	}()
+
+	go func() {
+		// lwip tcp
+
+		for {
+			conn, err := stack.TCP().Accept()
+			if err != nil {
+				return
+			}
 
-		handleUDP(payload, endpoint, sender)
-	})
+			source := conn.LocalAddr().(*net.TCPAddr)
+			target := conn.RemoteAddr().(*net.TCPAddr)
 
-	t.Start()
+			if shouldHijackDns(dnsIP, target.IP, target.Port) {
+				hijackTCPDns(conn)
 
-	tun = t
+				continue
+			}
+
+			handleTCP(conn, source, target)
+		}
+	}()
+
+	go func() {
+		// lwip udp
+
+		for {
+			buf := allocUDP(mtu)
+
+			n, lAddr, rAddr, err := stack.UDP().ReadFrom(buf)
+			if err != nil {
+				return
+			}
+
+			source := lAddr.(*net.UDPAddr)
+			target := rAddr.(*net.UDPAddr)
+
+			if n == 0 {
+				continue
+			}
+
+			if shouldHijackDns(dnsIP, target.IP, target.Port) {
+				hijackUDPDns(buf[:n], source, target, stack.UDP())
+
+				continue
+			}
+
+			handleUDP(buf[:n], source, target, stack.UDP())
+		}
+	}()
+
+	tun = &context{
+		stack:  stack,
+		device: device,
+	}
 
 	return nil
 }
@@ -77,7 +136,8 @@ func Stop() {
 
 func stopLocked() {
 	if tun != nil {
-		tun.Close()
+		tun.device.Close()
+		tun.stack.Close()
 	}
 
 	tun = nil

core/src/main/golang/tun/udp.go

@@ -1,12 +1,10 @@
 package tun
 
 import (
-	"io"
 	"net"
 
 	"github.com/Dreamacro/clash/transport/socks5"
-	"github.com/kr328/tun2socket/binding"
-	"github.com/kr328/tun2socket/redirect"
+	"github.com/kr328/tun2socket/bridge"
 
 	adapters "github.com/Dreamacro/clash/adapters/inbound"
 	"github.com/Dreamacro/clash/common/pool"
@@ -15,10 +13,9 @@ import (
 )
 
 type udpPacket struct {
-	metadata *C.Metadata
-	source   binding.Address
+	source   *net.UDPAddr
 	data     []byte
-	send     redirect.UDPSender
+	udp      bridge.UDP
 }
 
 func (u *udpPacket) Data() []byte {
@@ -26,15 +23,7 @@ func (u *udpPacket) Data() []byte {
 }
 
 func (u *udpPacket) WriteBack(b []byte, addr net.Addr) (n int, err error) {
-	uAddr, ok := addr.(*net.UDPAddr)
-	if !ok {
-		return 0, io.ErrClosedPipe
-	}
-
-	return len(b), u.send(b, &binding.Endpoint{
-		Source: binding.Address{IP: uAddr.IP, Port: uint16(uAddr.Port)},
-		Target: u.source,
-	})
+	return u.udp.WriteTo(b, u.source, addr)
 }
 
 func (u *udpPacket) Drop() {
@@ -49,20 +38,14 @@ func (u *udpPacket) LocalAddr() net.Addr {
 	}
 }
 
-func handleUDP(payload []byte, endpoint *binding.Endpoint, sender redirect.UDPSender) {
+func handleUDP(payload []byte, source *net.UDPAddr, target *net.UDPAddr, udp bridge.UDP) {
 	pkt := &udpPacket{
-		source: endpoint.Source,
-		data:   payload,
-		send:   sender,
-	}
-
-	rAddr := &net.UDPAddr{
-		IP:   endpoint.Target.IP,
-		Port: int(endpoint.Target.Port),
-		Zone: "",
+		source:   source,
+		data:     payload,
+		udp:      udp,
 	}
 
-	adapter := adapters.NewPacket(socks5.ParseAddrToSocksAddr(rAddr), pkt, C.SOCKS)
+	adapter := adapters.NewPacket(socks5.ParseAddrToSocksAddr(target), pkt, C.SOCKS)
 
 	tunnel.AddPacket(adapter)
 }

core/src/main/golang/tun2socket

@@ -0,0 +1 @@
+Subproject commit a57caac68ee30407ceecc6d1c5f8f495bcb4f4fb

core/src/main/java/com/github/kr328/clash/core/Clash.kt

@@ -61,13 +61,11 @@ object Clash {
     fun startTun(
         fd: Int,
         mtu: Int,
-        gateway: String,
-        mirror: String,
         dns: String,
         markSocket: (Int) -> Boolean,
         querySocketUid: (protocol: Int, source: InetSocketAddress, target: InetSocketAddress) -> Int
     ) {
-        Bridge.nativeStartTun(fd, mtu, gateway, mirror, "$dns:53", object : TunInterface {
+        Bridge.nativeStartTun(fd, mtu, dns, object : TunInterface {
             override fun markSocket(fd: Int) {
                 markSocket(fd)
             }

core/src/main/java/com/github/kr328/clash/core/bridge/Bridge.kt

@@ -17,15 +17,7 @@ object Bridge {
     external fun nativeQueryTrafficTotal(): Long
     external fun nativeNotifyDnsChanged(dnsList: String)
     external fun nativeNotifyInstalledAppChanged(uidList: String)
-    external fun nativeStartTun(
-        fd: Int,
-        mtu: Int,
-        gateway: String,
-        mirror: String,
-        dns: String,
-        cb: TunInterface
-    )
-
+    external fun nativeStartTun(fd: Int, mtu: Int, dns: String, cb: TunInterface)
     external fun nativeStopTun()
     external fun nativeStartHttp(listenAt: String): String?
     external fun nativeStopHttp()

service/src/main/java/com/github/kr328/clash/service/TunService.kt

@@ -217,8 +217,6 @@ class TunService : VpnService(), CoroutineScope by CoroutineScope(Dispatchers.De
                 fd = establish()?.detachFd()
                     ?: throw NullPointerException("Establish VPN rejected by system"),
                 mtu = TUN_MTU,
-                gateway = TUN_GATEWAY,
-                mirror = TUN_MIRROR,
                 dns = if (store.dnsHijacking) NET_ANY else TUN_DNS,
             )
         }
@@ -230,7 +228,6 @@ class TunService : VpnService(), CoroutineScope by CoroutineScope(Dispatchers.De
         private const val TUN_MTU = 9000
         private const val TUN_SUBNET_PREFIX = 30
         private const val TUN_GATEWAY = "172.31.255.253"
-        private const val TUN_MIRROR = "172.31.255.254"
         private const val TUN_DNS = "198.18.0.1"
         private const val NET_ANY = "0.0.0.0"
     }

service/src/main/java/com/github/kr328/clash/service/clash/module/TunModule.kt

@@ -16,8 +16,6 @@ class TunModule(private val vpn: VpnService) : Module<Unit>(vpn) {
     data class TunDevice(
         val fd: Int,
         val mtu: Int,
-        val gateway: String,
-        val mirror: String,
         val dns: String
     )
 
@@ -58,8 +56,6 @@ class TunModule(private val vpn: VpnService) : Module<Unit>(vpn) {
         Clash.startTun(
             fd = device.fd,
             mtu = device.mtu,
-            gateway = device.gateway,
-            mirror = device.mirror,
             dns = device.dns,
             markSocket = vpn::protect,
             querySocketUid = this::queryUid