Fix some flaky tests related to pick port (#985)

* Fix some tests to PickPort correctly * Add retry logic for two socks tests Two socks tests listen for both TCP and UDP with same port number, in some cases the port is not available Add retry logic for server core, start server core and client core separately Extract a common method * Add retry logic for two dokodemo tests Two dokodemo tests listen for 5 ports, in some cases they are not available Add retry logic for client core, start server and client separately
2024-11-22 15:03:02 +00:00 · 2022-03-12 11:56:40 -05:00 · 2022-03-12 11:56:40 -05:00 · 94c249a8c8
commit 94c249a8c8
parent c1a54ae58e
6 changed files with 241 additions and 159 deletions
--- a/core/xray_test.go
+++ b/core/xray_test.go
@ -58,7 +58,7 @@ func TestXrayClose(t *testing.T) {
 					Address: net.NewIPOrDomain(net.LocalHostIP),
 					Port:    uint32(0),
 					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP, net.Network_UDP},
+						Network: []net.Network{net.Network_TCP},
 					},
 				}),
 			},
--- a/testing/scenarios/common.go
+++ b/testing/scenarios/common.go
@ -11,6 +11,7 @@ import (
 	"runtime"
 	"sync"
 	"syscall"
+	"testing"
 	"time"

 	"github.com/golang/protobuf/proto"
@ -235,3 +236,20 @@ func testTCPConn2(conn net.Conn, payloadSize int, timeout time.Duration) func()
 		return nil
 	}
 }
+
+func WaitConnAvailableWithTest(t *testing.T, testFunc func() error) bool {
+	for i := 1; ; i++ {
+		if i > 10 {
+			t.Log("All attempts failed to test tcp conn")
+			return false
+		}
+		time.Sleep(time.Millisecond * 10)
+		if err := testFunc(); err != nil {
+			t.Log("err ", err)
+		} else {
+			t.Log("success with", i, "attempts")
+			break
+		}
+	}
+	return true
+}
--- a/testing/scenarios/dokodemo_test.go
+++ b/testing/scenarios/dokodemo_test.go
@ -64,55 +64,68 @@ func TestDokodemoTCP(t *testing.T) {
 			},
 		},
 	}
+	server, err := InitializeServerConfig(serverConfig)
+	common.Must(err)
+	defer CloseServer(server)

-	clientPort := uint32(tcp.PickPort())
 	clientPortRange := uint32(5)
-	clientConfig := &core.Config{
-		App: []*serial.TypedMessage{
-			serial.ToTypedMessage(&log.Config{
-				ErrorLogLevel: clog.Severity_Debug,
-				ErrorLogType:  log.LogType_Console,
-			}),
-		},
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
-					Address: net.NewIPOrDomain(dest.Address),
-					Port:    uint32(dest.Port),
-					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP},
-					},
+	retry := 1
+	clientPort := uint32(tcp.PickPort())
+	for {
+		clientConfig := &core.Config{
+			App: []*serial.TypedMessage{
+				serial.ToTypedMessage(&log.Config{
+					ErrorLogLevel: clog.Severity_Debug,
+					ErrorLogType:  log.LogType_Console,
 				}),
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&outbound.Config{
-					Receiver: []*protocol.ServerEndpoint{
-						{
-							Address: net.NewIPOrDomain(net.LocalHostIP),
-							Port:    uint32(serverPort),
-							User: []*protocol.User{
-								{
-									Account: serial.ToTypedMessage(&vmess.Account{
-										Id: userID.String(),
-									}),
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_TCP},
+						},
+					}),
+				},
+			},
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&outbound.Config{
+						Receiver: []*protocol.ServerEndpoint{
+							{
+								Address: net.NewIPOrDomain(net.LocalHostIP),
+								Port:    uint32(serverPort),
+								User: []*protocol.User{
+									{
+										Account: serial.ToTypedMessage(&vmess.Account{
+											Id: userID.String(),
+										}),
+									},
 								},
 							},
 						},
-					},
-				}),
+					}),
+				},
 			},
-		},
-	}
+		}

-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
-	common.Must(err)
-	defer CloseAllServers(servers)
+		server, _ := InitializeServerConfig(clientConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testTCPConn(net.Port(clientPort), 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start client")
+		}
+		clientPort = uint32(tcp.PickPort())
+	}

 	for port := clientPort; port <= clientPort+clientPortRange; port++ {
 		if err := testTCPConn(net.Port(port), 1024, time.Second*2)(); err != nil {
@ -155,49 +168,62 @@ func TestDokodemoUDP(t *testing.T) {
 			},
 		},
 	}
+	server, err := InitializeServerConfig(serverConfig)
+	common.Must(err)
+	defer CloseServer(server)

-	clientPort := uint32(udp.PickPort())
 	clientPortRange := uint32(5)
-	clientConfig := &core.Config{
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
-					Address: net.NewIPOrDomain(dest.Address),
-					Port:    uint32(dest.Port),
-					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_UDP},
-					},
-				}),
+	retry := 1
+	clientPort := uint32(udp.PickPort())
+	for {
+		clientConfig := &core.Config{
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{{From: clientPort, To: clientPort + clientPortRange}}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_UDP},
+						},
+					}),
+				},
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&outbound.Config{
-					Receiver: []*protocol.ServerEndpoint{
-						{
-							Address: net.NewIPOrDomain(net.LocalHostIP),
-							Port:    uint32(serverPort),
-							User: []*protocol.User{
-								{
-									Account: serial.ToTypedMessage(&vmess.Account{
-										Id: userID.String(),
-									}),
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&outbound.Config{
+						Receiver: []*protocol.ServerEndpoint{
+							{
+								Address: net.NewIPOrDomain(net.LocalHostIP),
+								Port:    uint32(serverPort),
+								User: []*protocol.User{
+									{
+										Account: serial.ToTypedMessage(&vmess.Account{
+											Id: userID.String(),
+										}),
+									},
 								},
 							},
 						},
-					},
-				}),
+					}),
+				},
 			},
-		},
-	}
+		}

-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
-	common.Must(err)
-	defer CloseAllServers(servers)
+		server, _ := InitializeServerConfig(clientConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testUDPConn(net.Port(clientPort), 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start client")
+		}
+		clientPort = uint32(udp.PickPort())
+	}

 	var errg errgroup.Group
 	for port := clientPort; port <= clientPort+clientPortRange; port++ {
--- a/testing/scenarios/feature_test.go
+++ b/testing/scenarios/feature_test.go
@ -234,7 +234,7 @@ func TestProxyOverKCP(t *testing.T) {
 	defer tcpServer.Close()

 	serverUserID := protocol.NewID(uuid.New())
-	serverPort := tcp.PickPort()
+	serverPort := udp.PickPort()
 	serverConfig := &core.Config{
 		Inbound: []*core.InboundHandlerConfig{
 			{
--- a/testing/scenarios/socks_test.go
+++ b/testing/scenarios/socks_test.go
@ -111,32 +111,59 @@ func TestSocksBridageUDP(t *testing.T) {
 	common.Must(err)
 	defer udpServer.Close()

+	retry := 1
 	serverPort := tcp.PickPort()
-	serverConfig := &core.Config{
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
-					AuthType: socks.AuthType_PASSWORD,
-					Accounts: map[string]string{
-						"Test Account": "Test Password",
-					},
-					Address:    net.NewIPOrDomain(net.LocalHostIP),
-					UdpEnabled: true,
-				}),
+	for {
+		serverConfig := &core.Config{
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
+						AuthType: socks.AuthType_PASSWORD,
+						Accounts: map[string]string{
+							"Test Account": "Test Password",
+						},
+						Address:    net.NewIPOrDomain(net.LocalHostIP),
+						UdpEnabled: true,
+					}),
+				},
+				{
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort + 1)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_UDP},
+						},
+					}),
+				},
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+				},
 			},
-		},
+		}
+
+		server, _ := InitializeServerConfig(serverConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testUDPConn(serverPort+1, 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start server")
+		}
+		serverPort = tcp.PickPort()
 	}

-	clientPort := tcp.PickPort()
+	clientPort := udp.PickPort()
 	clientConfig := &core.Config{
 		Inbound: []*core.InboundHandlerConfig{
 			{
@ -148,7 +175,7 @@ func TestSocksBridageUDP(t *testing.T) {
 					Address: net.NewIPOrDomain(dest.Address),
 					Port:    uint32(dest.Port),
 					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP, net.Network_UDP},
+						Network: []net.Network{net.Network_UDP},
 					},
 				}),
 			},
@ -175,12 +202,12 @@ func TestSocksBridageUDP(t *testing.T) {
 		},
 	}

-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
+	server, err := InitializeServerConfig(clientConfig)
 	common.Must(err)
-	defer CloseAllServers(servers)
+	defer CloseServer(server)

-	if err := testUDPConn(clientPort, 1024, time.Second*5)(); err != nil {
-		t.Error(err)
+	if !WaitConnAvailableWithTest(t, testUDPConn(clientPort, 1024, time.Second*2)) {
+		t.Fail()
 	}
 }

@ -192,46 +219,74 @@ func TestSocksBridageUDPWithRouting(t *testing.T) {
 	common.Must(err)
 	defer udpServer.Close()

+	retry := 1
 	serverPort := tcp.PickPort()
-	serverConfig := &core.Config{
-		App: []*serial.TypedMessage{
-			serial.ToTypedMessage(&router.Config{
-				Rule: []*router.RoutingRule{
-					{
-						TargetTag: &router.RoutingRule_Tag{
-							Tag: "out",
+	for {
+		serverConfig := &core.Config{
+			App: []*serial.TypedMessage{
+				serial.ToTypedMessage(&router.Config{
+					Rule: []*router.RoutingRule{
+						{
+							TargetTag: &router.RoutingRule_Tag{
+								Tag: "out",
+							},
+							InboundTag: []string{"socks", "dokodemo"},
 						},
-						InboundTag: []string{"socks"},
 					},
+				}),
+			},
+			Inbound: []*core.InboundHandlerConfig{
+				{
+					Tag: "socks",
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
+						AuthType:   socks.AuthType_NO_AUTH,
+						Address:    net.NewIPOrDomain(net.LocalHostIP),
+						UdpEnabled: true,
+					}),
+				},
+				{
+					Tag: "dokodemo",
+					ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
+						PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort + 1)}},
+						Listen:   net.NewIPOrDomain(net.LocalHostIP),
+					}),
+					ProxySettings: serial.ToTypedMessage(&dokodemo.Config{
+						Address: net.NewIPOrDomain(dest.Address),
+						Port:    uint32(dest.Port),
+						NetworkList: &net.NetworkList{
+							Network: []net.Network{net.Network_UDP},
+						},
+					}),
 				},
-			}),
-		},
-		Inbound: []*core.InboundHandlerConfig{
-			{
-				Tag: "socks",
-				ReceiverSettings: serial.ToTypedMessage(&proxyman.ReceiverConfig{
-					PortList: &net.PortList{Range: []*net.PortRange{net.SinglePortRange(serverPort)}},
-					Listen:   net.NewIPOrDomain(net.LocalHostIP),
-				}),
-				ProxySettings: serial.ToTypedMessage(&socks.ServerConfig{
-					AuthType:   socks.AuthType_NO_AUTH,
-					Address:    net.NewIPOrDomain(net.LocalHostIP),
-					UdpEnabled: true,
-				}),
 			},
-		},
-		Outbound: []*core.OutboundHandlerConfig{
-			{
-				ProxySettings: serial.ToTypedMessage(&blackhole.Config{}),
+			Outbound: []*core.OutboundHandlerConfig{
+				{
+					ProxySettings: serial.ToTypedMessage(&blackhole.Config{}),
+				},
+				{
+					Tag:           "out",
+					ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
+				},
 			},
-			{
-				Tag:           "out",
-				ProxySettings: serial.ToTypedMessage(&freedom.Config{}),
-			},
-		},
+		}
+
+		server, _ := InitializeServerConfig(serverConfig)
+		if server != nil && WaitConnAvailableWithTest(t, testUDPConn(serverPort+1, 1024, time.Second*2)) {
+			defer CloseServer(server)
+			break
+		}
+		retry++
+		if retry > 5 {
+			t.Fatal("All attempts failed to start server")
+		}
+		serverPort = tcp.PickPort()
 	}

-	clientPort := tcp.PickPort()
+	clientPort := udp.PickPort()
 	clientConfig := &core.Config{
 		Inbound: []*core.InboundHandlerConfig{
 			{
@ -243,7 +298,7 @@ func TestSocksBridageUDPWithRouting(t *testing.T) {
 					Address: net.NewIPOrDomain(dest.Address),
 					Port:    uint32(dest.Port),
 					NetworkList: &net.NetworkList{
-						Network: []net.Network{net.Network_TCP, net.Network_UDP},
+						Network: []net.Network{net.Network_UDP},
 					},
 				}),
 			},
@ -262,12 +317,12 @@ func TestSocksBridageUDPWithRouting(t *testing.T) {
 		},
 	}

-	servers, err := InitializeServerConfigs(serverConfig, clientConfig)
+	server, err := InitializeServerConfig(clientConfig)
 	common.Must(err)
-	defer CloseAllServers(servers)
+	defer CloseServer(server)

-	if err := testUDPConn(clientPort, 1024, time.Second*5)(); err != nil {
-		t.Error(err)
+	if !WaitConnAvailableWithTest(t, testUDPConn(clientPort, 1024, time.Second*2)) {
+		t.Fail()
 	}
 }

--- a/testing/scenarios/vmess_test.go
+++ b/testing/scenarios/vmess_test.go
@ -108,7 +108,7 @@ func TestVMessDynamicPort(t *testing.T) {
 		}

 		server, _ := InitializeServerConfig(serverConfig)
-		if server != nil && tcpConnAvailableAtPort(t, serverPort+100) {
+		if server != nil && WaitConnAvailableWithTest(t, testTCPConn(serverPort+100, 1024, time.Second*2)) {
 			defer CloseServer(server)
 			break
 		}
@ -167,28 +167,11 @@ func TestVMessDynamicPort(t *testing.T) {
 	common.Must(err)
 	defer CloseServer(server)

-	if !tcpConnAvailableAtPort(t, clientPort) {
+	if !WaitConnAvailableWithTest(t, testTCPConn(clientPort, 1024, time.Second*2)) {
 		t.Fail()
 	}
 }

-func tcpConnAvailableAtPort(t *testing.T, port net.Port) bool {
-	for i := 1; ; i++ {
-		if i > 10 {
-			t.Log("All attempts failed to test tcp conn")
-			return false
-		}
-		time.Sleep(time.Millisecond * 10)
-		if err := testTCPConn(port, 1024, time.Second*2)(); err != nil {
-			t.Log("err ", err)
-		} else {
-			t.Log("success with", i, "attempts")
-			break
-		}
-	}
-	return true
-}
-
 func TestVMessGCM(t *testing.T) {
 	tcpServer := tcp.Server{
 		MsgProcessor: xor,