Why accepted two same 5-tuple socket when concurrent connect to the server?

Question

server.go

package main

import (
    "fmt"
    "io"
    "io/ioutil"
    "log"
    "net"
    "net/http"
    _ "net/http/pprof"
    "sync"
    "syscall"
)

type ConnSet struct {
    data  map[int]net.Conn
    mutex sync.Mutex
}

func (m *ConnSet) Update(id int, conn net.Conn) error {
    m.mutex.Lock()
    defer m.mutex.Unlock()
    if _, ok := m.data[id]; ok {
        fmt.Printf("add: key %d existed \n", id)
        return fmt.Errorf("add: key %d existed \n", id)
    }
    m.data[id] = conn
    return nil
}

var connSet = &ConnSet{
    data: make(map[int]net.Conn),
}

func main() {
    setLimit()

    ln, err := net.Listen("tcp", ":12345")
    if err != nil {
        panic(err)
    }

    go func() {
        if err := http.ListenAndServe(":6060", nil); err != nil {
            log.Fatalf("pprof failed: %v", err)
        }
    }()

    var connections []net.Conn
    defer func() {
        for _, conn := range connections {
            conn.Close()
        }
    }()

    for {
        conn, e := ln.Accept()
        if e != nil {
            if ne, ok := e.(net.Error); ok && ne.Temporary() {
                log.Printf("accept temp err: %v", ne)
                continue
            }

            log.Printf("accept err: %v", e)
            return
        }
        port := conn.RemoteAddr().(*net.TCPAddr).Port
        connSet.Update(port, conn)
        go handleConn(conn)
        connections = append(connections, conn)
        if len(connections)%100 == 0 {
            log.Printf("total number of connections: %v", len(connections))
        }
    }
}

func handleConn(conn net.Conn) {
    io.Copy(ioutil.Discard, conn)
}

func setLimit() {
    var rLimit syscall.Rlimit
    if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
        panic(err)
    }
    rLimit.Cur = rLimit.Max
    if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
        panic(err)
    }

    log.Printf("set cur limit: %d", rLimit.Cur)
}

client.go

package main

import (
    "bytes"
    "flag"
    "fmt"
    "io"
    "log"
    "net"
    "os"
    "strconv"
    "sync"
    "syscall"
    "time"
)

var portFlag = flag.Int("port", 12345, "port")

type ConnSet struct {
    data  map[int]net.Conn
    mutex sync.Mutex
}

func (m *ConnSet) Update(id int, conn net.Conn) error {
    m.mutex.Lock()
    defer m.mutex.Unlock()
    if _, ok := m.data[id]; ok {
        fmt.Printf("add: key %d existed \n", id)
        return fmt.Errorf("add: key %d existed \n", id)
    }
    m.data[id] = conn
    return nil
}

var connSet = &ConnSet{
    data: make(map[int]net.Conn),
}

func echoClient() {
    addr := fmt.Sprintf("127.0.0.1:%d", *portFlag)
    dialer := net.Dialer{}
    conn, err := dialer.Dial("tcp", addr)
    if err != nil {
        fmt.Println("ERROR", err)
        os.Exit(1)
    }
    port := conn.LocalAddr().(*net.TCPAddr).Port
    connSet.Update(port, conn)
    defer conn.Close()

    for i := 0; i < 10; i++ {
        s := fmt.Sprintf("%s", strconv.Itoa(i))
        _, err := conn.Write([]byte(s))
        if err != nil {
            log.Println("write error: ", err)
        }
        b := make([]byte, 1024)
        _, err = conn.Read(b)
        switch err {
        case nil:
            if string(bytes.Trim(b, "\x00")) != s {
                log.Printf("resp req not equal, req: %d, res: %s", i, string(bytes.Trim(b, "\x00")))
            }
        case io.EOF:
            fmt.Println("eof")
            break
        default:
            fmt.Println("ERROR", err)
            break
        }
    }
    time.Sleep(time.Hour)
    if err := conn.Close(); err != nil {
        log.Printf("client conn close err: %s", err)
    }
}

func main() {
    flag.Parse()
    setLimit()
    before := time.Now()
    var wg sync.WaitGroup
    for i := 0; i < 20000; i++ {
        wg.Add(1)
        go func() {
            defer wg.Done()
            echoClient()
        }()
    }
    wg.Wait()
    fmt.Println(time.Now().Sub(before))
}

func setLimit() {
    var rLimit syscall.Rlimit
    if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
        panic(err)
    }
    rLimit.Cur = rLimit.Max
    if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
        panic(err)
    }

    log.Printf("set cur limit: %d", rLimit.Cur)
}

running command

go run server.go
---
go run client.go

server running screenshot

The client simultaneously initiates 20,000 connections to the server, and the server accepted two remotePort connections that are exactly the same (in a extremely short period of time).

I try to use tcpconn.py from bcc (patched from tcpconnect.py by add skc_num(aka: local_port))

tcpaccept.py tracing the connection, and also finds that the remote port is duplicated on the server side when there is no duplicate on the client side

In my understanding, the 5-tuple of the socket will not be duplicated, Why the server accepted two sockets with exactly the same remote port?

My test environment:

Fedora 31, kernel version 5.3.15 x86_64 and Ubuntu 18.04.3 LTS, kernel version 4.19.1 x86_64

go version go1.13.5 linux/amd64

wireshark: server TCP Keep-Alive to both ACK & PSH+ACK

server TCP Keep-Alive to PSH+ACK only

Answer 1

Connection is added to the map data map[int]net.Conn when it is established, but when connection is closed it is not removed from the map. So if connection gets closed its port become free and could be reused by an Operation System for next connection. That's a reason why you can see duplicate ports.

Try to remove port from map when they are get closed.