Connect socket using secondary IP address

Question

Not too long ago, I was able to connect to a given destination address from either a primary or secondary IP address on a given interface the following steps.

1. add a secondary IP address to a given interface using ip addr add ...,
2. bind a socket to the primary or secondary address, as desired,
3. connect to destination ip address using primary or secondary address, as bound in #2.

I recently upgraded a machine to Linux 3.3.6 and this no longer works, but I don't remember the last version I tried it on that was actually working. Does anyone know how to do the same kind of thing in newer kernels? I just verified the same code on 2.6.23 on a different machine.

UPDATE 2 This seems to be related to the NIC driver. With 3.3.6 and 8139too, everything is fine. Same machine, but using a different NIC and via-rhine, then the problem shows up.

Here's what happens when I run the code (source at end) on 3.3.6 (packet capture output from tshark running in the background):

$ ./bind_connect 10.0.1.124 10.0.1.120
Bound socket: 10.0.1.120 [10.0.1.120->10.0.1.124]

#           wrong source IP (should use 10.0.1.120 from bind)
#                      |
#                      v
$ 121.051052   10.0.1.220 -> 10.0.1.124   50418 80 TCP 74 50418 > 80 [SYN] Seq=3358582895 Win=14600 Len=0 MSS=1460 SACK_PERM=1 TSval=30589624 TSecr=0 WS=128
121.051428   10.0.1.124 -> 10.0.1.220   80 50418 TCP 74 80 > 50418 [SYN, ACK] Seq=1815118993 Ack=3358582896 Win=5792 Len=0 MSS=1460 SACK_PERM=1 TSval=8157158 TSecr=30589624 WS=2
121.051475   10.0.1.220 -> 10.0.1.124   50418 80 TCP 66 50418 > 80 [ACK] Seq=3358582896 Ack=1815118994 Win=14720 Len=0 TSval=30589624 TSecr=8157158
121.051504   10.0.1.220 -> 10.0.1.124   50418 80 TCP 66 50418 > 80 [FIN, ACK] Seq=3358582896 Ack=1815118994 Win=14720 Len=0 TSval=30589624 TSecr=8157158
121.051768   10.0.1.124 -> 10.0.1.220   80 50418 TCP 66 80 > 50418 [ACK] Seq=1815118994 Ack=3358582897 Win=5792 Len=0 TSval=8157158 TSecr=30589624
121.051913   10.0.1.124 -> 10.0.1.220   80 50418 TCP 66 80 > 50418 [FIN, ACK] Seq=1815118994 Ack=3358582897 Win=5792 Len=0 TSval=8157158 TSecr=30589624
121.051941   10.0.1.220 -> 10.0.1.124   50418 80 TCP 66 50418 > 80 [ACK] Seq=3358582897 Ack=1815118995 Win=14720 Len=0 TSval=30589625 TSecr=8157158


$ ./bind_connect 10.0.1.124 10.0.1.220
Bound socket: 10.0.1.220 [10.0.1.220->10.0.1.124]
$ 124.139966   10.0.1.220 -> 10.0.1.124   41470 80 TCP 74 41470 > 80 [SYN] Seq=1404133303 Win=14600 Len=0 MSS=1460 SACK_PERM=1 TSval=30590551 TSecr=0 WS=128
124.140421   10.0.1.124 -> 10.0.1.220   80 41470 TCP 74 80 > 41470 [SYN, ACK] Seq=1918490481 Ack=1404133304 Win=5792 Len=0 MSS=1460 SACK_PERM=1 TSval=8157467 TSecr=30590551 WS=2
124.140478   10.0.1.220 -> 10.0.1.124   41470 80 TCP 66 41470 > 80 [ACK] Seq=1404133304 Ack=1918490482 Win=14720 Len=0 TSval=30590551 TSecr=8157467
124.140553   10.0.1.220 -> 10.0.1.124   41470 80 TCP 66 41470 > 80 [FIN, ACK] Seq=1404133304 Ack=1918490482 Win=14720 Len=0 TSval=30590551 TSecr=8157467
124.140934   10.0.1.124 -> 10.0.1.220   80 41470 TCP 66 80 > 41470 [ACK] Seq=1918490482 Ack=1404133305 Win=5792 Len=0 TSval=8157467 TSecr=30590551
124.140976   10.0.1.124 -> 10.0.1.220   80 41470 TCP 66 80 > 41470 [FIN, ACK] Seq=1918490482 Ack=1404133305 Win=5792 Len=0 TSval=8157467 TSecr=30590551
124.140995   10.0.1.220 -> 10.0.1.124   41470 80 TCP 66 41470 > 80 [ACK] Seq=1404133305 Ack=1918490483 Win=14720 Len=0 TSval=30590551 TSecr=8157467

$ uname -a
Linux erebus.mn.ca 3.3.6-1-ARCH #1 SMP PREEMPT Sun May 13 09:59:18 UTC 2012 i686 GNU/Linux
$ ip route get 10.0.1.124
10.0.1.124 dev eth0.894  src 10.0.1.220 
    cache  ipid 0x61ca rtt 3ms rttvar 3ms cwnd 10

And then the same on 2.6.23:

$ ./bind_connect 10.0.0.123 10.0.0.226                                                                                                                                           >
Bound socket: 10.0.0.226 [10.0.0.226->10.0.0.123]
$ 231.566278   10.0.0.226 -> 10.0.0.123   TCP 54109 > http [SYN] Seq=0 Win=5840 Len=0 MSS=1460 TSV=99922052 TSER=0 WS=5
231.566448   10.0.0.123 -> 10.0.0.226   TCP http > 54109 [SYN, ACK] Seq=0 Ack=1 Win=5792 Len=0 MSS=1460 TSV=7060755 TSER=99922052 WS=1
231.566463   10.0.0.226 -> 10.0.0.123   TCP 54109 > http [ACK] Seq=1 Ack=1 Win=5856 Len=0 TSV=99922052 TSER=7060755
231.566510   10.0.0.226 -> 10.0.0.123   TCP 54109 > http [FIN, ACK] Seq=1 Ack=1 Win=5856 Len=0 TSV=99922052 TSER=7060755
231.566593   10.0.0.123 -> 10.0.0.226   TCP http > 54109 [ACK] Seq=1 Ack=2 Win=5792 Len=0 TSV=7060755 TSER=99922052
231.566704   10.0.0.123 -> 10.0.0.226   TCP http > 54109 [FIN, ACK] Seq=1 Ack=2 Win=5792 Len=0 TSV=7060755 TSER=99922052
231.566737   10.0.0.226 -> 10.0.0.123   TCP 54109 > http [ACK] Seq=2 Ack=2 Win=5856 Len=0 TSV=99922052 TSER=7060755

$ ./bind_connect 10.0.0.123 10.0.0.126                                                                                                                                           >
Bound socket: 10.0.0.126 [10.0.0.126->10.0.0.123]
$ 235.824867   10.0.0.126 -> 10.0.0.123   TCP 34228 > http [SYN] Seq=0 Win=5840 Len=0 MSS=1460 TSV=99926310 TSER=0 WS=5
235.825185   10.0.0.123 -> 10.0.0.126   TCP http > 34228 [SYN, ACK] Seq=0 Ack=1 Win=5792 Len=0 MSS=1460 TSV=7061180 TSER=99926310 WS=1
235.825236   10.0.0.126 -> 10.0.0.123   TCP 34228 > http [ACK] Seq=1 Ack=1 Win=5856 Len=0 TSV=99926311 TSER=7061180
235.825273   10.0.0.126 -> 10.0.0.123   TCP 34228 > http [FIN, ACK] Seq=1 Ack=1 Win=5856 Len=0 TSV=99926311 TSER=7061180
235.825721   10.0.0.123 -> 10.0.0.126   TCP http > 34228 [ACK] Seq=1 Ack=2 Win=5792 Len=0 TSV=7061180 TSER=99926311
235.825722   10.0.0.123 -> 10.0.0.126   TCP http > 34228 [FIN, ACK] Seq=1 Ack=2 Win=5792 Len=0 TSV=7061180 TSER=99926311
235.825756   10.0.0.126 -> 10.0.0.123   TCP 34228 > http [ACK] Seq=2 Ack=2 Win=5856 Len=0 TSV=99926311 TSER=7061180

$ uname -a
Linux gaia.mn.ca 2.6.23.17-88.fc7 #1 SMP Thu May 15 00:35:10 EDT 2008 i686 i686 i386 GNU/Linux
$ ip route get 10.0.0.123
10.0.0.123 dev eth0  src 10.0.0.126
    cache  mtu 1500 advmss 1460 hoplimit 64

Code for bind_connect:

#include <arpa/inet.h>
#include <errno.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <stdio.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>

int main(int argc, char **argv)
{
    struct sockaddr_in sn;
    socklen_t sn_len;
    struct sockaddr_in src;
    struct sockaddr_in dst;
    int sock;
    char b1[INET_ADDRSTRLEN];
    char b2[INET_ADDRSTRLEN];
    char b3[INET_ADDRSTRLEN];

    src.sin_family = dst.sin_family = AF_INET;
    src.sin_port = 0;
    dst.sin_port = htons(80);

    if (argc < 3) {
        printf("missing argument\n");
        return 1;
    }

    if (inet_pton(AF_INET, argv[1], &dst.sin_addr) != 1) {
        perror("pton");
        return -errno;
    }

    if (inet_pton(AF_INET, argv[2], &src.sin_addr) != 1) {
        perror("pton");
        return -errno;
    }

    sock = socket(AF_INET, SOCK_STREAM, 0);
    if (sock < 0) {
        perror("socket");
        return -errno;
    }

    if (bind(sock, (struct sockaddr*)&src, sizeof src) != 0) {
        perror("bind");
        return -errno;
    }

    sn_len = sizeof sn;
    if (getsockname(sock, (struct sockaddr*)&sn, &sn_len) != 0) {
        perror("getsockname");
        return -errno;
    }

    printf("Bound socket: %s [%s->%s]\n",
           inet_ntop(AF_INET, &sn.sin_addr, b1, sizeof b1),
           inet_ntop(AF_INET, &src.sin_addr, b2, sizeof b2),
           inet_ntop(AF_INET, &dst.sin_addr, b3, sizeof b3));

    if (connect(sock, (struct sockaddr*)&dst, sizeof dst) != 0) {
        perror("connect");
        return -errno;
    }

    close(sock);

    return 0;
}

Answer 1

The problem turns out to be that I had set up a nat POSTROUTING rule with iptables, i.e.

Chain POSTROUTING (policy ACCEPT 11 packets, 637 bytes)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MASQUERADE  all  --  any    eth0.894  anywhere             anywhere            

Once I removed this rule:

$ sudo iptables -t nat -F

Things started to work again. I had set up the nat rule a while ago while testing another problem, and then forgotten to remove it afterward.