Search code examples
dockerudptcpreplay

Docker bridge does not transmit from tcpreplay IPv6/UDP/GTP traffic

I have a pcap file with IPv6/UDP/TCP packets. I want to reproduce it using the tcpreplay tool from one Docker container (A) to a second container (B). Containers connected to the one bridge. But the packet is not transferred to container B.

Pcap file

 tcpdump -envvv -r one_icmp_100b-1709738246.pcap

reading from file one_icmp_100b-1709738246.pcap, link-type EN10MB (Ethernet), snapshot length 262144
18:17:37.074830 02:42:00:a2:11:04 > 02:42:ac:1c:00:04, ethertype IPv6 (0x86dd), length 168: (flowlabel 0xe4a49, hlim 63, next-header UDP (17) payload length: 114) 2001::5.40915 > 2011::3.2152: [bad udp cksum 0x409d -> 0xd5bb!] UDP, length 106
18:17:37.074869 02:42:00:a2:11:04 > 02:42:ac:1c:00:04, ethertype IPv6 (0x86dd), length 113: (flowlabel 0xe4a49, hlim 63, next-header UDP (17) payload length: 59) 2001::5.40915 > 2011::3.2152: [bad udp cksum 0x4066 -> 0x4d5b!] UDP, length 51

Host server

Ubuntu 22.04.3

Linux msk-dev-gnb-07 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

$ docker version
Client: Docker Engine - Community
 Version:           24.0.7
 API version:       1.43
 Go version:        go1.20.10
 Git commit:        afdd53b
 Built:             Thu Oct 26 09:07:41 2023
 OS/Arch:           linux/amd64
 Context:           default

Server: Docker Engine - Community
 Engine:
  Version:          24.0.7
  API version:      1.43 (minimum version 1.12)
  Go version:       go1.20.10
  Git commit:       311b9ff
  Built:            Thu Oct 26 09:07:41 2023
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.6.26
  GitCommit:        3dd1e886e55dd695541fdcd67420c2888645a495
 runc:
  Version:          1.1.10
  GitCommit:        v1.1.10-0-g18a0cb0
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

~$ docker compose version
Docker Compose version v2.21.0

Container A

Debain 11

tcpreplay version: 4.3.3 (build git:v4.3.3) (debug)

Interface Aa (from it will transmit pcap)

# ip a sho dev eth0
59616: eth0@if59617: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 10000 qdisc noqueue state UP group default
    link/ether 02:42:ac:1d:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 172.29.0.2/16 brd 172.29.255.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 2025::62/64 scope global nodad
       valid_lft forever preferred_lft forever
    inet6 fe80::42:acff:fe1d:2/64 scope link
       valid_lft forever preferred_lft forever

Container B

Ubuntu 20.04.6

Interface Bb

root@331af0a8df97:/app# ip a sh eth0
59622: eth0@if59623: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 10000 qdisc noqueue state UP group default
    link/ether 02:42:ac:1c:00:04 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 172.29.0.3/16 brd 172.29.255.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 2025::52/64 scope global nodad
       valid_lft forever preferred_lft forever
    inet6 fe80::42:acff:fe1c:4/64 scope link
       valid_lft forever preferred_lft forever

Interafeces of both Containers connected to one bridge by docker compose file

# clipping from a docker compose file
  lan1_up_net:
    name: localstand_uplink_lan1_up_net
    driver: bridge
    driver_opts:
      com.docker.network.driver.mtu: "10000"
    ipam:
      driver: default
      config:
      - subnet: 2025::/64
    enable_ipv6: true

# 
$ docker network inspect localstand_uplink_lan1_up_net
[
    {
        "Name": "localstand_uplink_lan1_up_net",
        "Id": "ebf078ab6c21802cbcc5caa54978badd8ee6e328c61016b08e61765f55eb26b9",
        "Created": "2024-03-15T17:09:55.937604118+03:00",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": true,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "172.29.0.0/16",
                    "Gateway": "172.29.0.1"
                },
                {
                    "Subnet": "2025::/64"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "331af0a8df97bc502dcf00d57921f45bf05f14d46878b7a597bbc83c0086ca61": {
                "Name": "localstand_uplink-app-1",
                "EndpointID": "316b94f2dee3a0e8bf604688fe7420c5acbc69f3e3315fe864b66e4f5f89a57c",
                "MacAddress": "02:42:ac:1c:00:04",
                "IPv4Address": "172.29.0.3/16",
                "IPv6Address": "2025::52/64"
            },
            "4fdb4f71a5e7aca6f485bb4ceb6b76e85a8bafdae8f0379c1c9bde97de0c1a96": {
                "Name": "localstand_uplink-test-scripts-1",
                "EndpointID": "a165ea13ff6900b10fd7a65aa0737660a69ec43cf10d02ba76336d898da1cd05",
                "MacAddress": "02:42:ac:1d:00:02",
                "IPv4Address": "172.29.0.2/16",
                "IPv6Address": "2025::62/64"
            }
        },
        "Options": {
            "com.docker.network.driver.mtu": "10000"
        },
        "Labels": {
            "com.docker.compose.network": "lan1_up_net",
            "com.docker.compose.project": "localstand_uplink",
            "com.docker.compose.version": "2.21.0"
        }
    }
]

The host machine has default settings configured by docker

for bridge, iptables, forwarding

# on host server
$ ip -6 route | grep 2025
2025::/64 dev br-ebf078ab6c21 proto kernel metric 256 pref medium

$ bridge link | grep br-ebf078ab6c21
59617: veth7e48772@if59616: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 10000 master br-ebf078ab6c21 state forwarding priority 32 cost 2
59623: veth8622553@if59622: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 10000 master br-ebf078ab6c21 state forwarding priority 32 cost 2

$ sudo iptables-save | grep br-ebf078ab6c21
-A FORWARD -o br-ebf078ab6c21 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o br-ebf078ab6c21 -j DOCKER
-A FORWARD -i br-ebf078ab6c21 ! -o br-ebf078ab6c21 -j ACCEPT
-A FORWARD -i br-ebf078ab6c21 -o br-ebf078ab6c21 -j ACCEPT
-A DOCKER-ISOLATION-STAGE-1 -i br-ebf078ab6c21 ! -o br-ebf078ab6c21 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-2 -o br-ebf078ab6c21 -j DROP
-A POSTROUTING -s 172.29.0.0/16 ! -o br-ebf078ab6c21 -j MASQUERADE
-A DOCKER -i br-ebf078ab6c21 -j RETURN


$ sudo sysctl -a | grep ipv6 | grep forwa|grep all
net.ipv6.conf.all.forwarding = 1
net.ipv6.conf.all.mc_forwarding = 0

$ sudo sysctl -a | grep ipv6 | grep forwa|grep ebf078ab6c21
net.ipv6.conf.br-ebf078ab6c21.forwarding = 1
net.ipv6.conf.br-ebf078ab6c21.mc_forwarding = 0

I am trying to play a pcap file from container A and expect to receive network packets from pcap in container B.

on Container A

# tcpreplay-edit --enet-dmac=02:42:ac:1c:00:04 -S [2001::5]:[2025::62] -D [2011::3]:[2025::52] -i eth0 one_icmp_100b-1709738246.pcap

Actual: 2 packets (281 bytes) sent in 0.000105 seconds
Rated: 2676190.4 Bps, 21.40 Mbps, 19047.61 pps
Statistics for network device: eth0
        Successful packets:        2
        Failed packets:            0
        Truncated packets:         0
        Retried packets (ENOBUFS): 0
        Retried packets (EAGAIN):  0

traffic in bridge on host machine

$ tcpdump -i  br-ebf078ab6c21 -en
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on br-ebf078ab6c21, link-type EN10MB (Ethernet), snapshot length 262144 bytes

23:48:14.802476 02:42:00:a2:11:04 > 02:42:ac:1c:00:04, ethertype IPv6 (0x86dd), length 168: 2025::62.40915 > 2025::52.2152: UDP, length 106
23:48:14.802555 02:42:00:a2:11:04 > 02:42:ac:1c:00:04, ethertype IPv6 (0x86dd), length 113: 2025::62.40915 > 2025::52.2152: UDP, length 51

on Container B

nothing

# tcpdump -i eth0 -en
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes

Test connectivity by netcat tool

# step 1
# on Container B
netcat -6 -u -l 5000

# step 2
# on Container A
echo 'HELLO,TEST UDP' | netcat -6 -s 2025::62 -w1 -u 2025::52 5000

# step 3
# on Container B
root@331af0a8df97:/app# netcat -6 -u -l 5000
HELLO,TEST UDP


# in bridge on host machine 
$ sudo tcpdump -i  br-ebf078ab6c21 -en
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on br-ebf078ab6c21, link-type EN10MB (Ethernet), snapshot length 262144 bytes

00:43:49.647306 02:42:ac:1d:00:02 > 02:42:ac:1c:00:04, ethertype IPv6 (0x86dd), length 77: 2025::62.48947 > 2025::52.5000: UDP, length 15
        0x0000:  6000 0a71 0017 1140 2025 0000 0000 0000
        0x0010:  0000 0000 0000 0062 2025 0000 0000 0000
        0x0020:  0000 0000 0000 0052 bf33 1388 0017 4126
        0x0030:  4845 4c4c 4f2c 5445 5354 2055 4450 0a

# on Container B
root@331af0a8df97:/app# tcpdump -i eth0 -en -X

00:43:49.647333 02:42:ac:1d:00:02 > 02:42:ac:1c:00:04, ethertype IPv6 (0x86dd), length 77: 2025::62.48947 > 2025::52.5000: UDP, length 15
        0x0000:  6000 0a71 0017 1140 2025 0000 0000 0000  `..q...@.%......
        0x0010:  0000 0000 0000 0062 2025 0000 0000 0000  .......b.%......
        0x0020:  0000 0000 0000 0052 bf33 1388 0017 4126  .......R.3....A&
        0x0030:  4845 4c4c 4f2c 5445 5354 2055 4450 0a    HELLO,TEST.UDP.

What else can I check?

Example pcap file: You can download the file here: one_icmp_100b-1709738246.pcap https://disk.yandex.ru/d/mVF7rH3i7Wkk7A

Solution

  • Solved.

    After restarting the host machine and executing the command docker network prune, the network packet from the pcap file began to reach container B.