431 lines
14 KiB
Python
Executable File
431 lines
14 KiB
Python
Executable File
#!/usr/bin/python
|
|
# @lint-avoid-python-3-compatibility-imports
|
|
#
|
|
# tcpretrans Trace or count TCP retransmits and TLPs.
|
|
# For Linux, uses BCC, eBPF. Embedded C.
|
|
#
|
|
# USAGE: tcpretrans [-c] [-h] [-l] [-4 | -6]
|
|
#
|
|
# This uses dynamic tracing of kernel functions, and will need to be updated
|
|
# to match kernel changes.
|
|
#
|
|
# Copyright 2016 Netflix, Inc.
|
|
# Licensed under the Apache License, Version 2.0 (the "License")
|
|
#
|
|
# 14-Feb-2016 Brendan Gregg Created this.
|
|
# 03-Nov-2017 Matthias Tafelmeier Extended this.
|
|
|
|
from __future__ import print_function
|
|
from bcc import BPF
|
|
import argparse
|
|
from time import strftime
|
|
from socket import inet_ntop, AF_INET, AF_INET6
|
|
from struct import pack
|
|
from time import sleep
|
|
|
|
# arguments
|
|
examples = """examples:
|
|
./tcpretrans # trace TCP retransmits
|
|
./tcpretrans -l # include TLP attempts
|
|
./tcpretrans -4 # trace IPv4 family only
|
|
./tcpretrans -6 # trace IPv6 family only
|
|
"""
|
|
parser = argparse.ArgumentParser(
|
|
description="Trace TCP retransmits",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog=examples)
|
|
parser.add_argument("-s", "--sequence", action="store_true",
|
|
help="display TCP sequence numbers")
|
|
parser.add_argument("-l", "--lossprobe", action="store_true",
|
|
help="include tail loss probe attempts")
|
|
parser.add_argument("-c", "--count", action="store_true",
|
|
help="count occurred retransmits per flow")
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument("-4", "--ipv4", action="store_true",
|
|
help="trace IPv4 family only")
|
|
group.add_argument("-6", "--ipv6", action="store_true",
|
|
help="trace IPv6 family only")
|
|
parser.add_argument("--ebpf", action="store_true",
|
|
help=argparse.SUPPRESS)
|
|
args = parser.parse_args()
|
|
debug = 0
|
|
|
|
# define BPF program
|
|
bpf_text = """
|
|
#include <uapi/linux/ptrace.h>
|
|
#include <net/sock.h>
|
|
#include <net/tcp.h>
|
|
#include <bcc/proto.h>
|
|
|
|
#define RETRANSMIT 1
|
|
#define TLP 2
|
|
|
|
// separate data structs for ipv4 and ipv6
|
|
struct ipv4_data_t {
|
|
u32 pid;
|
|
u64 ip;
|
|
u32 seq;
|
|
u32 saddr;
|
|
u32 daddr;
|
|
u16 lport;
|
|
u16 dport;
|
|
u64 state;
|
|
u64 type;
|
|
};
|
|
BPF_PERF_OUTPUT(ipv4_events);
|
|
|
|
struct ipv6_data_t {
|
|
u32 pid;
|
|
u32 seq;
|
|
u64 ip;
|
|
unsigned __int128 saddr;
|
|
unsigned __int128 daddr;
|
|
u16 lport;
|
|
u16 dport;
|
|
u64 state;
|
|
u64 type;
|
|
};
|
|
BPF_PERF_OUTPUT(ipv6_events);
|
|
|
|
// separate flow keys per address family
|
|
struct ipv4_flow_key_t {
|
|
u32 saddr;
|
|
u32 daddr;
|
|
u16 lport;
|
|
u16 dport;
|
|
};
|
|
BPF_HASH(ipv4_count, struct ipv4_flow_key_t);
|
|
|
|
struct ipv6_flow_key_t {
|
|
unsigned __int128 saddr;
|
|
unsigned __int128 daddr;
|
|
u16 lport;
|
|
u16 dport;
|
|
};
|
|
BPF_HASH(ipv6_count, struct ipv6_flow_key_t);
|
|
"""
|
|
|
|
bpf_text_kprobe = """
|
|
static int trace_event(struct pt_regs *ctx, struct sock *skp, struct sk_buff *skb, int type)
|
|
{
|
|
struct tcp_skb_cb *tcb;
|
|
u32 seq;
|
|
|
|
if (skp == NULL)
|
|
return 0;
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
|
|
// pull in details
|
|
u16 family = skp->__sk_common.skc_family;
|
|
u16 lport = skp->__sk_common.skc_num;
|
|
u16 dport = skp->__sk_common.skc_dport;
|
|
char state = skp->__sk_common.skc_state;
|
|
|
|
seq = 0;
|
|
if (skb) {
|
|
/* macro TCP_SKB_CB from net/tcp.h */
|
|
tcb = ((struct tcp_skb_cb *)&((skb)->cb[0]));
|
|
seq = tcb->seq;
|
|
}
|
|
|
|
FILTER_FAMILY
|
|
|
|
if (family == AF_INET) {
|
|
IPV4_INIT
|
|
IPV4_CORE
|
|
} else if (family == AF_INET6) {
|
|
IPV6_INIT
|
|
IPV6_CORE
|
|
}
|
|
// else drop
|
|
|
|
return 0;
|
|
}
|
|
"""
|
|
|
|
bpf_text_kprobe_retransmit = """
|
|
int trace_retransmit(struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb)
|
|
{
|
|
trace_event(ctx, sk, skb, RETRANSMIT);
|
|
return 0;
|
|
}
|
|
"""
|
|
|
|
bpf_text_kprobe_tlp = """
|
|
int trace_tlp(struct pt_regs *ctx, struct sock *sk)
|
|
{
|
|
trace_event(ctx, sk, NULL, TLP);
|
|
return 0;
|
|
}
|
|
"""
|
|
|
|
bpf_text_tracepoint = """
|
|
TRACEPOINT_PROBE(tcp, tcp_retransmit_skb)
|
|
{
|
|
struct tcp_skb_cb *tcb;
|
|
u32 seq;
|
|
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
const struct sock *skp = (const struct sock *)args->skaddr;
|
|
const struct sk_buff *skb = (const struct sk_buff *)args->skbaddr;
|
|
u16 lport = args->sport;
|
|
u16 dport = args->dport;
|
|
char state = skp->__sk_common.skc_state;
|
|
u16 family = skp->__sk_common.skc_family;
|
|
|
|
seq = 0;
|
|
if (skb) {
|
|
/* macro TCP_SKB_CB from net/tcp.h */
|
|
tcb = ((struct tcp_skb_cb *)&((skb)->cb[0]));
|
|
seq = tcb->seq;
|
|
}
|
|
|
|
FILTER_FAMILY
|
|
|
|
if (family == AF_INET) {
|
|
IPV4_CODE
|
|
} else if (family == AF_INET6) {
|
|
IPV6_CODE
|
|
}
|
|
return 0;
|
|
}
|
|
"""
|
|
|
|
struct_init = { 'ipv4':
|
|
{ 'count' :
|
|
"""
|
|
struct ipv4_flow_key_t flow_key = {};
|
|
flow_key.saddr = skp->__sk_common.skc_rcv_saddr;
|
|
flow_key.daddr = skp->__sk_common.skc_daddr;
|
|
// lport is host order
|
|
flow_key.lport = lport;
|
|
flow_key.dport = ntohs(dport);""",
|
|
'trace' :
|
|
"""
|
|
struct ipv4_data_t data4 = {};
|
|
data4.pid = pid;
|
|
data4.ip = 4;
|
|
data4.seq = seq;
|
|
data4.type = type;
|
|
data4.saddr = skp->__sk_common.skc_rcv_saddr;
|
|
data4.daddr = skp->__sk_common.skc_daddr;
|
|
// lport is host order
|
|
data4.lport = lport;
|
|
data4.dport = ntohs(dport);
|
|
data4.state = state; """
|
|
},
|
|
'ipv6':
|
|
{ 'count' :
|
|
"""
|
|
struct ipv6_flow_key_t flow_key = {};
|
|
bpf_probe_read_kernel(&flow_key.saddr, sizeof(flow_key.saddr),
|
|
skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32);
|
|
bpf_probe_read_kernel(&flow_key.daddr, sizeof(flow_key.daddr),
|
|
skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32);
|
|
// lport is host order
|
|
flow_key.lport = lport;
|
|
flow_key.dport = ntohs(dport);""",
|
|
'trace' : """
|
|
struct ipv6_data_t data6 = {};
|
|
data6.pid = pid;
|
|
data6.ip = 6;
|
|
data6.seq = seq;
|
|
data6.type = type;
|
|
bpf_probe_read_kernel(&data6.saddr, sizeof(data6.saddr),
|
|
skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32);
|
|
bpf_probe_read_kernel(&data6.daddr, sizeof(data6.daddr),
|
|
skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32);
|
|
// lport is host order
|
|
data6.lport = lport;
|
|
data6.dport = ntohs(dport);
|
|
data6.state = state;"""
|
|
}
|
|
}
|
|
|
|
struct_init_tracepoint = { 'ipv4':
|
|
{ 'count' : """
|
|
struct ipv4_flow_key_t flow_key = {};
|
|
__builtin_memcpy(&flow_key.saddr, args->saddr, sizeof(flow_key.saddr));
|
|
__builtin_memcpy(&flow_key.daddr, args->daddr, sizeof(flow_key.daddr));
|
|
flow_key.lport = lport;
|
|
flow_key.dport = dport;
|
|
ipv4_count.increment(flow_key);
|
|
""",
|
|
'trace' : """
|
|
struct ipv4_data_t data4 = {};
|
|
data4.pid = pid;
|
|
data4.lport = lport;
|
|
data4.dport = dport;
|
|
data4.type = RETRANSMIT;
|
|
data4.ip = 4;
|
|
data4.seq = seq;
|
|
data4.state = state;
|
|
__builtin_memcpy(&data4.saddr, args->saddr, sizeof(data4.saddr));
|
|
__builtin_memcpy(&data4.daddr, args->daddr, sizeof(data4.daddr));
|
|
ipv4_events.perf_submit(args, &data4, sizeof(data4));
|
|
"""
|
|
},
|
|
'ipv6':
|
|
{ 'count' : """
|
|
struct ipv6_flow_key_t flow_key = {};
|
|
__builtin_memcpy(&flow_key.saddr, args->saddr_v6, sizeof(flow_key.saddr));
|
|
__builtin_memcpy(&flow_key.daddr, args->daddr_v6, sizeof(flow_key.daddr));
|
|
flow_key.lport = lport;
|
|
flow_key.dport = dport;
|
|
ipv6_count.increment(flow_key);
|
|
""",
|
|
'trace' : """
|
|
struct ipv6_data_t data6 = {};
|
|
data6.pid = pid;
|
|
data6.lport = lport;
|
|
data6.dport = dport;
|
|
data6.type = RETRANSMIT;
|
|
data6.ip = 6;
|
|
data6.seq = seq;
|
|
data6.state = state;
|
|
__builtin_memcpy(&data6.saddr, args->saddr_v6, sizeof(data6.saddr));
|
|
__builtin_memcpy(&data6.daddr, args->daddr_v6, sizeof(data6.daddr));
|
|
ipv6_events.perf_submit(args, &data6, sizeof(data6));
|
|
"""
|
|
}
|
|
}
|
|
|
|
count_core_base = """
|
|
COUNT_STRUCT.increment(flow_key);
|
|
"""
|
|
|
|
if BPF.tracepoint_exists("tcp", "tcp_retransmit_skb"):
|
|
if args.count:
|
|
bpf_text_tracepoint = bpf_text_tracepoint.replace("IPV4_CODE", struct_init_tracepoint['ipv4']['count'])
|
|
bpf_text_tracepoint = bpf_text_tracepoint.replace("IPV6_CODE", struct_init_tracepoint['ipv6']['count'])
|
|
else:
|
|
bpf_text_tracepoint = bpf_text_tracepoint.replace("IPV4_CODE", struct_init_tracepoint['ipv4']['trace'])
|
|
bpf_text_tracepoint = bpf_text_tracepoint.replace("IPV6_CODE", struct_init_tracepoint['ipv6']['trace'])
|
|
bpf_text += bpf_text_tracepoint
|
|
|
|
if args.lossprobe or not BPF.tracepoint_exists("tcp", "tcp_retransmit_skb"):
|
|
bpf_text += bpf_text_kprobe
|
|
if args.count:
|
|
bpf_text = bpf_text.replace("IPV4_INIT", struct_init['ipv4']['count'])
|
|
bpf_text = bpf_text.replace("IPV6_INIT", struct_init['ipv6']['count'])
|
|
bpf_text = bpf_text.replace("IPV4_CORE", count_core_base.replace("COUNT_STRUCT", 'ipv4_count'))
|
|
bpf_text = bpf_text.replace("IPV6_CORE", count_core_base.replace("COUNT_STRUCT", 'ipv6_count'))
|
|
else:
|
|
bpf_text = bpf_text.replace("IPV4_INIT", struct_init['ipv4']['trace'])
|
|
bpf_text = bpf_text.replace("IPV6_INIT", struct_init['ipv6']['trace'])
|
|
bpf_text = bpf_text.replace("IPV4_CORE", "ipv4_events.perf_submit(ctx, &data4, sizeof(data4));")
|
|
bpf_text = bpf_text.replace("IPV6_CORE", "ipv6_events.perf_submit(ctx, &data6, sizeof(data6));")
|
|
if args.lossprobe:
|
|
bpf_text += bpf_text_kprobe_tlp
|
|
if not BPF.tracepoint_exists("tcp", "tcp_retransmit_skb"):
|
|
bpf_text += bpf_text_kprobe_retransmit
|
|
if args.ipv4:
|
|
bpf_text = bpf_text.replace('FILTER_FAMILY',
|
|
'if (family != AF_INET) { return 0; }')
|
|
elif args.ipv6:
|
|
bpf_text = bpf_text.replace('FILTER_FAMILY',
|
|
'if (family != AF_INET6) { return 0; }')
|
|
else:
|
|
bpf_text = bpf_text.replace('FILTER_FAMILY', '')
|
|
if debug or args.ebpf:
|
|
print(bpf_text)
|
|
if args.ebpf:
|
|
exit()
|
|
|
|
# from bpf_text:
|
|
type = {}
|
|
type[1] = 'R'
|
|
type[2] = 'L'
|
|
|
|
# from include/net/tcp_states.h:
|
|
tcpstate = {}
|
|
tcpstate[1] = 'ESTABLISHED'
|
|
tcpstate[2] = 'SYN_SENT'
|
|
tcpstate[3] = 'SYN_RECV'
|
|
tcpstate[4] = 'FIN_WAIT1'
|
|
tcpstate[5] = 'FIN_WAIT2'
|
|
tcpstate[6] = 'TIME_WAIT'
|
|
tcpstate[7] = 'CLOSE'
|
|
tcpstate[8] = 'CLOSE_WAIT'
|
|
tcpstate[9] = 'LAST_ACK'
|
|
tcpstate[10] = 'LISTEN'
|
|
tcpstate[11] = 'CLOSING'
|
|
tcpstate[12] = 'NEW_SYN_RECV'
|
|
|
|
# process event
|
|
def print_ipv4_event(cpu, data, size):
|
|
event = b["ipv4_events"].event(data)
|
|
print("%-8s %-7d %-2d %-20s %1s> %-20s" % (
|
|
strftime("%H:%M:%S"), event.pid, event.ip,
|
|
"%s:%d" % (inet_ntop(AF_INET, pack('I', event.saddr)), event.lport),
|
|
type[event.type],
|
|
"%s:%s" % (inet_ntop(AF_INET, pack('I', event.daddr)), event.dport)),
|
|
end='')
|
|
if args.sequence:
|
|
print(" %-12s %s" % (tcpstate[event.state], event.seq))
|
|
else:
|
|
print(" %s" % (tcpstate[event.state]))
|
|
|
|
def print_ipv6_event(cpu, data, size):
|
|
event = b["ipv6_events"].event(data)
|
|
print("%-8s %-7d %-2d %-20s %1s> %-20s" % (
|
|
strftime("%H:%M:%S"), event.pid, event.ip,
|
|
"%s:%d" % (inet_ntop(AF_INET6, event.saddr), event.lport),
|
|
type[event.type],
|
|
"%s:%d" % (inet_ntop(AF_INET6, event.daddr), event.dport)),
|
|
end='')
|
|
if args.sequence:
|
|
print(" %-12s %s" % (tcpstate[event.state], event.seq))
|
|
else:
|
|
print(" %s" % (tcpstate[event.state]))
|
|
|
|
def depict_cnt(counts_tab, l3prot='ipv4'):
|
|
for k, v in sorted(counts_tab.items(), key=lambda counts: counts[1].value):
|
|
depict_key = ""
|
|
ep_fmt = "[%s]#%d"
|
|
if l3prot == 'ipv4':
|
|
depict_key = "%-20s <-> %-20s" % (ep_fmt % (inet_ntop(AF_INET, pack('I', k.saddr)), k.lport),
|
|
ep_fmt % (inet_ntop(AF_INET, pack('I', k.daddr)), k.dport))
|
|
else:
|
|
depict_key = "%-20s <-> %-20s" % (ep_fmt % (inet_ntop(AF_INET6, k.saddr), k.lport),
|
|
ep_fmt % (inet_ntop(AF_INET6, k.daddr), k.dport))
|
|
|
|
print ("%s %10d" % (depict_key, v.value))
|
|
|
|
# initialize BPF
|
|
b = BPF(text=bpf_text)
|
|
if not BPF.tracepoint_exists("tcp", "tcp_retransmit_skb"):
|
|
b.attach_kprobe(event="tcp_retransmit_skb", fn_name="trace_retransmit")
|
|
if args.lossprobe:
|
|
b.attach_kprobe(event="tcp_send_loss_probe", fn_name="trace_tlp")
|
|
|
|
print("Tracing retransmits ... Hit Ctrl-C to end")
|
|
if args.count:
|
|
try:
|
|
while 1:
|
|
sleep(99999999)
|
|
except BaseException:
|
|
pass
|
|
|
|
# header
|
|
print("\n%-25s %-25s %-10s" % (
|
|
"LADDR:LPORT", "RADDR:RPORT", "RETRANSMITS"))
|
|
depict_cnt(b.get_table("ipv4_count"))
|
|
depict_cnt(b.get_table("ipv6_count"), l3prot='ipv6')
|
|
# read events
|
|
else:
|
|
# header
|
|
print("%-8s %-7s %-2s %-20s %1s> %-20s" % ("TIME", "PID", "IP",
|
|
"LADDR:LPORT", "T", "RADDR:RPORT"), end='')
|
|
if args.sequence:
|
|
print(" %-12s %-10s" % ("STATE", "SEQ"))
|
|
else:
|
|
print(" %-4s" % ("STATE"))
|
|
b["ipv4_events"].open_perf_buffer(print_ipv4_event)
|
|
b["ipv6_events"].open_perf_buffer(print_ipv6_event)
|
|
while 1:
|
|
try:
|
|
b.perf_buffer_poll()
|
|
except KeyboardInterrupt:
|
|
exit()
|