ovs conntrack based firewall driver (by quqi99)

作者：张华  发表于：2016-04-20
版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明

( http://blog.csdn.net/quqi99 )

我们知道，Neutron security group特性是基于iptables实现的，iptables规则只能作用于linux bridge，不能作用于ovs bridge上，所以在VM port和ovs br-int之前又多弄了一个linux bridge (qbr-xxx)，这会极大影响性能。如今openvswitch 2.5 （需使用linux kernel 4.3+) （sudo add-apt-repository cloud-archive:mitaka && sudo apt-get install openvswitch-switch）已经支持conntract特性，neutron也在Mitaka中实现了这一特性[1]. 创建两个虚机之后查看它的流表如下，解释见内联注释。pls also refer - OpenStack Security (quqi99)_技术并艺术着-CSDN博客

• 注意: DVR的流规则是设置在br-tun中, OVS Firewall的流规则在br-int中

cookie=0xb7d7ed46110fd50e, duration=10510.153s, table=0, n_packets=6, n_bytes=582, idle_age=886, priority=2,in_port=1 actions=drop

# Table 0是分类表，reg5用于存储port_id (出口流量使用port_id标识，入口流量采用mac_address标识。出口与入口以虚机为基准）, reg6用于存储zone避免不同的port可能出现conntrack参数相同的情况。出口流量转到table 71, 入口流量转到table 81
 cookie=0xb7d7ed46110fd50e, duration=10155.140s, table=0, n_packets=25, n_bytes=2332, idle_age=9619, priority=100,in_port=13 actions=load:0xd->NXM_NX_REG5[],load:0x1->NXM_NX_REG6[],resubmit(,71)
 cookie=0xb7d7ed46110fd50e, duration=10155.041s, table=0, n_packets=97, n_bytes=12752, idle_age=9617, priority=100,in_port=10 actions=load:0xa->NXM_NX_REG5[],load:0x1->NXM_NX_REG6[],resubmit(,71)
 cookie=0xb7d7ed46110fd50e, duration=10155.140s, table=0, n_packets=12, n_bytes=1489, idle_age=10143, priority=90,dl_dst=fa:16:3e:e9:f9:c8 actions=load:0xd->NXM_NX_REG5[],load:0x1->NXM_NX_REG6[],resubmit(,81)
 cookie=0xb7d7ed46110fd50e, duration=10155.040s, table=0, n_packets=118, n_bytes=21821, idle_age=9617, priority=90,dl_dst=fa:16:3e:5c:25:9d actions=load:0xa->NXM_NX_REG5[],load:0x1->NXM_NX_REG6[],resubmit(,81)

# 对从int-br-phy (在br-int与br-phy中的一个ovs patch port)进br-int的入虚机流量将vlan 1053换成local vlan 1.
 cookie=0xb7d7ed46110fd50e, duration=10447.209s, table=0, n_packets=0, n_bytes=0, idle_age=10447, priority=3,in_port=1,dl_vlan=1053 actions=mod_vlan_vid:1,NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10510.260s, table=0, n_packets=19, n_bytes=1554, idle_age=10383, priority=0 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10510.252s, table=23, n_packets=0, n_bytes=0, idle_age=10510, priority=0 actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10510.244s, table=24, n_packets=0, n_bytes=0, idle_age=10510, priority=0 actions=drop

# Allow ICMPv6 traffic for multicast listeners, neighbour solicitation and neighbour advertisement for egress flow.
 cookie=0xb7d7ed46110fd50e, duration=10155.140s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xd,in_port=13,icmp_type=130 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.140s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xd,in_port=13,icmp_type=131 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.139s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xd,in_port=13,icmp_type=132 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.139s, table=71, n_packets=1, n_bytes=78, idle_age=10147, priority=95,icmp6,reg5=0xd,in_port=13,icmp_type=135 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.139s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xd,in_port=13,icmp_type=136 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.040s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xa,in_port=10,icmp_type=130 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.039s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xa,in_port=10,icmp_type=131 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.039s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xa,in_port=10,icmp_type=132 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.039s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xa,in_port=10,icmp_type=135 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.039s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=95,icmp6,reg5=0xa,in_port=10,icmp_type=136 actions=NORMAL

# 在table 71中对出虚机的流量做arp spoofing protection
 cookie=0xb7d7ed46110fd50e, duration=10155.139s, table=71, n_packets=7, n_bytes=294, idle_age=9619, priority=95,arp,reg5=0xd,in_port=13,dl_src=fa:16:3e:e9:f9:c8,arp_spa=10.0.1.8 actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.039s, table=71, n_packets=7, n_bytes=294, idle_age=9617, priority=95,arp,reg5=0xa,in_port=10,dl_src=fa:16:3e:5c:25:9d,arp_spa=10.0.1.7 actions=NORMAL

# 充许端口为68,67, 546, 547(dhcp, dhcpv6, slaas, ndp)的流量出虚机, but DHCP servers are blocked on instances.
 cookie=0xb7d7ed46110fd50e, duration=10155.138s, table=71, n_packets=2, n_bytes=668, idle_age=10148, priority=80,udp,reg5=0xd,in_port=13,tp_src=68,tp_dst=67 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.138s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=80,udp6,reg5=0xd,in_port=13,tp_src=546,tp_dst=547 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.137s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=70,udp,reg5=0xd,in_port=13,tp_src=67,tp_dst=68 actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.137s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=70,udp6,reg5=0xd,in_port=13,tp_src=547,tp_dst=546 actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.038s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=80,udp,reg5=0xa,in_port=10,tp_src=68,tp_dst=67 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.038s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=80,udp6,reg5=0xa,in_port=10,tp_src=546,tp_dst=547 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.037s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=70,udp,reg5=0xa,in_port=10,tp_src=67,tp_dst=68 actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.037s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=70,udp6,reg5=0xa,in_port=10,tp_src=547,tp_dst=546 actions=drop

# 对-trk状态的出虚机流量转到table 72继续处理，并使用ip+mac从内核的conntrack中获取conntrack的相关信息，drop其他流量
 cookie=0xb7d7ed46110fd50e, duration=10155.138s, table=71, n_packets=10, n_bytes=902, idle_age=9619, priority=65,ct_state=-trk,ip,reg5=0xd,in_port=13,dl_src=fa:16:3e:e9:f9:c8,nw_src=10.0.1.8 actions=ct(table=72,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.039s, table=71, n_packets=90, n_bytes=12458, idle_age=9619, priority=65,ct_state=-trk,ip,reg5=0xa,in_port=10,dl_src=fa:16:3e:5c:25:9d,nw_src=10.0.1.7 actions=ct(table=72,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.138s, table=71, n_packets=4, n_bytes=300, idle_age=10138, priority=65,ct_state=-trk,ipv6,reg5=0xd,in_port=13,dl_src=fa:16:3e:e9:f9:c8,ipv6_src=fe80::f816:3eff:fee9:f9c8 actions=ct(table=72,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.038s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=65,ct_state=-trk,ipv6,reg5=0xa,in_port=10,dl_src=fa:16:3e:5c:25:9d,ipv6_src=fe80::f816:3eff:fe5c:259d actions=ct(table=72,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.136s, table=71, n_packets=1, n_bytes=90, idle_age=10148, priority=10,ct_state=-trk,reg5=0xd,in_port=13 actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.037s, table=71, n_packets=0, n_bytes=0, idle_age=10155, priority=10,ct_state=-trk,reg5=0xa,in_port=10 actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10509.934s, table=71, n_packets=0, n_bytes=0, idle_age=10509, priority=0 actions=drop

# tables 72表接受established (est-rel-rpl) or related (-new-est+rel-inv, +est-rel+rpl) or new (new-est)状态的出虚机流量, drop掉invalid (inv+trk, ct_mark=0x1)状态的流量.并且这里实现用户自定义的security group规则。 
 cookie=0xb7d7ed46110fd50e, duration=10155.129s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,ipv6,reg5=0xd,dl_src=fa:16:3e:e9:f9:c8 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.129s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,ip,reg5=0xd,dl_src=fa:16:3e:e9:f9:c8 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.029s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,ipv6,reg5=0xa,dl_src=fa:16:3e:5c:25:9d actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.028s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,ip,reg5=0xa,dl_src=fa:16:3e:5c:25:9d actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.129s, table=72, n_packets=4, n_bytes=300, idle_age=10138, 
priority=70,ct_state=+new-est,ipv6,reg5=0xd,dl_src=fa:16:3e:e9:f9:c8 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.129s, table=72, n_packets=7, n_bytes=608, idle_age=10148, priority=70,ct_state=+new-est,ip,reg5=0xd,dl_src=fa:16:3e:e9:f9:c8 actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.029s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+new-est,ipv6,reg5=0xa,dl_src=fa:16:3e:5c:25:9d actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.028s, table=72, n_packets=3, n_bytes=294, idle_age=9619, priority=70,ct_state=+new-est,ip,reg5=0xa,dl_src=fa:16:3e:5c:25:9d actions=resubmit(,73)
 cookie=0xb7d7ed46110fd50e, duration=10155.031s, table=72, n_packets=0, n_bytes=0, idle_age=10155, 
priority=50,ct_state=+inv+trk actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.131s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_mark=0x1,reg5=0xd actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.031s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_mark=0x1,reg5=0xa actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.131s, table=72, n_packets=3, n_bytes=294, idle_age=9619, priority=50,ct_state=+est-rel+rpl,ct_zone=1,ct_mark=0,reg5=0xd actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.031s, table=72, n_packets=87, n_bytes=12164, idle_age=9619, priority=50,ct_state=+est-rel+rpl,ct_zone=1,ct_mark=0,reg5=0xa actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.130s, table=72, n_packets=0, n_bytes=0, idle_age=10155, 
priority=50,ct_state=-new-est+rel-inv,ct_zone=1,ct_mark=0,reg5=0xd actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.030s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_state=-new-est+rel-inv,ct_zone=1,ct_mark=0,reg5=0xa actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.130s, table=72, n_packets=0, n_bytes=0, idle_age=10155, 
priority=40,ct_state=-est,reg5=0xd actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.130s, table=72, n_packets=0, n_bytes=0, idle_age=10155, 

# In the following flows are marked established connections that weren’t matched in the previous flows, which means they don’t have accepting security group rule anymore.
priority=40,ct_state=+est,reg5=0xd actions=ct(commit,zone=NXM_NX_REG6[0..15],exec(load:0x1->NXM_NX_CT_MARK[]))
 cookie=0xb7d7ed46110fd50e, duration=10155.030s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=40,ct_state=-est,reg5=0xa actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.030s, table=72, n_packets=0, n_bytes=0, idle_age=10155, priority=40,ct_state=+est,reg5=0xa actions=ct(commit,zone=NXM_NX_REG6[0..15],exec(load:0x1->NXM_NX_CT_MARK[]))
 cookie=0xb7d7ed46110fd50e, duration=10509.925s, table=72, n_packets=0, n_bytes=0, idle_age=10509, priority=0 actions=drop

# 处理入口流量
 cookie=0xb7d7ed46110fd50e, duration=10155.136s, table=73, n_packets=3, n_bytes=294, idle_age=9619, priority=100,dl_dst=fa:16:3e:e9:f9:c8 actions=load:0xd->NXM_NX_REG5[],resubmit(,81)
 cookie=0xb7d7ed46110fd50e, duration=10155.037s, table=73, n_packets=0, n_bytes=0, idle_age=10155, priority=100,dl_dst=fa:16:3e:5c:25:9d actions=load:0xa->NXM_NX_REG5[],resubmit(,81)
 cookie=0xb7d7ed46110fd50e, duration=10155.136s, table=73, n_packets=11, n_bytes=908, idle_age=10138, priority=90,ct_state=+new-est,reg5=0xd actions=ct(commit,zone=NXM_NX_REG6[0..15]),NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.036s, table=73, n_packets=0, n_bytes=0, idle_age=10155, priority=90,ct_state=+new-est,reg5=0xa actions=ct(commit,zone=NXM_NX_REG6[0..15]),NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.136s, table=73, n_packets=2, n_bytes=668, idle_age=10148, priority=80,reg5=0xd actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10155.036s, table=73, n_packets=0, n_bytes=0, idle_age=10155, priority=80,reg5=0xa actions=NORMAL
 cookie=0xb7d7ed46110fd50e, duration=10509.917s, table=73, n_packets=0, n_bytes=0, idle_age=10509, priority=0 actions=drop

# Table 81 is for ingress traffic, accepts arp response, icmp6 response and udp response.
 cookie=0xb7d7ed46110fd50e, duration=10155.135s, table=81, n_packets=3, n_bytes=126, idle_age=10143, priority=100,arp,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.035s, table=81, n_packets=3, n_bytes=126, idle_age=9617, priority=100,arp,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.135s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,icmp_type=130 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.135s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,icmp_type=131 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.135s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,icmp_type=132 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.135s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,icmp_type=135 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.134s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,icmp_type=136 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.035s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,icmp_type=130 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.035s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,icmp_type=131 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.034s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,icmp_type=132 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.033s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,icmp_type=135 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.033s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=100,icmp6,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,icmp_type=136 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.133s, table=81, n_packets=2, n_bytes=755, idle_age=10148, 
priority=95,udp,reg5=0xd,tp_src=67,tp_dst=68 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.133s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=95,udp6,reg5=0xd,tp_src=547,tp_dst=546 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.033s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=95,udp,reg5=0xa,tp_src=67,tp_dst=68 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.033s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=95,udp6,reg5=0xa,tp_src=547,tp_dst=546 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.133s, table=81, n_packets=7, n_bytes=608, idle_age=10148, 

# Table 81 is for ingress traffic, indentifies not tracked ingress connections.
priority=90,ct_state=-trk,ip,reg5=0xd actions=ct(table=82,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.133s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=90,ct_state=-trk,ipv6,reg5=0xd actions=ct(table=82,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.033s, table=81, n_packets=115, n_bytes=21695, idle_age=9619, priority=90,ct_state=-trk,ip,reg5=0xa actions=ct(table=82,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=90,ct_state=-trk,ipv6,reg5=0xa actions=ct(table=82,zone=NXM_NX_REG6[0..15])
 cookie=0xb7d7ed46110fd50e, duration=10155.132s, table=81, n_packets=3, n_bytes=294, idle_age=9619, priority=80,ct_state=+trk,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8 actions=resubmit(,82)
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=81, n_packets=0, n_bytes=0, idle_age=10155, priority=80,ct_state=+trk,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d actions=resubmit(,82)
 cookie=0xb7d7ed46110fd50e, duration=10509.910s, table=81, n_packets=0, n_bytes=0, idle_age=10509, priority=0 actions=drop

# Table 82接受new (new-est) and established (est-rel-rpl, +est) and related (est-rel+rpl, -new-est+rel-inv)状态的入口流量. 也包括用户自定义的一些流量，如 (nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0).
 cookie=0xb7d7ed46110fd50e, duration=10155.130s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,tcp,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,tp_dst=0x16/0xfffe actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.030s, table=82, n_packets=112, n_bytes=21473, idle_age=9619, priority=70,ct_state=+est-rel-rpl,tcp,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,tp_dst=0x16/0xfffe actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.130s, table=82, n_packets=0, n_bytes=0, idle_age=10155, 
priority=70,ct_state=+new-est,tcp,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,tp_dst=0x16/0xfffe actions=ct(commit,zone=NXM_NX_REG6[0..15]),strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.030s, table=82, n_packets=3, n_bytes=222, idle_age=9622, priority=70,ct_state=+new-est,tcp,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,tp_dst=0x16/0xfffe actions=ct(commit,zone=NXM_NX_REG6[0..15]),strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.129s, table=82, n_packets=0, n_bytes=0, idle_age=10155, 
priority=70,ct_state=+est-rel-rpl,icmp,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.028s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,icmp,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.129s, table=82, n_packets=3, n_bytes=294, idle_age=9619, priority=70,ct_state=+new-est,icmp,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8 actions=ct(commit,zone=NXM_NX_REG6[0..15]),strip_vlan,output:13
cookie=0xb7d7ed46110fd50e, duration=10155.028s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+new-est,icmp,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d actions=ct(commit,zone=NXM_NX_REG6[0..15]),strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.128s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,ip,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,nw_src=10.0.1.7 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.028s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+est-rel-rpl,ip,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,nw_src=10.0.1.8 actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.128s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+new-est,ip,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8,nw_src=10.0.1.7 actions=ct(commit,zone=NXM_NX_REG6[0..15]),strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.028s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=70,ct_state=+new-est,ip,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d,nw_src=10.0.1.8 actions=ct(commit,zone=NXM_NX_REG6[0..15]),strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.132s, table=82, n_packets=7, n_bytes=608, idle_age=10148, priority=50,ct_state=+est-rel+rpl,ct_zone=1,ct_mark=0,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_state=+est-rel+rpl,ct_zone=1,ct_mark=0,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.132s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_state=-new-est+rel-inv,ct_zone=1,ct_mark=0,reg5=0xd,dl_dst=fa:16:3e:e9:f9:c8 actions=strip_vlan,output:13
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_state=-new-est+rel-inv,ct_zone=1,ct_mark=0,reg5=0xa,dl_dst=fa:16:3e:5c:25:9d actions=strip_vlan,output:10
 cookie=0xb7d7ed46110fd50e, duration=10155.131s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=40,ct_state=-est,reg5=0xd actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.131s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=40,ct_state=+est,reg5=0xd actions=ct(commit,zone=NXM_NX_REG6[0..15],exec(load:0x1->NXM_NX_CT_MARK[]))
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=40,ct_state=-est,reg5=0xa actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.031s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=40,ct_state=+est,reg5=0xa actions=ct(commit,zone=NXM_NX_REG6[0..15],exec(load:0x1->NXM_NX_CT_MARK[]))

# Table 82 accepts drops invalid ingress connections.
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_state=+inv+trk actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.132s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_mark=0x1,reg5=0xd actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10155.032s, table=82, n_packets=0, n_bytes=0, idle_age=10155, priority=50,ct_mark=0x1,reg5=0xa actions=drop
 cookie=0xb7d7ed46110fd50e, duration=10509.902s, table=82, n_packets=0, n_bytes=0, idle_age=10509, priority=0 actions=drop

附录 - ARP流分析实例

service vm: host=vcd41026, VIP=192.168.3.187, internal-ubtmgmt=192.168.3.193, MAC=fa:16:3e:3b:ef:ba; lb-mgmt-net=192.168.254.85
backend vm: host=vcd41021, internal-ubtmgmt=192.168.3.132, FIP=10.246.80.221, MAC=fa:16:3e:ad:1a:1c1, vcd41026# egress flow - arp spoofing protectioncookie=0xe6e269027e4ec7ad, duration=96905.204s, table=71, n_packets=1185, n_bytes=49770, idle_age=763, hard_age=65534, priority=95,arp,reg5=0x4b,in_port=75,dl_src=fa:16:3e:3b:ef:ba,arp_spa=192.168.3.193 actions=resubmit(,94)
cookie=0xe6e269027e4ec7ad, duration=721987.518s, table=94, n_packets=73782147, n_bytes=28359003552, idle_age=0, hard_age=65534, priority=1 actions=NORMAL# ingress flow - allow arp (port-id=reg5=0x4b for 192.168.3.193)cookie=0xe6e269027e4ec7ad, duration=96905.204s, table=60, n_packets=336, n_bytes=31745, idle_age=65534, hard_age=65534, priority=90,dl_vlan=10,dl_dst=fa:16:3e:3b:ef:ba actions=load:0x4b->NXM_NX_REG5[],load:0xa->NXM_NX_REG6[],strip_vlan,resubmit(,81)cookie=0xe6e269027e4ec7ad, duration=96905.205s, table=81, n_packets=41, n_bytes=1722, idle_age=65534, hard_age=65534, priority=100,arp,reg5=0x4b actions=output:752, vcd41021# egress flow - arp spoofing protectioncookie=0xa4e060f3d74213af, duration=265333.510s, table=71, n_packets=1447, n_bytes=60774, idle_age=264, hard_age=65534, priority=95,arp,reg5=0x67,in_port=103,dl_src=fa:16:3e:ad:1a:1c,arp_spa=192.168.3.132 actions=resubmit(,94)# ingress flow - allow arp (port-id=reg5=0x67 for 192.168.3.132)cookie=0xa4e060f3d74213af, duration=265333.510s, table=60, n_packets=6344, n_bytes=31182693, idle_age=264, hard_age=65534, priority=90,dl_vlan=28,dl_dst=fa:16:3e:ad:1a:1c actions=load:0x67->NXM_NX_REG5[],load:0x1c->NXM_NX_REG6[],strip_vlan,resubmit(,81)cookie=0xa4e060f3d74213af, duration=265333.511s, table=81, n_packets=2445, n_bytes=102690, idle_age=264, hard_age=65534, priority=100,arp,reg5=0x67 actions=output:103

附录：VM走br-tun路径

vm -> qrouter-xxx -> br-int -> br-tun -> vxlan-0a050015 (compute side)1, vm -> qrouter-xxx, vm goes to it's local gw qrouter-xxx2, qrouter-xxx -> br-int, SG, egress flow - arp spoofing protection# ovs-ofctl dump-flows br-int |grep '192.168.21.151' |grep arpcookie=0x8a4738b01717a42e, duration=10015.875s, table=71, n_packets=217, n_bytes=9114, idle_age=0, priority=95,arp,reg5=0x9,in_port=9,dl_src=fa:16:3e:d3:6f:80,arp_spa=192.168.21.151 actions=resubmit(,94)3, br-tun -> vxlan-0a050015, DVR, egressroot@juju-21f0ba-focal-10:/home/ubuntu# ovs-vsctl -- --columns=name,ofport list Interface vxlan-0a050015
name                : vxlan-0a050015
ofport              : 2
root@juju-21f0ba-focal-10:/home/ubuntu# ovs-ofctl dump-flows br-tun |grep 'output:2'cookie=0x52f592e5ba6d5a89, duration=9573.753s, table=20, n_packets=0, n_bytes=0, idle_age=59831, priority=2,dl_vlan=1,dl_dst=fa:16:3e:48:14:aa actions=strip_vlan,load:0x3e9->NXM_NX_TUN_ID[],output:2cookie=0x52f592e5ba6d5a89, duration=59798.694s, table=20, n_packets=0, n_bytes=0, hard_timeout=300, idle_age=59798, hard_age=1, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:48:14:aa actions=load:0->NXM_OF_VLAN_TCI[],load:0x3e9->NXM_NX_TUN_ID[],output:2cookie=0x52f592e5ba6d5a89, duration=6848.986s, table=22, n_packets=53, n_bytes=5531, idle_age=413, priority=1,dl_vlan=3 actions=strip_vlan,load:0x4d1->NXM_NX_TUN_ID[],output:2,output:3cookie=0x52f592e5ba6d5a89, duration=6848.040s, table=22, n_packets=8, n_bytes=432, idle_age=9731, priority=1,dl_vlan=1 actions=strip_vlan,load:0x3e9->NXM_NX_TUN_ID[],output:2,output:34, br-int -> vm  - ingress - SG# ovs-vsctl -- --columns=name,ofport list Interface tapeb832f99-97
name                : tapeb832f99-97
ofport              : 9# ovs-ofctl dump-flows br-int |grep 'strip_vlan' |grep 'output:9'cookie=0x8a4738b01717a42e, duration=10371.693s, table=60, n_packets=0, n_bytes=0, idle_age=10393, priority=20,dl_vlan=3,dl_dst=fa:16:3e:d3:6f:80 actions=strip_vlan,output:95, arp responder# ovs-ofctl dump-flows br-tun |grep -i 'arp'cookie=0x52f592e5ba6d5a89, duration=63966.670s, table=2, n_packets=18072, n_bytes=759024, idle_age=3, priority=1,arp,dl_dst=ff:ff:ff:ff:ff:ff actions=resubmit(,21)cookie=0x52f592e5ba6d5a89, duration=251.974s, table=21, n_packets=0, n_bytes=0, idle_age=61836, priority=1,arp,dl_vlan=1,arp_tpa=169.254.194.177 actions=load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e4814aa->NXM_NX_ARP_SHA[],load:0xa9fec2b1->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:fa:16:3e:48:14:aa,IN_PORTcookie=0x52f592e5ba6d5a89, duration=251.969s, table=21, n_packets=0, n_bytes=0, idle_age=61835, priority=1,arp,dl_vlan=1,arp_tpa=169.254.192.40 actions=load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e8b3310->NXM_NX_ARP_SHA[],load:0xa9fec028->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:fa:16:3e:8b:33:10,IN_PORTcookie=0x52f592e5ba6d5a89, duration=258.822s, table=22, n_packets=5, n_bytes=210, idle_age=11422, priority=3,arp,dl_vlan=3,arp_tpa=192.168.21.1 actions=drop

[1] https://review.openstack.org/#/c/302766/