2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/skbuff.h>
35 #include <linux/if_arp.h>
36 #include <linux/netdevice.h>
38 #include <linux/if_vlan.h>
39 #include <net/udp_tunnel.h>
40 #include <net/sch_generic.h>
41 #include <linux/netfilter.h>
42 #include <rdma/ib_addr.h>
48 static LIST_HEAD(rxe_dev_list
);
49 static DEFINE_SPINLOCK(dev_list_lock
); /* spinlock for device list */
51 struct rxe_dev
*net_to_rxe(struct net_device
*ndev
)
54 struct rxe_dev
*found
= NULL
;
56 spin_lock_bh(&dev_list_lock
);
57 list_for_each_entry(rxe
, &rxe_dev_list
, list
) {
58 if (rxe
->ndev
== ndev
) {
63 spin_unlock_bh(&dev_list_lock
);
68 struct rxe_dev
*get_rxe_by_name(const char *name
)
71 struct rxe_dev
*found
= NULL
;
73 spin_lock_bh(&dev_list_lock
);
74 list_for_each_entry(rxe
, &rxe_dev_list
, list
) {
75 if (!strcmp(name
, rxe
->ib_dev
.name
)) {
80 spin_unlock_bh(&dev_list_lock
);
85 static struct rxe_recv_sockets recv_sockets
;
87 struct device
*rxe_dma_device(struct rxe_dev
*rxe
)
89 struct net_device
*ndev
;
93 if (is_vlan_dev(ndev
))
94 ndev
= vlan_dev_real_dev(ndev
);
96 return ndev
->dev
.parent
;
99 int rxe_mcast_add(struct rxe_dev
*rxe
, union ib_gid
*mgid
)
102 unsigned char ll_addr
[ETH_ALEN
];
104 ipv6_eth_mc_map((struct in6_addr
*)mgid
->raw
, ll_addr
);
105 err
= dev_mc_add(rxe
->ndev
, ll_addr
);
110 int rxe_mcast_delete(struct rxe_dev
*rxe
, union ib_gid
*mgid
)
113 unsigned char ll_addr
[ETH_ALEN
];
115 ipv6_eth_mc_map((struct in6_addr
*)mgid
->raw
, ll_addr
);
116 err
= dev_mc_del(rxe
->ndev
, ll_addr
);
121 static struct dst_entry
*rxe_find_route4(struct net_device
*ndev
,
122 struct in_addr
*saddr
,
123 struct in_addr
*daddr
)
126 struct flowi4 fl
= { { 0 } };
128 memset(&fl
, 0, sizeof(fl
));
129 fl
.flowi4_oif
= ndev
->ifindex
;
130 memcpy(&fl
.saddr
, saddr
, sizeof(*saddr
));
131 memcpy(&fl
.daddr
, daddr
, sizeof(*daddr
));
132 fl
.flowi4_proto
= IPPROTO_UDP
;
134 rt
= ip_route_output_key(&init_net
, &fl
);
136 pr_err_ratelimited("no route to %pI4\n", &daddr
->s_addr
);
143 #if IS_ENABLED(CONFIG_IPV6)
144 static struct dst_entry
*rxe_find_route6(struct net_device
*ndev
,
145 struct in6_addr
*saddr
,
146 struct in6_addr
*daddr
)
148 struct dst_entry
*ndst
;
149 struct flowi6 fl6
= { { 0 } };
151 memset(&fl6
, 0, sizeof(fl6
));
152 fl6
.flowi6_oif
= ndev
->ifindex
;
153 memcpy(&fl6
.saddr
, saddr
, sizeof(*saddr
));
154 memcpy(&fl6
.daddr
, daddr
, sizeof(*daddr
));
155 fl6
.flowi6_proto
= IPPROTO_UDP
;
157 ndst
= ipv6_stub
->ipv6_dst_lookup_flow(sock_net(recv_sockets
.sk6
->sk
),
158 recv_sockets
.sk6
->sk
, &fl6
,
160 if (unlikely(IS_ERR(ndst
))) {
161 pr_err_ratelimited("no route to %pI6\n", daddr
);
165 if (unlikely(ndst
->error
)) {
166 pr_err("no route to %pI6\n", daddr
);
178 static struct dst_entry
*rxe_find_route6(struct net_device
*ndev
,
179 struct in6_addr
*saddr
,
180 struct in6_addr
*daddr
)
187 static struct dst_entry
*rxe_find_route(struct rxe_dev
*rxe
,
191 const struct ib_gid_attr
*attr
;
192 struct dst_entry
*dst
= NULL
;
193 struct net_device
*ndev
;
195 attr
= rdma_get_gid_attr(&rxe
->ib_dev
, qp
->attr
.port_num
,
201 if (qp_type(qp
) == IB_QPT_RC
)
202 dst
= sk_dst_get(qp
->sk
->sk
);
204 if (!dst
|| !dst_check(dst
, qp
->dst_cookie
)) {
208 if (av
->network_type
== RDMA_NETWORK_IPV4
) {
209 struct in_addr
*saddr
;
210 struct in_addr
*daddr
;
212 saddr
= &av
->sgid_addr
._sockaddr_in
.sin_addr
;
213 daddr
= &av
->dgid_addr
._sockaddr_in
.sin_addr
;
214 dst
= rxe_find_route4(ndev
, saddr
, daddr
);
215 } else if (av
->network_type
== RDMA_NETWORK_IPV6
) {
216 struct in6_addr
*saddr6
;
217 struct in6_addr
*daddr6
;
219 saddr6
= &av
->sgid_addr
._sockaddr_in6
.sin6_addr
;
220 daddr6
= &av
->dgid_addr
._sockaddr_in6
.sin6_addr
;
221 dst
= rxe_find_route6(ndev
, saddr6
, daddr6
);
222 #if IS_ENABLED(CONFIG_IPV6)
225 rt6_get_cookie((struct rt6_info
*)dst
);
229 if (dst
&& (qp_type(qp
) == IB_QPT_RC
)) {
231 sk_dst_set(qp
->sk
->sk
, dst
);
234 rdma_put_gid_attr(attr
);
238 static int rxe_udp_encap_recv(struct sock
*sk
, struct sk_buff
*skb
)
241 struct net_device
*ndev
= skb
->dev
;
242 struct net_device
*rdev
= ndev
;
243 struct rxe_dev
*rxe
= net_to_rxe(ndev
);
244 struct rxe_pkt_info
*pkt
= SKB_TO_PKT(skb
);
246 if (!rxe
&& is_vlan_dev(rdev
)) {
247 rdev
= vlan_dev_real_dev(ndev
);
248 rxe
= net_to_rxe(rdev
);
253 if (skb_linearize(skb
)) {
254 pr_err("skb_linearize failed\n");
261 pkt
->hdr
= (u8
*)(udph
+ 1);
262 pkt
->mask
= RXE_GRH_MASK
;
263 pkt
->paylen
= be16_to_cpu(udph
->len
) - sizeof(*udph
);
274 static struct socket
*rxe_setup_udp_tunnel(struct net
*net
, __be16 port
,
279 struct udp_port_cfg udp_cfg
= { };
280 struct udp_tunnel_sock_cfg tnl_cfg
= { };
283 udp_cfg
.family
= AF_INET6
;
284 udp_cfg
.ipv6_v6only
= 1;
286 udp_cfg
.family
= AF_INET
;
289 udp_cfg
.local_udp_port
= port
;
291 /* Create UDP socket */
292 err
= udp_sock_create(net
, &udp_cfg
, &sock
);
294 pr_err("failed to create udp socket. err = %d\n", err
);
298 tnl_cfg
.encap_type
= 1;
299 tnl_cfg
.encap_rcv
= rxe_udp_encap_recv
;
301 /* Setup UDP tunnel */
302 setup_udp_tunnel_sock(net
, sock
, &tnl_cfg
);
307 static void rxe_release_udp_tunnel(struct socket
*sk
)
310 udp_tunnel_sock_release(sk
);
313 static void prepare_udp_hdr(struct sk_buff
*skb
, __be16 src_port
,
318 __skb_push(skb
, sizeof(*udph
));
319 skb_reset_transport_header(skb
);
322 udph
->dest
= dst_port
;
323 udph
->source
= src_port
;
324 udph
->len
= htons(skb
->len
);
328 static void prepare_ipv4_hdr(struct dst_entry
*dst
, struct sk_buff
*skb
,
329 __be32 saddr
, __be32 daddr
, __u8 proto
,
330 __u8 tos
, __u8 ttl
, __be16 df
, bool xnet
)
334 skb_scrub_packet(skb
, xnet
);
337 skb_dst_set(skb
, dst_clone(dst
));
338 memset(IPCB(skb
), 0, sizeof(*IPCB(skb
)));
340 skb_push(skb
, sizeof(struct iphdr
));
341 skb_reset_network_header(skb
);
345 iph
->version
= IPVERSION
;
346 iph
->ihl
= sizeof(struct iphdr
) >> 2;
348 iph
->protocol
= proto
;
353 __ip_select_ident(dev_net(dst
->dev
), iph
,
354 skb_shinfo(skb
)->gso_segs
?: 1);
355 iph
->tot_len
= htons(skb
->len
);
359 static void prepare_ipv6_hdr(struct dst_entry
*dst
, struct sk_buff
*skb
,
360 struct in6_addr
*saddr
, struct in6_addr
*daddr
,
361 __u8 proto
, __u8 prio
, __u8 ttl
)
363 struct ipv6hdr
*ip6h
;
365 memset(&(IPCB(skb
)->opt
), 0, sizeof(IPCB(skb
)->opt
));
366 IPCB(skb
)->flags
&= ~(IPSKB_XFRM_TUNNEL_SIZE
| IPSKB_XFRM_TRANSFORMED
368 skb_dst_set(skb
, dst_clone(dst
));
370 __skb_push(skb
, sizeof(*ip6h
));
371 skb_reset_network_header(skb
);
372 ip6h
= ipv6_hdr(skb
);
373 ip6_flow_hdr(ip6h
, prio
, htonl(0));
374 ip6h
->payload_len
= htons(skb
->len
);
375 ip6h
->nexthdr
= proto
;
376 ip6h
->hop_limit
= ttl
;
377 ip6h
->daddr
= *daddr
;
378 ip6h
->saddr
= *saddr
;
379 ip6h
->payload_len
= htons(skb
->len
- sizeof(*ip6h
));
382 static int prepare4(struct rxe_dev
*rxe
, struct rxe_pkt_info
*pkt
,
383 struct sk_buff
*skb
, struct rxe_av
*av
)
385 struct rxe_qp
*qp
= pkt
->qp
;
386 struct dst_entry
*dst
;
388 __be16 df
= htons(IP_DF
);
389 struct in_addr
*saddr
= &av
->sgid_addr
._sockaddr_in
.sin_addr
;
390 struct in_addr
*daddr
= &av
->dgid_addr
._sockaddr_in
.sin_addr
;
392 dst
= rxe_find_route(rxe
, qp
, av
);
394 pr_err("Host not reachable\n");
395 return -EHOSTUNREACH
;
398 if (!memcmp(saddr
, daddr
, sizeof(*daddr
)))
399 pkt
->mask
|= RXE_LOOPBACK_MASK
;
401 prepare_udp_hdr(skb
, htons(RXE_ROCE_V2_SPORT
),
402 htons(ROCE_V2_UDP_DPORT
));
404 prepare_ipv4_hdr(dst
, skb
, saddr
->s_addr
, daddr
->s_addr
, IPPROTO_UDP
,
405 av
->grh
.traffic_class
, av
->grh
.hop_limit
, df
, xnet
);
411 static int prepare6(struct rxe_dev
*rxe
, struct rxe_pkt_info
*pkt
,
412 struct sk_buff
*skb
, struct rxe_av
*av
)
414 struct rxe_qp
*qp
= pkt
->qp
;
415 struct dst_entry
*dst
;
416 struct in6_addr
*saddr
= &av
->sgid_addr
._sockaddr_in6
.sin6_addr
;
417 struct in6_addr
*daddr
= &av
->dgid_addr
._sockaddr_in6
.sin6_addr
;
419 dst
= rxe_find_route(rxe
, qp
, av
);
421 pr_err("Host not reachable\n");
422 return -EHOSTUNREACH
;
425 if (!memcmp(saddr
, daddr
, sizeof(*daddr
)))
426 pkt
->mask
|= RXE_LOOPBACK_MASK
;
428 prepare_udp_hdr(skb
, htons(RXE_ROCE_V2_SPORT
),
429 htons(ROCE_V2_UDP_DPORT
));
431 prepare_ipv6_hdr(dst
, skb
, saddr
, daddr
, IPPROTO_UDP
,
432 av
->grh
.traffic_class
,
439 int rxe_prepare(struct rxe_dev
*rxe
, struct rxe_pkt_info
*pkt
,
440 struct sk_buff
*skb
, u32
*crc
)
443 struct rxe_av
*av
= rxe_get_av(pkt
);
445 if (av
->network_type
== RDMA_NETWORK_IPV4
)
446 err
= prepare4(rxe
, pkt
, skb
, av
);
447 else if (av
->network_type
== RDMA_NETWORK_IPV6
)
448 err
= prepare6(rxe
, pkt
, skb
, av
);
450 *crc
= rxe_icrc_hdr(pkt
, skb
);
455 static void rxe_skb_tx_dtor(struct sk_buff
*skb
)
457 struct sock
*sk
= skb
->sk
;
458 struct rxe_qp
*qp
= sk
->sk_user_data
;
459 int skb_out
= atomic_dec_return(&qp
->skb_out
);
461 if (unlikely(qp
->need_req_skb
&&
462 skb_out
< RXE_INFLIGHT_SKBS_PER_QP_LOW
))
463 rxe_run_task(&qp
->req
.task
, 1);
468 int rxe_send(struct rxe_pkt_info
*pkt
, struct sk_buff
*skb
)
473 av
= rxe_get_av(pkt
);
475 skb
->destructor
= rxe_skb_tx_dtor
;
476 skb
->sk
= pkt
->qp
->sk
->sk
;
478 rxe_add_ref(pkt
->qp
);
479 atomic_inc(&pkt
->qp
->skb_out
);
481 if (av
->network_type
== RDMA_NETWORK_IPV4
) {
482 err
= ip_local_out(dev_net(skb_dst(skb
)->dev
), skb
->sk
, skb
);
483 } else if (av
->network_type
== RDMA_NETWORK_IPV6
) {
484 err
= ip6_local_out(dev_net(skb_dst(skb
)->dev
), skb
->sk
, skb
);
486 pr_err("Unknown layer 3 protocol: %d\n", av
->network_type
);
487 atomic_dec(&pkt
->qp
->skb_out
);
488 rxe_drop_ref(pkt
->qp
);
493 if (unlikely(net_xmit_eval(err
))) {
494 pr_debug("error sending packet: %d\n", err
);
501 void rxe_loopback(struct sk_buff
*skb
)
506 static inline int addr_same(struct rxe_dev
*rxe
, struct rxe_av
*av
)
508 return rxe
->port
.port_guid
== av
->grh
.dgid
.global
.interface_id
;
511 struct sk_buff
*rxe_init_packet(struct rxe_dev
*rxe
, struct rxe_av
*av
,
512 int paylen
, struct rxe_pkt_info
*pkt
)
514 unsigned int hdr_len
;
516 struct net_device
*ndev
;
517 const struct ib_gid_attr
*attr
;
518 const int port_num
= 1;
520 attr
= rdma_get_gid_attr(&rxe
->ib_dev
, port_num
, av
->grh
.sgid_index
);
525 if (av
->network_type
== RDMA_NETWORK_IPV4
)
526 hdr_len
= ETH_HLEN
+ sizeof(struct udphdr
) +
527 sizeof(struct iphdr
);
529 hdr_len
= ETH_HLEN
+ sizeof(struct udphdr
) +
530 sizeof(struct ipv6hdr
);
532 skb
= alloc_skb(paylen
+ hdr_len
+ LL_RESERVED_SPACE(ndev
),
538 skb_reserve(skb
, hdr_len
+ LL_RESERVED_SPACE(ndev
));
540 /* FIXME: hold reference to this netdev until life of this skb. */
542 if (av
->network_type
== RDMA_NETWORK_IPV4
)
543 skb
->protocol
= htons(ETH_P_IP
);
545 skb
->protocol
= htons(ETH_P_IPV6
);
548 pkt
->port_num
= port_num
;
549 pkt
->hdr
= skb_put_zero(skb
, paylen
);
550 pkt
->mask
|= RXE_GRH_MASK
;
553 rdma_put_gid_attr(attr
);
558 * this is required by rxe_cfg to match rxe devices in
559 * /sys/class/infiniband up with their underlying ethernet devices
561 const char *rxe_parent_name(struct rxe_dev
*rxe
, unsigned int port_num
)
563 return rxe
->ndev
->name
;
566 enum rdma_link_layer
rxe_link_layer(struct rxe_dev
*rxe
, unsigned int port_num
)
568 return IB_LINK_LAYER_ETHERNET
;
571 struct rxe_dev
*rxe_net_add(struct net_device
*ndev
)
574 struct rxe_dev
*rxe
= NULL
;
576 rxe
= (struct rxe_dev
*)ib_alloc_device(sizeof(*rxe
));
582 err
= rxe_add(rxe
, ndev
->mtu
);
584 ib_dealloc_device(&rxe
->ib_dev
);
588 spin_lock_bh(&dev_list_lock
);
589 list_add_tail(&rxe
->list
, &rxe_dev_list
);
590 spin_unlock_bh(&dev_list_lock
);
594 void rxe_remove_all(void)
596 spin_lock_bh(&dev_list_lock
);
597 while (!list_empty(&rxe_dev_list
)) {
598 struct rxe_dev
*rxe
=
599 list_first_entry(&rxe_dev_list
, struct rxe_dev
, list
);
601 list_del(&rxe
->list
);
602 spin_unlock_bh(&dev_list_lock
);
604 spin_lock_bh(&dev_list_lock
);
606 spin_unlock_bh(&dev_list_lock
);
609 static void rxe_port_event(struct rxe_dev
*rxe
,
610 enum ib_event_type event
)
614 ev
.device
= &rxe
->ib_dev
;
615 ev
.element
.port_num
= 1;
618 ib_dispatch_event(&ev
);
621 /* Caller must hold net_info_lock */
622 void rxe_port_up(struct rxe_dev
*rxe
)
624 struct rxe_port
*port
;
627 port
->attr
.state
= IB_PORT_ACTIVE
;
628 port
->attr
.phys_state
= IB_PHYS_STATE_LINK_UP
;
630 rxe_port_event(rxe
, IB_EVENT_PORT_ACTIVE
);
631 pr_info("set %s active\n", rxe
->ib_dev
.name
);
634 /* Caller must hold net_info_lock */
635 void rxe_port_down(struct rxe_dev
*rxe
)
637 struct rxe_port
*port
;
640 port
->attr
.state
= IB_PORT_DOWN
;
641 port
->attr
.phys_state
= IB_PHYS_STATE_LINK_DOWN
;
643 rxe_port_event(rxe
, IB_EVENT_PORT_ERR
);
644 pr_info("set %s down\n", rxe
->ib_dev
.name
);
647 static int rxe_notify(struct notifier_block
*not_blk
,
651 struct net_device
*ndev
= netdev_notifier_info_to_dev(arg
);
652 struct rxe_dev
*rxe
= net_to_rxe(ndev
);
658 case NETDEV_UNREGISTER
:
659 list_del(&rxe
->list
);
668 case NETDEV_CHANGEMTU
:
669 pr_info("%s changed mtu to %d\n", ndev
->name
, ndev
->mtu
);
670 rxe_set_mtu(rxe
, ndev
->mtu
);
673 if (netif_running(ndev
) && netif_carrier_ok(ndev
))
679 case NETDEV_GOING_DOWN
:
680 case NETDEV_CHANGEADDR
:
681 case NETDEV_CHANGENAME
:
682 case NETDEV_FEAT_CHANGE
:
684 pr_info("ignoring netdev event = %ld for %s\n",
692 static struct notifier_block rxe_net_notifier
= {
693 .notifier_call
= rxe_notify
,
696 static int rxe_net_ipv4_init(void)
698 recv_sockets
.sk4
= rxe_setup_udp_tunnel(&init_net
,
699 htons(ROCE_V2_UDP_DPORT
), false);
700 if (IS_ERR(recv_sockets
.sk4
)) {
701 recv_sockets
.sk4
= NULL
;
702 pr_err("Failed to create IPv4 UDP tunnel\n");
709 static int rxe_net_ipv6_init(void)
711 #if IS_ENABLED(CONFIG_IPV6)
713 recv_sockets
.sk6
= rxe_setup_udp_tunnel(&init_net
,
714 htons(ROCE_V2_UDP_DPORT
), true);
715 if (IS_ERR(recv_sockets
.sk6
)) {
716 recv_sockets
.sk6
= NULL
;
717 pr_err("Failed to create IPv6 UDP tunnel\n");
724 void rxe_net_exit(void)
726 rxe_release_udp_tunnel(recv_sockets
.sk6
);
727 rxe_release_udp_tunnel(recv_sockets
.sk4
);
728 unregister_netdevice_notifier(&rxe_net_notifier
);
731 int rxe_net_init(void)
735 recv_sockets
.sk6
= NULL
;
737 err
= rxe_net_ipv4_init();
740 err
= rxe_net_ipv6_init();
743 err
= register_netdevice_notifier(&rxe_net_notifier
);
745 pr_err("Failed to register netdev notifier\n");