1 /* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 * Copyright 2017 Mellanox Technologies, Ltd 3 */ 4 5 #include <stdint.h> 6 #include <stdbool.h> 7 #include <sys/types.h> 8 #include <sys/socket.h> 9 #include <asm/types.h> 10 #include <linux/in.h> 11 #include <linux/if.h> 12 #include <linux/if_ether.h> 13 #include <linux/ip.h> 14 #include <linux/ipv6.h> 15 #include <linux/if_tunnel.h> 16 #include <linux/filter.h> 17 #include <linux/bpf.h> 18 19 #include "tap_rss.h" 20 21 /** Create IPv4 address */ 22 #define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) | \ 23 (((b) & 0xff) << 16) | \ 24 (((c) & 0xff) << 8) | \ 25 ((d) & 0xff)) 26 27 #define PORT(a, b) ((__u16)(((a) & 0xff) << 8) | \ 28 ((b) & 0xff)) 29 30 /* 31 * The queue number is offset by a unique QUEUE_OFFSET, to distinguish 32 * packets that have gone through this rule (skb->cb[1] != 0) from others. 33 */ 34 #define QUEUE_OFFSET 0x7cafe800 35 #define PIN_GLOBAL_NS 2 36 37 #define KEY_IDX 0 38 #define BPF_MAP_ID_KEY 1 39 40 struct vlan_hdr { 41 __be16 proto; 42 __be16 tci; 43 }; 44 45 struct bpf_elf_map __attribute__((section("maps"), used)) 46 map_keys = { 47 .type = BPF_MAP_TYPE_HASH, 48 .id = BPF_MAP_ID_KEY, 49 .size_key = sizeof(__u32), 50 .size_value = sizeof(struct rss_key), 51 .max_elem = 256, 52 .pinning = PIN_GLOBAL_NS, 53 }; 54 55 __section("cls_q") int 56 match_q(struct __sk_buff *skb) 57 { 58 __u32 queue = skb->cb[1]; 59 volatile __u32 q = 0xdeadbeef; 60 __u32 match_queue = QUEUE_OFFSET + q; 61 62 /* printt("match_q$i() queue = %d\n", queue); */ 63 64 if (queue != match_queue) 65 return TC_ACT_OK; 66 67 /* queue match */ 68 skb->cb[1] = 0; 69 return TC_ACT_UNSPEC; 70 } 71 72 73 struct ipv4_l3_l4_tuple { 74 __u32 src_addr; 75 __u32 dst_addr; 76 __u16 dport; 77 __u16 sport; 78 } __rte_packed; 79 80 struct ipv6_l3_l4_tuple { 81 __u8 src_addr[16]; 82 __u8 dst_addr[16]; 83 __u16 dport; 84 __u16 sport; 85 } __rte_packed; 86 87 static const __u8 def_rss_key[TAP_RSS_HASH_KEY_SIZE] = { 88 0xd1, 0x81, 0xc6, 0x2c, 89 0xf7, 0xf4, 0xdb, 0x5b, 90 0x19, 0x83, 0xa2, 0xfc, 91 0x94, 0x3e, 0x1a, 0xdb, 92 0xd9, 0x38, 0x9e, 0x6b, 93 0xd1, 0x03, 0x9c, 0x2c, 94 0xa7, 0x44, 0x99, 0xad, 95 0x59, 0x3d, 0x56, 0xd9, 96 0xf3, 0x25, 0x3c, 0x06, 97 0x2a, 0xdc, 0x1f, 0xfc, 98 }; 99 100 static __u32 __attribute__((always_inline)) 101 rte_softrss_be(const __u32 *input_tuple, const uint8_t *rss_key, 102 __u8 input_len) 103 { 104 __u32 i, j, hash = 0; 105 #pragma unroll 106 for (j = 0; j < input_len; j++) { 107 #pragma unroll 108 for (i = 0; i < 32; i++) { 109 if (input_tuple[j] & (1U << (31 - i))) { 110 hash ^= ((const __u32 *)def_rss_key)[j] << i | 111 (__u32)((uint64_t) 112 (((const __u32 *)def_rss_key)[j + 1]) 113 >> (32 - i)); 114 } 115 } 116 } 117 return hash; 118 } 119 120 static int __attribute__((always_inline)) 121 rss_l3_l4(struct __sk_buff *skb) 122 { 123 void *data_end = (void *)(long)skb->data_end; 124 void *data = (void *)(long)skb->data; 125 __u16 proto = (__u16)skb->protocol; 126 __u32 key_idx = 0xdeadbeef; 127 __u32 hash; 128 struct rss_key *rsskey; 129 __u64 off = ETH_HLEN; 130 int j; 131 __u8 *key = 0; 132 __u32 len; 133 __u32 queue = 0; 134 135 rsskey = map_lookup_elem(&map_keys, &key_idx); 136 if (!rsskey) { 137 printt("hash(): rss key is not configured\n"); 138 return TC_ACT_OK; 139 } 140 key = (__u8 *)rsskey->key; 141 142 /* Get correct proto for 802.1ad */ 143 if (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) { 144 if (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) + 145 sizeof(proto) > data_end) 146 return TC_ACT_OK; 147 proto = *(__u16 *)(data + ETH_ALEN * 2 + 148 sizeof(struct vlan_hdr)); 149 off += sizeof(struct vlan_hdr); 150 } 151 152 if (proto == htons(ETH_P_IP)) { 153 if (data + off + sizeof(struct iphdr) + sizeof(__u32) 154 > data_end) 155 return TC_ACT_OK; 156 157 __u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr); 158 __u8 *src_dst_port = data + off + sizeof(struct iphdr); 159 struct ipv4_l3_l4_tuple v4_tuple = { 160 .src_addr = IPv4(*(src_dst_addr + 0), 161 *(src_dst_addr + 1), 162 *(src_dst_addr + 2), 163 *(src_dst_addr + 3)), 164 .dst_addr = IPv4(*(src_dst_addr + 4), 165 *(src_dst_addr + 5), 166 *(src_dst_addr + 6), 167 *(src_dst_addr + 7)), 168 .sport = PORT(*(src_dst_port + 0), 169 *(src_dst_port + 1)), 170 .dport = PORT(*(src_dst_port + 2), 171 *(src_dst_port + 3)), 172 }; 173 __u8 input_len = sizeof(v4_tuple) / sizeof(__u32); 174 if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3)) 175 input_len--; 176 hash = rte_softrss_be((__u32 *)&v4_tuple, key, 3); 177 } else if (proto == htons(ETH_P_IPV6)) { 178 if (data + off + sizeof(struct ipv6hdr) + 179 sizeof(__u32) > data_end) 180 return TC_ACT_OK; 181 __u8 *src_dst_addr = data + off + 182 offsetof(struct ipv6hdr, saddr); 183 __u8 *src_dst_port = data + off + 184 sizeof(struct ipv6hdr); 185 struct ipv6_l3_l4_tuple v6_tuple; 186 for (j = 0; j < 4; j++) 187 *((uint32_t *)&v6_tuple.src_addr + j) = 188 __builtin_bswap32(*((uint32_t *) 189 src_dst_addr + j)); 190 for (j = 0; j < 4; j++) 191 *((uint32_t *)&v6_tuple.dst_addr + j) = 192 __builtin_bswap32(*((uint32_t *) 193 src_dst_addr + 4 + j)); 194 v6_tuple.sport = PORT(*(src_dst_port + 0), 195 *(src_dst_port + 1)); 196 v6_tuple.dport = PORT(*(src_dst_port + 2), 197 *(src_dst_port + 3)); 198 199 __u8 input_len = sizeof(v6_tuple) / sizeof(__u32); 200 if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3)) 201 input_len--; 202 hash = rte_softrss_be((__u32 *)&v6_tuple, key, 9); 203 } else { 204 return TC_ACT_PIPE; 205 } 206 207 queue = rsskey->queues[(hash % rsskey->nb_queues) & 208 (TAP_MAX_QUEUES - 1)]; 209 skb->cb[1] = QUEUE_OFFSET + queue; 210 /* printt(">>>>> rss_l3_l4 hash=0x%x queue=%u\n", hash, queue); */ 211 212 return TC_ACT_RECLASSIFY; 213 } 214 215 #define RSS(L) \ 216 __section(#L) int \ 217 L ## _hash(struct __sk_buff *skb) \ 218 { \ 219 return rss_ ## L (skb); \ 220 } 221 222 RSS(l3_l4) 223 224 BPF_LICENSE("Dual BSD/GPL"); 225