?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
spi-topcliff-pch: supports a spi mode setup and bit order setup by IO control
[zen-stable.git] / drivers / net / ethernet / sfc / filter.c
blob1fbbbee7b1ae225bdd54433ec04e354f5ff1182f
1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
8 */
9
10 #include <linux/in.h>
11 #include <net/ip.h>
12 #include "efx.h"
13 #include "filter.h"
14 #include "io.h"
15 #include "nic.h"
16 #include "regs.h"
17
18 /* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
21 */
22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
24
25 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
27 * table is full.
28 */
29 #define FILTER_CTL_SRCH_MAX 200
30
31 /* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33 #define FILTER_CTL_SRCH_HINT_MAX 5
34
35 enum efx_filter_table_id {
36 EFX_FILTER_TABLE_RX_IP = 0,
37 EFX_FILTER_TABLE_RX_MAC,
38 EFX_FILTER_TABLE_COUNT,
39 };
40
41 struct efx_filter_table {
42 enum efx_filter_table_id id;
43 u32 offset; /* address of table relative to BAR */
44 unsigned size; /* number of entries */
45 unsigned step; /* step between entries */
46 unsigned used; /* number currently used */
47 unsigned long *used_bitmap;
48 struct efx_filter_spec *spec;
49 unsigned search_depth[EFX_FILTER_TYPE_COUNT];
50 };
51
52 struct efx_filter_state {
53 spinlock_t lock;
54 struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
55 #ifdef CONFIG_RFS_ACCEL
56 u32 *rps_flow_id;
57 unsigned rps_expire_index;
58 #endif
59 };
60
61 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
62 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
63 static u16 efx_filter_hash(u32 key)
64 {
65 u16 tmp;
66
67 /* First 16 rounds */
68 tmp = 0x1fff ^ key >> 16;
69 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
70 tmp = tmp ^ tmp >> 9;
71 /* Last 16 rounds */
72 tmp = tmp ^ tmp << 13 ^ key;
73 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
74 return tmp ^ tmp >> 9;
75 }
76
77 /* To allow for hash collisions, filter search continues at these
78 * increments from the first possible entry selected by the hash. */
79 static u16 efx_filter_increment(u32 key)
80 {
81 return key * 2 - 1;
82 }
83
84 static enum efx_filter_table_id
85 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
86 {
87 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
88 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
89 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
90 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
91 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
92 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
93 EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
94 return spec->type >> 2;
95 }
96
97 static struct efx_filter_table *
98 efx_filter_spec_table(struct efx_filter_state *state,
99 const struct efx_filter_spec *spec)
100 {
101 if (spec->type == EFX_FILTER_UNSPEC)
102 return NULL;
103 else
104 return &state->table[efx_filter_spec_table_id(spec)];
105 }
106
107 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
108 {
109 memset(table->search_depth, 0, sizeof(table->search_depth));
110 }
111
112 static void efx_filter_push_rx_limits(struct efx_nic *efx)
113 {
114 struct efx_filter_state *state = efx->filter_state;
115 struct efx_filter_table *table;
116 efx_oword_t filter_ctl;
117
118 efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
119
120 table = &state->table[EFX_FILTER_TABLE_RX_IP];
121 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
122 table->search_depth[EFX_FILTER_TCP_FULL] +
123 FILTER_CTL_SRCH_FUDGE_FULL);
124 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
125 table->search_depth[EFX_FILTER_TCP_WILD] +
126 FILTER_CTL_SRCH_FUDGE_WILD);
127 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
128 table->search_depth[EFX_FILTER_UDP_FULL] +
129 FILTER_CTL_SRCH_FUDGE_FULL);
130 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
131 table->search_depth[EFX_FILTER_UDP_WILD] +
132 FILTER_CTL_SRCH_FUDGE_WILD);
133
134 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
135 if (table->size) {
136 EFX_SET_OWORD_FIELD(
137 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
138 table->search_depth[EFX_FILTER_MAC_FULL] +
139 FILTER_CTL_SRCH_FUDGE_FULL);
140 EFX_SET_OWORD_FIELD(
141 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
142 table->search_depth[EFX_FILTER_MAC_WILD] +
143 FILTER_CTL_SRCH_FUDGE_WILD);
144 }
145
146 efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
147 }
148
149 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
150 __be32 host1, __be16 port1,
151 __be32 host2, __be16 port2)
152 {
153 spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
154 spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
155 spec->data[2] = ntohl(host2);
156 }
157
158 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec,
159 __be32 *host1, __be16 *port1,
160 __be32 *host2, __be16 *port2)
161 {
162 *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
163 *port1 = htons(spec->data[0]);
164 *host2 = htonl(spec->data[2]);
165 *port2 = htons(spec->data[1] >> 16);
166 }
167
168 /**
169 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
170 * @spec: Specification to initialise
171 * @proto: Transport layer protocol number
172 * @host: Local host address (network byte order)
173 * @port: Local port (network byte order)
174 */
175 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
176 __be32 host, __be16 port)
177 {
178 __be32 host1;
179 __be16 port1;
180
181 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
182
183 /* This cannot currently be combined with other filtering */
184 if (spec->type != EFX_FILTER_UNSPEC)
185 return -EPROTONOSUPPORT;
186
187 if (port == 0)
188 return -EINVAL;
189
190 switch (proto) {
191 case IPPROTO_TCP:
192 spec->type = EFX_FILTER_TCP_WILD;
193 break;
194 case IPPROTO_UDP:
195 spec->type = EFX_FILTER_UDP_WILD;
196 break;
197 default:
198 return -EPROTONOSUPPORT;
199 }
200
201 /* Filter is constructed in terms of source and destination,
202 * with the odd wrinkle that the ports are swapped in a UDP
203 * wildcard filter. We need to convert from local and remote
204 * (= zero for wildcard) addresses.
205 */
206 host1 = 0;
207 if (proto != IPPROTO_UDP) {
208 port1 = 0;
209 } else {
210 port1 = port;
211 port = 0;
212 }
213
214 __efx_filter_set_ipv4(spec, host1, port1, host, port);
215 return 0;
216 }
217
218 int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec,
219 u8 *proto, __be32 *host, __be16 *port)
220 {
221 __be32 host1;
222 __be16 port1;
223
224 switch (spec->type) {
225 case EFX_FILTER_TCP_WILD:
226 *proto = IPPROTO_TCP;
227 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
228 return 0;
229 case EFX_FILTER_UDP_WILD:
230 *proto = IPPROTO_UDP;
231 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
232 return 0;
233 default:
234 return -EINVAL;
235 }
236 }
237
238 /**
239 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
240 * @spec: Specification to initialise
241 * @proto: Transport layer protocol number
242 * @host: Local host address (network byte order)
243 * @port: Local port (network byte order)
244 * @rhost: Remote host address (network byte order)
245 * @rport: Remote port (network byte order)
246 */
247 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
248 __be32 host, __be16 port,
249 __be32 rhost, __be16 rport)
250 {
251 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
252
253 /* This cannot currently be combined with other filtering */
254 if (spec->type != EFX_FILTER_UNSPEC)
255 return -EPROTONOSUPPORT;
256
257 if (port == 0 || rport == 0)
258 return -EINVAL;
259
260 switch (proto) {
261 case IPPROTO_TCP:
262 spec->type = EFX_FILTER_TCP_FULL;
263 break;
264 case IPPROTO_UDP:
265 spec->type = EFX_FILTER_UDP_FULL;
266 break;
267 default:
268 return -EPROTONOSUPPORT;
269 }
270
271 __efx_filter_set_ipv4(spec, rhost, rport, host, port);
272 return 0;
273 }
274
275 int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec,
276 u8 *proto, __be32 *host, __be16 *port,
277 __be32 *rhost, __be16 *rport)
278 {
279 switch (spec->type) {
280 case EFX_FILTER_TCP_FULL:
281 *proto = IPPROTO_TCP;
282 break;
283 case EFX_FILTER_UDP_FULL:
284 *proto = IPPROTO_UDP;
285 break;
286 default:
287 return -EINVAL;
288 }
289
290 __efx_filter_get_ipv4(spec, rhost, rport, host, port);
291 return 0;
292 }
293
294 /**
295 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
296 * @spec: Specification to initialise
297 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
298 * @addr: Local Ethernet MAC address
299 */
300 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
301 u16 vid, const u8 *addr)
302 {
303 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
304
305 /* This cannot currently be combined with other filtering */
306 if (spec->type != EFX_FILTER_UNSPEC)
307 return -EPROTONOSUPPORT;
308
309 if (vid == EFX_FILTER_VID_UNSPEC) {
310 spec->type = EFX_FILTER_MAC_WILD;
311 spec->data[0] = 0;
312 } else {
313 spec->type = EFX_FILTER_MAC_FULL;
314 spec->data[0] = vid;
315 }
316
317 spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
318 spec->data[2] = addr[0] << 8 | addr[1];
319 return 0;
320 }
321
322 int efx_filter_get_eth_local(const struct efx_filter_spec *spec,
323 u16 *vid, u8 *addr)
324 {
325 switch (spec->type) {
326 case EFX_FILTER_MAC_WILD:
327 *vid = EFX_FILTER_VID_UNSPEC;
328 break;
329 case EFX_FILTER_MAC_FULL:
330 *vid = spec->data[0];
331 break;
332 default:
333 return -EINVAL;
334 }
335
336 addr[0] = spec->data[2] >> 8;
337 addr[1] = spec->data[2];
338 addr[2] = spec->data[1] >> 24;
339 addr[3] = spec->data[1] >> 16;
340 addr[4] = spec->data[1] >> 8;
341 addr[5] = spec->data[1];
342 return 0;
343 }
344
345 /* Build a filter entry and return its n-tuple key. */
346 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
347 {
348 u32 data3;
349
350 switch (efx_filter_spec_table_id(spec)) {
351 case EFX_FILTER_TABLE_RX_IP: {
352 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
353 spec->type == EFX_FILTER_UDP_WILD);
354 EFX_POPULATE_OWORD_7(
355 *filter,
356 FRF_BZ_RSS_EN,
357 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
358 FRF_BZ_SCATTER_EN,
359 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
360 FRF_BZ_TCP_UDP, is_udp,
361 FRF_BZ_RXQ_ID, spec->dmaq_id,
362 EFX_DWORD_2, spec->data[2],
363 EFX_DWORD_1, spec->data[1],
364 EFX_DWORD_0, spec->data[0]);
365 data3 = is_udp;
366 break;
367 }
368
369 case EFX_FILTER_TABLE_RX_MAC: {
370 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
371 EFX_POPULATE_OWORD_8(
372 *filter,
373 FRF_CZ_RMFT_RSS_EN,
374 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
375 FRF_CZ_RMFT_SCATTER_EN,
376 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
377 FRF_CZ_RMFT_IP_OVERRIDE,
378 !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
379 FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
380 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
381 FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
382 FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
383 FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
384 data3 = is_wild;
385 break;
386 }
387
388 default:
389 BUG();
390 }
391
392 return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
393 }
394
395 static bool efx_filter_equal(const struct efx_filter_spec *left,
396 const struct efx_filter_spec *right)
397 {
398 if (left->type != right->type ||
399 memcmp(left->data, right->data, sizeof(left->data)))
400 return false;
401
402 return true;
403 }
404
405 static int efx_filter_search(struct efx_filter_table *table,
406 struct efx_filter_spec *spec, u32 key,
407 bool for_insert, unsigned int *depth_required)
408 {
409 unsigned hash, incr, filter_idx, depth, depth_max;
410
411 hash = efx_filter_hash(key);
412 incr = efx_filter_increment(key);
413
414 filter_idx = hash & (table->size - 1);
415 depth = 1;
416 depth_max = (for_insert ?
417 (spec->priority <= EFX_FILTER_PRI_HINT ?
418 FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX) :
419 table->search_depth[spec->type]);
420
421 for (;;) {
422 /* Return success if entry is used and matches this spec
423 * or entry is unused and we are trying to insert.
424 */
425 if (test_bit(filter_idx, table->used_bitmap) ?
426 efx_filter_equal(spec, &table->spec[filter_idx]) :
427 for_insert) {
428 *depth_required = depth;
429 return filter_idx;
430 }
431
432 /* Return failure if we reached the maximum search depth */
433 if (depth == depth_max)
434 return for_insert ? -EBUSY : -ENOENT;
435
436 filter_idx = (filter_idx + incr) & (table->size - 1);
437 ++depth;
438 }
439 }
440
441 /*
442 * Construct/deconstruct external filter IDs. These must be ordered
443 * by matching priority, for RX NFC semantics.
444 *
445 * Each RX MAC filter entry has a flag for whether it can override an
446 * RX IP filter that also matches. So we assign locations for MAC
447 * filters with overriding behaviour, then for IP filters, then for
448 * MAC filters without overriding behaviour.
449 */
450
451 #define EFX_FILTER_INDEX_WIDTH 13
452 #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
453
454 static inline u32 efx_filter_make_id(enum efx_filter_table_id table_id,
455 unsigned int index, u8 flags)
456 {
457 return (table_id == EFX_FILTER_TABLE_RX_MAC &&
458 flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP) ?
459 index :
460 (table_id + 1) << EFX_FILTER_INDEX_WIDTH | index;
461 }
462
463 static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id)
464 {
465 return (id <= EFX_FILTER_INDEX_MASK) ?
466 EFX_FILTER_TABLE_RX_MAC :
467 (id >> EFX_FILTER_INDEX_WIDTH) - 1;
468 }
469
470 static inline unsigned int efx_filter_id_index(u32 id)
471 {
472 return id & EFX_FILTER_INDEX_MASK;
473 }
474
475 static inline u8 efx_filter_id_flags(u32 id)
476 {
477 return (id <= EFX_FILTER_INDEX_MASK) ?
478 EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_OVERRIDE_IP :
479 EFX_FILTER_FLAG_RX;
480 }
481
482 u32 efx_filter_get_rx_id_limit(struct efx_nic *efx)
483 {
484 struct efx_filter_state *state = efx->filter_state;
485
486 if (state->table[EFX_FILTER_TABLE_RX_MAC].size != 0)
487 return ((EFX_FILTER_TABLE_RX_MAC + 1) << EFX_FILTER_INDEX_WIDTH)
488 + state->table[EFX_FILTER_TABLE_RX_MAC].size;
489 else if (state->table[EFX_FILTER_TABLE_RX_IP].size != 0)
490 return ((EFX_FILTER_TABLE_RX_IP + 1) << EFX_FILTER_INDEX_WIDTH)
491 + state->table[EFX_FILTER_TABLE_RX_IP].size;
492 else
493 return 0;
494 }
495
496 /**
497 * efx_filter_insert_filter - add or replace a filter
498 * @efx: NIC in which to insert the filter
499 * @spec: Specification for the filter
500 * @replace: Flag for whether the specified filter may replace a filter
501 * with an identical match expression and equal or lower priority
502 *
503 * On success, return the filter ID.
504 * On failure, return a negative error code.
505 */
506 s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
507 bool replace)
508 {
509 struct efx_filter_state *state = efx->filter_state;
510 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
511 struct efx_filter_spec *saved_spec;
512 efx_oword_t filter;
513 unsigned int filter_idx, depth;
514 u32 key;
515 int rc;
516
517 if (!table || table->size == 0)
518 return -EINVAL;
519
520 key = efx_filter_build(&filter, spec);
521
522 netif_vdbg(efx, hw, efx->net_dev,
523 "%s: type %d search_depth=%d", __func__, spec->type,
524 table->search_depth[spec->type]);
525
526 spin_lock_bh(&state->lock);
527
528 rc = efx_filter_search(table, spec, key, true, &depth);
529 if (rc < 0)
530 goto out;
531 filter_idx = rc;
532 BUG_ON(filter_idx >= table->size);
533 saved_spec = &table->spec[filter_idx];
534
535 if (test_bit(filter_idx, table->used_bitmap)) {
536 /* Should we replace the existing filter? */
537 if (!replace) {
538 rc = -EEXIST;
539 goto out;
540 }
541 if (spec->priority < saved_spec->priority) {
542 rc = -EPERM;
543 goto out;
544 }
545 } else {
546 __set_bit(filter_idx, table->used_bitmap);
547 ++table->used;
548 }
549 *saved_spec = *spec;
550
551 if (table->search_depth[spec->type] < depth) {
552 table->search_depth[spec->type] = depth;
553 efx_filter_push_rx_limits(efx);
554 }
555
556 efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
557
558 netif_vdbg(efx, hw, efx->net_dev,
559 "%s: filter type %d index %d rxq %u set",
560 __func__, spec->type, filter_idx, spec->dmaq_id);
561 rc = efx_filter_make_id(table->id, filter_idx, spec->flags);
562
563 out:
564 spin_unlock_bh(&state->lock);
565 return rc;
566 }
567
568 static void efx_filter_table_clear_entry(struct efx_nic *efx,
569 struct efx_filter_table *table,
570 unsigned int filter_idx)
571 {
572 static efx_oword_t filter;
573
574 if (test_bit(filter_idx, table->used_bitmap)) {
575 __clear_bit(filter_idx, table->used_bitmap);
576 --table->used;
577 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
578
579 efx_writeo(efx, &filter,
580 table->offset + table->step * filter_idx);
581 }
582 }
583
584 /**
585 * efx_filter_remove_id_safe - remove a filter by ID, carefully
586 * @efx: NIC from which to remove the filter
587 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
588 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
589 *
590 * This function will range-check @filter_id, so it is safe to call
591 * with a value passed from userland.
592 */
593 int efx_filter_remove_id_safe(struct efx_nic *efx,
594 enum efx_filter_priority priority,
595 u32 filter_id)
596 {
597 struct efx_filter_state *state = efx->filter_state;
598 enum efx_filter_table_id table_id;
599 struct efx_filter_table *table;
600 unsigned int filter_idx;
601 struct efx_filter_spec *spec;
602 u8 filter_flags;
603 int rc;
604
605 table_id = efx_filter_id_table_id(filter_id);
606 if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
607 return -ENOENT;
608 table = &state->table[table_id];
609
610 filter_idx = efx_filter_id_index(filter_id);
611 if (filter_idx >= table->size)
612 return -ENOENT;
613 spec = &table->spec[filter_idx];
614
615 filter_flags = efx_filter_id_flags(filter_id);
616
617 spin_lock_bh(&state->lock);
618
619 if (test_bit(filter_idx, table->used_bitmap) &&
620 spec->priority == priority && spec->flags == filter_flags) {
621 efx_filter_table_clear_entry(efx, table, filter_idx);
622 if (table->used == 0)
623 efx_filter_table_reset_search_depth(table);
624 rc = 0;
625 } else {
626 rc = -ENOENT;
627 }
628
629 spin_unlock_bh(&state->lock);
630
631 return rc;
632 }
633
634 /**
635 * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
636 * @efx: NIC from which to remove the filter
637 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
638 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
639 * @spec: Buffer in which to store filter specification
640 *
641 * This function will range-check @filter_id, so it is safe to call
642 * with a value passed from userland.
643 */
644 int efx_filter_get_filter_safe(struct efx_nic *efx,
645 enum efx_filter_priority priority,
646 u32 filter_id, struct efx_filter_spec *spec_buf)
647 {
648 struct efx_filter_state *state = efx->filter_state;
649 enum efx_filter_table_id table_id;
650 struct efx_filter_table *table;
651 struct efx_filter_spec *spec;
652 unsigned int filter_idx;
653 u8 filter_flags;
654 int rc;
655
656 table_id = efx_filter_id_table_id(filter_id);
657 if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
658 return -ENOENT;
659 table = &state->table[table_id];
660
661 filter_idx = efx_filter_id_index(filter_id);
662 if (filter_idx >= table->size)
663 return -ENOENT;
664 spec = &table->spec[filter_idx];
665
666 filter_flags = efx_filter_id_flags(filter_id);
667
668 spin_lock_bh(&state->lock);
669
670 if (test_bit(filter_idx, table->used_bitmap) &&
671 spec->priority == priority && spec->flags == filter_flags) {
672 *spec_buf = *spec;
673 rc = 0;
674 } else {
675 rc = -ENOENT;
676 }
677
678 spin_unlock_bh(&state->lock);
679
680 return rc;
681 }
682
683 static void efx_filter_table_clear(struct efx_nic *efx,
684 enum efx_filter_table_id table_id,
685 enum efx_filter_priority priority)
686 {
687 struct efx_filter_state *state = efx->filter_state;
688 struct efx_filter_table *table = &state->table[table_id];
689 unsigned int filter_idx;
690
691 spin_lock_bh(&state->lock);
692
693 for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
694 if (table->spec[filter_idx].priority <= priority)
695 efx_filter_table_clear_entry(efx, table, filter_idx);
696 if (table->used == 0)
697 efx_filter_table_reset_search_depth(table);
698
699 spin_unlock_bh(&state->lock);
700 }
701
702 /**
703 * efx_filter_clear_rx - remove RX filters by priority
704 * @efx: NIC from which to remove the filters
705 * @priority: Maximum priority to remove
706 */
707 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
708 {
709 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
710 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
711 }
712
713 u32 efx_filter_count_rx_used(struct efx_nic *efx,
714 enum efx_filter_priority priority)
715 {
716 struct efx_filter_state *state = efx->filter_state;
717 enum efx_filter_table_id table_id;
718 struct efx_filter_table *table;
719 unsigned int filter_idx;
720 u32 count = 0;
721
722 spin_lock_bh(&state->lock);
723
724 for (table_id = EFX_FILTER_TABLE_RX_IP;
725 table_id <= EFX_FILTER_TABLE_RX_MAC;
726 table_id++) {
727 table = &state->table[table_id];
728 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
729 if (test_bit(filter_idx, table->used_bitmap) &&
730 table->spec[filter_idx].priority == priority)
731 ++count;
732 }
733 }
734
735 spin_unlock_bh(&state->lock);
736
737 return count;
738 }
739
740 s32 efx_filter_get_rx_ids(struct efx_nic *efx,
741 enum efx_filter_priority priority,
742 u32 *buf, u32 size)
743 {
744 struct efx_filter_state *state = efx->filter_state;
745 enum efx_filter_table_id table_id;
746 struct efx_filter_table *table;
747 unsigned int filter_idx;
748 s32 count = 0;
749
750 spin_lock_bh(&state->lock);
751
752 for (table_id = EFX_FILTER_TABLE_RX_IP;
753 table_id <= EFX_FILTER_TABLE_RX_MAC;
754 table_id++) {
755 table = &state->table[table_id];
756 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
757 if (test_bit(filter_idx, table->used_bitmap) &&
758 table->spec[filter_idx].priority == priority) {
759 if (count == size) {
760 count = -EMSGSIZE;
761 goto out;
762 }
763 buf[count++] = efx_filter_make_id(
764 table_id, filter_idx,
765 table->spec[filter_idx].flags);
766 }
767 }
768 }
769 out:
770 spin_unlock_bh(&state->lock);
771
772 return count;
773 }
774
775 /* Restore filter stater after reset */
776 void efx_restore_filters(struct efx_nic *efx)
777 {
778 struct efx_filter_state *state = efx->filter_state;
779 enum efx_filter_table_id table_id;
780 struct efx_filter_table *table;
781 efx_oword_t filter;
782 unsigned int filter_idx;
783
784 spin_lock_bh(&state->lock);
785
786 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
787 table = &state->table[table_id];
788 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
789 if (!test_bit(filter_idx, table->used_bitmap))
790 continue;
791 efx_filter_build(&filter, &table->spec[filter_idx]);
792 efx_writeo(efx, &filter,
793 table->offset + table->step * filter_idx);
794 }
795 }
796
797 efx_filter_push_rx_limits(efx);
798
799 spin_unlock_bh(&state->lock);
800 }
801
802 int efx_probe_filters(struct efx_nic *efx)
803 {
804 struct efx_filter_state *state;
805 struct efx_filter_table *table;
806 unsigned table_id;
807
808 state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
809 if (!state)
810 return -ENOMEM;
811 efx->filter_state = state;
812
813 spin_lock_init(&state->lock);
814
815 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
816 #ifdef CONFIG_RFS_ACCEL
817 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
818 sizeof(*state->rps_flow_id),
819 GFP_KERNEL);
820 if (!state->rps_flow_id)
821 goto fail;
822 #endif
823 table = &state->table[EFX_FILTER_TABLE_RX_IP];
824 table->id = EFX_FILTER_TABLE_RX_IP;
825 table->offset = FR_BZ_RX_FILTER_TBL0;
826 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
827 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
828 }
829
830 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
831 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
832 table->id = EFX_FILTER_TABLE_RX_MAC;
833 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
834 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
835 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
836 }
837
838 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
839 table = &state->table[table_id];
840 if (table->size == 0)
841 continue;
842 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
843 sizeof(unsigned long),
844 GFP_KERNEL);
845 if (!table->used_bitmap)
846 goto fail;
847 table->spec = vzalloc(table->size * sizeof(*table->spec));
848 if (!table->spec)
849 goto fail;
850 }
851
852 return 0;
853
854 fail:
855 efx_remove_filters(efx);
856 return -ENOMEM;
857 }
858
859 void efx_remove_filters(struct efx_nic *efx)
860 {
861 struct efx_filter_state *state = efx->filter_state;
862 enum efx_filter_table_id table_id;
863
864 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
865 kfree(state->table[table_id].used_bitmap);
866 vfree(state->table[table_id].spec);
867 }
868 #ifdef CONFIG_RFS_ACCEL
869 kfree(state->rps_flow_id);
870 #endif
871 kfree(state);
872 }
873
874 #ifdef CONFIG_RFS_ACCEL
875
876 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
877 u16 rxq_index, u32 flow_id)
878 {
879 struct efx_nic *efx = netdev_priv(net_dev);
880 struct efx_channel *channel;
881 struct efx_filter_state *state = efx->filter_state;
882 struct efx_filter_spec spec;
883 const struct iphdr *ip;
884 const __be16 *ports;
885 int nhoff;
886 int rc;
887
888 nhoff = skb_network_offset(skb);
889
890 if (skb->protocol != htons(ETH_P_IP))
891 return -EPROTONOSUPPORT;
892
893 /* RFS must validate the IP header length before calling us */
894 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
895 ip = (const struct iphdr *)(skb->data + nhoff);
896 if (ip_is_fragment(ip))
897 return -EPROTONOSUPPORT;
898 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
899 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
900
901 efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
902 rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
903 ip->daddr, ports[1], ip->saddr, ports[0]);
904 if (rc)
905 return rc;
906
907 rc = efx_filter_insert_filter(efx, &spec, true);
908 if (rc < 0)
909 return rc;
910
911 /* Remember this so we can check whether to expire the filter later */
912 state->rps_flow_id[rc] = flow_id;
913 channel = efx_get_channel(efx, skb_get_rx_queue(skb));
914 ++channel->rfs_filters_added;
915
916 netif_info(efx, rx_status, efx->net_dev,
917 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
918 (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
919 &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
920 rxq_index, flow_id, rc);
921
922 return rc;
923 }
924
925 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
926 {
927 struct efx_filter_state *state = efx->filter_state;
928 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
929 unsigned mask = table->size - 1;
930 unsigned index;
931 unsigned stop;
932
933 if (!spin_trylock_bh(&state->lock))
934 return false;
935
936 index = state->rps_expire_index;
937 stop = (index + quota) & mask;
938
939 while (index != stop) {
940 if (test_bit(index, table->used_bitmap) &&
941 table->spec[index].priority == EFX_FILTER_PRI_HINT &&
942 rps_may_expire_flow(efx->net_dev,
943 table->spec[index].dmaq_id,
944 state->rps_flow_id[index], index)) {
945 netif_info(efx, rx_status, efx->net_dev,
946 "expiring filter %d [flow %u]\n",
947 index, state->rps_flow_id[index]);
948 efx_filter_table_clear_entry(efx, table, index);
949 }
950 index = (index + 1) & mask;
951 }
952
953 state->rps_expire_index = stop;
954 if (table->used == 0)
955 efx_filter_table_reset_search_depth(table);
956
957 spin_unlock_bh(&state->lock);
958 return true;
959 }
960
961 #endif /* CONFIG_RFS_ACCEL */