?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'cleancache/linux-next'
[linux-2.6/next.git] / drivers / net / sfc / filter.c
blob95a980fd63d5d9c195c683512808fa43d6d91a2d
1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
8 */
9
10 #include <linux/in.h>
11 #include <net/ip.h>
12 #include "efx.h"
13 #include "filter.h"
14 #include "io.h"
15 #include "nic.h"
16 #include "regs.h"
17
18 /* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
21 */
22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
24
25 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
27 * table is full.
28 */
29 #define FILTER_CTL_SRCH_MAX 200
30
31 /* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33 #define FILTER_CTL_SRCH_HINT_MAX 5
34
35 enum efx_filter_table_id {
36 EFX_FILTER_TABLE_RX_IP = 0,
37 EFX_FILTER_TABLE_RX_MAC,
38 EFX_FILTER_TABLE_COUNT,
39 };
40
41 struct efx_filter_table {
42 enum efx_filter_table_id id;
43 u32 offset; /* address of table relative to BAR */
44 unsigned size; /* number of entries */
45 unsigned step; /* step between entries */
46 unsigned used; /* number currently used */
47 unsigned long *used_bitmap;
48 struct efx_filter_spec *spec;
49 unsigned search_depth[EFX_FILTER_TYPE_COUNT];
50 };
51
52 struct efx_filter_state {
53 spinlock_t lock;
54 struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
55 #ifdef CONFIG_RFS_ACCEL
56 u32 *rps_flow_id;
57 unsigned rps_expire_index;
58 #endif
59 };
60
61 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
62 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
63 static u16 efx_filter_hash(u32 key)
64 {
65 u16 tmp;
66
67 /* First 16 rounds */
68 tmp = 0x1fff ^ key >> 16;
69 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
70 tmp = tmp ^ tmp >> 9;
71 /* Last 16 rounds */
72 tmp = tmp ^ tmp << 13 ^ key;
73 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
74 return tmp ^ tmp >> 9;
75 }
76
77 /* To allow for hash collisions, filter search continues at these
78 * increments from the first possible entry selected by the hash. */
79 static u16 efx_filter_increment(u32 key)
80 {
81 return key * 2 - 1;
82 }
83
84 static enum efx_filter_table_id
85 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
86 {
87 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
88 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
89 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
90 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
91 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
92 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
93 EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
94 return spec->type >> 2;
95 }
96
97 static struct efx_filter_table *
98 efx_filter_spec_table(struct efx_filter_state *state,
99 const struct efx_filter_spec *spec)
100 {
101 if (spec->type == EFX_FILTER_UNSPEC)
102 return NULL;
103 else
104 return &state->table[efx_filter_spec_table_id(spec)];
105 }
106
107 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
108 {
109 memset(table->search_depth, 0, sizeof(table->search_depth));
110 }
111
112 static void efx_filter_push_rx_limits(struct efx_nic *efx)
113 {
114 struct efx_filter_state *state = efx->filter_state;
115 struct efx_filter_table *table;
116 efx_oword_t filter_ctl;
117
118 efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
119
120 table = &state->table[EFX_FILTER_TABLE_RX_IP];
121 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
122 table->search_depth[EFX_FILTER_TCP_FULL] +
123 FILTER_CTL_SRCH_FUDGE_FULL);
124 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
125 table->search_depth[EFX_FILTER_TCP_WILD] +
126 FILTER_CTL_SRCH_FUDGE_WILD);
127 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
128 table->search_depth[EFX_FILTER_UDP_FULL] +
129 FILTER_CTL_SRCH_FUDGE_FULL);
130 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
131 table->search_depth[EFX_FILTER_UDP_WILD] +
132 FILTER_CTL_SRCH_FUDGE_WILD);
133
134 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
135 if (table->size) {
136 EFX_SET_OWORD_FIELD(
137 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
138 table->search_depth[EFX_FILTER_MAC_FULL] +
139 FILTER_CTL_SRCH_FUDGE_FULL);
140 EFX_SET_OWORD_FIELD(
141 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
142 table->search_depth[EFX_FILTER_MAC_WILD] +
143 FILTER_CTL_SRCH_FUDGE_WILD);
144 }
145
146 efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
147 }
148
149 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
150 __be32 host1, __be16 port1,
151 __be32 host2, __be16 port2)
152 {
153 spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
154 spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
155 spec->data[2] = ntohl(host2);
156 }
157
158 /**
159 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
160 * @spec: Specification to initialise
161 * @proto: Transport layer protocol number
162 * @host: Local host address (network byte order)
163 * @port: Local port (network byte order)
164 */
165 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
166 __be32 host, __be16 port)
167 {
168 __be32 host1;
169 __be16 port1;
170
171 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
172
173 /* This cannot currently be combined with other filtering */
174 if (spec->type != EFX_FILTER_UNSPEC)
175 return -EPROTONOSUPPORT;
176
177 if (port == 0)
178 return -EINVAL;
179
180 switch (proto) {
181 case IPPROTO_TCP:
182 spec->type = EFX_FILTER_TCP_WILD;
183 break;
184 case IPPROTO_UDP:
185 spec->type = EFX_FILTER_UDP_WILD;
186 break;
187 default:
188 return -EPROTONOSUPPORT;
189 }
190
191 /* Filter is constructed in terms of source and destination,
192 * with the odd wrinkle that the ports are swapped in a UDP
193 * wildcard filter. We need to convert from local and remote
194 * (= zero for wildcard) addresses.
195 */
196 host1 = 0;
197 if (proto != IPPROTO_UDP) {
198 port1 = 0;
199 } else {
200 port1 = port;
201 port = 0;
202 }
203
204 __efx_filter_set_ipv4(spec, host1, port1, host, port);
205 return 0;
206 }
207
208 /**
209 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
210 * @spec: Specification to initialise
211 * @proto: Transport layer protocol number
212 * @host: Local host address (network byte order)
213 * @port: Local port (network byte order)
214 * @rhost: Remote host address (network byte order)
215 * @rport: Remote port (network byte order)
216 */
217 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
218 __be32 host, __be16 port,
219 __be32 rhost, __be16 rport)
220 {
221 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
222
223 /* This cannot currently be combined with other filtering */
224 if (spec->type != EFX_FILTER_UNSPEC)
225 return -EPROTONOSUPPORT;
226
227 if (port == 0 || rport == 0)
228 return -EINVAL;
229
230 switch (proto) {
231 case IPPROTO_TCP:
232 spec->type = EFX_FILTER_TCP_FULL;
233 break;
234 case IPPROTO_UDP:
235 spec->type = EFX_FILTER_UDP_FULL;
236 break;
237 default:
238 return -EPROTONOSUPPORT;
239 }
240
241 __efx_filter_set_ipv4(spec, rhost, rport, host, port);
242 return 0;
243 }
244
245 /**
246 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
247 * @spec: Specification to initialise
248 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
249 * @addr: Local Ethernet MAC address
250 */
251 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
252 u16 vid, const u8 *addr)
253 {
254 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
255
256 /* This cannot currently be combined with other filtering */
257 if (spec->type != EFX_FILTER_UNSPEC)
258 return -EPROTONOSUPPORT;
259
260 if (vid == EFX_FILTER_VID_UNSPEC) {
261 spec->type = EFX_FILTER_MAC_WILD;
262 spec->data[0] = 0;
263 } else {
264 spec->type = EFX_FILTER_MAC_FULL;
265 spec->data[0] = vid;
266 }
267
268 spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
269 spec->data[2] = addr[0] << 8 | addr[1];
270 return 0;
271 }
272
273 /* Build a filter entry and return its n-tuple key. */
274 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
275 {
276 u32 data3;
277
278 switch (efx_filter_spec_table_id(spec)) {
279 case EFX_FILTER_TABLE_RX_IP: {
280 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
281 spec->type == EFX_FILTER_UDP_WILD);
282 EFX_POPULATE_OWORD_7(
283 *filter,
284 FRF_BZ_RSS_EN,
285 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
286 FRF_BZ_SCATTER_EN,
287 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
288 FRF_BZ_TCP_UDP, is_udp,
289 FRF_BZ_RXQ_ID, spec->dmaq_id,
290 EFX_DWORD_2, spec->data[2],
291 EFX_DWORD_1, spec->data[1],
292 EFX_DWORD_0, spec->data[0]);
293 data3 = is_udp;
294 break;
295 }
296
297 case EFX_FILTER_TABLE_RX_MAC: {
298 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
299 EFX_POPULATE_OWORD_8(
300 *filter,
301 FRF_CZ_RMFT_RSS_EN,
302 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
303 FRF_CZ_RMFT_SCATTER_EN,
304 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
305 FRF_CZ_RMFT_IP_OVERRIDE,
306 !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
307 FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
308 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
309 FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
310 FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
311 FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
312 data3 = is_wild;
313 break;
314 }
315
316 default:
317 BUG();
318 }
319
320 return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
321 }
322
323 static bool efx_filter_equal(const struct efx_filter_spec *left,
324 const struct efx_filter_spec *right)
325 {
326 if (left->type != right->type ||
327 memcmp(left->data, right->data, sizeof(left->data)))
328 return false;
329
330 return true;
331 }
332
333 static int efx_filter_search(struct efx_filter_table *table,
334 struct efx_filter_spec *spec, u32 key,
335 bool for_insert, int *depth_required)
336 {
337 unsigned hash, incr, filter_idx, depth, depth_max;
338 struct efx_filter_spec *cmp;
339
340 hash = efx_filter_hash(key);
341 incr = efx_filter_increment(key);
342 depth_max = (spec->priority <= EFX_FILTER_PRI_HINT ?
343 FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX);
344
345 for (depth = 1, filter_idx = hash & (table->size - 1);
346 depth <= depth_max && test_bit(filter_idx, table->used_bitmap);
347 ++depth) {
348 cmp = &table->spec[filter_idx];
349 if (efx_filter_equal(spec, cmp))
350 goto found;
351 filter_idx = (filter_idx + incr) & (table->size - 1);
352 }
353 if (!for_insert)
354 return -ENOENT;
355 if (depth > depth_max)
356 return -EBUSY;
357 found:
358 *depth_required = depth;
359 return filter_idx;
360 }
361
362 /* Construct/deconstruct external filter IDs */
363
364 static inline int
365 efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
366 {
367 return table_id << 16 | index;
368 }
369
370 /**
371 * efx_filter_insert_filter - add or replace a filter
372 * @efx: NIC in which to insert the filter
373 * @spec: Specification for the filter
374 * @replace: Flag for whether the specified filter may replace a filter
375 * with an identical match expression and equal or lower priority
376 *
377 * On success, return the filter ID.
378 * On failure, return a negative error code.
379 */
380 int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
381 bool replace)
382 {
383 struct efx_filter_state *state = efx->filter_state;
384 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
385 struct efx_filter_spec *saved_spec;
386 efx_oword_t filter;
387 int filter_idx, depth;
388 u32 key;
389 int rc;
390
391 if (!table || table->size == 0)
392 return -EINVAL;
393
394 key = efx_filter_build(&filter, spec);
395
396 netif_vdbg(efx, hw, efx->net_dev,
397 "%s: type %d search_depth=%d", __func__, spec->type,
398 table->search_depth[spec->type]);
399
400 spin_lock_bh(&state->lock);
401
402 rc = efx_filter_search(table, spec, key, true, &depth);
403 if (rc < 0)
404 goto out;
405 filter_idx = rc;
406 BUG_ON(filter_idx >= table->size);
407 saved_spec = &table->spec[filter_idx];
408
409 if (test_bit(filter_idx, table->used_bitmap)) {
410 /* Should we replace the existing filter? */
411 if (!replace) {
412 rc = -EEXIST;
413 goto out;
414 }
415 if (spec->priority < saved_spec->priority) {
416 rc = -EPERM;
417 goto out;
418 }
419 } else {
420 __set_bit(filter_idx, table->used_bitmap);
421 ++table->used;
422 }
423 *saved_spec = *spec;
424
425 if (table->search_depth[spec->type] < depth) {
426 table->search_depth[spec->type] = depth;
427 efx_filter_push_rx_limits(efx);
428 }
429
430 efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
431
432 netif_vdbg(efx, hw, efx->net_dev,
433 "%s: filter type %d index %d rxq %u set",
434 __func__, spec->type, filter_idx, spec->dmaq_id);
435 rc = efx_filter_make_id(table->id, filter_idx);
436
437 out:
438 spin_unlock_bh(&state->lock);
439 return rc;
440 }
441
442 static void efx_filter_table_clear_entry(struct efx_nic *efx,
443 struct efx_filter_table *table,
444 int filter_idx)
445 {
446 static efx_oword_t filter;
447
448 if (test_bit(filter_idx, table->used_bitmap)) {
449 __clear_bit(filter_idx, table->used_bitmap);
450 --table->used;
451 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
452
453 efx_writeo(efx, &filter,
454 table->offset + table->step * filter_idx);
455 }
456 }
457
458 /**
459 * efx_filter_remove_filter - remove a filter by specification
460 * @efx: NIC from which to remove the filter
461 * @spec: Specification for the filter
462 *
463 * On success, return zero.
464 * On failure, return a negative error code.
465 */
466 int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
467 {
468 struct efx_filter_state *state = efx->filter_state;
469 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
470 struct efx_filter_spec *saved_spec;
471 efx_oword_t filter;
472 int filter_idx, depth;
473 u32 key;
474 int rc;
475
476 if (!table)
477 return -EINVAL;
478
479 key = efx_filter_build(&filter, spec);
480
481 spin_lock_bh(&state->lock);
482
483 rc = efx_filter_search(table, spec, key, false, &depth);
484 if (rc < 0)
485 goto out;
486 filter_idx = rc;
487 saved_spec = &table->spec[filter_idx];
488
489 if (spec->priority < saved_spec->priority) {
490 rc = -EPERM;
491 goto out;
492 }
493
494 efx_filter_table_clear_entry(efx, table, filter_idx);
495 if (table->used == 0)
496 efx_filter_table_reset_search_depth(table);
497 rc = 0;
498
499 out:
500 spin_unlock_bh(&state->lock);
501 return rc;
502 }
503
504 static void efx_filter_table_clear(struct efx_nic *efx,
505 enum efx_filter_table_id table_id,
506 enum efx_filter_priority priority)
507 {
508 struct efx_filter_state *state = efx->filter_state;
509 struct efx_filter_table *table = &state->table[table_id];
510 int filter_idx;
511
512 spin_lock_bh(&state->lock);
513
514 for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
515 if (table->spec[filter_idx].priority <= priority)
516 efx_filter_table_clear_entry(efx, table, filter_idx);
517 if (table->used == 0)
518 efx_filter_table_reset_search_depth(table);
519
520 spin_unlock_bh(&state->lock);
521 }
522
523 /**
524 * efx_filter_clear_rx - remove RX filters by priority
525 * @efx: NIC from which to remove the filters
526 * @priority: Maximum priority to remove
527 */
528 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
529 {
530 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
531 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
532 }
533
534 /* Restore filter stater after reset */
535 void efx_restore_filters(struct efx_nic *efx)
536 {
537 struct efx_filter_state *state = efx->filter_state;
538 enum efx_filter_table_id table_id;
539 struct efx_filter_table *table;
540 efx_oword_t filter;
541 int filter_idx;
542
543 spin_lock_bh(&state->lock);
544
545 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
546 table = &state->table[table_id];
547 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
548 if (!test_bit(filter_idx, table->used_bitmap))
549 continue;
550 efx_filter_build(&filter, &table->spec[filter_idx]);
551 efx_writeo(efx, &filter,
552 table->offset + table->step * filter_idx);
553 }
554 }
555
556 efx_filter_push_rx_limits(efx);
557
558 spin_unlock_bh(&state->lock);
559 }
560
561 int efx_probe_filters(struct efx_nic *efx)
562 {
563 struct efx_filter_state *state;
564 struct efx_filter_table *table;
565 unsigned table_id;
566
567 state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
568 if (!state)
569 return -ENOMEM;
570 efx->filter_state = state;
571
572 spin_lock_init(&state->lock);
573
574 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
575 #ifdef CONFIG_RFS_ACCEL
576 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
577 sizeof(*state->rps_flow_id),
578 GFP_KERNEL);
579 if (!state->rps_flow_id)
580 goto fail;
581 #endif
582 table = &state->table[EFX_FILTER_TABLE_RX_IP];
583 table->id = EFX_FILTER_TABLE_RX_IP;
584 table->offset = FR_BZ_RX_FILTER_TBL0;
585 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
586 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
587 }
588
589 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
590 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
591 table->id = EFX_FILTER_TABLE_RX_MAC;
592 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
593 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
594 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
595 }
596
597 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
598 table = &state->table[table_id];
599 if (table->size == 0)
600 continue;
601 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
602 sizeof(unsigned long),
603 GFP_KERNEL);
604 if (!table->used_bitmap)
605 goto fail;
606 table->spec = vzalloc(table->size * sizeof(*table->spec));
607 if (!table->spec)
608 goto fail;
609 }
610
611 return 0;
612
613 fail:
614 efx_remove_filters(efx);
615 return -ENOMEM;
616 }
617
618 void efx_remove_filters(struct efx_nic *efx)
619 {
620 struct efx_filter_state *state = efx->filter_state;
621 enum efx_filter_table_id table_id;
622
623 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
624 kfree(state->table[table_id].used_bitmap);
625 vfree(state->table[table_id].spec);
626 }
627 #ifdef CONFIG_RFS_ACCEL
628 kfree(state->rps_flow_id);
629 #endif
630 kfree(state);
631 }
632
633 #ifdef CONFIG_RFS_ACCEL
634
635 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
636 u16 rxq_index, u32 flow_id)
637 {
638 struct efx_nic *efx = netdev_priv(net_dev);
639 struct efx_channel *channel;
640 struct efx_filter_state *state = efx->filter_state;
641 struct efx_filter_spec spec;
642 const struct iphdr *ip;
643 const __be16 *ports;
644 int nhoff;
645 int rc;
646
647 nhoff = skb_network_offset(skb);
648
649 if (skb->protocol != htons(ETH_P_IP))
650 return -EPROTONOSUPPORT;
651
652 /* RFS must validate the IP header length before calling us */
653 EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + sizeof(*ip)));
654 ip = (const struct iphdr *)(skb->data + nhoff);
655 if (ip->frag_off & htons(IP_MF | IP_OFFSET))
656 return -EPROTONOSUPPORT;
657 EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + 4 * ip->ihl + 4));
658 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
659
660 efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
661 rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
662 ip->daddr, ports[1], ip->saddr, ports[0]);
663 if (rc)
664 return rc;
665
666 rc = efx_filter_insert_filter(efx, &spec, true);
667 if (rc < 0)
668 return rc;
669
670 /* Remember this so we can check whether to expire the filter later */
671 state->rps_flow_id[rc] = flow_id;
672 channel = efx_get_channel(efx, skb_get_rx_queue(skb));
673 ++channel->rfs_filters_added;
674
675 netif_info(efx, rx_status, efx->net_dev,
676 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
677 (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
678 &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
679 rxq_index, flow_id, rc);
680
681 return rc;
682 }
683
684 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
685 {
686 struct efx_filter_state *state = efx->filter_state;
687 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
688 unsigned mask = table->size - 1;
689 unsigned index;
690 unsigned stop;
691
692 if (!spin_trylock_bh(&state->lock))
693 return false;
694
695 index = state->rps_expire_index;
696 stop = (index + quota) & mask;
697
698 while (index != stop) {
699 if (test_bit(index, table->used_bitmap) &&
700 table->spec[index].priority == EFX_FILTER_PRI_HINT &&
701 rps_may_expire_flow(efx->net_dev,
702 table->spec[index].dmaq_id,
703 state->rps_flow_id[index], index)) {
704 netif_info(efx, rx_status, efx->net_dev,
705 "expiring filter %d [flow %u]\n",
706 index, state->rps_flow_id[index]);
707 efx_filter_table_clear_entry(efx, table, index);
708 }
709 index = (index + 1) & mask;
710 }
711
712 state->rps_expire_index = stop;
713 if (table->used == 0)
714 efx_filter_table_reset_search_depth(table);
715
716 spin_unlock_bh(&state->lock);
717 return true;
718 }
719
720 #endif /* CONFIG_RFS_ACCEL */
linux-next