2 * net/sched/sch_sfb.c Stochastic Fair Blue
4 * Copyright (c) 2008-2011 Juliusz Chroboczek <jch@pps.jussieu.fr>
5 * Copyright (c) 2011 Eric Dumazet <eric.dumazet@gmail.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * version 2 as published by the Free Software Foundation.
11 * W. Feng, D. Kandlur, D. Saha, K. Shin. Blue:
12 * A New Class of Active Queue Management Algorithms.
13 * U. Michigan CSE-TR-387-99, April 1999.
15 * http://www.thefengs.com/wuchang/blue/CSE-TR-387-99.pdf
19 #include <linux/module.h>
20 #include <linux/types.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/skbuff.h>
24 #include <linux/random.h>
25 #include <linux/jhash.h>
27 #include <net/pkt_sched.h>
28 #include <net/pkt_cls.h>
29 #include <net/inet_ecn.h>
32 * SFB uses two B[l][n] : L x N arrays of bins (L levels, N bins per level)
33 * This implementation uses L = 8 and N = 16
34 * This permits us to split one 32bit hash (provided per packet by rxhash or
35 * external classifier) into 8 subhashes of 4 bits.
37 #define SFB_BUCKET_SHIFT 4
38 #define SFB_NUMBUCKETS (1 << SFB_BUCKET_SHIFT) /* N bins per Level */
39 #define SFB_BUCKET_MASK (SFB_NUMBUCKETS - 1)
40 #define SFB_LEVELS (32 / SFB_BUCKET_SHIFT) /* L */
42 /* SFB algo uses a virtual queue, named "bin" */
44 u16 qlen
; /* length of virtual queue */
45 u16 p_mark
; /* marking probability */
48 /* We use a double buffering right before hash change
49 * (Section 4.4 of SFB reference : moving hash functions)
52 u32 perturbation
; /* jhash perturbation */
53 struct sfb_bucket bins
[SFB_LEVELS
][SFB_NUMBUCKETS
];
56 struct sfb_sched_data
{
58 struct tcf_proto __rcu
*filter_list
;
59 struct tcf_block
*block
;
60 unsigned long rehash_interval
;
61 unsigned long warmup_time
; /* double buffering warmup time in jiffies */
63 u32 bin_size
; /* maximum queue length per bin */
64 u32 increment
; /* d1 */
65 u32 decrement
; /* d2 */
66 u32 limit
; /* HARD maximal queue length */
70 unsigned long rehash_time
;
71 unsigned long token_time
;
73 u8 slot
; /* current active bins (0 or 1) */
74 bool double_buffering
;
75 struct sfb_bins bins
[2];
82 u32 childdrop
; /* drops in child qdisc */
83 u32 marked
; /* ECN mark */
88 * Each queued skb might be hashed on one or two bins
89 * We store in skb_cb the two hash values.
90 * (A zero value means double buffering was not used)
96 static inline struct sfb_skb_cb
*sfb_skb_cb(const struct sk_buff
*skb
)
98 qdisc_cb_private_validate(skb
, sizeof(struct sfb_skb_cb
));
99 return (struct sfb_skb_cb
*)qdisc_skb_cb(skb
)->data
;
103 * If using 'internal' SFB flow classifier, hash comes from skb rxhash
104 * If using external classifier, hash comes from the classid.
106 static u32
sfb_hash(const struct sk_buff
*skb
, u32 slot
)
108 return sfb_skb_cb(skb
)->hashes
[slot
];
111 /* Probabilities are coded as Q0.16 fixed-point values,
112 * with 0xFFFF representing 65535/65536 (almost 1.0)
113 * Addition and subtraction are saturating in [0, 65535]
115 static u32
prob_plus(u32 p1
, u32 p2
)
119 return min_t(u32
, res
, SFB_MAX_PROB
);
122 static u32
prob_minus(u32 p1
, u32 p2
)
124 return p1
> p2
? p1
- p2
: 0;
127 static void increment_one_qlen(u32 sfbhash
, u32 slot
, struct sfb_sched_data
*q
)
130 struct sfb_bucket
*b
= &q
->bins
[slot
].bins
[0][0];
132 for (i
= 0; i
< SFB_LEVELS
; i
++) {
133 u32 hash
= sfbhash
& SFB_BUCKET_MASK
;
135 sfbhash
>>= SFB_BUCKET_SHIFT
;
136 if (b
[hash
].qlen
< 0xFFFF)
138 b
+= SFB_NUMBUCKETS
; /* next level */
142 static void increment_qlen(const struct sk_buff
*skb
, struct sfb_sched_data
*q
)
146 sfbhash
= sfb_hash(skb
, 0);
148 increment_one_qlen(sfbhash
, 0, q
);
150 sfbhash
= sfb_hash(skb
, 1);
152 increment_one_qlen(sfbhash
, 1, q
);
155 static void decrement_one_qlen(u32 sfbhash
, u32 slot
,
156 struct sfb_sched_data
*q
)
159 struct sfb_bucket
*b
= &q
->bins
[slot
].bins
[0][0];
161 for (i
= 0; i
< SFB_LEVELS
; i
++) {
162 u32 hash
= sfbhash
& SFB_BUCKET_MASK
;
164 sfbhash
>>= SFB_BUCKET_SHIFT
;
165 if (b
[hash
].qlen
> 0)
167 b
+= SFB_NUMBUCKETS
; /* next level */
171 static void decrement_qlen(const struct sk_buff
*skb
, struct sfb_sched_data
*q
)
175 sfbhash
= sfb_hash(skb
, 0);
177 decrement_one_qlen(sfbhash
, 0, q
);
179 sfbhash
= sfb_hash(skb
, 1);
181 decrement_one_qlen(sfbhash
, 1, q
);
184 static void decrement_prob(struct sfb_bucket
*b
, struct sfb_sched_data
*q
)
186 b
->p_mark
= prob_minus(b
->p_mark
, q
->decrement
);
189 static void increment_prob(struct sfb_bucket
*b
, struct sfb_sched_data
*q
)
191 b
->p_mark
= prob_plus(b
->p_mark
, q
->increment
);
194 static void sfb_zero_all_buckets(struct sfb_sched_data
*q
)
196 memset(&q
->bins
, 0, sizeof(q
->bins
));
200 * compute max qlen, max p_mark, and avg p_mark
202 static u32
sfb_compute_qlen(u32
*prob_r
, u32
*avgpm_r
, const struct sfb_sched_data
*q
)
205 u32 qlen
= 0, prob
= 0, totalpm
= 0;
206 const struct sfb_bucket
*b
= &q
->bins
[q
->slot
].bins
[0][0];
208 for (i
= 0; i
< SFB_LEVELS
* SFB_NUMBUCKETS
; i
++) {
211 totalpm
+= b
->p_mark
;
212 if (prob
< b
->p_mark
)
217 *avgpm_r
= totalpm
/ (SFB_LEVELS
* SFB_NUMBUCKETS
);
222 static void sfb_init_perturbation(u32 slot
, struct sfb_sched_data
*q
)
224 q
->bins
[slot
].perturbation
= prandom_u32();
227 static void sfb_swap_slot(struct sfb_sched_data
*q
)
229 sfb_init_perturbation(q
->slot
, q
);
231 q
->double_buffering
= false;
234 /* Non elastic flows are allowed to use part of the bandwidth, expressed
235 * in "penalty_rate" packets per second, with "penalty_burst" burst
237 static bool sfb_rate_limit(struct sk_buff
*skb
, struct sfb_sched_data
*q
)
239 if (q
->penalty_rate
== 0 || q
->penalty_burst
== 0)
242 if (q
->tokens_avail
< 1) {
243 unsigned long age
= min(10UL * HZ
, jiffies
- q
->token_time
);
245 q
->tokens_avail
= (age
* q
->penalty_rate
) / HZ
;
246 if (q
->tokens_avail
> q
->penalty_burst
)
247 q
->tokens_avail
= q
->penalty_burst
;
248 q
->token_time
= jiffies
;
249 if (q
->tokens_avail
< 1)
257 static bool sfb_classify(struct sk_buff
*skb
, struct tcf_proto
*fl
,
258 int *qerr
, u32
*salt
)
260 struct tcf_result res
;
263 result
= tcf_classify(skb
, fl
, &res
, false);
265 #ifdef CONFIG_NET_CLS_ACT
270 *qerr
= NET_XMIT_SUCCESS
| __NET_XMIT_STOLEN
;
276 *salt
= TC_H_MIN(res
.classid
);
282 static int sfb_enqueue(struct sk_buff
*skb
, struct Qdisc
*sch
,
283 struct sk_buff
**to_free
)
286 struct sfb_sched_data
*q
= qdisc_priv(sch
);
287 struct Qdisc
*child
= q
->qdisc
;
288 struct tcf_proto
*fl
;
294 int ret
= NET_XMIT_SUCCESS
| __NET_XMIT_BYPASS
;
296 if (unlikely(sch
->q
.qlen
>= q
->limit
)) {
297 qdisc_qstats_overlimit(sch
);
298 q
->stats
.queuedrop
++;
302 if (q
->rehash_interval
> 0) {
303 unsigned long limit
= q
->rehash_time
+ q
->rehash_interval
;
305 if (unlikely(time_after(jiffies
, limit
))) {
307 q
->rehash_time
= jiffies
;
308 } else if (unlikely(!q
->double_buffering
&& q
->warmup_time
> 0 &&
309 time_after(jiffies
, limit
- q
->warmup_time
))) {
310 q
->double_buffering
= true;
314 fl
= rcu_dereference_bh(q
->filter_list
);
318 /* If using external classifiers, get result and record it. */
319 if (!sfb_classify(skb
, fl
, &ret
, &salt
))
321 sfbhash
= jhash_1word(salt
, q
->bins
[slot
].perturbation
);
323 sfbhash
= skb_get_hash_perturb(skb
, q
->bins
[slot
].perturbation
);
329 sfb_skb_cb(skb
)->hashes
[slot
] = sfbhash
;
331 for (i
= 0; i
< SFB_LEVELS
; i
++) {
332 u32 hash
= sfbhash
& SFB_BUCKET_MASK
;
333 struct sfb_bucket
*b
= &q
->bins
[slot
].bins
[i
][hash
];
335 sfbhash
>>= SFB_BUCKET_SHIFT
;
337 decrement_prob(b
, q
);
338 else if (b
->qlen
>= q
->bin_size
)
339 increment_prob(b
, q
);
340 if (minqlen
> b
->qlen
)
342 if (p_min
> b
->p_mark
)
347 sfb_skb_cb(skb
)->hashes
[slot
] = 0;
349 if (unlikely(minqlen
>= q
->max
)) {
350 qdisc_qstats_overlimit(sch
);
351 q
->stats
.bucketdrop
++;
355 if (unlikely(p_min
>= SFB_MAX_PROB
)) {
357 if (q
->double_buffering
) {
358 sfbhash
= skb_get_hash_perturb(skb
,
359 q
->bins
[slot
].perturbation
);
362 sfb_skb_cb(skb
)->hashes
[slot
] = sfbhash
;
364 for (i
= 0; i
< SFB_LEVELS
; i
++) {
365 u32 hash
= sfbhash
& SFB_BUCKET_MASK
;
366 struct sfb_bucket
*b
= &q
->bins
[slot
].bins
[i
][hash
];
368 sfbhash
>>= SFB_BUCKET_SHIFT
;
370 decrement_prob(b
, q
);
371 else if (b
->qlen
>= q
->bin_size
)
372 increment_prob(b
, q
);
375 if (sfb_rate_limit(skb
, q
)) {
376 qdisc_qstats_overlimit(sch
);
377 q
->stats
.penaltydrop
++;
383 r
= prandom_u32() & SFB_MAX_PROB
;
385 if (unlikely(r
< p_min
)) {
386 if (unlikely(p_min
> SFB_MAX_PROB
/ 2)) {
387 /* If we're marking that many packets, then either
388 * this flow is unresponsive, or we're badly congested.
389 * In either case, we want to start dropping packets.
391 if (r
< (p_min
- SFB_MAX_PROB
/ 2) * 2) {
392 q
->stats
.earlydrop
++;
396 if (INET_ECN_set_ce(skb
)) {
399 q
->stats
.earlydrop
++;
405 ret
= qdisc_enqueue(skb
, child
, to_free
);
406 if (likely(ret
== NET_XMIT_SUCCESS
)) {
407 qdisc_qstats_backlog_inc(sch
, skb
);
409 increment_qlen(skb
, q
);
410 } else if (net_xmit_drop_count(ret
)) {
411 q
->stats
.childdrop
++;
412 qdisc_qstats_drop(sch
);
417 qdisc_drop(skb
, sch
, to_free
);
420 if (ret
& __NET_XMIT_BYPASS
)
421 qdisc_qstats_drop(sch
);
426 static struct sk_buff
*sfb_dequeue(struct Qdisc
*sch
)
428 struct sfb_sched_data
*q
= qdisc_priv(sch
);
429 struct Qdisc
*child
= q
->qdisc
;
432 skb
= child
->dequeue(q
->qdisc
);
435 qdisc_bstats_update(sch
, skb
);
436 qdisc_qstats_backlog_dec(sch
, skb
);
438 decrement_qlen(skb
, q
);
444 static struct sk_buff
*sfb_peek(struct Qdisc
*sch
)
446 struct sfb_sched_data
*q
= qdisc_priv(sch
);
447 struct Qdisc
*child
= q
->qdisc
;
449 return child
->ops
->peek(child
);
452 /* No sfb_drop -- impossible since the child doesn't return the dropped skb. */
454 static void sfb_reset(struct Qdisc
*sch
)
456 struct sfb_sched_data
*q
= qdisc_priv(sch
);
458 qdisc_reset(q
->qdisc
);
459 sch
->qstats
.backlog
= 0;
462 q
->double_buffering
= false;
463 sfb_zero_all_buckets(q
);
464 sfb_init_perturbation(0, q
);
467 static void sfb_destroy(struct Qdisc
*sch
)
469 struct sfb_sched_data
*q
= qdisc_priv(sch
);
471 tcf_block_put(q
->block
);
475 static const struct nla_policy sfb_policy
[TCA_SFB_MAX
+ 1] = {
476 [TCA_SFB_PARMS
] = { .len
= sizeof(struct tc_sfb_qopt
) },
479 static const struct tc_sfb_qopt sfb_default_ops
= {
480 .rehash_interval
= 600 * MSEC_PER_SEC
,
481 .warmup_time
= 60 * MSEC_PER_SEC
,
485 .increment
= (SFB_MAX_PROB
+ 500) / 1000, /* 0.1 % */
486 .decrement
= (SFB_MAX_PROB
+ 3000) / 6000,
491 static int sfb_change(struct Qdisc
*sch
, struct nlattr
*opt
,
492 struct netlink_ext_ack
*extack
)
494 struct sfb_sched_data
*q
= qdisc_priv(sch
);
496 struct nlattr
*tb
[TCA_SFB_MAX
+ 1];
497 const struct tc_sfb_qopt
*ctl
= &sfb_default_ops
;
502 err
= nla_parse_nested(tb
, TCA_SFB_MAX
, opt
, sfb_policy
, NULL
);
506 if (tb
[TCA_SFB_PARMS
] == NULL
)
509 ctl
= nla_data(tb
[TCA_SFB_PARMS
]);
514 limit
= qdisc_dev(sch
)->tx_queue_len
;
516 child
= fifo_create_dflt(sch
, &pfifo_qdisc_ops
, limit
, extack
);
518 return PTR_ERR(child
);
520 if (child
!= &noop_qdisc
)
521 qdisc_hash_add(child
, true);
524 qdisc_tree_reduce_backlog(q
->qdisc
, q
->qdisc
->q
.qlen
,
525 q
->qdisc
->qstats
.backlog
);
529 q
->rehash_interval
= msecs_to_jiffies(ctl
->rehash_interval
);
530 q
->warmup_time
= msecs_to_jiffies(ctl
->warmup_time
);
531 q
->rehash_time
= jiffies
;
533 q
->increment
= ctl
->increment
;
534 q
->decrement
= ctl
->decrement
;
536 q
->bin_size
= ctl
->bin_size
;
537 q
->penalty_rate
= ctl
->penalty_rate
;
538 q
->penalty_burst
= ctl
->penalty_burst
;
539 q
->tokens_avail
= ctl
->penalty_burst
;
540 q
->token_time
= jiffies
;
543 q
->double_buffering
= false;
544 sfb_zero_all_buckets(q
);
545 sfb_init_perturbation(0, q
);
546 sfb_init_perturbation(1, q
);
548 sch_tree_unlock(sch
);
553 static int sfb_init(struct Qdisc
*sch
, struct nlattr
*opt
,
554 struct netlink_ext_ack
*extack
)
556 struct sfb_sched_data
*q
= qdisc_priv(sch
);
559 err
= tcf_block_get(&q
->block
, &q
->filter_list
, sch
, extack
);
563 q
->qdisc
= &noop_qdisc
;
564 return sfb_change(sch
, opt
, extack
);
567 static int sfb_dump(struct Qdisc
*sch
, struct sk_buff
*skb
)
569 struct sfb_sched_data
*q
= qdisc_priv(sch
);
571 struct tc_sfb_qopt opt
= {
572 .rehash_interval
= jiffies_to_msecs(q
->rehash_interval
),
573 .warmup_time
= jiffies_to_msecs(q
->warmup_time
),
576 .bin_size
= q
->bin_size
,
577 .increment
= q
->increment
,
578 .decrement
= q
->decrement
,
579 .penalty_rate
= q
->penalty_rate
,
580 .penalty_burst
= q
->penalty_burst
,
583 sch
->qstats
.backlog
= q
->qdisc
->qstats
.backlog
;
584 opts
= nla_nest_start(skb
, TCA_OPTIONS
);
586 goto nla_put_failure
;
587 if (nla_put(skb
, TCA_SFB_PARMS
, sizeof(opt
), &opt
))
588 goto nla_put_failure
;
589 return nla_nest_end(skb
, opts
);
592 nla_nest_cancel(skb
, opts
);
596 static int sfb_dump_stats(struct Qdisc
*sch
, struct gnet_dump
*d
)
598 struct sfb_sched_data
*q
= qdisc_priv(sch
);
599 struct tc_sfb_xstats st
= {
600 .earlydrop
= q
->stats
.earlydrop
,
601 .penaltydrop
= q
->stats
.penaltydrop
,
602 .bucketdrop
= q
->stats
.bucketdrop
,
603 .queuedrop
= q
->stats
.queuedrop
,
604 .childdrop
= q
->stats
.childdrop
,
605 .marked
= q
->stats
.marked
,
608 st
.maxqlen
= sfb_compute_qlen(&st
.maxprob
, &st
.avgprob
, q
);
610 return gnet_stats_copy_app(d
, &st
, sizeof(st
));
613 static int sfb_dump_class(struct Qdisc
*sch
, unsigned long cl
,
614 struct sk_buff
*skb
, struct tcmsg
*tcm
)
619 static int sfb_graft(struct Qdisc
*sch
, unsigned long arg
, struct Qdisc
*new,
620 struct Qdisc
**old
, struct netlink_ext_ack
*extack
)
622 struct sfb_sched_data
*q
= qdisc_priv(sch
);
627 *old
= qdisc_replace(sch
, new, &q
->qdisc
);
631 static struct Qdisc
*sfb_leaf(struct Qdisc
*sch
, unsigned long arg
)
633 struct sfb_sched_data
*q
= qdisc_priv(sch
);
638 static unsigned long sfb_find(struct Qdisc
*sch
, u32 classid
)
643 static void sfb_unbind(struct Qdisc
*sch
, unsigned long arg
)
647 static int sfb_change_class(struct Qdisc
*sch
, u32 classid
, u32 parentid
,
648 struct nlattr
**tca
, unsigned long *arg
,
649 struct netlink_ext_ack
*extack
)
654 static int sfb_delete(struct Qdisc
*sch
, unsigned long cl
)
659 static void sfb_walk(struct Qdisc
*sch
, struct qdisc_walker
*walker
)
662 if (walker
->count
>= walker
->skip
)
663 if (walker
->fn(sch
, 1, walker
) < 0) {
671 static struct tcf_block
*sfb_tcf_block(struct Qdisc
*sch
, unsigned long cl
,
672 struct netlink_ext_ack
*extack
)
674 struct sfb_sched_data
*q
= qdisc_priv(sch
);
681 static unsigned long sfb_bind(struct Qdisc
*sch
, unsigned long parent
,
688 static const struct Qdisc_class_ops sfb_class_ops
= {
692 .change
= sfb_change_class
,
693 .delete = sfb_delete
,
695 .tcf_block
= sfb_tcf_block
,
696 .bind_tcf
= sfb_bind
,
697 .unbind_tcf
= sfb_unbind
,
698 .dump
= sfb_dump_class
,
701 static struct Qdisc_ops sfb_qdisc_ops __read_mostly
= {
703 .priv_size
= sizeof(struct sfb_sched_data
),
704 .cl_ops
= &sfb_class_ops
,
705 .enqueue
= sfb_enqueue
,
706 .dequeue
= sfb_dequeue
,
710 .destroy
= sfb_destroy
,
711 .change
= sfb_change
,
713 .dump_stats
= sfb_dump_stats
,
714 .owner
= THIS_MODULE
,
717 static int __init
sfb_module_init(void)
719 return register_qdisc(&sfb_qdisc_ops
);
722 static void __exit
sfb_module_exit(void)
724 unregister_qdisc(&sfb_qdisc_ops
);
727 module_init(sfb_module_init
)
728 module_exit(sfb_module_exit
)
730 MODULE_DESCRIPTION("Stochastic Fair Blue queue discipline");
731 MODULE_AUTHOR("Juliusz Chroboczek");
732 MODULE_AUTHOR("Eric Dumazet");
733 MODULE_LICENSE("GPL");