2 * Fair Queue CoDel discipline
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com>
12 #include <linux/module.h>
13 #include <linux/types.h>
14 #include <linux/kernel.h>
15 #include <linux/jiffies.h>
16 #include <linux/string.h>
18 #include <linux/errno.h>
19 #include <linux/init.h>
20 #include <linux/skbuff.h>
21 #include <linux/jhash.h>
22 #include <linux/slab.h>
23 #include <linux/vmalloc.h>
24 #include <net/netlink.h>
25 #include <net/pkt_sched.h>
26 #include <net/codel.h>
31 * Packets are classified (internal classifier or external) on flows.
32 * This is a Stochastic model (as we use a hash, several flows
33 * might be hashed on same slot)
34 * Each flow has a CoDel managed queue.
35 * Flows are linked onto two (Round Robin) lists,
36 * so that new flows have priority on old ones.
38 * For a given flow, packets are not reordered (CoDel uses a FIFO)
40 * ECN capability is on by default.
41 * Low memory footprint (64 bytes per flow)
44 struct fq_codel_flow
{
47 struct list_head flowchain
;
49 u32 dropped
; /* number of drops (or ECN marks) on this flow */
50 struct codel_vars cvars
;
51 }; /* please try to keep this structure <= 64 bytes */
53 struct fq_codel_sched_data
{
54 struct tcf_proto __rcu
*filter_list
; /* optional external classifier */
55 struct fq_codel_flow
*flows
; /* Flows table [flows_cnt] */
56 u32
*backlogs
; /* backlog table [flows_cnt] */
57 u32 flows_cnt
; /* number of flows */
58 u32 perturbation
; /* hash perturbation */
59 u32 quantum
; /* psched_mtu(qdisc_dev(sch)); */
60 struct codel_params cparams
;
61 struct codel_stats cstats
;
65 struct list_head new_flows
; /* list of new flows */
66 struct list_head old_flows
; /* list of old flows */
69 static unsigned int fq_codel_hash(const struct fq_codel_sched_data
*q
,
72 u32 hash
= skb_get_hash_perturb(skb
, q
->perturbation
);
74 return reciprocal_scale(hash
, q
->flows_cnt
);
77 static unsigned int fq_codel_classify(struct sk_buff
*skb
, struct Qdisc
*sch
,
80 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
81 struct tcf_proto
*filter
;
82 struct tcf_result res
;
85 if (TC_H_MAJ(skb
->priority
) == sch
->handle
&&
86 TC_H_MIN(skb
->priority
) > 0 &&
87 TC_H_MIN(skb
->priority
) <= q
->flows_cnt
)
88 return TC_H_MIN(skb
->priority
);
90 filter
= rcu_dereference_bh(q
->filter_list
);
92 return fq_codel_hash(q
, skb
) + 1;
94 *qerr
= NET_XMIT_SUCCESS
| __NET_XMIT_BYPASS
;
95 result
= tc_classify(skb
, filter
, &res
, false);
97 #ifdef CONFIG_NET_CLS_ACT
101 *qerr
= NET_XMIT_SUCCESS
| __NET_XMIT_STOLEN
;
106 if (TC_H_MIN(res
.classid
) <= q
->flows_cnt
)
107 return TC_H_MIN(res
.classid
);
112 /* helper functions : might be changed when/if skb use a standard list_head */
114 /* remove one skb from head of slot queue */
115 static inline struct sk_buff
*dequeue_head(struct fq_codel_flow
*flow
)
117 struct sk_buff
*skb
= flow
->head
;
119 flow
->head
= skb
->next
;
124 /* add skb to flow queue (tail add) */
125 static inline void flow_queue_add(struct fq_codel_flow
*flow
,
128 if (flow
->head
== NULL
)
131 flow
->tail
->next
= skb
;
136 static unsigned int fq_codel_drop(struct Qdisc
*sch
)
138 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
140 unsigned int maxbacklog
= 0, idx
= 0, i
, len
;
141 struct fq_codel_flow
*flow
;
143 /* Queue is full! Find the fat flow and drop packet from it.
144 * This might sound expensive, but with 1024 flows, we scan
145 * 4KB of memory, and we dont need to handle a complex tree
146 * in fast path (packet queue/enqueue) with many cache misses.
148 for (i
= 0; i
< q
->flows_cnt
; i
++) {
149 if (q
->backlogs
[i
] > maxbacklog
) {
150 maxbacklog
= q
->backlogs
[i
];
154 flow
= &q
->flows
[idx
];
155 skb
= dequeue_head(flow
);
156 len
= qdisc_pkt_len(skb
);
157 q
->backlogs
[idx
] -= len
;
159 qdisc_qstats_drop(sch
);
160 qdisc_qstats_backlog_dec(sch
, skb
);
166 static unsigned int fq_codel_qdisc_drop(struct Qdisc
*sch
)
168 unsigned int prev_backlog
;
170 prev_backlog
= sch
->qstats
.backlog
;
172 return prev_backlog
- sch
->qstats
.backlog
;
175 static int fq_codel_enqueue(struct sk_buff
*skb
, struct Qdisc
*sch
)
177 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
179 struct fq_codel_flow
*flow
;
180 int uninitialized_var(ret
);
182 idx
= fq_codel_classify(skb
, sch
, &ret
);
184 if (ret
& __NET_XMIT_BYPASS
)
185 qdisc_qstats_drop(sch
);
191 codel_set_enqueue_time(skb
);
192 flow
= &q
->flows
[idx
];
193 flow_queue_add(flow
, skb
);
194 q
->backlogs
[idx
] += qdisc_pkt_len(skb
);
195 qdisc_qstats_backlog_inc(sch
, skb
);
197 if (list_empty(&flow
->flowchain
)) {
198 list_add_tail(&flow
->flowchain
, &q
->new_flows
);
200 flow
->deficit
= q
->quantum
;
203 if (++sch
->q
.qlen
<= sch
->limit
)
204 return NET_XMIT_SUCCESS
;
207 /* Return Congestion Notification only if we dropped a packet
210 if (fq_codel_drop(sch
) == idx
)
213 /* As we dropped a packet, better let upper stack know this */
214 qdisc_tree_decrease_qlen(sch
, 1);
215 return NET_XMIT_SUCCESS
;
218 /* This is the specific function called from codel_dequeue()
219 * to dequeue a packet from queue. Note: backlog is handled in
220 * codel, we dont need to reduce it here.
222 static struct sk_buff
*dequeue(struct codel_vars
*vars
, struct Qdisc
*sch
)
224 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
225 struct fq_codel_flow
*flow
;
226 struct sk_buff
*skb
= NULL
;
228 flow
= container_of(vars
, struct fq_codel_flow
, cvars
);
230 skb
= dequeue_head(flow
);
231 q
->backlogs
[flow
- q
->flows
] -= qdisc_pkt_len(skb
);
237 static struct sk_buff
*fq_codel_dequeue(struct Qdisc
*sch
)
239 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
241 struct fq_codel_flow
*flow
;
242 struct list_head
*head
;
243 u32 prev_drop_count
, prev_ecn_mark
;
246 head
= &q
->new_flows
;
247 if (list_empty(head
)) {
248 head
= &q
->old_flows
;
249 if (list_empty(head
))
252 flow
= list_first_entry(head
, struct fq_codel_flow
, flowchain
);
254 if (flow
->deficit
<= 0) {
255 flow
->deficit
+= q
->quantum
;
256 list_move_tail(&flow
->flowchain
, &q
->old_flows
);
260 prev_drop_count
= q
->cstats
.drop_count
;
261 prev_ecn_mark
= q
->cstats
.ecn_mark
;
263 skb
= codel_dequeue(sch
, &q
->cparams
, &flow
->cvars
, &q
->cstats
,
266 flow
->dropped
+= q
->cstats
.drop_count
- prev_drop_count
;
267 flow
->dropped
+= q
->cstats
.ecn_mark
- prev_ecn_mark
;
270 /* force a pass through old_flows to prevent starvation */
271 if ((head
== &q
->new_flows
) && !list_empty(&q
->old_flows
))
272 list_move_tail(&flow
->flowchain
, &q
->old_flows
);
274 list_del_init(&flow
->flowchain
);
277 qdisc_bstats_update(sch
, skb
);
278 flow
->deficit
-= qdisc_pkt_len(skb
);
279 /* We cant call qdisc_tree_decrease_qlen() if our qlen is 0,
280 * or HTB crashes. Defer it for next round.
282 if (q
->cstats
.drop_count
&& sch
->q
.qlen
) {
283 qdisc_tree_decrease_qlen(sch
, q
->cstats
.drop_count
);
284 q
->cstats
.drop_count
= 0;
289 static void fq_codel_reset(struct Qdisc
*sch
)
291 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
294 INIT_LIST_HEAD(&q
->new_flows
);
295 INIT_LIST_HEAD(&q
->old_flows
);
296 for (i
= 0; i
< q
->flows_cnt
; i
++) {
297 struct fq_codel_flow
*flow
= q
->flows
+ i
;
300 struct sk_buff
*skb
= dequeue_head(flow
);
302 qdisc_qstats_backlog_dec(sch
, skb
);
306 INIT_LIST_HEAD(&flow
->flowchain
);
307 codel_vars_init(&flow
->cvars
);
309 memset(q
->backlogs
, 0, q
->flows_cnt
* sizeof(u32
));
313 static const struct nla_policy fq_codel_policy
[TCA_FQ_CODEL_MAX
+ 1] = {
314 [TCA_FQ_CODEL_TARGET
] = { .type
= NLA_U32
},
315 [TCA_FQ_CODEL_LIMIT
] = { .type
= NLA_U32
},
316 [TCA_FQ_CODEL_INTERVAL
] = { .type
= NLA_U32
},
317 [TCA_FQ_CODEL_ECN
] = { .type
= NLA_U32
},
318 [TCA_FQ_CODEL_FLOWS
] = { .type
= NLA_U32
},
319 [TCA_FQ_CODEL_QUANTUM
] = { .type
= NLA_U32
},
320 [TCA_FQ_CODEL_CE_THRESHOLD
] = { .type
= NLA_U32
},
323 static int fq_codel_change(struct Qdisc
*sch
, struct nlattr
*opt
)
325 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
326 struct nlattr
*tb
[TCA_FQ_CODEL_MAX
+ 1];
332 err
= nla_parse_nested(tb
, TCA_FQ_CODEL_MAX
, opt
, fq_codel_policy
);
335 if (tb
[TCA_FQ_CODEL_FLOWS
]) {
338 q
->flows_cnt
= nla_get_u32(tb
[TCA_FQ_CODEL_FLOWS
]);
340 q
->flows_cnt
> 65536)
345 if (tb
[TCA_FQ_CODEL_TARGET
]) {
346 u64 target
= nla_get_u32(tb
[TCA_FQ_CODEL_TARGET
]);
348 q
->cparams
.target
= (target
* NSEC_PER_USEC
) >> CODEL_SHIFT
;
351 if (tb
[TCA_FQ_CODEL_CE_THRESHOLD
]) {
352 u64 val
= nla_get_u32(tb
[TCA_FQ_CODEL_CE_THRESHOLD
]);
354 q
->cparams
.ce_threshold
= (val
* NSEC_PER_USEC
) >> CODEL_SHIFT
;
357 if (tb
[TCA_FQ_CODEL_INTERVAL
]) {
358 u64 interval
= nla_get_u32(tb
[TCA_FQ_CODEL_INTERVAL
]);
360 q
->cparams
.interval
= (interval
* NSEC_PER_USEC
) >> CODEL_SHIFT
;
363 if (tb
[TCA_FQ_CODEL_LIMIT
])
364 sch
->limit
= nla_get_u32(tb
[TCA_FQ_CODEL_LIMIT
]);
366 if (tb
[TCA_FQ_CODEL_ECN
])
367 q
->cparams
.ecn
= !!nla_get_u32(tb
[TCA_FQ_CODEL_ECN
]);
369 if (tb
[TCA_FQ_CODEL_QUANTUM
])
370 q
->quantum
= max(256U, nla_get_u32(tb
[TCA_FQ_CODEL_QUANTUM
]));
372 while (sch
->q
.qlen
> sch
->limit
) {
373 struct sk_buff
*skb
= fq_codel_dequeue(sch
);
376 q
->cstats
.drop_count
++;
378 qdisc_tree_decrease_qlen(sch
, q
->cstats
.drop_count
);
379 q
->cstats
.drop_count
= 0;
381 sch_tree_unlock(sch
);
385 static void *fq_codel_zalloc(size_t sz
)
387 void *ptr
= kzalloc(sz
, GFP_KERNEL
| __GFP_NOWARN
);
394 static void fq_codel_free(void *addr
)
399 static void fq_codel_destroy(struct Qdisc
*sch
)
401 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
403 tcf_destroy_chain(&q
->filter_list
);
404 fq_codel_free(q
->backlogs
);
405 fq_codel_free(q
->flows
);
408 static int fq_codel_init(struct Qdisc
*sch
, struct nlattr
*opt
)
410 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
413 sch
->limit
= 10*1024;
415 q
->quantum
= psched_mtu(qdisc_dev(sch
));
416 q
->perturbation
= prandom_u32();
417 INIT_LIST_HEAD(&q
->new_flows
);
418 INIT_LIST_HEAD(&q
->old_flows
);
419 codel_params_init(&q
->cparams
, sch
);
420 codel_stats_init(&q
->cstats
);
421 q
->cparams
.ecn
= true;
424 int err
= fq_codel_change(sch
, opt
);
430 q
->flows
= fq_codel_zalloc(q
->flows_cnt
*
431 sizeof(struct fq_codel_flow
));
434 q
->backlogs
= fq_codel_zalloc(q
->flows_cnt
* sizeof(u32
));
436 fq_codel_free(q
->flows
);
439 for (i
= 0; i
< q
->flows_cnt
; i
++) {
440 struct fq_codel_flow
*flow
= q
->flows
+ i
;
442 INIT_LIST_HEAD(&flow
->flowchain
);
443 codel_vars_init(&flow
->cvars
);
447 sch
->flags
|= TCQ_F_CAN_BYPASS
;
449 sch
->flags
&= ~TCQ_F_CAN_BYPASS
;
453 static int fq_codel_dump(struct Qdisc
*sch
, struct sk_buff
*skb
)
455 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
458 opts
= nla_nest_start(skb
, TCA_OPTIONS
);
460 goto nla_put_failure
;
462 if (nla_put_u32(skb
, TCA_FQ_CODEL_TARGET
,
463 codel_time_to_us(q
->cparams
.target
)) ||
464 nla_put_u32(skb
, TCA_FQ_CODEL_LIMIT
,
466 nla_put_u32(skb
, TCA_FQ_CODEL_INTERVAL
,
467 codel_time_to_us(q
->cparams
.interval
)) ||
468 nla_put_u32(skb
, TCA_FQ_CODEL_ECN
,
470 nla_put_u32(skb
, TCA_FQ_CODEL_QUANTUM
,
472 nla_put_u32(skb
, TCA_FQ_CODEL_FLOWS
,
474 goto nla_put_failure
;
476 if (q
->cparams
.ce_threshold
!= CODEL_DISABLED_THRESHOLD
&&
477 nla_put_u32(skb
, TCA_FQ_CODEL_CE_THRESHOLD
,
478 codel_time_to_us(q
->cparams
.ce_threshold
)))
479 goto nla_put_failure
;
481 return nla_nest_end(skb
, opts
);
487 static int fq_codel_dump_stats(struct Qdisc
*sch
, struct gnet_dump
*d
)
489 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
490 struct tc_fq_codel_xstats st
= {
491 .type
= TCA_FQ_CODEL_XSTATS_QDISC
,
493 struct list_head
*pos
;
495 st
.qdisc_stats
.maxpacket
= q
->cstats
.maxpacket
;
496 st
.qdisc_stats
.drop_overlimit
= q
->drop_overlimit
;
497 st
.qdisc_stats
.ecn_mark
= q
->cstats
.ecn_mark
;
498 st
.qdisc_stats
.new_flow_count
= q
->new_flow_count
;
499 st
.qdisc_stats
.ce_mark
= q
->cstats
.ce_mark
;
501 list_for_each(pos
, &q
->new_flows
)
502 st
.qdisc_stats
.new_flows_len
++;
504 list_for_each(pos
, &q
->old_flows
)
505 st
.qdisc_stats
.old_flows_len
++;
507 return gnet_stats_copy_app(d
, &st
, sizeof(st
));
510 static struct Qdisc
*fq_codel_leaf(struct Qdisc
*sch
, unsigned long arg
)
515 static unsigned long fq_codel_get(struct Qdisc
*sch
, u32 classid
)
520 static unsigned long fq_codel_bind(struct Qdisc
*sch
, unsigned long parent
,
523 /* we cannot bypass queue discipline anymore */
524 sch
->flags
&= ~TCQ_F_CAN_BYPASS
;
528 static void fq_codel_put(struct Qdisc
*q
, unsigned long cl
)
532 static struct tcf_proto __rcu
**fq_codel_find_tcf(struct Qdisc
*sch
,
535 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
539 return &q
->filter_list
;
542 static int fq_codel_dump_class(struct Qdisc
*sch
, unsigned long cl
,
543 struct sk_buff
*skb
, struct tcmsg
*tcm
)
545 tcm
->tcm_handle
|= TC_H_MIN(cl
);
549 static int fq_codel_dump_class_stats(struct Qdisc
*sch
, unsigned long cl
,
552 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
554 struct gnet_stats_queue qs
= { 0 };
555 struct tc_fq_codel_xstats xstats
;
557 if (idx
< q
->flows_cnt
) {
558 const struct fq_codel_flow
*flow
= &q
->flows
[idx
];
559 const struct sk_buff
*skb
= flow
->head
;
561 memset(&xstats
, 0, sizeof(xstats
));
562 xstats
.type
= TCA_FQ_CODEL_XSTATS_CLASS
;
563 xstats
.class_stats
.deficit
= flow
->deficit
;
564 xstats
.class_stats
.ldelay
=
565 codel_time_to_us(flow
->cvars
.ldelay
);
566 xstats
.class_stats
.count
= flow
->cvars
.count
;
567 xstats
.class_stats
.lastcount
= flow
->cvars
.lastcount
;
568 xstats
.class_stats
.dropping
= flow
->cvars
.dropping
;
569 if (flow
->cvars
.dropping
) {
570 codel_tdiff_t delta
= flow
->cvars
.drop_next
-
573 xstats
.class_stats
.drop_next
= (delta
>= 0) ?
574 codel_time_to_us(delta
) :
575 -codel_time_to_us(-delta
);
581 qs
.backlog
= q
->backlogs
[idx
];
582 qs
.drops
= flow
->dropped
;
584 if (gnet_stats_copy_queue(d
, NULL
, &qs
, 0) < 0)
586 if (idx
< q
->flows_cnt
)
587 return gnet_stats_copy_app(d
, &xstats
, sizeof(xstats
));
591 static void fq_codel_walk(struct Qdisc
*sch
, struct qdisc_walker
*arg
)
593 struct fq_codel_sched_data
*q
= qdisc_priv(sch
);
599 for (i
= 0; i
< q
->flows_cnt
; i
++) {
600 if (list_empty(&q
->flows
[i
].flowchain
) ||
601 arg
->count
< arg
->skip
) {
605 if (arg
->fn(sch
, i
+ 1, arg
) < 0) {
613 static const struct Qdisc_class_ops fq_codel_class_ops
= {
614 .leaf
= fq_codel_leaf
,
617 .tcf_chain
= fq_codel_find_tcf
,
618 .bind_tcf
= fq_codel_bind
,
619 .unbind_tcf
= fq_codel_put
,
620 .dump
= fq_codel_dump_class
,
621 .dump_stats
= fq_codel_dump_class_stats
,
622 .walk
= fq_codel_walk
,
625 static struct Qdisc_ops fq_codel_qdisc_ops __read_mostly
= {
626 .cl_ops
= &fq_codel_class_ops
,
628 .priv_size
= sizeof(struct fq_codel_sched_data
),
629 .enqueue
= fq_codel_enqueue
,
630 .dequeue
= fq_codel_dequeue
,
631 .peek
= qdisc_peek_dequeued
,
632 .drop
= fq_codel_qdisc_drop
,
633 .init
= fq_codel_init
,
634 .reset
= fq_codel_reset
,
635 .destroy
= fq_codel_destroy
,
636 .change
= fq_codel_change
,
637 .dump
= fq_codel_dump
,
638 .dump_stats
= fq_codel_dump_stats
,
639 .owner
= THIS_MODULE
,
642 static int __init
fq_codel_module_init(void)
644 return register_qdisc(&fq_codel_qdisc_ops
);
647 static void __exit
fq_codel_module_exit(void)
649 unregister_qdisc(&fq_codel_qdisc_ops
);
652 module_init(fq_codel_module_init
)
653 module_exit(fq_codel_module_exit
)
654 MODULE_AUTHOR("Eric Dumazet");
655 MODULE_LICENSE("GPL");