?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
writeback: split writeback_inodes_wb
[linux-2.6/next.git] / net / sched / sch_api.c
blobb9e8c3b7d406aacd9cc2319ddbffe0440cc6c311
1 /*
2 * net/sched/sch_api.c Packet scheduler API.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10 *
11 * Fixes:
12 *
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
16 */
17
18 #include <linux/module.h>
19 #include <linux/types.h>
20 #include <linux/kernel.h>
21 #include <linux/string.h>
22 #include <linux/errno.h>
23 #include <linux/skbuff.h>
24 #include <linux/init.h>
25 #include <linux/proc_fs.h>
26 #include <linux/seq_file.h>
27 #include <linux/kmod.h>
28 #include <linux/list.h>
29 #include <linux/hrtimer.h>
30 #include <linux/lockdep.h>
31 #include <linux/slab.h>
32
33 #include <net/net_namespace.h>
34 #include <net/sock.h>
35 #include <net/netlink.h>
36 #include <net/pkt_sched.h>
37
38 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
39 struct nlmsghdr *n, u32 clid,
40 struct Qdisc *old, struct Qdisc *new);
41 static int tclass_notify(struct net *net, struct sk_buff *oskb,
42 struct nlmsghdr *n, struct Qdisc *q,
43 unsigned long cl, int event);
44
45 /*
46
47 Short review.
48 -------------
49
50 This file consists of two interrelated parts:
51
52 1. queueing disciplines manager frontend.
53 2. traffic classes manager frontend.
54
55 Generally, queueing discipline ("qdisc") is a black box,
56 which is able to enqueue packets and to dequeue them (when
57 device is ready to send something) in order and at times
58 determined by algorithm hidden in it.
59
60 qdisc's are divided to two categories:
61 - "queues", which have no internal structure visible from outside.
62 - "schedulers", which split all the packets to "traffic classes",
63 using "packet classifiers" (look at cls_api.c)
64
65 In turn, classes may have child qdiscs (as rule, queues)
66 attached to them etc. etc. etc.
67
68 The goal of the routines in this file is to translate
69 information supplied by user in the form of handles
70 to more intelligible for kernel form, to make some sanity
71 checks and part of work, which is common to all qdiscs
72 and to provide rtnetlink notifications.
73
74 All real intelligent work is done inside qdisc modules.
75
76
77
78 Every discipline has two major routines: enqueue and dequeue.
79
80 ---dequeue
81
82 dequeue usually returns a skb to send. It is allowed to return NULL,
83 but it does not mean that queue is empty, it just means that
84 discipline does not want to send anything this time.
85 Queue is really empty if q->q.qlen == 0.
86 For complicated disciplines with multiple queues q->q is not
87 real packet queue, but however q->q.qlen must be valid.
88
89 ---enqueue
90
91 enqueue returns 0, if packet was enqueued successfully.
92 If packet (this one or another one) was dropped, it returns
93 not zero error code.
94 NET_XMIT_DROP - this packet dropped
95 Expected action: do not backoff, but wait until queue will clear.
96 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
97 Expected action: backoff or ignore
98 NET_XMIT_POLICED - dropped by police.
99 Expected action: backoff or error to real-time apps.
100
101 Auxiliary routines:
102
103 ---peek
104
105 like dequeue but without removing a packet from the queue
106
107 ---reset
108
109 returns qdisc to initial state: purge all buffers, clear all
110 timers, counters (except for statistics) etc.
111
112 ---init
113
114 initializes newly created qdisc.
115
116 ---destroy
117
118 destroys resources allocated by init and during lifetime of qdisc.
119
120 ---change
121
122 changes qdisc parameters.
123 */
124
125 /* Protects list of registered TC modules. It is pure SMP lock. */
126 static DEFINE_RWLOCK(qdisc_mod_lock);
127
128
129 /************************************************
130 * Queueing disciplines manipulation. *
131 ************************************************/
132
133
134 /* The list of all installed queueing disciplines. */
135
136 static struct Qdisc_ops *qdisc_base;
137
138 /* Register/uregister queueing discipline */
139
140 int register_qdisc(struct Qdisc_ops *qops)
141 {
142 struct Qdisc_ops *q, **qp;
143 int rc = -EEXIST;
144
145 write_lock(&qdisc_mod_lock);
146 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
147 if (!strcmp(qops->id, q->id))
148 goto out;
149
150 if (qops->enqueue == NULL)
151 qops->enqueue = noop_qdisc_ops.enqueue;
152 if (qops->peek == NULL) {
153 if (qops->dequeue == NULL) {
154 qops->peek = noop_qdisc_ops.peek;
155 } else {
156 rc = -EINVAL;
157 goto out;
158 }
159 }
160 if (qops->dequeue == NULL)
161 qops->dequeue = noop_qdisc_ops.dequeue;
162
163 qops->next = NULL;
164 *qp = qops;
165 rc = 0;
166 out:
167 write_unlock(&qdisc_mod_lock);
168 return rc;
169 }
170 EXPORT_SYMBOL(register_qdisc);
171
172 int unregister_qdisc(struct Qdisc_ops *qops)
173 {
174 struct Qdisc_ops *q, **qp;
175 int err = -ENOENT;
176
177 write_lock(&qdisc_mod_lock);
178 for (qp = &qdisc_base; (q=*qp)!=NULL; qp = &q->next)
179 if (q == qops)
180 break;
181 if (q) {
182 *qp = q->next;
183 q->next = NULL;
184 err = 0;
185 }
186 write_unlock(&qdisc_mod_lock);
187 return err;
188 }
189 EXPORT_SYMBOL(unregister_qdisc);
190
191 /* We know handle. Find qdisc among all qdisc's attached to device
192 (root qdisc, all its children, children of children etc.)
193 */
194
195 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
196 {
197 struct Qdisc *q;
198
199 if (!(root->flags & TCQ_F_BUILTIN) &&
200 root->handle == handle)
201 return root;
202
203 list_for_each_entry(q, &root->list, list) {
204 if (q->handle == handle)
205 return q;
206 }
207 return NULL;
208 }
209
210 static void qdisc_list_add(struct Qdisc *q)
211 {
212 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS))
213 list_add_tail(&q->list, &qdisc_dev(q)->qdisc->list);
214 }
215
216 void qdisc_list_del(struct Qdisc *q)
217 {
218 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS))
219 list_del(&q->list);
220 }
221 EXPORT_SYMBOL(qdisc_list_del);
222
223 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
224 {
225 struct Qdisc *q;
226
227 q = qdisc_match_from_root(dev->qdisc, handle);
228 if (q)
229 goto out;
230
231 q = qdisc_match_from_root(dev->rx_queue.qdisc_sleeping, handle);
232 out:
233 return q;
234 }
235
236 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
237 {
238 unsigned long cl;
239 struct Qdisc *leaf;
240 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
241
242 if (cops == NULL)
243 return NULL;
244 cl = cops->get(p, classid);
245
246 if (cl == 0)
247 return NULL;
248 leaf = cops->leaf(p, cl);
249 cops->put(p, cl);
250 return leaf;
251 }
252
253 /* Find queueing discipline by name */
254
255 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
256 {
257 struct Qdisc_ops *q = NULL;
258
259 if (kind) {
260 read_lock(&qdisc_mod_lock);
261 for (q = qdisc_base; q; q = q->next) {
262 if (nla_strcmp(kind, q->id) == 0) {
263 if (!try_module_get(q->owner))
264 q = NULL;
265 break;
266 }
267 }
268 read_unlock(&qdisc_mod_lock);
269 }
270 return q;
271 }
272
273 static struct qdisc_rate_table *qdisc_rtab_list;
274
275 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
276 {
277 struct qdisc_rate_table *rtab;
278
279 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
280 if (memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) == 0) {
281 rtab->refcnt++;
282 return rtab;
283 }
284 }
285
286 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
287 nla_len(tab) != TC_RTAB_SIZE)
288 return NULL;
289
290 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
291 if (rtab) {
292 rtab->rate = *r;
293 rtab->refcnt = 1;
294 memcpy(rtab->data, nla_data(tab), 1024);
295 rtab->next = qdisc_rtab_list;
296 qdisc_rtab_list = rtab;
297 }
298 return rtab;
299 }
300 EXPORT_SYMBOL(qdisc_get_rtab);
301
302 void qdisc_put_rtab(struct qdisc_rate_table *tab)
303 {
304 struct qdisc_rate_table *rtab, **rtabp;
305
306 if (!tab || --tab->refcnt)
307 return;
308
309 for (rtabp = &qdisc_rtab_list; (rtab=*rtabp) != NULL; rtabp = &rtab->next) {
310 if (rtab == tab) {
311 *rtabp = rtab->next;
312 kfree(rtab);
313 return;
314 }
315 }
316 }
317 EXPORT_SYMBOL(qdisc_put_rtab);
318
319 static LIST_HEAD(qdisc_stab_list);
320 static DEFINE_SPINLOCK(qdisc_stab_lock);
321
322 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
323 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
324 [TCA_STAB_DATA] = { .type = NLA_BINARY },
325 };
326
327 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt)
328 {
329 struct nlattr *tb[TCA_STAB_MAX + 1];
330 struct qdisc_size_table *stab;
331 struct tc_sizespec *s;
332 unsigned int tsize = 0;
333 u16 *tab = NULL;
334 int err;
335
336 err = nla_parse_nested(tb, TCA_STAB_MAX, opt, stab_policy);
337 if (err < 0)
338 return ERR_PTR(err);
339 if (!tb[TCA_STAB_BASE])
340 return ERR_PTR(-EINVAL);
341
342 s = nla_data(tb[TCA_STAB_BASE]);
343
344 if (s->tsize > 0) {
345 if (!tb[TCA_STAB_DATA])
346 return ERR_PTR(-EINVAL);
347 tab = nla_data(tb[TCA_STAB_DATA]);
348 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
349 }
350
351 if (!s || tsize != s->tsize || (!tab && tsize > 0))
352 return ERR_PTR(-EINVAL);
353
354 spin_lock(&qdisc_stab_lock);
355
356 list_for_each_entry(stab, &qdisc_stab_list, list) {
357 if (memcmp(&stab->szopts, s, sizeof(*s)))
358 continue;
359 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
360 continue;
361 stab->refcnt++;
362 spin_unlock(&qdisc_stab_lock);
363 return stab;
364 }
365
366 spin_unlock(&qdisc_stab_lock);
367
368 stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
369 if (!stab)
370 return ERR_PTR(-ENOMEM);
371
372 stab->refcnt = 1;
373 stab->szopts = *s;
374 if (tsize > 0)
375 memcpy(stab->data, tab, tsize * sizeof(u16));
376
377 spin_lock(&qdisc_stab_lock);
378 list_add_tail(&stab->list, &qdisc_stab_list);
379 spin_unlock(&qdisc_stab_lock);
380
381 return stab;
382 }
383
384 void qdisc_put_stab(struct qdisc_size_table *tab)
385 {
386 if (!tab)
387 return;
388
389 spin_lock(&qdisc_stab_lock);
390
391 if (--tab->refcnt == 0) {
392 list_del(&tab->list);
393 kfree(tab);
394 }
395
396 spin_unlock(&qdisc_stab_lock);
397 }
398 EXPORT_SYMBOL(qdisc_put_stab);
399
400 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
401 {
402 struct nlattr *nest;
403
404 nest = nla_nest_start(skb, TCA_STAB);
405 if (nest == NULL)
406 goto nla_put_failure;
407 NLA_PUT(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts);
408 nla_nest_end(skb, nest);
409
410 return skb->len;
411
412 nla_put_failure:
413 return -1;
414 }
415
416 void qdisc_calculate_pkt_len(struct sk_buff *skb, struct qdisc_size_table *stab)
417 {
418 int pkt_len, slot;
419
420 pkt_len = skb->len + stab->szopts.overhead;
421 if (unlikely(!stab->szopts.tsize))
422 goto out;
423
424 slot = pkt_len + stab->szopts.cell_align;
425 if (unlikely(slot < 0))
426 slot = 0;
427
428 slot >>= stab->szopts.cell_log;
429 if (likely(slot < stab->szopts.tsize))
430 pkt_len = stab->data[slot];
431 else
432 pkt_len = stab->data[stab->szopts.tsize - 1] *
433 (slot / stab->szopts.tsize) +
434 stab->data[slot % stab->szopts.tsize];
435
436 pkt_len <<= stab->szopts.size_log;
437 out:
438 if (unlikely(pkt_len < 1))
439 pkt_len = 1;
440 qdisc_skb_cb(skb)->pkt_len = pkt_len;
441 }
442 EXPORT_SYMBOL(qdisc_calculate_pkt_len);
443
444 void qdisc_warn_nonwc(char *txt, struct Qdisc *qdisc)
445 {
446 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
447 printk(KERN_WARNING
448 "%s: %s qdisc %X: is non-work-conserving?\n",
449 txt, qdisc->ops->id, qdisc->handle >> 16);
450 qdisc->flags |= TCQ_F_WARN_NONWC;
451 }
452 }
453 EXPORT_SYMBOL(qdisc_warn_nonwc);
454
455 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
456 {
457 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
458 timer);
459
460 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
461 __netif_schedule(qdisc_root(wd->qdisc));
462
463 return HRTIMER_NORESTART;
464 }
465
466 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
467 {
468 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
469 wd->timer.function = qdisc_watchdog;
470 wd->qdisc = qdisc;
471 }
472 EXPORT_SYMBOL(qdisc_watchdog_init);
473
474 void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, psched_time_t expires)
475 {
476 ktime_t time;
477
478 if (test_bit(__QDISC_STATE_DEACTIVATED,
479 &qdisc_root_sleeping(wd->qdisc)->state))
480 return;
481
482 wd->qdisc->flags |= TCQ_F_THROTTLED;
483 time = ktime_set(0, 0);
484 time = ktime_add_ns(time, PSCHED_TICKS2NS(expires));
485 hrtimer_start(&wd->timer, time, HRTIMER_MODE_ABS);
486 }
487 EXPORT_SYMBOL(qdisc_watchdog_schedule);
488
489 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
490 {
491 hrtimer_cancel(&wd->timer);
492 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
493 }
494 EXPORT_SYMBOL(qdisc_watchdog_cancel);
495
496 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
497 {
498 unsigned int size = n * sizeof(struct hlist_head), i;
499 struct hlist_head *h;
500
501 if (size <= PAGE_SIZE)
502 h = kmalloc(size, GFP_KERNEL);
503 else
504 h = (struct hlist_head *)
505 __get_free_pages(GFP_KERNEL, get_order(size));
506
507 if (h != NULL) {
508 for (i = 0; i < n; i++)
509 INIT_HLIST_HEAD(&h[i]);
510 }
511 return h;
512 }
513
514 static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
515 {
516 unsigned int size = n * sizeof(struct hlist_head);
517
518 if (size <= PAGE_SIZE)
519 kfree(h);
520 else
521 free_pages((unsigned long)h, get_order(size));
522 }
523
524 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
525 {
526 struct Qdisc_class_common *cl;
527 struct hlist_node *n, *next;
528 struct hlist_head *nhash, *ohash;
529 unsigned int nsize, nmask, osize;
530 unsigned int i, h;
531
532 /* Rehash when load factor exceeds 0.75 */
533 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
534 return;
535 nsize = clhash->hashsize * 2;
536 nmask = nsize - 1;
537 nhash = qdisc_class_hash_alloc(nsize);
538 if (nhash == NULL)
539 return;
540
541 ohash = clhash->hash;
542 osize = clhash->hashsize;
543
544 sch_tree_lock(sch);
545 for (i = 0; i < osize; i++) {
546 hlist_for_each_entry_safe(cl, n, next, &ohash[i], hnode) {
547 h = qdisc_class_hash(cl->classid, nmask);
548 hlist_add_head(&cl->hnode, &nhash[h]);
549 }
550 }
551 clhash->hash = nhash;
552 clhash->hashsize = nsize;
553 clhash->hashmask = nmask;
554 sch_tree_unlock(sch);
555
556 qdisc_class_hash_free(ohash, osize);
557 }
558 EXPORT_SYMBOL(qdisc_class_hash_grow);
559
560 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
561 {
562 unsigned int size = 4;
563
564 clhash->hash = qdisc_class_hash_alloc(size);
565 if (clhash->hash == NULL)
566 return -ENOMEM;
567 clhash->hashsize = size;
568 clhash->hashmask = size - 1;
569 clhash->hashelems = 0;
570 return 0;
571 }
572 EXPORT_SYMBOL(qdisc_class_hash_init);
573
574 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
575 {
576 qdisc_class_hash_free(clhash->hash, clhash->hashsize);
577 }
578 EXPORT_SYMBOL(qdisc_class_hash_destroy);
579
580 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
581 struct Qdisc_class_common *cl)
582 {
583 unsigned int h;
584
585 INIT_HLIST_NODE(&cl->hnode);
586 h = qdisc_class_hash(cl->classid, clhash->hashmask);
587 hlist_add_head(&cl->hnode, &clhash->hash[h]);
588 clhash->hashelems++;
589 }
590 EXPORT_SYMBOL(qdisc_class_hash_insert);
591
592 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
593 struct Qdisc_class_common *cl)
594 {
595 hlist_del(&cl->hnode);
596 clhash->hashelems--;
597 }
598 EXPORT_SYMBOL(qdisc_class_hash_remove);
599
600 /* Allocate an unique handle from space managed by kernel */
601
602 static u32 qdisc_alloc_handle(struct net_device *dev)
603 {
604 int i = 0x10000;
605 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
606
607 do {
608 autohandle += TC_H_MAKE(0x10000U, 0);
609 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
610 autohandle = TC_H_MAKE(0x80000000U, 0);
611 } while (qdisc_lookup(dev, autohandle) && --i > 0);
612
613 return i>0 ? autohandle : 0;
614 }
615
616 void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n)
617 {
618 const struct Qdisc_class_ops *cops;
619 unsigned long cl;
620 u32 parentid;
621
622 if (n == 0)
623 return;
624 while ((parentid = sch->parent)) {
625 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
626 return;
627
628 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
629 if (sch == NULL) {
630 WARN_ON(parentid != TC_H_ROOT);
631 return;
632 }
633 cops = sch->ops->cl_ops;
634 if (cops->qlen_notify) {
635 cl = cops->get(sch, parentid);
636 cops->qlen_notify(sch, cl);
637 cops->put(sch, cl);
638 }
639 sch->q.qlen -= n;
640 }
641 }
642 EXPORT_SYMBOL(qdisc_tree_decrease_qlen);
643
644 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
645 struct nlmsghdr *n, u32 clid,
646 struct Qdisc *old, struct Qdisc *new)
647 {
648 if (new || old)
649 qdisc_notify(net, skb, n, clid, old, new);
650
651 if (old)
652 qdisc_destroy(old);
653 }
654
655 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
656 * to device "dev".
657 *
658 * When appropriate send a netlink notification using 'skb'
659 * and "n".
660 *
661 * On success, destroy old qdisc.
662 */
663
664 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
665 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
666 struct Qdisc *new, struct Qdisc *old)
667 {
668 struct Qdisc *q = old;
669 struct net *net = dev_net(dev);
670 int err = 0;
671
672 if (parent == NULL) {
673 unsigned int i, num_q, ingress;
674
675 ingress = 0;
676 num_q = dev->num_tx_queues;
677 if ((q && q->flags & TCQ_F_INGRESS) ||
678 (new && new->flags & TCQ_F_INGRESS)) {
679 num_q = 1;
680 ingress = 1;
681 }
682
683 if (dev->flags & IFF_UP)
684 dev_deactivate(dev);
685
686 if (new && new->ops->attach) {
687 new->ops->attach(new);
688 num_q = 0;
689 }
690
691 for (i = 0; i < num_q; i++) {
692 struct netdev_queue *dev_queue = &dev->rx_queue;
693
694 if (!ingress)
695 dev_queue = netdev_get_tx_queue(dev, i);
696
697 old = dev_graft_qdisc(dev_queue, new);
698 if (new && i > 0)
699 atomic_inc(&new->refcnt);
700
701 if (!ingress)
702 qdisc_destroy(old);
703 }
704
705 if (!ingress) {
706 notify_and_destroy(net, skb, n, classid,
707 dev->qdisc, new);
708 if (new && !new->ops->attach)
709 atomic_inc(&new->refcnt);
710 dev->qdisc = new ? : &noop_qdisc;
711 } else {
712 notify_and_destroy(net, skb, n, classid, old, new);
713 }
714
715 if (dev->flags & IFF_UP)
716 dev_activate(dev);
717 } else {
718 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
719
720 err = -EOPNOTSUPP;
721 if (cops && cops->graft) {
722 unsigned long cl = cops->get(parent, classid);
723 if (cl) {
724 err = cops->graft(parent, cl, new, &old);
725 cops->put(parent, cl);
726 } else
727 err = -ENOENT;
728 }
729 if (!err)
730 notify_and_destroy(net, skb, n, classid, old, new);
731 }
732 return err;
733 }
734
735 /* lockdep annotation is needed for ingress; egress gets it only for name */
736 static struct lock_class_key qdisc_tx_lock;
737 static struct lock_class_key qdisc_rx_lock;
738
739 /*
740 Allocate and initialize new qdisc.
741
742 Parameters are passed via opt.
743 */
744
745 static struct Qdisc *
746 qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
747 struct Qdisc *p, u32 parent, u32 handle,
748 struct nlattr **tca, int *errp)
749 {
750 int err;
751 struct nlattr *kind = tca[TCA_KIND];
752 struct Qdisc *sch;
753 struct Qdisc_ops *ops;
754 struct qdisc_size_table *stab;
755
756 ops = qdisc_lookup_ops(kind);
757 #ifdef CONFIG_MODULES
758 if (ops == NULL && kind != NULL) {
759 char name[IFNAMSIZ];
760 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
761 /* We dropped the RTNL semaphore in order to
762 * perform the module load. So, even if we
763 * succeeded in loading the module we have to
764 * tell the caller to replay the request. We
765 * indicate this using -EAGAIN.
766 * We replay the request because the device may
767 * go away in the mean time.
768 */
769 rtnl_unlock();
770 request_module("sch_%s", name);
771 rtnl_lock();
772 ops = qdisc_lookup_ops(kind);
773 if (ops != NULL) {
774 /* We will try again qdisc_lookup_ops,
775 * so don't keep a reference.
776 */
777 module_put(ops->owner);
778 err = -EAGAIN;
779 goto err_out;
780 }
781 }
782 }
783 #endif
784
785 err = -ENOENT;
786 if (ops == NULL)
787 goto err_out;
788
789 sch = qdisc_alloc(dev_queue, ops);
790 if (IS_ERR(sch)) {
791 err = PTR_ERR(sch);
792 goto err_out2;
793 }
794
795 sch->parent = parent;
796
797 if (handle == TC_H_INGRESS) {
798 sch->flags |= TCQ_F_INGRESS;
799 handle = TC_H_MAKE(TC_H_INGRESS, 0);
800 lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock);
801 } else {
802 if (handle == 0) {
803 handle = qdisc_alloc_handle(dev);
804 err = -ENOMEM;
805 if (handle == 0)
806 goto err_out3;
807 }
808 lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock);
809 }
810
811 sch->handle = handle;
812
813 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
814 if (tca[TCA_STAB]) {
815 stab = qdisc_get_stab(tca[TCA_STAB]);
816 if (IS_ERR(stab)) {
817 err = PTR_ERR(stab);
818 goto err_out4;
819 }
820 sch->stab = stab;
821 }
822 if (tca[TCA_RATE]) {
823 spinlock_t *root_lock;
824
825 err = -EOPNOTSUPP;
826 if (sch->flags & TCQ_F_MQROOT)
827 goto err_out4;
828
829 if ((sch->parent != TC_H_ROOT) &&
830 !(sch->flags & TCQ_F_INGRESS) &&
831 (!p || !(p->flags & TCQ_F_MQROOT)))
832 root_lock = qdisc_root_sleeping_lock(sch);
833 else
834 root_lock = qdisc_lock(sch);
835
836 err = gen_new_estimator(&sch->bstats, &sch->rate_est,
837 root_lock, tca[TCA_RATE]);
838 if (err)
839 goto err_out4;
840 }
841
842 qdisc_list_add(sch);
843
844 return sch;
845 }
846 err_out3:
847 dev_put(dev);
848 kfree((char *) sch - sch->padded);
849 err_out2:
850 module_put(ops->owner);
851 err_out:
852 *errp = err;
853 return NULL;
854
855 err_out4:
856 /*
857 * Any broken qdiscs that would require a ops->reset() here?
858 * The qdisc was never in action so it shouldn't be necessary.
859 */
860 qdisc_put_stab(sch->stab);
861 if (ops->destroy)
862 ops->destroy(sch);
863 goto err_out3;
864 }
865
866 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
867 {
868 struct qdisc_size_table *stab = NULL;
869 int err = 0;
870
871 if (tca[TCA_OPTIONS]) {
872 if (sch->ops->change == NULL)
873 return -EINVAL;
874 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
875 if (err)
876 return err;
877 }
878
879 if (tca[TCA_STAB]) {
880 stab = qdisc_get_stab(tca[TCA_STAB]);
881 if (IS_ERR(stab))
882 return PTR_ERR(stab);
883 }
884
885 qdisc_put_stab(sch->stab);
886 sch->stab = stab;
887
888 if (tca[TCA_RATE]) {
889 /* NB: ignores errors from replace_estimator
890 because change can't be undone. */
891 if (sch->flags & TCQ_F_MQROOT)
892 goto out;
893 gen_replace_estimator(&sch->bstats, &sch->rate_est,
894 qdisc_root_sleeping_lock(sch),
895 tca[TCA_RATE]);
896 }
897 out:
898 return 0;
899 }
900
901 struct check_loop_arg
902 {
903 struct qdisc_walker w;
904 struct Qdisc *p;
905 int depth;
906 };
907
908 static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
909
910 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
911 {
912 struct check_loop_arg arg;
913
914 if (q->ops->cl_ops == NULL)
915 return 0;
916
917 arg.w.stop = arg.w.skip = arg.w.count = 0;
918 arg.w.fn = check_loop_fn;
919 arg.depth = depth;
920 arg.p = p;
921 q->ops->cl_ops->walk(q, &arg.w);
922 return arg.w.stop ? -ELOOP : 0;
923 }
924
925 static int
926 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
927 {
928 struct Qdisc *leaf;
929 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
930 struct check_loop_arg *arg = (struct check_loop_arg *)w;
931
932 leaf = cops->leaf(q, cl);
933 if (leaf) {
934 if (leaf == arg->p || arg->depth > 7)
935 return -ELOOP;
936 return check_loop(leaf, arg->p, arg->depth + 1);
937 }
938 return 0;
939 }
940
941 /*
942 * Delete/get qdisc.
943 */
944
945 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
946 {
947 struct net *net = sock_net(skb->sk);
948 struct tcmsg *tcm = NLMSG_DATA(n);
949 struct nlattr *tca[TCA_MAX + 1];
950 struct net_device *dev;
951 u32 clid = tcm->tcm_parent;
952 struct Qdisc *q = NULL;
953 struct Qdisc *p = NULL;
954 int err;
955
956 if ((dev = __dev_get_by_index(net, tcm->tcm_ifindex)) == NULL)
957 return -ENODEV;
958
959 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
960 if (err < 0)
961 return err;
962
963 if (clid) {
964 if (clid != TC_H_ROOT) {
965 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
966 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
967 return -ENOENT;
968 q = qdisc_leaf(p, clid);
969 } else { /* ingress */
970 q = dev->rx_queue.qdisc_sleeping;
971 }
972 } else {
973 q = dev->qdisc;
974 }
975 if (!q)
976 return -ENOENT;
977
978 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
979 return -EINVAL;
980 } else {
981 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
982 return -ENOENT;
983 }
984
985 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
986 return -EINVAL;
987
988 if (n->nlmsg_type == RTM_DELQDISC) {
989 if (!clid)
990 return -EINVAL;
991 if (q->handle == 0)
992 return -ENOENT;
993 if ((err = qdisc_graft(dev, p, skb, n, clid, NULL, q)) != 0)
994 return err;
995 } else {
996 qdisc_notify(net, skb, n, clid, NULL, q);
997 }
998 return 0;
999 }
1000
1001 /*
1002 Create/change qdisc.
1003 */
1004
1005 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1006 {
1007 struct net *net = sock_net(skb->sk);
1008 struct tcmsg *tcm;
1009 struct nlattr *tca[TCA_MAX + 1];
1010 struct net_device *dev;
1011 u32 clid;
1012 struct Qdisc *q, *p;
1013 int err;
1014
1015 replay:
1016 /* Reinit, just in case something touches this. */
1017 tcm = NLMSG_DATA(n);
1018 clid = tcm->tcm_parent;
1019 q = p = NULL;
1020
1021 if ((dev = __dev_get_by_index(net, tcm->tcm_ifindex)) == NULL)
1022 return -ENODEV;
1023
1024 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1025 if (err < 0)
1026 return err;
1027
1028 if (clid) {
1029 if (clid != TC_H_ROOT) {
1030 if (clid != TC_H_INGRESS) {
1031 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
1032 return -ENOENT;
1033 q = qdisc_leaf(p, clid);
1034 } else { /*ingress */
1035 q = dev->rx_queue.qdisc_sleeping;
1036 }
1037 } else {
1038 q = dev->qdisc;
1039 }
1040
1041 /* It may be default qdisc, ignore it */
1042 if (q && q->handle == 0)
1043 q = NULL;
1044
1045 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1046 if (tcm->tcm_handle) {
1047 if (q && !(n->nlmsg_flags&NLM_F_REPLACE))
1048 return -EEXIST;
1049 if (TC_H_MIN(tcm->tcm_handle))
1050 return -EINVAL;
1051 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
1052 goto create_n_graft;
1053 if (n->nlmsg_flags&NLM_F_EXCL)
1054 return -EEXIST;
1055 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1056 return -EINVAL;
1057 if (q == p ||
1058 (p && check_loop(q, p, 0)))
1059 return -ELOOP;
1060 atomic_inc(&q->refcnt);
1061 goto graft;
1062 } else {
1063 if (q == NULL)
1064 goto create_n_graft;
1065
1066 /* This magic test requires explanation.
1067 *
1068 * We know, that some child q is already
1069 * attached to this parent and have choice:
1070 * either to change it or to create/graft new one.
1071 *
1072 * 1. We are allowed to create/graft only
1073 * if CREATE and REPLACE flags are set.
1074 *
1075 * 2. If EXCL is set, requestor wanted to say,
1076 * that qdisc tcm_handle is not expected
1077 * to exist, so that we choose create/graft too.
1078 *
1079 * 3. The last case is when no flags are set.
1080 * Alas, it is sort of hole in API, we
1081 * cannot decide what to do unambiguously.
1082 * For now we select create/graft, if
1083 * user gave KIND, which does not match existing.
1084 */
1085 if ((n->nlmsg_flags&NLM_F_CREATE) &&
1086 (n->nlmsg_flags&NLM_F_REPLACE) &&
1087 ((n->nlmsg_flags&NLM_F_EXCL) ||
1088 (tca[TCA_KIND] &&
1089 nla_strcmp(tca[TCA_KIND], q->ops->id))))
1090 goto create_n_graft;
1091 }
1092 }
1093 } else {
1094 if (!tcm->tcm_handle)
1095 return -EINVAL;
1096 q = qdisc_lookup(dev, tcm->tcm_handle);
1097 }
1098
1099 /* Change qdisc parameters */
1100 if (q == NULL)
1101 return -ENOENT;
1102 if (n->nlmsg_flags&NLM_F_EXCL)
1103 return -EEXIST;
1104 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1105 return -EINVAL;
1106 err = qdisc_change(q, tca);
1107 if (err == 0)
1108 qdisc_notify(net, skb, n, clid, NULL, q);
1109 return err;
1110
1111 create_n_graft:
1112 if (!(n->nlmsg_flags&NLM_F_CREATE))
1113 return -ENOENT;
1114 if (clid == TC_H_INGRESS)
1115 q = qdisc_create(dev, &dev->rx_queue, p,
1116 tcm->tcm_parent, tcm->tcm_parent,
1117 tca, &err);
1118 else {
1119 struct netdev_queue *dev_queue;
1120
1121 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1122 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1123 else if (p)
1124 dev_queue = p->dev_queue;
1125 else
1126 dev_queue = netdev_get_tx_queue(dev, 0);
1127
1128 q = qdisc_create(dev, dev_queue, p,
1129 tcm->tcm_parent, tcm->tcm_handle,
1130 tca, &err);
1131 }
1132 if (q == NULL) {
1133 if (err == -EAGAIN)
1134 goto replay;
1135 return err;
1136 }
1137
1138 graft:
1139 err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
1140 if (err) {
1141 if (q)
1142 qdisc_destroy(q);
1143 return err;
1144 }
1145
1146 return 0;
1147 }
1148
1149 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1150 u32 pid, u32 seq, u16 flags, int event)
1151 {
1152 struct tcmsg *tcm;
1153 struct nlmsghdr *nlh;
1154 unsigned char *b = skb_tail_pointer(skb);
1155 struct gnet_dump d;
1156
1157 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
1158 tcm = NLMSG_DATA(nlh);
1159 tcm->tcm_family = AF_UNSPEC;
1160 tcm->tcm__pad1 = 0;
1161 tcm->tcm__pad2 = 0;
1162 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1163 tcm->tcm_parent = clid;
1164 tcm->tcm_handle = q->handle;
1165 tcm->tcm_info = atomic_read(&q->refcnt);
1166 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
1167 if (q->ops->dump && q->ops->dump(q, skb) < 0)
1168 goto nla_put_failure;
1169 q->qstats.qlen = q->q.qlen;
1170
1171 if (q->stab && qdisc_dump_stab(skb, q->stab) < 0)
1172 goto nla_put_failure;
1173
1174 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1175 qdisc_root_sleeping_lock(q), &d) < 0)
1176 goto nla_put_failure;
1177
1178 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
1179 goto nla_put_failure;
1180
1181 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
1182 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
1183 gnet_stats_copy_queue(&d, &q->qstats) < 0)
1184 goto nla_put_failure;
1185
1186 if (gnet_stats_finish_copy(&d) < 0)
1187 goto nla_put_failure;
1188
1189 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1190 return skb->len;
1191
1192 nlmsg_failure:
1193 nla_put_failure:
1194 nlmsg_trim(skb, b);
1195 return -1;
1196 }
1197
1198 static bool tc_qdisc_dump_ignore(struct Qdisc *q)
1199 {
1200 return (q->flags & TCQ_F_BUILTIN) ? true : false;
1201 }
1202
1203 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
1204 struct nlmsghdr *n, u32 clid,
1205 struct Qdisc *old, struct Qdisc *new)
1206 {
1207 struct sk_buff *skb;
1208 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1209
1210 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1211 if (!skb)
1212 return -ENOBUFS;
1213
1214 if (old && !tc_qdisc_dump_ignore(old)) {
1215 if (tc_fill_qdisc(skb, old, clid, pid, n->nlmsg_seq, 0, RTM_DELQDISC) < 0)
1216 goto err_out;
1217 }
1218 if (new && !tc_qdisc_dump_ignore(new)) {
1219 if (tc_fill_qdisc(skb, new, clid, pid, n->nlmsg_seq, old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1220 goto err_out;
1221 }
1222
1223 if (skb->len)
1224 return rtnetlink_send(skb, net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
1225
1226 err_out:
1227 kfree_skb(skb);
1228 return -EINVAL;
1229 }
1230
1231 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1232 struct netlink_callback *cb,
1233 int *q_idx_p, int s_q_idx)
1234 {
1235 int ret = 0, q_idx = *q_idx_p;
1236 struct Qdisc *q;
1237
1238 if (!root)
1239 return 0;
1240
1241 q = root;
1242 if (q_idx < s_q_idx) {
1243 q_idx++;
1244 } else {
1245 if (!tc_qdisc_dump_ignore(q) &&
1246 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
1247 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1248 goto done;
1249 q_idx++;
1250 }
1251 list_for_each_entry(q, &root->list, list) {
1252 if (q_idx < s_q_idx) {
1253 q_idx++;
1254 continue;
1255 }
1256 if (!tc_qdisc_dump_ignore(q) &&
1257 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
1258 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1259 goto done;
1260 q_idx++;
1261 }
1262
1263 out:
1264 *q_idx_p = q_idx;
1265 return ret;
1266 done:
1267 ret = -1;
1268 goto out;
1269 }
1270
1271 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1272 {
1273 struct net *net = sock_net(skb->sk);
1274 int idx, q_idx;
1275 int s_idx, s_q_idx;
1276 struct net_device *dev;
1277
1278 s_idx = cb->args[0];
1279 s_q_idx = q_idx = cb->args[1];
1280
1281 rcu_read_lock();
1282 idx = 0;
1283 for_each_netdev_rcu(net, dev) {
1284 struct netdev_queue *dev_queue;
1285
1286 if (idx < s_idx)
1287 goto cont;
1288 if (idx > s_idx)
1289 s_q_idx = 0;
1290 q_idx = 0;
1291
1292 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx) < 0)
1293 goto done;
1294
1295 dev_queue = &dev->rx_queue;
1296 if (tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb, &q_idx, s_q_idx) < 0)
1297 goto done;
1298
1299 cont:
1300 idx++;
1301 }
1302
1303 done:
1304 rcu_read_unlock();
1305
1306 cb->args[0] = idx;
1307 cb->args[1] = q_idx;
1308
1309 return skb->len;
1310 }
1311
1312
1313
1314 /************************************************
1315 * Traffic classes manipulation. *
1316 ************************************************/
1317
1318
1319
1320 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1321 {
1322 struct net *net = sock_net(skb->sk);
1323 struct tcmsg *tcm = NLMSG_DATA(n);
1324 struct nlattr *tca[TCA_MAX + 1];
1325 struct net_device *dev;
1326 struct Qdisc *q = NULL;
1327 const struct Qdisc_class_ops *cops;
1328 unsigned long cl = 0;
1329 unsigned long new_cl;
1330 u32 pid = tcm->tcm_parent;
1331 u32 clid = tcm->tcm_handle;
1332 u32 qid = TC_H_MAJ(clid);
1333 int err;
1334
1335 if ((dev = __dev_get_by_index(net, tcm->tcm_ifindex)) == NULL)
1336 return -ENODEV;
1337
1338 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1339 if (err < 0)
1340 return err;
1341
1342 /*
1343 parent == TC_H_UNSPEC - unspecified parent.
1344 parent == TC_H_ROOT - class is root, which has no parent.
1345 parent == X:0 - parent is root class.
1346 parent == X:Y - parent is a node in hierarchy.
1347 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1348
1349 handle == 0:0 - generate handle from kernel pool.
1350 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1351 handle == X:Y - clear.
1352 handle == X:0 - root class.
1353 */
1354
1355 /* Step 1. Determine qdisc handle X:0 */
1356
1357 if (pid != TC_H_ROOT) {
1358 u32 qid1 = TC_H_MAJ(pid);
1359
1360 if (qid && qid1) {
1361 /* If both majors are known, they must be identical. */
1362 if (qid != qid1)
1363 return -EINVAL;
1364 } else if (qid1) {
1365 qid = qid1;
1366 } else if (qid == 0)
1367 qid = dev->qdisc->handle;
1368
1369 /* Now qid is genuine qdisc handle consistent
1370 both with parent and child.
1371
1372 TC_H_MAJ(pid) still may be unspecified, complete it now.
1373 */
1374 if (pid)
1375 pid = TC_H_MAKE(qid, pid);
1376 } else {
1377 if (qid == 0)
1378 qid = dev->qdisc->handle;
1379 }
1380
1381 /* OK. Locate qdisc */
1382 if ((q = qdisc_lookup(dev, qid)) == NULL)
1383 return -ENOENT;
1384
1385 /* An check that it supports classes */
1386 cops = q->ops->cl_ops;
1387 if (cops == NULL)
1388 return -EINVAL;
1389
1390 /* Now try to get class */
1391 if (clid == 0) {
1392 if (pid == TC_H_ROOT)
1393 clid = qid;
1394 } else
1395 clid = TC_H_MAKE(qid, clid);
1396
1397 if (clid)
1398 cl = cops->get(q, clid);
1399
1400 if (cl == 0) {
1401 err = -ENOENT;
1402 if (n->nlmsg_type != RTM_NEWTCLASS || !(n->nlmsg_flags&NLM_F_CREATE))
1403 goto out;
1404 } else {
1405 switch (n->nlmsg_type) {
1406 case RTM_NEWTCLASS:
1407 err = -EEXIST;
1408 if (n->nlmsg_flags&NLM_F_EXCL)
1409 goto out;
1410 break;
1411 case RTM_DELTCLASS:
1412 err = -EOPNOTSUPP;
1413 if (cops->delete)
1414 err = cops->delete(q, cl);
1415 if (err == 0)
1416 tclass_notify(net, skb, n, q, cl, RTM_DELTCLASS);
1417 goto out;
1418 case RTM_GETTCLASS:
1419 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
1420 goto out;
1421 default:
1422 err = -EINVAL;
1423 goto out;
1424 }
1425 }
1426
1427 new_cl = cl;
1428 err = -EOPNOTSUPP;
1429 if (cops->change)
1430 err = cops->change(q, clid, pid, tca, &new_cl);
1431 if (err == 0)
1432 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
1433
1434 out:
1435 if (cl)
1436 cops->put(q, cl);
1437
1438 return err;
1439 }
1440
1441
1442 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1443 unsigned long cl,
1444 u32 pid, u32 seq, u16 flags, int event)
1445 {
1446 struct tcmsg *tcm;
1447 struct nlmsghdr *nlh;
1448 unsigned char *b = skb_tail_pointer(skb);
1449 struct gnet_dump d;
1450 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1451
1452 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
1453 tcm = NLMSG_DATA(nlh);
1454 tcm->tcm_family = AF_UNSPEC;
1455 tcm->tcm__pad1 = 0;
1456 tcm->tcm__pad2 = 0;
1457 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1458 tcm->tcm_parent = q->handle;
1459 tcm->tcm_handle = q->handle;
1460 tcm->tcm_info = 0;
1461 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
1462 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1463 goto nla_put_failure;
1464
1465 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1466 qdisc_root_sleeping_lock(q), &d) < 0)
1467 goto nla_put_failure;
1468
1469 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1470 goto nla_put_failure;
1471
1472 if (gnet_stats_finish_copy(&d) < 0)
1473 goto nla_put_failure;
1474
1475 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1476 return skb->len;
1477
1478 nlmsg_failure:
1479 nla_put_failure:
1480 nlmsg_trim(skb, b);
1481 return -1;
1482 }
1483
1484 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1485 struct nlmsghdr *n, struct Qdisc *q,
1486 unsigned long cl, int event)
1487 {
1488 struct sk_buff *skb;
1489 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1490
1491 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1492 if (!skb)
1493 return -ENOBUFS;
1494
1495 if (tc_fill_tclass(skb, q, cl, pid, n->nlmsg_seq, 0, event) < 0) {
1496 kfree_skb(skb);
1497 return -EINVAL;
1498 }
1499
1500 return rtnetlink_send(skb, net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
1501 }
1502
1503 struct qdisc_dump_args
1504 {
1505 struct qdisc_walker w;
1506 struct sk_buff *skb;
1507 struct netlink_callback *cb;
1508 };
1509
1510 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1511 {
1512 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1513
1514 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).pid,
1515 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1516 }
1517
1518 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
1519 struct tcmsg *tcm, struct netlink_callback *cb,
1520 int *t_p, int s_t)
1521 {
1522 struct qdisc_dump_args arg;
1523
1524 if (tc_qdisc_dump_ignore(q) ||
1525 *t_p < s_t || !q->ops->cl_ops ||
1526 (tcm->tcm_parent &&
1527 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1528 (*t_p)++;
1529 return 0;
1530 }
1531 if (*t_p > s_t)
1532 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1533 arg.w.fn = qdisc_class_dump;
1534 arg.skb = skb;
1535 arg.cb = cb;
1536 arg.w.stop = 0;
1537 arg.w.skip = cb->args[1];
1538 arg.w.count = 0;
1539 q->ops->cl_ops->walk(q, &arg.w);
1540 cb->args[1] = arg.w.count;
1541 if (arg.w.stop)
1542 return -1;
1543 (*t_p)++;
1544 return 0;
1545 }
1546
1547 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
1548 struct tcmsg *tcm, struct netlink_callback *cb,
1549 int *t_p, int s_t)
1550 {
1551 struct Qdisc *q;
1552
1553 if (!root)
1554 return 0;
1555
1556 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
1557 return -1;
1558
1559 list_for_each_entry(q, &root->list, list) {
1560 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
1561 return -1;
1562 }
1563
1564 return 0;
1565 }
1566
1567 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1568 {
1569 struct tcmsg *tcm = (struct tcmsg*)NLMSG_DATA(cb->nlh);
1570 struct net *net = sock_net(skb->sk);
1571 struct netdev_queue *dev_queue;
1572 struct net_device *dev;
1573 int t, s_t;
1574
1575 if (cb->nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*tcm)))
1576 return 0;
1577 if ((dev = dev_get_by_index(net, tcm->tcm_ifindex)) == NULL)
1578 return 0;
1579
1580 s_t = cb->args[0];
1581 t = 0;
1582
1583 if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0)
1584 goto done;
1585
1586 dev_queue = &dev->rx_queue;
1587 if (tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb, &t, s_t) < 0)
1588 goto done;
1589
1590 done:
1591 cb->args[0] = t;
1592
1593 dev_put(dev);
1594 return skb->len;
1595 }
1596
1597 /* Main classifier routine: scans classifier chain attached
1598 to this qdisc, (optionally) tests for protocol and asks
1599 specific classifiers.
1600 */
1601 int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp,
1602 struct tcf_result *res)
1603 {
1604 __be16 protocol = skb->protocol;
1605 int err = 0;
1606
1607 for (; tp; tp = tp->next) {
1608 if ((tp->protocol == protocol ||
1609 tp->protocol == htons(ETH_P_ALL)) &&
1610 (err = tp->classify(skb, tp, res)) >= 0) {
1611 #ifdef CONFIG_NET_CLS_ACT
1612 if (err != TC_ACT_RECLASSIFY && skb->tc_verd)
1613 skb->tc_verd = SET_TC_VERD(skb->tc_verd, 0);
1614 #endif
1615 return err;
1616 }
1617 }
1618 return -1;
1619 }
1620 EXPORT_SYMBOL(tc_classify_compat);
1621
1622 int tc_classify(struct sk_buff *skb, struct tcf_proto *tp,
1623 struct tcf_result *res)
1624 {
1625 int err = 0;
1626 __be16 protocol;
1627 #ifdef CONFIG_NET_CLS_ACT
1628 struct tcf_proto *otp = tp;
1629 reclassify:
1630 #endif
1631 protocol = skb->protocol;
1632
1633 err = tc_classify_compat(skb, tp, res);
1634 #ifdef CONFIG_NET_CLS_ACT
1635 if (err == TC_ACT_RECLASSIFY) {
1636 u32 verd = G_TC_VERD(skb->tc_verd);
1637 tp = otp;
1638
1639 if (verd++ >= MAX_REC_LOOP) {
1640 if (net_ratelimit())
1641 printk(KERN_NOTICE
1642 "%s: packet reclassify loop"
1643 " rule prio %u protocol %02x\n",
1644 tp->q->ops->id,
1645 tp->prio & 0xffff, ntohs(tp->protocol));
1646 return TC_ACT_SHOT;
1647 }
1648 skb->tc_verd = SET_TC_VERD(skb->tc_verd, verd);
1649 goto reclassify;
1650 }
1651 #endif
1652 return err;
1653 }
1654 EXPORT_SYMBOL(tc_classify);
1655
1656 void tcf_destroy(struct tcf_proto *tp)
1657 {
1658 tp->ops->destroy(tp);
1659 module_put(tp->ops->owner);
1660 kfree(tp);
1661 }
1662
1663 void tcf_destroy_chain(struct tcf_proto **fl)
1664 {
1665 struct tcf_proto *tp;
1666
1667 while ((tp = *fl) != NULL) {
1668 *fl = tp->next;
1669 tcf_destroy(tp);
1670 }
1671 }
1672 EXPORT_SYMBOL(tcf_destroy_chain);
1673
1674 #ifdef CONFIG_PROC_FS
1675 static int psched_show(struct seq_file *seq, void *v)
1676 {
1677 struct timespec ts;
1678
1679 hrtimer_get_res(CLOCK_MONOTONIC, &ts);
1680 seq_printf(seq, "%08x %08x %08x %08x\n",
1681 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
1682 1000000,
1683 (u32)NSEC_PER_SEC/(u32)ktime_to_ns(timespec_to_ktime(ts)));
1684
1685 return 0;
1686 }
1687
1688 static int psched_open(struct inode *inode, struct file *file)
1689 {
1690 return single_open(file, psched_show, NULL);
1691 }
1692
1693 static const struct file_operations psched_fops = {
1694 .owner = THIS_MODULE,
1695 .open = psched_open,
1696 .read = seq_read,
1697 .llseek = seq_lseek,
1698 .release = single_release,
1699 };
1700
1701 static int __net_init psched_net_init(struct net *net)
1702 {
1703 struct proc_dir_entry *e;
1704
1705 e = proc_net_fops_create(net, "psched", 0, &psched_fops);
1706 if (e == NULL)
1707 return -ENOMEM;
1708
1709 return 0;
1710 }
1711
1712 static void __net_exit psched_net_exit(struct net *net)
1713 {
1714 proc_net_remove(net, "psched");
1715 }
1716 #else
1717 static int __net_init psched_net_init(struct net *net)
1718 {
1719 return 0;
1720 }
1721
1722 static void __net_exit psched_net_exit(struct net *net)
1723 {
1724 }
1725 #endif
1726
1727 static struct pernet_operations psched_net_ops = {
1728 .init = psched_net_init,
1729 .exit = psched_net_exit,
1730 };
1731
1732 static int __init pktsched_init(void)
1733 {
1734 int err;
1735
1736 err = register_pernet_subsys(&psched_net_ops);
1737 if (err) {
1738 printk(KERN_ERR "pktsched_init: "
1739 "cannot initialize per netns operations\n");
1740 return err;
1741 }
1742
1743 register_qdisc(&pfifo_qdisc_ops);
1744 register_qdisc(&bfifo_qdisc_ops);
1745 register_qdisc(&pfifo_head_drop_qdisc_ops);
1746 register_qdisc(&mq_qdisc_ops);
1747
1748 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL);
1749 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL);
1750 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc);
1751 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL);
1752 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL);
1753 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass);
1754
1755 return 0;
1756 }
1757
1758 subsys_initcall(pktsched_init);
linux-next