?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 3.11-rc3
[cris-mirror.git] / drivers / block / aoe / aoecmd.c
blob99cb944a002dc4dfd5f2010a6f2de9d75c09ebc8
1 /* Copyright (c) 2013 Coraid, Inc. See COPYING for GPL terms. */
2 /*
3 * aoecmd.c
4 * Filesystem request handling methods
5 */
6
7 #include <linux/ata.h>
8 #include <linux/slab.h>
9 #include <linux/hdreg.h>
10 #include <linux/blkdev.h>
11 #include <linux/skbuff.h>
12 #include <linux/netdevice.h>
13 #include <linux/genhd.h>
14 #include <linux/moduleparam.h>
15 #include <linux/workqueue.h>
16 #include <linux/kthread.h>
17 #include <net/net_namespace.h>
18 #include <asm/unaligned.h>
19 #include <linux/uio.h>
20 #include "aoe.h"
21
22 #define MAXIOC (8192) /* default meant to avoid most soft lockups */
23
24 static void ktcomplete(struct frame *, struct sk_buff *);
25 static int count_targets(struct aoedev *d, int *untainted);
26
27 static struct buf *nextbuf(struct aoedev *);
28
29 static int aoe_deadsecs = 60 * 3;
30 module_param(aoe_deadsecs, int, 0644);
31 MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
32
33 static int aoe_maxout = 64;
34 module_param(aoe_maxout, int, 0644);
35 MODULE_PARM_DESC(aoe_maxout,
36 "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
37
38 /* The number of online cpus during module initialization gives us a
39 * convenient heuristic cap on the parallelism used for ktio threads
40 * doing I/O completion. It is not important that the cap equal the
41 * actual number of running CPUs at any given time, but because of CPU
42 * hotplug, we take care to use ncpus instead of using
43 * num_online_cpus() after module initialization.
44 */
45 static int ncpus;
46
47 /* mutex lock used for synchronization while thread spawning */
48 static DEFINE_MUTEX(ktio_spawn_lock);
49
50 static wait_queue_head_t *ktiowq;
51 static struct ktstate *kts;
52
53 /* io completion queue */
54 struct iocq_ktio {
55 struct list_head head;
56 spinlock_t lock;
57 };
58 static struct iocq_ktio *iocq;
59
60 static struct page *empty_page;
61
62 static struct sk_buff *
63 new_skb(ulong len)
64 {
65 struct sk_buff *skb;
66
67 skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC);
68 if (skb) {
69 skb_reserve(skb, MAX_HEADER);
70 skb_reset_mac_header(skb);
71 skb_reset_network_header(skb);
72 skb->protocol = __constant_htons(ETH_P_AOE);
73 skb_checksum_none_assert(skb);
74 }
75 return skb;
76 }
77
78 static struct frame *
79 getframe_deferred(struct aoedev *d, u32 tag)
80 {
81 struct list_head *head, *pos, *nx;
82 struct frame *f;
83
84 head = &d->rexmitq;
85 list_for_each_safe(pos, nx, head) {
86 f = list_entry(pos, struct frame, head);
87 if (f->tag == tag) {
88 list_del(pos);
89 return f;
90 }
91 }
92 return NULL;
93 }
94
95 static struct frame *
96 getframe(struct aoedev *d, u32 tag)
97 {
98 struct frame *f;
99 struct list_head *head, *pos, *nx;
100 u32 n;
101
102 n = tag % NFACTIVE;
103 head = &d->factive[n];
104 list_for_each_safe(pos, nx, head) {
105 f = list_entry(pos, struct frame, head);
106 if (f->tag == tag) {
107 list_del(pos);
108 return f;
109 }
110 }
111 return NULL;
112 }
113
114 /*
115 * Leave the top bit clear so we have tagspace for userland.
116 * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
117 * This driver reserves tag -1 to mean "unused frame."
118 */
119 static int
120 newtag(struct aoedev *d)
121 {
122 register ulong n;
123
124 n = jiffies & 0xffff;
125 return n |= (++d->lasttag & 0x7fff) << 16;
126 }
127
128 static u32
129 aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
130 {
131 u32 host_tag = newtag(d);
132
133 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
134 memcpy(h->dst, t->addr, sizeof h->dst);
135 h->type = __constant_cpu_to_be16(ETH_P_AOE);
136 h->verfl = AOE_HVER;
137 h->major = cpu_to_be16(d->aoemajor);
138 h->minor = d->aoeminor;
139 h->cmd = AOECMD_ATA;
140 h->tag = cpu_to_be32(host_tag);
141
142 return host_tag;
143 }
144
145 static inline void
146 put_lba(struct aoe_atahdr *ah, sector_t lba)
147 {
148 ah->lba0 = lba;
149 ah->lba1 = lba >>= 8;
150 ah->lba2 = lba >>= 8;
151 ah->lba3 = lba >>= 8;
152 ah->lba4 = lba >>= 8;
153 ah->lba5 = lba >>= 8;
154 }
155
156 static struct aoeif *
157 ifrotate(struct aoetgt *t)
158 {
159 struct aoeif *ifp;
160
161 ifp = t->ifp;
162 ifp++;
163 if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
164 ifp = t->ifs;
165 if (ifp->nd == NULL)
166 return NULL;
167 return t->ifp = ifp;
168 }
169
170 static void
171 skb_pool_put(struct aoedev *d, struct sk_buff *skb)
172 {
173 __skb_queue_tail(&d->skbpool, skb);
174 }
175
176 static struct sk_buff *
177 skb_pool_get(struct aoedev *d)
178 {
179 struct sk_buff *skb = skb_peek(&d->skbpool);
180
181 if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) {
182 __skb_unlink(skb, &d->skbpool);
183 return skb;
184 }
185 if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX &&
186 (skb = new_skb(ETH_ZLEN)))
187 return skb;
188
189 return NULL;
190 }
191
192 void
193 aoe_freetframe(struct frame *f)
194 {
195 struct aoetgt *t;
196
197 t = f->t;
198 f->buf = NULL;
199 f->lba = 0;
200 f->bv = NULL;
201 f->r_skb = NULL;
202 f->flags = 0;
203 list_add(&f->head, &t->ffree);
204 }
205
206 static struct frame *
207 newtframe(struct aoedev *d, struct aoetgt *t)
208 {
209 struct frame *f;
210 struct sk_buff *skb;
211 struct list_head *pos;
212
213 if (list_empty(&t->ffree)) {
214 if (t->falloc >= NSKBPOOLMAX*2)
215 return NULL;
216 f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
217 if (f == NULL)
218 return NULL;
219 t->falloc++;
220 f->t = t;
221 } else {
222 pos = t->ffree.next;
223 list_del(pos);
224 f = list_entry(pos, struct frame, head);
225 }
226
227 skb = f->skb;
228 if (skb == NULL) {
229 f->skb = skb = new_skb(ETH_ZLEN);
230 if (!skb) {
231 bail: aoe_freetframe(f);
232 return NULL;
233 }
234 }
235
236 if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
237 skb = skb_pool_get(d);
238 if (skb == NULL)
239 goto bail;
240 skb_pool_put(d, f->skb);
241 f->skb = skb;
242 }
243
244 skb->truesize -= skb->data_len;
245 skb_shinfo(skb)->nr_frags = skb->data_len = 0;
246 skb_trim(skb, 0);
247 return f;
248 }
249
250 static struct frame *
251 newframe(struct aoedev *d)
252 {
253 struct frame *f;
254 struct aoetgt *t, **tt;
255 int totout = 0;
256 int use_tainted;
257 int has_untainted;
258
259 if (!d->targets || !d->targets[0]) {
260 printk(KERN_ERR "aoe: NULL TARGETS!\n");
261 return NULL;
262 }
263 tt = d->tgt; /* last used target */
264 for (use_tainted = 0, has_untainted = 0;;) {
265 tt++;
266 if (tt >= &d->targets[d->ntargets] || !*tt)
267 tt = d->targets;
268 t = *tt;
269 if (!t->taint) {
270 has_untainted = 1;
271 totout += t->nout;
272 }
273 if (t->nout < t->maxout
274 && (use_tainted || !t->taint)
275 && t->ifp->nd) {
276 f = newtframe(d, t);
277 if (f) {
278 ifrotate(t);
279 d->tgt = tt;
280 return f;
281 }
282 }
283 if (tt == d->tgt) { /* we've looped and found nada */
284 if (!use_tainted && !has_untainted)
285 use_tainted = 1;
286 else
287 break;
288 }
289 }
290 if (totout == 0) {
291 d->kicked++;
292 d->flags |= DEVFL_KICKME;
293 }
294 return NULL;
295 }
296
297 static void
298 skb_fillup(struct sk_buff *skb, struct bio_vec *bv, ulong off, ulong cnt)
299 {
300 int frag = 0;
301 ulong fcnt;
302 loop:
303 fcnt = bv->bv_len - (off - bv->bv_offset);
304 if (fcnt > cnt)
305 fcnt = cnt;
306 skb_fill_page_desc(skb, frag++, bv->bv_page, off, fcnt);
307 cnt -= fcnt;
308 if (cnt <= 0)
309 return;
310 bv++;
311 off = bv->bv_offset;
312 goto loop;
313 }
314
315 static void
316 fhash(struct frame *f)
317 {
318 struct aoedev *d = f->t->d;
319 u32 n;
320
321 n = f->tag % NFACTIVE;
322 list_add_tail(&f->head, &d->factive[n]);
323 }
324
325 static void
326 ata_rw_frameinit(struct frame *f)
327 {
328 struct aoetgt *t;
329 struct aoe_hdr *h;
330 struct aoe_atahdr *ah;
331 struct sk_buff *skb;
332 char writebit, extbit;
333
334 skb = f->skb;
335 h = (struct aoe_hdr *) skb_mac_header(skb);
336 ah = (struct aoe_atahdr *) (h + 1);
337 skb_put(skb, sizeof(*h) + sizeof(*ah));
338 memset(h, 0, skb->len);
339
340 writebit = 0x10;
341 extbit = 0x4;
342
343 t = f->t;
344 f->tag = aoehdr_atainit(t->d, t, h);
345 fhash(f);
346 t->nout++;
347 f->waited = 0;
348 f->waited_total = 0;
349 if (f->buf)
350 f->lba = f->buf->sector;
351
352 /* set up ata header */
353 ah->scnt = f->bcnt >> 9;
354 put_lba(ah, f->lba);
355 if (t->d->flags & DEVFL_EXT) {
356 ah->aflags |= AOEAFL_EXT;
357 } else {
358 extbit = 0;
359 ah->lba3 &= 0x0f;
360 ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */
361 }
362 if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
363 skb_fillup(skb, f->bv, f->bv_off, f->bcnt);
364 ah->aflags |= AOEAFL_WRITE;
365 skb->len += f->bcnt;
366 skb->data_len = f->bcnt;
367 skb->truesize += f->bcnt;
368 t->wpkts++;
369 } else {
370 t->rpkts++;
371 writebit = 0;
372 }
373
374 ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
375 skb->dev = t->ifp->nd;
376 }
377
378 static int
379 aoecmd_ata_rw(struct aoedev *d)
380 {
381 struct frame *f;
382 struct buf *buf;
383 struct aoetgt *t;
384 struct sk_buff *skb;
385 struct sk_buff_head queue;
386 ulong bcnt, fbcnt;
387
388 buf = nextbuf(d);
389 if (buf == NULL)
390 return 0;
391 f = newframe(d);
392 if (f == NULL)
393 return 0;
394 t = *d->tgt;
395 bcnt = d->maxbcnt;
396 if (bcnt == 0)
397 bcnt = DEFAULTBCNT;
398 if (bcnt > buf->resid)
399 bcnt = buf->resid;
400 fbcnt = bcnt;
401 f->bv = buf->bv;
402 f->bv_off = f->bv->bv_offset + (f->bv->bv_len - buf->bv_resid);
403 do {
404 if (fbcnt < buf->bv_resid) {
405 buf->bv_resid -= fbcnt;
406 buf->resid -= fbcnt;
407 break;
408 }
409 fbcnt -= buf->bv_resid;
410 buf->resid -= buf->bv_resid;
411 if (buf->resid == 0) {
412 d->ip.buf = NULL;
413 break;
414 }
415 buf->bv++;
416 buf->bv_resid = buf->bv->bv_len;
417 WARN_ON(buf->bv_resid == 0);
418 } while (fbcnt);
419
420 /* initialize the headers & frame */
421 f->buf = buf;
422 f->bcnt = bcnt;
423 ata_rw_frameinit(f);
424
425 /* mark all tracking fields and load out */
426 buf->nframesout += 1;
427 buf->sector += bcnt >> 9;
428
429 skb = skb_clone(f->skb, GFP_ATOMIC);
430 if (skb) {
431 do_gettimeofday(&f->sent);
432 f->sent_jiffs = (u32) jiffies;
433 __skb_queue_head_init(&queue);
434 __skb_queue_tail(&queue, skb);
435 aoenet_xmit(&queue);
436 }
437 return 1;
438 }
439
440 /* some callers cannot sleep, and they can call this function,
441 * transmitting the packets later, when interrupts are on
442 */
443 static void
444 aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
445 {
446 struct aoe_hdr *h;
447 struct aoe_cfghdr *ch;
448 struct sk_buff *skb;
449 struct net_device *ifp;
450
451 rcu_read_lock();
452 for_each_netdev_rcu(&init_net, ifp) {
453 dev_hold(ifp);
454 if (!is_aoe_netif(ifp))
455 goto cont;
456
457 skb = new_skb(sizeof *h + sizeof *ch);
458 if (skb == NULL) {
459 printk(KERN_INFO "aoe: skb alloc failure\n");
460 goto cont;
461 }
462 skb_put(skb, sizeof *h + sizeof *ch);
463 skb->dev = ifp;
464 __skb_queue_tail(queue, skb);
465 h = (struct aoe_hdr *) skb_mac_header(skb);
466 memset(h, 0, sizeof *h + sizeof *ch);
467
468 memset(h->dst, 0xff, sizeof h->dst);
469 memcpy(h->src, ifp->dev_addr, sizeof h->src);
470 h->type = __constant_cpu_to_be16(ETH_P_AOE);
471 h->verfl = AOE_HVER;
472 h->major = cpu_to_be16(aoemajor);
473 h->minor = aoeminor;
474 h->cmd = AOECMD_CFG;
475
476 cont:
477 dev_put(ifp);
478 }
479 rcu_read_unlock();
480 }
481
482 static void
483 resend(struct aoedev *d, struct frame *f)
484 {
485 struct sk_buff *skb;
486 struct sk_buff_head queue;
487 struct aoe_hdr *h;
488 struct aoe_atahdr *ah;
489 struct aoetgt *t;
490 char buf[128];
491 u32 n;
492
493 t = f->t;
494 n = newtag(d);
495 skb = f->skb;
496 if (ifrotate(t) == NULL) {
497 /* probably can't happen, but set it up to fail anyway */
498 pr_info("aoe: resend: no interfaces to rotate to.\n");
499 ktcomplete(f, NULL);
500 return;
501 }
502 h = (struct aoe_hdr *) skb_mac_header(skb);
503 ah = (struct aoe_atahdr *) (h+1);
504
505 if (!(f->flags & FFL_PROBE)) {
506 snprintf(buf, sizeof(buf),
507 "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
508 "retransmit", d->aoemajor, d->aoeminor,
509 f->tag, jiffies, n,
510 h->src, h->dst, t->nout);
511 aoechr_error(buf);
512 }
513
514 f->tag = n;
515 fhash(f);
516 h->tag = cpu_to_be32(n);
517 memcpy(h->dst, t->addr, sizeof h->dst);
518 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
519
520 skb->dev = t->ifp->nd;
521 skb = skb_clone(skb, GFP_ATOMIC);
522 if (skb == NULL)
523 return;
524 do_gettimeofday(&f->sent);
525 f->sent_jiffs = (u32) jiffies;
526 __skb_queue_head_init(&queue);
527 __skb_queue_tail(&queue, skb);
528 aoenet_xmit(&queue);
529 }
530
531 static int
532 tsince_hr(struct frame *f)
533 {
534 struct timeval now;
535 int n;
536
537 do_gettimeofday(&now);
538 n = now.tv_usec - f->sent.tv_usec;
539 n += (now.tv_sec - f->sent.tv_sec) * USEC_PER_SEC;
540
541 if (n < 0)
542 n = -n;
543
544 /* For relatively long periods, use jiffies to avoid
545 * discrepancies caused by updates to the system time.
546 *
547 * On system with HZ of 1000, 32-bits is over 49 days
548 * worth of jiffies, or over 71 minutes worth of usecs.
549 *
550 * Jiffies overflow is handled by subtraction of unsigned ints:
551 * (gdb) print (unsigned) 2 - (unsigned) 0xfffffffe
552 * $3 = 4
553 * (gdb)
554 */
555 if (n > USEC_PER_SEC / 4) {
556 n = ((u32) jiffies) - f->sent_jiffs;
557 n *= USEC_PER_SEC / HZ;
558 }
559
560 return n;
561 }
562
563 static int
564 tsince(u32 tag)
565 {
566 int n;
567
568 n = jiffies & 0xffff;
569 n -= tag & 0xffff;
570 if (n < 0)
571 n += 1<<16;
572 return jiffies_to_usecs(n + 1);
573 }
574
575 static struct aoeif *
576 getif(struct aoetgt *t, struct net_device *nd)
577 {
578 struct aoeif *p, *e;
579
580 p = t->ifs;
581 e = p + NAOEIFS;
582 for (; p < e; p++)
583 if (p->nd == nd)
584 return p;
585 return NULL;
586 }
587
588 static void
589 ejectif(struct aoetgt *t, struct aoeif *ifp)
590 {
591 struct aoeif *e;
592 struct net_device *nd;
593 ulong n;
594
595 nd = ifp->nd;
596 e = t->ifs + NAOEIFS - 1;
597 n = (e - ifp) * sizeof *ifp;
598 memmove(ifp, ifp+1, n);
599 e->nd = NULL;
600 dev_put(nd);
601 }
602
603 static struct frame *
604 reassign_frame(struct frame *f)
605 {
606 struct frame *nf;
607 struct sk_buff *skb;
608
609 nf = newframe(f->t->d);
610 if (!nf)
611 return NULL;
612 if (nf->t == f->t) {
613 aoe_freetframe(nf);
614 return NULL;
615 }
616
617 skb = nf->skb;
618 nf->skb = f->skb;
619 nf->buf = f->buf;
620 nf->bcnt = f->bcnt;
621 nf->lba = f->lba;
622 nf->bv = f->bv;
623 nf->bv_off = f->bv_off;
624 nf->waited = 0;
625 nf->waited_total = f->waited_total;
626 nf->sent = f->sent;
627 nf->sent_jiffs = f->sent_jiffs;
628 f->skb = skb;
629
630 return nf;
631 }
632
633 static void
634 probe(struct aoetgt *t)
635 {
636 struct aoedev *d;
637 struct frame *f;
638 struct sk_buff *skb;
639 struct sk_buff_head queue;
640 size_t n, m;
641 int frag;
642
643 d = t->d;
644 f = newtframe(d, t);
645 if (!f) {
646 pr_err("%s %pm for e%ld.%d: %s\n",
647 "aoe: cannot probe remote address",
648 t->addr,
649 (long) d->aoemajor, d->aoeminor,
650 "no frame available");
651 return;
652 }
653 f->flags |= FFL_PROBE;
654 ifrotate(t);
655 f->bcnt = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
656 ata_rw_frameinit(f);
657 skb = f->skb;
658 for (frag = 0, n = f->bcnt; n > 0; ++frag, n -= m) {
659 if (n < PAGE_SIZE)
660 m = n;
661 else
662 m = PAGE_SIZE;
663 skb_fill_page_desc(skb, frag, empty_page, 0, m);
664 }
665 skb->len += f->bcnt;
666 skb->data_len = f->bcnt;
667 skb->truesize += f->bcnt;
668
669 skb = skb_clone(f->skb, GFP_ATOMIC);
670 if (skb) {
671 do_gettimeofday(&f->sent);
672 f->sent_jiffs = (u32) jiffies;
673 __skb_queue_head_init(&queue);
674 __skb_queue_tail(&queue, skb);
675 aoenet_xmit(&queue);
676 }
677 }
678
679 static long
680 rto(struct aoedev *d)
681 {
682 long t;
683
684 t = 2 * d->rttavg >> RTTSCALE;
685 t += 8 * d->rttdev >> RTTDSCALE;
686 if (t == 0)
687 t = 1;
688
689 return t;
690 }
691
692 static void
693 rexmit_deferred(struct aoedev *d)
694 {
695 struct aoetgt *t;
696 struct frame *f;
697 struct frame *nf;
698 struct list_head *pos, *nx, *head;
699 int since;
700 int untainted;
701
702 count_targets(d, &untainted);
703
704 head = &d->rexmitq;
705 list_for_each_safe(pos, nx, head) {
706 f = list_entry(pos, struct frame, head);
707 t = f->t;
708 if (t->taint) {
709 if (!(f->flags & FFL_PROBE)) {
710 nf = reassign_frame(f);
711 if (nf) {
712 if (t->nout_probes == 0
713 && untainted > 0) {
714 probe(t);
715 t->nout_probes++;
716 }
717 list_replace(&f->head, &nf->head);
718 pos = &nf->head;
719 aoe_freetframe(f);
720 f = nf;
721 t = f->t;
722 }
723 } else if (untainted < 1) {
724 /* don't probe w/o other untainted aoetgts */
725 goto stop_probe;
726 } else if (tsince_hr(f) < t->taint * rto(d)) {
727 /* reprobe slowly when taint is high */
728 continue;
729 }
730 } else if (f->flags & FFL_PROBE) {
731 stop_probe: /* don't probe untainted aoetgts */
732 list_del(pos);
733 aoe_freetframe(f);
734 /* leaving d->kicked, because this is routine */
735 f->t->d->flags |= DEVFL_KICKME;
736 continue;
737 }
738 if (t->nout >= t->maxout)
739 continue;
740 list_del(pos);
741 t->nout++;
742 if (f->flags & FFL_PROBE)
743 t->nout_probes++;
744 since = tsince_hr(f);
745 f->waited += since;
746 f->waited_total += since;
747 resend(d, f);
748 }
749 }
750
751 /* An aoetgt accumulates demerits quickly, and successful
752 * probing redeems the aoetgt slowly.
753 */
754 static void
755 scorn(struct aoetgt *t)
756 {
757 int n;
758
759 n = t->taint++;
760 t->taint += t->taint * 2;
761 if (n > t->taint)
762 t->taint = n;
763 if (t->taint > MAX_TAINT)
764 t->taint = MAX_TAINT;
765 }
766
767 static int
768 count_targets(struct aoedev *d, int *untainted)
769 {
770 int i, good;
771
772 for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
773 if (d->targets[i]->taint == 0)
774 good++;
775
776 if (untainted)
777 *untainted = good;
778 return i;
779 }
780
781 static void
782 rexmit_timer(ulong vp)
783 {
784 struct aoedev *d;
785 struct aoetgt *t;
786 struct aoeif *ifp;
787 struct frame *f;
788 struct list_head *head, *pos, *nx;
789 LIST_HEAD(flist);
790 register long timeout;
791 ulong flags, n;
792 int i;
793 int utgts; /* number of aoetgt descriptors (not slots) */
794 int since;
795
796 d = (struct aoedev *) vp;
797
798 spin_lock_irqsave(&d->lock, flags);
799
800 /* timeout based on observed timings and variations */
801 timeout = rto(d);
802
803 utgts = count_targets(d, NULL);
804
805 if (d->flags & DEVFL_TKILL) {
806 spin_unlock_irqrestore(&d->lock, flags);
807 return;
808 }
809
810 /* collect all frames to rexmit into flist */
811 for (i = 0; i < NFACTIVE; i++) {
812 head = &d->factive[i];
813 list_for_each_safe(pos, nx, head) {
814 f = list_entry(pos, struct frame, head);
815 if (tsince_hr(f) < timeout)
816 break; /* end of expired frames */
817 /* move to flist for later processing */
818 list_move_tail(pos, &flist);
819 }
820 }
821
822 /* process expired frames */
823 while (!list_empty(&flist)) {
824 pos = flist.next;
825 f = list_entry(pos, struct frame, head);
826 since = tsince_hr(f);
827 n = f->waited_total + since;
828 n /= USEC_PER_SEC;
829 if (aoe_deadsecs
830 && n > aoe_deadsecs
831 && !(f->flags & FFL_PROBE)) {
832 /* Waited too long. Device failure.
833 * Hang all frames on first hash bucket for downdev
834 * to clean up.
835 */
836 list_splice(&flist, &d->factive[0]);
837 aoedev_downdev(d);
838 goto out;
839 }
840
841 t = f->t;
842 n = f->waited + since;
843 n /= USEC_PER_SEC;
844 if (aoe_deadsecs && utgts > 0
845 && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
846 scorn(t); /* avoid this target */
847
848 if (t->maxout != 1) {
849 t->ssthresh = t->maxout / 2;
850 t->maxout = 1;
851 }
852
853 if (f->flags & FFL_PROBE) {
854 t->nout_probes--;
855 } else {
856 ifp = getif(t, f->skb->dev);
857 if (ifp && ++ifp->lost > (t->nframes << 1)
858 && (ifp != t->ifs || t->ifs[1].nd)) {
859 ejectif(t, ifp);
860 ifp = NULL;
861 }
862 }
863 list_move_tail(pos, &d->rexmitq);
864 t->nout--;
865 }
866 rexmit_deferred(d);
867
868 out:
869 if ((d->flags & DEVFL_KICKME) && d->blkq) {
870 d->flags &= ~DEVFL_KICKME;
871 d->blkq->request_fn(d->blkq);
872 }
873
874 d->timer.expires = jiffies + TIMERTICK;
875 add_timer(&d->timer);
876
877 spin_unlock_irqrestore(&d->lock, flags);
878 }
879
880 static unsigned long
881 rqbiocnt(struct request *r)
882 {
883 struct bio *bio;
884 unsigned long n = 0;
885
886 __rq_for_each_bio(bio, r)
887 n++;
888 return n;
889 }
890
891 /* This can be removed if we are certain that no users of the block
892 * layer will ever use zero-count pages in bios. Otherwise we have to
893 * protect against the put_page sometimes done by the network layer.
894 *
895 * See http://oss.sgi.com/archives/xfs/2007-01/msg00594.html for
896 * discussion.
897 *
898 * We cannot use get_page in the workaround, because it insists on a
899 * positive page count as a precondition. So we use _count directly.
900 */
901 static void
902 bio_pageinc(struct bio *bio)
903 {
904 struct bio_vec *bv;
905 struct page *page;
906 int i;
907
908 bio_for_each_segment(bv, bio, i) {
909 page = bv->bv_page;
910 /* Non-zero page count for non-head members of
911 * compound pages is no longer allowed by the kernel,
912 * but this has never been seen here.
913 */
914 if (unlikely(PageCompound(page)))
915 if (compound_trans_head(page) != page) {
916 pr_crit("page tail used for block I/O\n");
917 BUG();
918 }
919 atomic_inc(&page->_count);
920 }
921 }
922
923 static void
924 bio_pagedec(struct bio *bio)
925 {
926 struct bio_vec *bv;
927 int i;
928
929 bio_for_each_segment(bv, bio, i)
930 atomic_dec(&bv->bv_page->_count);
931 }
932
933 static void
934 bufinit(struct buf *buf, struct request *rq, struct bio *bio)
935 {
936 memset(buf, 0, sizeof(*buf));
937 buf->rq = rq;
938 buf->bio = bio;
939 buf->resid = bio->bi_size;
940 buf->sector = bio->bi_sector;
941 bio_pageinc(bio);
942 buf->bv = bio_iovec(bio);
943 buf->bv_resid = buf->bv->bv_len;
944 WARN_ON(buf->bv_resid == 0);
945 }
946
947 static struct buf *
948 nextbuf(struct aoedev *d)
949 {
950 struct request *rq;
951 struct request_queue *q;
952 struct buf *buf;
953 struct bio *bio;
954
955 q = d->blkq;
956 if (q == NULL)
957 return NULL; /* initializing */
958 if (d->ip.buf)
959 return d->ip.buf;
960 rq = d->ip.rq;
961 if (rq == NULL) {
962 rq = blk_peek_request(q);
963 if (rq == NULL)
964 return NULL;
965 blk_start_request(rq);
966 d->ip.rq = rq;
967 d->ip.nxbio = rq->bio;
968 rq->special = (void *) rqbiocnt(rq);
969 }
970 buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
971 if (buf == NULL) {
972 pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
973 return NULL;
974 }
975 bio = d->ip.nxbio;
976 bufinit(buf, rq, bio);
977 bio = bio->bi_next;
978 d->ip.nxbio = bio;
979 if (bio == NULL)
980 d->ip.rq = NULL;
981 return d->ip.buf = buf;
982 }
983
984 /* enters with d->lock held */
985 void
986 aoecmd_work(struct aoedev *d)
987 {
988 rexmit_deferred(d);
989 while (aoecmd_ata_rw(d))
990 ;
991 }
992
993 /* this function performs work that has been deferred until sleeping is OK
994 */
995 void
996 aoecmd_sleepwork(struct work_struct *work)
997 {
998 struct aoedev *d = container_of(work, struct aoedev, work);
999 struct block_device *bd;
1000 u64 ssize;
1001
1002 if (d->flags & DEVFL_GDALLOC)
1003 aoeblk_gdalloc(d);
1004
1005 if (d->flags & DEVFL_NEWSIZE) {
1006 ssize = get_capacity(d->gd);
1007 bd = bdget_disk(d->gd, 0);
1008 if (bd) {
1009 mutex_lock(&bd->bd_inode->i_mutex);
1010 i_size_write(bd->bd_inode, (loff_t)ssize<<9);
1011 mutex_unlock(&bd->bd_inode->i_mutex);
1012 bdput(bd);
1013 }
1014 spin_lock_irq(&d->lock);
1015 d->flags |= DEVFL_UP;
1016 d->flags &= ~DEVFL_NEWSIZE;
1017 spin_unlock_irq(&d->lock);
1018 }
1019 }
1020
1021 static void
1022 ata_ident_fixstring(u16 *id, int ns)
1023 {
1024 u16 s;
1025
1026 while (ns-- > 0) {
1027 s = *id;
1028 *id++ = s >> 8 | s << 8;
1029 }
1030 }
1031
1032 static void
1033 ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
1034 {
1035 u64 ssize;
1036 u16 n;
1037
1038 /* word 83: command set supported */
1039 n = get_unaligned_le16(&id[83 << 1]);
1040
1041 /* word 86: command set/feature enabled */
1042 n |= get_unaligned_le16(&id[86 << 1]);
1043
1044 if (n & (1<<10)) { /* bit 10: LBA 48 */
1045 d->flags |= DEVFL_EXT;
1046
1047 /* word 100: number lba48 sectors */
1048 ssize = get_unaligned_le64(&id[100 << 1]);
1049
1050 /* set as in ide-disk.c:init_idedisk_capacity */
1051 d->geo.cylinders = ssize;
1052 d->geo.cylinders /= (255 * 63);
1053 d->geo.heads = 255;
1054 d->geo.sectors = 63;
1055 } else {
1056 d->flags &= ~DEVFL_EXT;
1057
1058 /* number lba28 sectors */
1059 ssize = get_unaligned_le32(&id[60 << 1]);
1060
1061 /* NOTE: obsolete in ATA 6 */
1062 d->geo.cylinders = get_unaligned_le16(&id[54 << 1]);
1063 d->geo.heads = get_unaligned_le16(&id[55 << 1]);
1064 d->geo.sectors = get_unaligned_le16(&id[56 << 1]);
1065 }
1066
1067 ata_ident_fixstring((u16 *) &id[10<<1], 10); /* serial */
1068 ata_ident_fixstring((u16 *) &id[23<<1], 4); /* firmware */
1069 ata_ident_fixstring((u16 *) &id[27<<1], 20); /* model */
1070 memcpy(d->ident, id, sizeof(d->ident));
1071
1072 if (d->ssize != ssize)
1073 printk(KERN_INFO
1074 "aoe: %pm e%ld.%d v%04x has %llu sectors\n",
1075 t->addr,
1076 d->aoemajor, d->aoeminor,
1077 d->fw_ver, (long long)ssize);
1078 d->ssize = ssize;
1079 d->geo.start = 0;
1080 if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
1081 return;
1082 if (d->gd != NULL) {
1083 set_capacity(d->gd, ssize);
1084 d->flags |= DEVFL_NEWSIZE;
1085 } else
1086 d->flags |= DEVFL_GDALLOC;
1087 schedule_work(&d->work);
1088 }
1089
1090 static void
1091 calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
1092 {
1093 register long n;
1094
1095 n = rtt;
1096
1097 /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
1098 n -= d->rttavg >> RTTSCALE;
1099 d->rttavg += n;
1100 if (n < 0)
1101 n = -n;
1102 n -= d->rttdev >> RTTDSCALE;
1103 d->rttdev += n;
1104
1105 if (!t || t->maxout >= t->nframes)
1106 return;
1107 if (t->maxout < t->ssthresh)
1108 t->maxout += 1;
1109 else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
1110 t->maxout += 1;
1111 t->next_cwnd = t->maxout;
1112 }
1113 }
1114
1115 static struct aoetgt *
1116 gettgt(struct aoedev *d, char *addr)
1117 {
1118 struct aoetgt **t, **e;
1119
1120 t = d->targets;
1121 e = t + d->ntargets;
1122 for (; t < e && *t; t++)
1123 if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0)
1124 return *t;
1125 return NULL;
1126 }
1127
1128 static void
1129 bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, long cnt)
1130 {
1131 ulong fcnt;
1132 char *p;
1133 int soff = 0;
1134 loop:
1135 fcnt = bv->bv_len - (off - bv->bv_offset);
1136 if (fcnt > cnt)
1137 fcnt = cnt;
1138 p = page_address(bv->bv_page) + off;
1139 skb_copy_bits(skb, soff, p, fcnt);
1140 soff += fcnt;
1141 cnt -= fcnt;
1142 if (cnt <= 0)
1143 return;
1144 bv++;
1145 off = bv->bv_offset;
1146 goto loop;
1147 }
1148
1149 void
1150 aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1151 {
1152 struct bio *bio;
1153 int bok;
1154 struct request_queue *q;
1155
1156 q = d->blkq;
1157 if (rq == d->ip.rq)
1158 d->ip.rq = NULL;
1159 do {
1160 bio = rq->bio;
1161 bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags);
1162 } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_size));
1163
1164 /* cf. http://lkml.org/lkml/2006/10/31/28 */
1165 if (!fastfail)
1166 __blk_run_queue(q);
1167 }
1168
1169 static void
1170 aoe_end_buf(struct aoedev *d, struct buf *buf)
1171 {
1172 struct request *rq;
1173 unsigned long n;
1174
1175 if (buf == d->ip.buf)
1176 d->ip.buf = NULL;
1177 rq = buf->rq;
1178 bio_pagedec(buf->bio);
1179 mempool_free(buf, d->bufpool);
1180 n = (unsigned long) rq->special;
1181 rq->special = (void *) --n;
1182 if (n == 0)
1183 aoe_end_request(d, rq, 0);
1184 }
1185
1186 static void
1187 ktiocomplete(struct frame *f)
1188 {
1189 struct aoe_hdr *hin, *hout;
1190 struct aoe_atahdr *ahin, *ahout;
1191 struct buf *buf;
1192 struct sk_buff *skb;
1193 struct aoetgt *t;
1194 struct aoeif *ifp;
1195 struct aoedev *d;
1196 long n;
1197 int untainted;
1198
1199 if (f == NULL)
1200 return;
1201
1202 t = f->t;
1203 d = t->d;
1204 skb = f->r_skb;
1205 buf = f->buf;
1206 if (f->flags & FFL_PROBE)
1207 goto out;
1208 if (!skb) /* just fail the buf. */
1209 goto noskb;
1210
1211 hout = (struct aoe_hdr *) skb_mac_header(f->skb);
1212 ahout = (struct aoe_atahdr *) (hout+1);
1213
1214 hin = (struct aoe_hdr *) skb->data;
1215 skb_pull(skb, sizeof(*hin));
1216 ahin = (struct aoe_atahdr *) skb->data;
1217 skb_pull(skb, sizeof(*ahin));
1218 if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */
1219 pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1220 ahout->cmdstat, ahin->cmdstat,
1221 d->aoemajor, d->aoeminor);
1222 noskb: if (buf)
1223 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
1224 goto out;
1225 }
1226
1227 n = ahout->scnt << 9;
1228 switch (ahout->cmdstat) {
1229 case ATA_CMD_PIO_READ:
1230 case ATA_CMD_PIO_READ_EXT:
1231 if (skb->len < n) {
1232 pr_err("%s e%ld.%d. skb->len=%d need=%ld\n",
1233 "aoe: runt data size in read from",
1234 (long) d->aoemajor, d->aoeminor,
1235 skb->len, n);
1236 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
1237 break;
1238 }
1239 bvcpy(f->bv, f->bv_off, skb, n);
1240 case ATA_CMD_PIO_WRITE:
1241 case ATA_CMD_PIO_WRITE_EXT:
1242 spin_lock_irq(&d->lock);
1243 ifp = getif(t, skb->dev);
1244 if (ifp)
1245 ifp->lost = 0;
1246 spin_unlock_irq(&d->lock);
1247 break;
1248 case ATA_CMD_ID_ATA:
1249 if (skb->len < 512) {
1250 pr_info("%s e%ld.%d. skb->len=%d need=512\n",
1251 "aoe: runt data size in ataid from",
1252 (long) d->aoemajor, d->aoeminor,
1253 skb->len);
1254 break;
1255 }
1256 if (skb_linearize(skb))
1257 break;
1258 spin_lock_irq(&d->lock);
1259 ataid_complete(d, t, skb->data);
1260 spin_unlock_irq(&d->lock);
1261 break;
1262 default:
1263 pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1264 ahout->cmdstat,
1265 be16_to_cpu(get_unaligned(&hin->major)),
1266 hin->minor);
1267 }
1268 out:
1269 spin_lock_irq(&d->lock);
1270 if (t->taint > 0
1271 && --t->taint > 0
1272 && t->nout_probes == 0) {
1273 count_targets(d, &untainted);
1274 if (untainted > 0) {
1275 probe(t);
1276 t->nout_probes++;
1277 }
1278 }
1279
1280 aoe_freetframe(f);
1281
1282 if (buf && --buf->nframesout == 0 && buf->resid == 0)
1283 aoe_end_buf(d, buf);
1284
1285 spin_unlock_irq(&d->lock);
1286 aoedev_put(d);
1287 dev_kfree_skb(skb);
1288 }
1289
1290 /* Enters with iocq.lock held.
1291 * Returns true iff responses needing processing remain.
1292 */
1293 static int
1294 ktio(int id)
1295 {
1296 struct frame *f;
1297 struct list_head *pos;
1298 int i;
1299 int actual_id;
1300
1301 for (i = 0; ; ++i) {
1302 if (i == MAXIOC)
1303 return 1;
1304 if (list_empty(&iocq[id].head))
1305 return 0;
1306 pos = iocq[id].head.next;
1307 list_del(pos);
1308 f = list_entry(pos, struct frame, head);
1309 spin_unlock_irq(&iocq[id].lock);
1310 ktiocomplete(f);
1311
1312 /* Figure out if extra threads are required. */
1313 actual_id = f->t->d->aoeminor % ncpus;
1314
1315 if (!kts[actual_id].active) {
1316 BUG_ON(id != 0);
1317 mutex_lock(&ktio_spawn_lock);
1318 if (!kts[actual_id].active
1319 && aoe_ktstart(&kts[actual_id]) == 0)
1320 kts[actual_id].active = 1;
1321 mutex_unlock(&ktio_spawn_lock);
1322 }
1323 spin_lock_irq(&iocq[id].lock);
1324 }
1325 }
1326
1327 static int
1328 kthread(void *vp)
1329 {
1330 struct ktstate *k;
1331 DECLARE_WAITQUEUE(wait, current);
1332 int more;
1333
1334 k = vp;
1335 current->flags |= PF_NOFREEZE;
1336 set_user_nice(current, -10);
1337 complete(&k->rendez); /* tell spawner we're running */
1338 do {
1339 spin_lock_irq(k->lock);
1340 more = k->fn(k->id);
1341 if (!more) {
1342 add_wait_queue(k->waitq, &wait);
1343 __set_current_state(TASK_INTERRUPTIBLE);
1344 }
1345 spin_unlock_irq(k->lock);
1346 if (!more) {
1347 schedule();
1348 remove_wait_queue(k->waitq, &wait);
1349 } else
1350 cond_resched();
1351 } while (!kthread_should_stop());
1352 complete(&k->rendez); /* tell spawner we're stopping */
1353 return 0;
1354 }
1355
1356 void
1357 aoe_ktstop(struct ktstate *k)
1358 {
1359 kthread_stop(k->task);
1360 wait_for_completion(&k->rendez);
1361 }
1362
1363 int
1364 aoe_ktstart(struct ktstate *k)
1365 {
1366 struct task_struct *task;
1367
1368 init_completion(&k->rendez);
1369 task = kthread_run(kthread, k, "%s", k->name);
1370 if (task == NULL || IS_ERR(task))
1371 return -ENOMEM;
1372 k->task = task;
1373 wait_for_completion(&k->rendez); /* allow kthread to start */
1374 init_completion(&k->rendez); /* for waiting for exit later */
1375 return 0;
1376 }
1377
1378 /* pass it off to kthreads for processing */
1379 static void
1380 ktcomplete(struct frame *f, struct sk_buff *skb)
1381 {
1382 int id;
1383 ulong flags;
1384
1385 f->r_skb = skb;
1386 id = f->t->d->aoeminor % ncpus;
1387 spin_lock_irqsave(&iocq[id].lock, flags);
1388 if (!kts[id].active) {
1389 spin_unlock_irqrestore(&iocq[id].lock, flags);
1390 /* The thread with id has not been spawned yet,
1391 * so delegate the work to the main thread and
1392 * try spawning a new thread.
1393 */
1394 id = 0;
1395 spin_lock_irqsave(&iocq[id].lock, flags);
1396 }
1397 list_add_tail(&f->head, &iocq[id].head);
1398 spin_unlock_irqrestore(&iocq[id].lock, flags);
1399 wake_up(&ktiowq[id]);
1400 }
1401
1402 struct sk_buff *
1403 aoecmd_ata_rsp(struct sk_buff *skb)
1404 {
1405 struct aoedev *d;
1406 struct aoe_hdr *h;
1407 struct frame *f;
1408 u32 n;
1409 ulong flags;
1410 char ebuf[128];
1411 u16 aoemajor;
1412
1413 h = (struct aoe_hdr *) skb->data;
1414 aoemajor = be16_to_cpu(get_unaligned(&h->major));
1415 d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
1416 if (d == NULL) {
1417 snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
1418 "for unknown device %d.%d\n",
1419 aoemajor, h->minor);
1420 aoechr_error(ebuf);
1421 return skb;
1422 }
1423
1424 spin_lock_irqsave(&d->lock, flags);
1425
1426 n = be32_to_cpu(get_unaligned(&h->tag));
1427 f = getframe(d, n);
1428 if (f) {
1429 calc_rttavg(d, f->t, tsince_hr(f));
1430 f->t->nout--;
1431 if (f->flags & FFL_PROBE)
1432 f->t->nout_probes--;
1433 } else {
1434 f = getframe_deferred(d, n);
1435 if (f) {
1436 calc_rttavg(d, NULL, tsince_hr(f));
1437 } else {
1438 calc_rttavg(d, NULL, tsince(n));
1439 spin_unlock_irqrestore(&d->lock, flags);
1440 aoedev_put(d);
1441 snprintf(ebuf, sizeof(ebuf),
1442 "%15s e%d.%d tag=%08x@%08lx s=%pm d=%pm\n",
1443 "unexpected rsp",
1444 get_unaligned_be16(&h->major),
1445 h->minor,
1446 get_unaligned_be32(&h->tag),
1447 jiffies,
1448 h->src,
1449 h->dst);
1450 aoechr_error(ebuf);
1451 return skb;
1452 }
1453 }
1454 aoecmd_work(d);
1455
1456 spin_unlock_irqrestore(&d->lock, flags);
1457
1458 ktcomplete(f, skb);
1459
1460 /*
1461 * Note here that we do not perform an aoedev_put, as we are
1462 * leaving this reference for the ktio to release.
1463 */
1464 return NULL;
1465 }
1466
1467 void
1468 aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
1469 {
1470 struct sk_buff_head queue;
1471
1472 __skb_queue_head_init(&queue);
1473 aoecmd_cfg_pkts(aoemajor, aoeminor, &queue);
1474 aoenet_xmit(&queue);
1475 }
1476
1477 struct sk_buff *
1478 aoecmd_ata_id(struct aoedev *d)
1479 {
1480 struct aoe_hdr *h;
1481 struct aoe_atahdr *ah;
1482 struct frame *f;
1483 struct sk_buff *skb;
1484 struct aoetgt *t;
1485
1486 f = newframe(d);
1487 if (f == NULL)
1488 return NULL;
1489
1490 t = *d->tgt;
1491
1492 /* initialize the headers & frame */
1493 skb = f->skb;
1494 h = (struct aoe_hdr *) skb_mac_header(skb);
1495 ah = (struct aoe_atahdr *) (h+1);
1496 skb_put(skb, sizeof *h + sizeof *ah);
1497 memset(h, 0, skb->len);
1498 f->tag = aoehdr_atainit(d, t, h);
1499 fhash(f);
1500 t->nout++;
1501 f->waited = 0;
1502 f->waited_total = 0;
1503
1504 /* set up ata header */
1505 ah->scnt = 1;
1506 ah->cmdstat = ATA_CMD_ID_ATA;
1507 ah->lba3 = 0xa0;
1508
1509 skb->dev = t->ifp->nd;
1510
1511 d->rttavg = RTTAVG_INIT;
1512 d->rttdev = RTTDEV_INIT;
1513 d->timer.function = rexmit_timer;
1514
1515 skb = skb_clone(skb, GFP_ATOMIC);
1516 if (skb) {
1517 do_gettimeofday(&f->sent);
1518 f->sent_jiffs = (u32) jiffies;
1519 }
1520
1521 return skb;
1522 }
1523
1524 static struct aoetgt **
1525 grow_targets(struct aoedev *d)
1526 {
1527 ulong oldn, newn;
1528 struct aoetgt **tt;
1529
1530 oldn = d->ntargets;
1531 newn = oldn * 2;
1532 tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
1533 if (!tt)
1534 return NULL;
1535 memmove(tt, d->targets, sizeof(*d->targets) * oldn);
1536 d->tgt = tt + (d->tgt - d->targets);
1537 kfree(d->targets);
1538 d->targets = tt;
1539 d->ntargets = newn;
1540
1541 return &d->targets[oldn];
1542 }
1543
1544 static struct aoetgt *
1545 addtgt(struct aoedev *d, char *addr, ulong nframes)
1546 {
1547 struct aoetgt *t, **tt, **te;
1548
1549 tt = d->targets;
1550 te = tt + d->ntargets;
1551 for (; tt < te && *tt; tt++)
1552 ;
1553
1554 if (tt == te) {
1555 tt = grow_targets(d);
1556 if (!tt)
1557 goto nomem;
1558 }
1559 t = kzalloc(sizeof(*t), GFP_ATOMIC);
1560 if (!t)
1561 goto nomem;
1562 t->nframes = nframes;
1563 t->d = d;
1564 memcpy(t->addr, addr, sizeof t->addr);
1565 t->ifp = t->ifs;
1566 aoecmd_wreset(t);
1567 t->maxout = t->nframes / 2;
1568 INIT_LIST_HEAD(&t->ffree);
1569 return *tt = t;
1570
1571 nomem:
1572 pr_info("aoe: cannot allocate memory to add target\n");
1573 return NULL;
1574 }
1575
1576 static void
1577 setdbcnt(struct aoedev *d)
1578 {
1579 struct aoetgt **t, **e;
1580 int bcnt = 0;
1581
1582 t = d->targets;
1583 e = t + d->ntargets;
1584 for (; t < e && *t; t++)
1585 if (bcnt == 0 || bcnt > (*t)->minbcnt)
1586 bcnt = (*t)->minbcnt;
1587 if (bcnt != d->maxbcnt) {
1588 d->maxbcnt = bcnt;
1589 pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
1590 d->aoemajor, d->aoeminor, bcnt);
1591 }
1592 }
1593
1594 static void
1595 setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
1596 {
1597 struct aoedev *d;
1598 struct aoeif *p, *e;
1599 int minbcnt;
1600
1601 d = t->d;
1602 minbcnt = bcnt;
1603 p = t->ifs;
1604 e = p + NAOEIFS;
1605 for (; p < e; p++) {
1606 if (p->nd == NULL)
1607 break; /* end of the valid interfaces */
1608 if (p->nd == nd) {
1609 p->bcnt = bcnt; /* we're updating */
1610 nd = NULL;
1611 } else if (minbcnt > p->bcnt)
1612 minbcnt = p->bcnt; /* find the min interface */
1613 }
1614 if (nd) {
1615 if (p == e) {
1616 pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
1617 return;
1618 }
1619 dev_hold(nd);
1620 p->nd = nd;
1621 p->bcnt = bcnt;
1622 }
1623 t->minbcnt = minbcnt;
1624 setdbcnt(d);
1625 }
1626
1627 void
1628 aoecmd_cfg_rsp(struct sk_buff *skb)
1629 {
1630 struct aoedev *d;
1631 struct aoe_hdr *h;
1632 struct aoe_cfghdr *ch;
1633 struct aoetgt *t;
1634 ulong flags, aoemajor;
1635 struct sk_buff *sl;
1636 struct sk_buff_head queue;
1637 u16 n;
1638
1639 sl = NULL;
1640 h = (struct aoe_hdr *) skb_mac_header(skb);
1641 ch = (struct aoe_cfghdr *) (h+1);
1642
1643 /*
1644 * Enough people have their dip switches set backwards to
1645 * warrant a loud message for this special case.
1646 */
1647 aoemajor = get_unaligned_be16(&h->major);
1648 if (aoemajor == 0xfff) {
1649 printk(KERN_ERR "aoe: Warning: shelf address is all ones. "
1650 "Check shelf dip switches.\n");
1651 return;
1652 }
1653 if (aoemajor == 0xffff) {
1654 pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
1655 aoemajor, (int) h->minor);
1656 return;
1657 }
1658 if (h->minor == 0xff) {
1659 pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
1660 aoemajor, (int) h->minor);
1661 return;
1662 }
1663
1664 n = be16_to_cpu(ch->bufcnt);
1665 if (n > aoe_maxout) /* keep it reasonable */
1666 n = aoe_maxout;
1667
1668 d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
1669 if (d == NULL) {
1670 pr_info("aoe: device allocation failure\n");
1671 return;
1672 }
1673
1674 spin_lock_irqsave(&d->lock, flags);
1675
1676 t = gettgt(d, h->src);
1677 if (t) {
1678 t->nframes = n;
1679 if (n < t->maxout)
1680 aoecmd_wreset(t);
1681 } else {
1682 t = addtgt(d, h->src, n);
1683 if (!t)
1684 goto bail;
1685 }
1686 n = skb->dev->mtu;
1687 n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
1688 n /= 512;
1689 if (n > ch->scnt)
1690 n = ch->scnt;
1691 n = n ? n * 512 : DEFAULTBCNT;
1692 setifbcnt(t, skb->dev, n);
1693
1694 /* don't change users' perspective */
1695 if (d->nopen == 0) {
1696 d->fw_ver = be16_to_cpu(ch->fwver);
1697 sl = aoecmd_ata_id(d);
1698 }
1699 bail:
1700 spin_unlock_irqrestore(&d->lock, flags);
1701 aoedev_put(d);
1702 if (sl) {
1703 __skb_queue_head_init(&queue);
1704 __skb_queue_tail(&queue, sl);
1705 aoenet_xmit(&queue);
1706 }
1707 }
1708
1709 void
1710 aoecmd_wreset(struct aoetgt *t)
1711 {
1712 t->maxout = 1;
1713 t->ssthresh = t->nframes / 2;
1714 t->next_cwnd = t->nframes;
1715 }
1716
1717 void
1718 aoecmd_cleanslate(struct aoedev *d)
1719 {
1720 struct aoetgt **t, **te;
1721
1722 d->rttavg = RTTAVG_INIT;
1723 d->rttdev = RTTDEV_INIT;
1724 d->maxbcnt = 0;
1725
1726 t = d->targets;
1727 te = t + d->ntargets;
1728 for (; t < te && *t; t++)
1729 aoecmd_wreset(*t);
1730 }
1731
1732 void
1733 aoe_failbuf(struct aoedev *d, struct buf *buf)
1734 {
1735 if (buf == NULL)
1736 return;
1737 buf->resid = 0;
1738 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
1739 if (buf->nframesout == 0)
1740 aoe_end_buf(d, buf);
1741 }
1742
1743 void
1744 aoe_flush_iocq(void)
1745 {
1746 int i;
1747
1748 for (i = 0; i < ncpus; i++) {
1749 if (kts[i].active)
1750 aoe_flush_iocq_by_index(i);
1751 }
1752 }
1753
1754 void
1755 aoe_flush_iocq_by_index(int id)
1756 {
1757 struct frame *f;
1758 struct aoedev *d;
1759 LIST_HEAD(flist);
1760 struct list_head *pos;
1761 struct sk_buff *skb;
1762 ulong flags;
1763
1764 spin_lock_irqsave(&iocq[id].lock, flags);
1765 list_splice_init(&iocq[id].head, &flist);
1766 spin_unlock_irqrestore(&iocq[id].lock, flags);
1767 while (!list_empty(&flist)) {
1768 pos = flist.next;
1769 list_del(pos);
1770 f = list_entry(pos, struct frame, head);
1771 d = f->t->d;
1772 skb = f->r_skb;
1773 spin_lock_irqsave(&d->lock, flags);
1774 if (f->buf) {
1775 f->buf->nframesout--;
1776 aoe_failbuf(d, f->buf);
1777 }
1778 aoe_freetframe(f);
1779 spin_unlock_irqrestore(&d->lock, flags);
1780 dev_kfree_skb(skb);
1781 aoedev_put(d);
1782 }
1783 }
1784
1785 int __init
1786 aoecmd_init(void)
1787 {
1788 void *p;
1789 int i;
1790 int ret;
1791
1792 /* get_zeroed_page returns page with ref count 1 */
1793 p = (void *) get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
1794 if (!p)
1795 return -ENOMEM;
1796 empty_page = virt_to_page(p);
1797
1798 ncpus = num_online_cpus();
1799
1800 iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
1801 if (!iocq)
1802 return -ENOMEM;
1803
1804 kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
1805 if (!kts) {
1806 ret = -ENOMEM;
1807 goto kts_fail;
1808 }
1809
1810 ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
1811 if (!ktiowq) {
1812 ret = -ENOMEM;
1813 goto ktiowq_fail;
1814 }
1815
1816 mutex_init(&ktio_spawn_lock);
1817
1818 for (i = 0; i < ncpus; i++) {
1819 INIT_LIST_HEAD(&iocq[i].head);
1820 spin_lock_init(&iocq[i].lock);
1821 init_waitqueue_head(&ktiowq[i]);
1822 snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i);
1823 kts[i].fn = ktio;
1824 kts[i].waitq = &ktiowq[i];
1825 kts[i].lock = &iocq[i].lock;
1826 kts[i].id = i;
1827 kts[i].active = 0;
1828 }
1829 kts[0].active = 1;
1830 if (aoe_ktstart(&kts[0])) {
1831 ret = -ENOMEM;
1832 goto ktstart_fail;
1833 }
1834 return 0;
1835
1836 ktstart_fail:
1837 kfree(ktiowq);
1838 ktiowq_fail:
1839 kfree(kts);
1840 kts_fail:
1841 kfree(iocq);
1842
1843 return ret;
1844 }
1845
1846 void
1847 aoecmd_exit(void)
1848 {
1849 int i;
1850
1851 for (i = 0; i < ncpus; i++)
1852 if (kts[i].active)
1853 aoe_ktstop(&kts[i]);
1854
1855 aoe_flush_iocq();
1856
1857 /* Free up the iocq and thread speicific configuration
1858 * allocated during startup.
1859 */
1860 kfree(iocq);
1861 kfree(kts);
1862 kfree(ktiowq);
1863
1864 free_page((unsigned long) page_address(empty_page));
1865 empty_page = NULL;
1866 }
CRIS linux kernel tree