?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
spi: sprd: adi: Add a reset reason for watchdog mode
[linux/fpc-iii.git] / drivers / dma / ioat / dma.c
blob1a422a8b43cfb29806026b0be2996dc1dee4aaa5
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel I/OAT DMA Linux driver
4 * Copyright(c) 2004 - 2015 Intel Corporation.
5 */
6
7 /*
8 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
9 * copy operations.
10 */
11
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/pci.h>
16 #include <linux/interrupt.h>
17 #include <linux/dmaengine.h>
18 #include <linux/delay.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/workqueue.h>
21 #include <linux/prefetch.h>
22 #include <linux/sizes.h>
23 #include "dma.h"
24 #include "registers.h"
25 #include "hw.h"
26
27 #include "../dmaengine.h"
28
29 static char *chanerr_str[] = {
30 "DMA Transfer Source Address Error",
31 "DMA Transfer Destination Address Error",
32 "Next Descriptor Address Error",
33 "Descriptor Error",
34 "Chan Address Value Error",
35 "CHANCMD Error",
36 "Chipset Uncorrectable Data Integrity Error",
37 "DMA Uncorrectable Data Integrity Error",
38 "Read Data Error",
39 "Write Data Error",
40 "Descriptor Control Error",
41 "Descriptor Transfer Size Error",
42 "Completion Address Error",
43 "Interrupt Configuration Error",
44 "Super extended descriptor Address Error",
45 "Unaffiliated Error",
46 "CRC or XOR P Error",
47 "XOR Q Error",
48 "Descriptor Count Error",
49 "DIF All F detect Error",
50 "Guard Tag verification Error",
51 "Application Tag verification Error",
52 "Reference Tag verification Error",
53 "Bundle Bit Error",
54 "Result DIF All F detect Error",
55 "Result Guard Tag verification Error",
56 "Result Application Tag verification Error",
57 "Result Reference Tag verification Error",
58 };
59
60 static void ioat_eh(struct ioatdma_chan *ioat_chan);
61
62 static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
63 {
64 int i;
65
66 for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) {
67 if ((chanerr >> i) & 1) {
68 dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
69 i, chanerr_str[i]);
70 }
71 }
72 }
73
74 /**
75 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
76 * @irq: interrupt id
77 * @data: interrupt data
78 */
79 irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
80 {
81 struct ioatdma_device *instance = data;
82 struct ioatdma_chan *ioat_chan;
83 unsigned long attnstatus;
84 int bit;
85 u8 intrctrl;
86
87 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
88
89 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
90 return IRQ_NONE;
91
92 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
93 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
94 return IRQ_NONE;
95 }
96
97 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
98 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
99 ioat_chan = ioat_chan_by_index(instance, bit);
100 if (test_bit(IOAT_RUN, &ioat_chan->state))
101 tasklet_schedule(&ioat_chan->cleanup_task);
102 }
103
104 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
105 return IRQ_HANDLED;
106 }
107
108 /**
109 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
110 * @irq: interrupt id
111 * @data: interrupt data
112 */
113 irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
114 {
115 struct ioatdma_chan *ioat_chan = data;
116
117 if (test_bit(IOAT_RUN, &ioat_chan->state))
118 tasklet_schedule(&ioat_chan->cleanup_task);
119
120 return IRQ_HANDLED;
121 }
122
123 void ioat_stop(struct ioatdma_chan *ioat_chan)
124 {
125 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
126 struct pci_dev *pdev = ioat_dma->pdev;
127 int chan_id = chan_num(ioat_chan);
128 struct msix_entry *msix;
129
130 /* 1/ stop irq from firing tasklets
131 * 2/ stop the tasklet from re-arming irqs
132 */
133 clear_bit(IOAT_RUN, &ioat_chan->state);
134
135 /* flush inflight interrupts */
136 switch (ioat_dma->irq_mode) {
137 case IOAT_MSIX:
138 msix = &ioat_dma->msix_entries[chan_id];
139 synchronize_irq(msix->vector);
140 break;
141 case IOAT_MSI:
142 case IOAT_INTX:
143 synchronize_irq(pdev->irq);
144 break;
145 default:
146 break;
147 }
148
149 /* flush inflight timers */
150 del_timer_sync(&ioat_chan->timer);
151
152 /* flush inflight tasklet runs */
153 tasklet_kill(&ioat_chan->cleanup_task);
154
155 /* final cleanup now that everything is quiesced and can't re-arm */
156 ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan);
157 }
158
159 static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
160 {
161 ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
162 ioat_chan->issued = ioat_chan->head;
163 writew(ioat_chan->dmacount,
164 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
165 dev_dbg(to_dev(ioat_chan),
166 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
167 __func__, ioat_chan->head, ioat_chan->tail,
168 ioat_chan->issued, ioat_chan->dmacount);
169 }
170
171 void ioat_issue_pending(struct dma_chan *c)
172 {
173 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
174
175 if (ioat_ring_pending(ioat_chan)) {
176 spin_lock_bh(&ioat_chan->prep_lock);
177 __ioat_issue_pending(ioat_chan);
178 spin_unlock_bh(&ioat_chan->prep_lock);
179 }
180 }
181
182 /**
183 * ioat_update_pending - log pending descriptors
184 * @ioat: ioat+ channel
185 *
186 * Check if the number of unsubmitted descriptors has exceeded the
187 * watermark. Called with prep_lock held
188 */
189 static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
190 {
191 if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
192 __ioat_issue_pending(ioat_chan);
193 }
194
195 static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
196 {
197 struct ioat_ring_ent *desc;
198 struct ioat_dma_descriptor *hw;
199
200 if (ioat_ring_space(ioat_chan) < 1) {
201 dev_err(to_dev(ioat_chan),
202 "Unable to start null desc - ring full\n");
203 return;
204 }
205
206 dev_dbg(to_dev(ioat_chan),
207 "%s: head: %#x tail: %#x issued: %#x\n",
208 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
209 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
210
211 hw = desc->hw;
212 hw->ctl = 0;
213 hw->ctl_f.null = 1;
214 hw->ctl_f.int_en = 1;
215 hw->ctl_f.compl_write = 1;
216 /* set size to non-zero value (channel returns error when size is 0) */
217 hw->size = NULL_DESC_BUFFER_SIZE;
218 hw->src_addr = 0;
219 hw->dst_addr = 0;
220 async_tx_ack(&desc->txd);
221 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
222 dump_desc_dbg(ioat_chan, desc);
223 /* make sure descriptors are written before we submit */
224 wmb();
225 ioat_chan->head += 1;
226 __ioat_issue_pending(ioat_chan);
227 }
228
229 void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
230 {
231 spin_lock_bh(&ioat_chan->prep_lock);
232 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
233 __ioat_start_null_desc(ioat_chan);
234 spin_unlock_bh(&ioat_chan->prep_lock);
235 }
236
237 static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
238 {
239 /* set the tail to be re-issued */
240 ioat_chan->issued = ioat_chan->tail;
241 ioat_chan->dmacount = 0;
242 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
243
244 dev_dbg(to_dev(ioat_chan),
245 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
246 __func__, ioat_chan->head, ioat_chan->tail,
247 ioat_chan->issued, ioat_chan->dmacount);
248
249 if (ioat_ring_pending(ioat_chan)) {
250 struct ioat_ring_ent *desc;
251
252 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
253 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
254 __ioat_issue_pending(ioat_chan);
255 } else
256 __ioat_start_null_desc(ioat_chan);
257 }
258
259 static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
260 {
261 unsigned long end = jiffies + tmo;
262 int err = 0;
263 u32 status;
264
265 status = ioat_chansts(ioat_chan);
266 if (is_ioat_active(status) || is_ioat_idle(status))
267 ioat_suspend(ioat_chan);
268 while (is_ioat_active(status) || is_ioat_idle(status)) {
269 if (tmo && time_after(jiffies, end)) {
270 err = -ETIMEDOUT;
271 break;
272 }
273 status = ioat_chansts(ioat_chan);
274 cpu_relax();
275 }
276
277 return err;
278 }
279
280 static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
281 {
282 unsigned long end = jiffies + tmo;
283 int err = 0;
284
285 ioat_reset(ioat_chan);
286 while (ioat_reset_pending(ioat_chan)) {
287 if (end && time_after(jiffies, end)) {
288 err = -ETIMEDOUT;
289 break;
290 }
291 cpu_relax();
292 }
293
294 return err;
295 }
296
297 static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
298 __releases(&ioat_chan->prep_lock)
299 {
300 struct dma_chan *c = tx->chan;
301 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
302 dma_cookie_t cookie;
303
304 cookie = dma_cookie_assign(tx);
305 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
306
307 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
308 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
309
310 /* make descriptor updates visible before advancing ioat->head,
311 * this is purposefully not smp_wmb() since we are also
312 * publishing the descriptor updates to a dma device
313 */
314 wmb();
315
316 ioat_chan->head += ioat_chan->produce;
317
318 ioat_update_pending(ioat_chan);
319 spin_unlock_bh(&ioat_chan->prep_lock);
320
321 return cookie;
322 }
323
324 static struct ioat_ring_ent *
325 ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
326 {
327 struct ioat_dma_descriptor *hw;
328 struct ioat_ring_ent *desc;
329 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
330 int chunk;
331 dma_addr_t phys;
332 u8 *pos;
333 off_t offs;
334
335 chunk = idx / IOAT_DESCS_PER_2M;
336 idx &= (IOAT_DESCS_PER_2M - 1);
337 offs = idx * IOAT_DESC_SZ;
338 pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
339 phys = ioat_chan->descs[chunk].hw + offs;
340 hw = (struct ioat_dma_descriptor *)pos;
341 memset(hw, 0, sizeof(*hw));
342
343 desc = kmem_cache_zalloc(ioat_cache, flags);
344 if (!desc)
345 return NULL;
346
347 dma_async_tx_descriptor_init(&desc->txd, chan);
348 desc->txd.tx_submit = ioat_tx_submit_unlock;
349 desc->hw = hw;
350 desc->txd.phys = phys;
351 return desc;
352 }
353
354 void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
355 {
356 kmem_cache_free(ioat_cache, desc);
357 }
358
359 struct ioat_ring_ent **
360 ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
361 {
362 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
363 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
364 struct ioat_ring_ent **ring;
365 int total_descs = 1 << order;
366 int i, chunks;
367
368 /* allocate the array to hold the software ring */
369 ring = kcalloc(total_descs, sizeof(*ring), flags);
370 if (!ring)
371 return NULL;
372
373 ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M;
374
375 for (i = 0; i < chunks; i++) {
376 struct ioat_descs *descs = &ioat_chan->descs[i];
377
378 descs->virt = dma_alloc_coherent(to_dev(ioat_chan),
379 SZ_2M, &descs->hw, flags);
380 if (!descs->virt && (i > 0)) {
381 int idx;
382
383 for (idx = 0; idx < i; idx++) {
384 dma_free_coherent(to_dev(ioat_chan), SZ_2M,
385 descs->virt, descs->hw);
386 descs->virt = NULL;
387 descs->hw = 0;
388 }
389
390 ioat_chan->desc_chunks = 0;
391 kfree(ring);
392 return NULL;
393 }
394 }
395
396 for (i = 0; i < total_descs; i++) {
397 ring[i] = ioat_alloc_ring_ent(c, i, flags);
398 if (!ring[i]) {
399 int idx;
400
401 while (i--)
402 ioat_free_ring_ent(ring[i], c);
403
404 for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
405 dma_free_coherent(to_dev(ioat_chan),
406 SZ_2M,
407 ioat_chan->descs[idx].virt,
408 ioat_chan->descs[idx].hw);
409 ioat_chan->descs[idx].virt = NULL;
410 ioat_chan->descs[idx].hw = 0;
411 }
412
413 ioat_chan->desc_chunks = 0;
414 kfree(ring);
415 return NULL;
416 }
417 set_desc_id(ring[i], i);
418 }
419
420 /* link descs */
421 for (i = 0; i < total_descs-1; i++) {
422 struct ioat_ring_ent *next = ring[i+1];
423 struct ioat_dma_descriptor *hw = ring[i]->hw;
424
425 hw->next = next->txd.phys;
426 }
427 ring[i]->hw->next = ring[0]->txd.phys;
428
429 /* setup descriptor pre-fetching for v3.4 */
430 if (ioat_dma->cap & IOAT_CAP_DPS) {
431 u16 drsctl = IOAT_CHAN_DRSZ_2MB | IOAT_CHAN_DRS_EN;
432
433 if (chunks == 1)
434 drsctl |= IOAT_CHAN_DRS_AUTOWRAP;
435
436 writew(drsctl, ioat_chan->reg_base + IOAT_CHAN_DRSCTL_OFFSET);
437
438 }
439
440 return ring;
441 }
442
443 /**
444 * ioat_check_space_lock - verify space and grab ring producer lock
445 * @ioat: ioat,3 channel (ring) to operate on
446 * @num_descs: allocation length
447 */
448 int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
449 __acquires(&ioat_chan->prep_lock)
450 {
451 spin_lock_bh(&ioat_chan->prep_lock);
452 /* never allow the last descriptor to be consumed, we need at
453 * least one free at all times to allow for on-the-fly ring
454 * resizing.
455 */
456 if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
457 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
458 __func__, num_descs, ioat_chan->head,
459 ioat_chan->tail, ioat_chan->issued);
460 ioat_chan->produce = num_descs;
461 return 0; /* with ioat->prep_lock held */
462 }
463 spin_unlock_bh(&ioat_chan->prep_lock);
464
465 dev_dbg_ratelimited(to_dev(ioat_chan),
466 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
467 __func__, num_descs, ioat_chan->head,
468 ioat_chan->tail, ioat_chan->issued);
469
470 /* progress reclaim in the allocation failure case we may be
471 * called under bh_disabled so we need to trigger the timer
472 * event directly
473 */
474 if (time_is_before_jiffies(ioat_chan->timer.expires)
475 && timer_pending(&ioat_chan->timer)) {
476 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
477 ioat_timer_event(&ioat_chan->timer);
478 }
479
480 return -ENOMEM;
481 }
482
483 static bool desc_has_ext(struct ioat_ring_ent *desc)
484 {
485 struct ioat_dma_descriptor *hw = desc->hw;
486
487 if (hw->ctl_f.op == IOAT_OP_XOR ||
488 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
489 struct ioat_xor_descriptor *xor = desc->xor;
490
491 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
492 return true;
493 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
494 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
495 struct ioat_pq_descriptor *pq = desc->pq;
496
497 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
498 return true;
499 }
500
501 return false;
502 }
503
504 static void
505 ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
506 {
507 if (!sed)
508 return;
509
510 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
511 kmem_cache_free(ioat_sed_cache, sed);
512 }
513
514 static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
515 {
516 u64 phys_complete;
517 u64 completion;
518
519 completion = *ioat_chan->completion;
520 phys_complete = ioat_chansts_to_addr(completion);
521
522 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
523 (unsigned long long) phys_complete);
524
525 return phys_complete;
526 }
527
528 static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
529 u64 *phys_complete)
530 {
531 *phys_complete = ioat_get_current_completion(ioat_chan);
532 if (*phys_complete == ioat_chan->last_completion)
533 return false;
534
535 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
536 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
537
538 return true;
539 }
540
541 static void
542 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
543 {
544 struct ioat_dma_descriptor *hw = desc->hw;
545
546 switch (hw->ctl_f.op) {
547 case IOAT_OP_PQ_VAL:
548 case IOAT_OP_PQ_VAL_16S:
549 {
550 struct ioat_pq_descriptor *pq = desc->pq;
551
552 /* check if there's error written */
553 if (!pq->dwbes_f.wbes)
554 return;
555
556 /* need to set a chanerr var for checking to clear later */
557
558 if (pq->dwbes_f.p_val_err)
559 *desc->result |= SUM_CHECK_P_RESULT;
560
561 if (pq->dwbes_f.q_val_err)
562 *desc->result |= SUM_CHECK_Q_RESULT;
563
564 return;
565 }
566 default:
567 return;
568 }
569 }
570
571 /**
572 * __cleanup - reclaim used descriptors
573 * @ioat: channel (ring) to clean
574 */
575 static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
576 {
577 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
578 struct ioat_ring_ent *desc;
579 bool seen_current = false;
580 int idx = ioat_chan->tail, i;
581 u16 active;
582
583 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
584 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
585
586 /*
587 * At restart of the channel, the completion address and the
588 * channel status will be 0 due to starting a new chain. Since
589 * it's new chain and the first descriptor "fails", there is
590 * nothing to clean up. We do not want to reap the entire submitted
591 * chain due to this 0 address value and then BUG.
592 */
593 if (!phys_complete)
594 return;
595
596 active = ioat_ring_active(ioat_chan);
597 for (i = 0; i < active && !seen_current; i++) {
598 struct dma_async_tx_descriptor *tx;
599
600 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
601 desc = ioat_get_ring_ent(ioat_chan, idx + i);
602 dump_desc_dbg(ioat_chan, desc);
603
604 /* set err stat if we are using dwbes */
605 if (ioat_dma->cap & IOAT_CAP_DWBES)
606 desc_get_errstat(ioat_chan, desc);
607
608 tx = &desc->txd;
609 if (tx->cookie) {
610 dma_cookie_complete(tx);
611 dma_descriptor_unmap(tx);
612 dmaengine_desc_get_callback_invoke(tx, NULL);
613 tx->callback = NULL;
614 tx->callback_result = NULL;
615 }
616
617 if (tx->phys == phys_complete)
618 seen_current = true;
619
620 /* skip extended descriptors */
621 if (desc_has_ext(desc)) {
622 BUG_ON(i + 1 >= active);
623 i++;
624 }
625
626 /* cleanup super extended descriptors */
627 if (desc->sed) {
628 ioat_free_sed(ioat_dma, desc->sed);
629 desc->sed = NULL;
630 }
631 }
632
633 /* finish all descriptor reads before incrementing tail */
634 smp_mb();
635 ioat_chan->tail = idx + i;
636 /* no active descs have written a completion? */
637 BUG_ON(active && !seen_current);
638 ioat_chan->last_completion = phys_complete;
639
640 if (active - i == 0) {
641 dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
642 __func__);
643 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
644 }
645
646 /* microsecond delay by sysfs variable per pending descriptor */
647 if (ioat_chan->intr_coalesce != ioat_chan->prev_intr_coalesce) {
648 writew(min((ioat_chan->intr_coalesce * (active - i)),
649 IOAT_INTRDELAY_MASK),
650 ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
651 ioat_chan->prev_intr_coalesce = ioat_chan->intr_coalesce;
652 }
653 }
654
655 static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
656 {
657 u64 phys_complete;
658
659 spin_lock_bh(&ioat_chan->cleanup_lock);
660
661 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
662 __cleanup(ioat_chan, phys_complete);
663
664 if (is_ioat_halted(*ioat_chan->completion)) {
665 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
666
667 if (chanerr &
668 (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) {
669 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
670 ioat_eh(ioat_chan);
671 }
672 }
673
674 spin_unlock_bh(&ioat_chan->cleanup_lock);
675 }
676
677 void ioat_cleanup_event(unsigned long data)
678 {
679 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
680
681 ioat_cleanup(ioat_chan);
682 if (!test_bit(IOAT_RUN, &ioat_chan->state))
683 return;
684 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
685 }
686
687 static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
688 {
689 u64 phys_complete;
690
691 /* set the completion address register again */
692 writel(lower_32_bits(ioat_chan->completion_dma),
693 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
694 writel(upper_32_bits(ioat_chan->completion_dma),
695 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
696
697 ioat_quiesce(ioat_chan, 0);
698 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
699 __cleanup(ioat_chan, phys_complete);
700
701 __ioat_restart_chan(ioat_chan);
702 }
703
704
705 static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
706 {
707 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
708 struct ioat_ring_ent *desc;
709 u16 active;
710 int idx = ioat_chan->tail, i;
711
712 /*
713 * We assume that the failed descriptor has been processed.
714 * Now we are just returning all the remaining submitted
715 * descriptors to abort.
716 */
717 active = ioat_ring_active(ioat_chan);
718
719 /* we skip the failed descriptor that tail points to */
720 for (i = 1; i < active; i++) {
721 struct dma_async_tx_descriptor *tx;
722
723 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
724 desc = ioat_get_ring_ent(ioat_chan, idx + i);
725
726 tx = &desc->txd;
727 if (tx->cookie) {
728 struct dmaengine_result res;
729
730 dma_cookie_complete(tx);
731 dma_descriptor_unmap(tx);
732 res.result = DMA_TRANS_ABORTED;
733 dmaengine_desc_get_callback_invoke(tx, &res);
734 tx->callback = NULL;
735 tx->callback_result = NULL;
736 }
737
738 /* skip extended descriptors */
739 if (desc_has_ext(desc)) {
740 WARN_ON(i + 1 >= active);
741 i++;
742 }
743
744 /* cleanup super extended descriptors */
745 if (desc->sed) {
746 ioat_free_sed(ioat_dma, desc->sed);
747 desc->sed = NULL;
748 }
749 }
750
751 smp_mb(); /* finish all descriptor reads before incrementing tail */
752 ioat_chan->tail = idx + active;
753
754 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
755 ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys;
756 }
757
758 static void ioat_eh(struct ioatdma_chan *ioat_chan)
759 {
760 struct pci_dev *pdev = to_pdev(ioat_chan);
761 struct ioat_dma_descriptor *hw;
762 struct dma_async_tx_descriptor *tx;
763 u64 phys_complete;
764 struct ioat_ring_ent *desc;
765 u32 err_handled = 0;
766 u32 chanerr_int;
767 u32 chanerr;
768 bool abort = false;
769 struct dmaengine_result res;
770
771 /* cleanup so tail points to descriptor that caused the error */
772 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
773 __cleanup(ioat_chan, phys_complete);
774
775 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
776 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
777
778 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
779 __func__, chanerr, chanerr_int);
780
781 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
782 hw = desc->hw;
783 dump_desc_dbg(ioat_chan, desc);
784
785 switch (hw->ctl_f.op) {
786 case IOAT_OP_XOR_VAL:
787 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
788 *desc->result |= SUM_CHECK_P_RESULT;
789 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
790 }
791 break;
792 case IOAT_OP_PQ_VAL:
793 case IOAT_OP_PQ_VAL_16S:
794 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
795 *desc->result |= SUM_CHECK_P_RESULT;
796 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
797 }
798 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
799 *desc->result |= SUM_CHECK_Q_RESULT;
800 err_handled |= IOAT_CHANERR_XOR_Q_ERR;
801 }
802 break;
803 }
804
805 if (chanerr & IOAT_CHANERR_RECOVER_MASK) {
806 if (chanerr & IOAT_CHANERR_READ_DATA_ERR) {
807 res.result = DMA_TRANS_READ_FAILED;
808 err_handled |= IOAT_CHANERR_READ_DATA_ERR;
809 } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) {
810 res.result = DMA_TRANS_WRITE_FAILED;
811 err_handled |= IOAT_CHANERR_WRITE_DATA_ERR;
812 }
813
814 abort = true;
815 } else
816 res.result = DMA_TRANS_NOERROR;
817
818 /* fault on unhandled error or spurious halt */
819 if (chanerr ^ err_handled || chanerr == 0) {
820 dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
821 __func__, chanerr, err_handled);
822 dev_err(to_dev(ioat_chan), "Errors handled:\n");
823 ioat_print_chanerrs(ioat_chan, err_handled);
824 dev_err(to_dev(ioat_chan), "Errors not handled:\n");
825 ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled));
826
827 BUG();
828 }
829
830 /* cleanup the faulty descriptor since we are continuing */
831 tx = &desc->txd;
832 if (tx->cookie) {
833 dma_cookie_complete(tx);
834 dma_descriptor_unmap(tx);
835 dmaengine_desc_get_callback_invoke(tx, &res);
836 tx->callback = NULL;
837 tx->callback_result = NULL;
838 }
839
840 /* mark faulting descriptor as complete */
841 *ioat_chan->completion = desc->txd.phys;
842
843 spin_lock_bh(&ioat_chan->prep_lock);
844 /* we need abort all descriptors */
845 if (abort) {
846 ioat_abort_descs(ioat_chan);
847 /* clean up the channel, we could be in weird state */
848 ioat_reset_hw(ioat_chan);
849 }
850
851 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
852 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
853
854 ioat_restart_channel(ioat_chan);
855 spin_unlock_bh(&ioat_chan->prep_lock);
856 }
857
858 static void check_active(struct ioatdma_chan *ioat_chan)
859 {
860 if (ioat_ring_active(ioat_chan)) {
861 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
862 return;
863 }
864
865 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
866 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
867 }
868
869 void ioat_timer_event(struct timer_list *t)
870 {
871 struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer);
872 dma_addr_t phys_complete;
873 u64 status;
874
875 status = ioat_chansts(ioat_chan);
876
877 /* when halted due to errors check for channel
878 * programming errors before advancing the completion state
879 */
880 if (is_ioat_halted(status)) {
881 u32 chanerr;
882
883 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
884 dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
885 __func__, chanerr);
886 dev_err(to_dev(ioat_chan), "Errors:\n");
887 ioat_print_chanerrs(ioat_chan, chanerr);
888
889 if (test_bit(IOAT_RUN, &ioat_chan->state)) {
890 spin_lock_bh(&ioat_chan->cleanup_lock);
891 spin_lock_bh(&ioat_chan->prep_lock);
892 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
893 spin_unlock_bh(&ioat_chan->prep_lock);
894
895 ioat_abort_descs(ioat_chan);
896 dev_warn(to_dev(ioat_chan), "Reset channel...\n");
897 ioat_reset_hw(ioat_chan);
898 dev_warn(to_dev(ioat_chan), "Restart channel...\n");
899 ioat_restart_channel(ioat_chan);
900
901 spin_lock_bh(&ioat_chan->prep_lock);
902 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
903 spin_unlock_bh(&ioat_chan->prep_lock);
904 spin_unlock_bh(&ioat_chan->cleanup_lock);
905 }
906
907 return;
908 }
909
910 spin_lock_bh(&ioat_chan->cleanup_lock);
911
912 /* handle the no-actives case */
913 if (!ioat_ring_active(ioat_chan)) {
914 spin_lock_bh(&ioat_chan->prep_lock);
915 check_active(ioat_chan);
916 spin_unlock_bh(&ioat_chan->prep_lock);
917 spin_unlock_bh(&ioat_chan->cleanup_lock);
918 return;
919 }
920
921 /* if we haven't made progress and we have already
922 * acknowledged a pending completion once, then be more
923 * forceful with a restart
924 */
925 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
926 __cleanup(ioat_chan, phys_complete);
927 else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
928 u32 chanerr;
929
930 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
931 dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n",
932 status, chanerr);
933 dev_err(to_dev(ioat_chan), "Errors:\n");
934 ioat_print_chanerrs(ioat_chan, chanerr);
935
936 dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
937 ioat_ring_active(ioat_chan));
938
939 spin_lock_bh(&ioat_chan->prep_lock);
940 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
941 spin_unlock_bh(&ioat_chan->prep_lock);
942
943 ioat_abort_descs(ioat_chan);
944 dev_warn(to_dev(ioat_chan), "Resetting channel...\n");
945 ioat_reset_hw(ioat_chan);
946 dev_warn(to_dev(ioat_chan), "Restarting channel...\n");
947 ioat_restart_channel(ioat_chan);
948
949 spin_lock_bh(&ioat_chan->prep_lock);
950 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
951 spin_unlock_bh(&ioat_chan->prep_lock);
952 spin_unlock_bh(&ioat_chan->cleanup_lock);
953 return;
954 } else
955 set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
956
957 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
958 spin_unlock_bh(&ioat_chan->cleanup_lock);
959 }
960
961 enum dma_status
962 ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
963 struct dma_tx_state *txstate)
964 {
965 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
966 enum dma_status ret;
967
968 ret = dma_cookie_status(c, cookie, txstate);
969 if (ret == DMA_COMPLETE)
970 return ret;
971
972 ioat_cleanup(ioat_chan);
973
974 return dma_cookie_status(c, cookie, txstate);
975 }
976
977 int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
978 {
979 /* throw away whatever the channel was doing and get it
980 * initialized, with ioat3 specific workarounds
981 */
982 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
983 struct pci_dev *pdev = ioat_dma->pdev;
984 u32 chanerr;
985 u16 dev_id;
986 int err;
987
988 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
989
990 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
991 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
992
993 if (ioat_dma->version < IOAT_VER_3_3) {
994 /* clear any pending errors */
995 err = pci_read_config_dword(pdev,
996 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
997 if (err) {
998 dev_err(&pdev->dev,
999 "channel error register unreachable\n");
1000 return err;
1001 }
1002 pci_write_config_dword(pdev,
1003 IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1004
1005 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1006 * (workaround for spurious config parity error after restart)
1007 */
1008 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1009 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
1010 pci_write_config_dword(pdev,
1011 IOAT_PCI_DMAUNCERRSTS_OFFSET,
1012 0x10);
1013 }
1014 }
1015
1016 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1017 ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000);
1018 ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008);
1019 ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800);
1020 }
1021
1022
1023 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
1024 if (!err) {
1025 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1026 writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000);
1027 writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008);
1028 writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800);
1029 }
1030 }
1031
1032 if (err)
1033 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1034
1035 return err;
1036 }
Linux with added FPC-III support