?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
HID: convert to dev_* prints
[linux-2.6/next.git] / drivers / dma / iop-adma.c
blob71fba82462cb98372568a2f2da6c8c325e1a10d8
1 /*
2 * offload engine driver for the Intel Xscale series of i/o processors
3 * Copyright © 2006, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 */
19
20 /*
21 * This driver supports the asynchrounous DMA copy and RAID engines available
22 * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
23 */
24
25 #include <linux/init.h>
26 #include <linux/module.h>
27 #include <linux/async_tx.h>
28 #include <linux/delay.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/spinlock.h>
31 #include <linux/interrupt.h>
32 #include <linux/platform_device.h>
33 #include <linux/memory.h>
34 #include <linux/ioport.h>
35
36 #include <mach/adma.h>
37
38 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
39 #define to_iop_adma_device(dev) \
40 container_of(dev, struct iop_adma_device, common)
41 #define tx_to_iop_adma_slot(tx) \
42 container_of(tx, struct iop_adma_desc_slot, async_tx)
43
44 /**
45 * iop_adma_free_slots - flags descriptor slots for reuse
46 * @slot: Slot to free
47 * Caller must hold &iop_chan->lock while calling this function
48 */
49 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
50 {
51 int stride = slot->slots_per_op;
52
53 while (stride--) {
54 slot->slots_per_op = 0;
55 slot = list_entry(slot->slot_node.next,
56 struct iop_adma_desc_slot,
57 slot_node);
58 }
59 }
60
61 static dma_cookie_t
62 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
63 struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
64 {
65 BUG_ON(desc->async_tx.cookie < 0);
66 if (desc->async_tx.cookie > 0) {
67 cookie = desc->async_tx.cookie;
68 desc->async_tx.cookie = 0;
69
70 /* call the callback (must not sleep or submit new
71 * operations to this channel)
72 */
73 if (desc->async_tx.callback)
74 desc->async_tx.callback(
75 desc->async_tx.callback_param);
76
77 /* unmap dma addresses
78 * (unmap_single vs unmap_page?)
79 */
80 if (desc->group_head && desc->unmap_len) {
81 struct iop_adma_desc_slot *unmap = desc->group_head;
82 struct device *dev =
83 &iop_chan->device->pdev->dev;
84 u32 len = unmap->unmap_len;
85 enum dma_ctrl_flags flags = desc->async_tx.flags;
86 u32 src_cnt;
87 dma_addr_t addr;
88
89 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
90 addr = iop_desc_get_dest_addr(unmap, iop_chan);
91 dma_unmap_page(dev, addr, len, DMA_FROM_DEVICE);
92 }
93
94 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
95 src_cnt = unmap->unmap_src_cnt;
96 while (src_cnt--) {
97 addr = iop_desc_get_src_addr(unmap,
98 iop_chan,
99 src_cnt);
100 dma_unmap_page(dev, addr, len,
101 DMA_TO_DEVICE);
102 }
103 }
104 desc->group_head = NULL;
105 }
106 }
107
108 /* run dependent operations */
109 async_tx_run_dependencies(&desc->async_tx);
110
111 return cookie;
112 }
113
114 static int
115 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
116 struct iop_adma_chan *iop_chan)
117 {
118 /* the client is allowed to attach dependent operations
119 * until 'ack' is set
120 */
121 if (!async_tx_test_ack(&desc->async_tx))
122 return 0;
123
124 /* leave the last descriptor in the chain
125 * so we can append to it
126 */
127 if (desc->chain_node.next == &iop_chan->chain)
128 return 1;
129
130 dev_dbg(iop_chan->device->common.dev,
131 "\tfree slot: %d slots_per_op: %d\n",
132 desc->idx, desc->slots_per_op);
133
134 list_del(&desc->chain_node);
135 iop_adma_free_slots(desc);
136
137 return 0;
138 }
139
140 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
141 {
142 struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
143 dma_cookie_t cookie = 0;
144 u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
145 int busy = iop_chan_is_busy(iop_chan);
146 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
147
148 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
149 /* free completed slots from the chain starting with
150 * the oldest descriptor
151 */
152 list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
153 chain_node) {
154 pr_debug("\tcookie: %d slot: %d busy: %d "
155 "this_desc: %#x next_desc: %#x ack: %d\n",
156 iter->async_tx.cookie, iter->idx, busy,
157 iter->async_tx.phys, iop_desc_get_next_desc(iter),
158 async_tx_test_ack(&iter->async_tx));
159 prefetch(_iter);
160 prefetch(&_iter->async_tx);
161
162 /* do not advance past the current descriptor loaded into the
163 * hardware channel, subsequent descriptors are either in
164 * process or have not been submitted
165 */
166 if (seen_current)
167 break;
168
169 /* stop the search if we reach the current descriptor and the
170 * channel is busy, or if it appears that the current descriptor
171 * needs to be re-read (i.e. has been appended to)
172 */
173 if (iter->async_tx.phys == current_desc) {
174 BUG_ON(seen_current++);
175 if (busy || iop_desc_get_next_desc(iter))
176 break;
177 }
178
179 /* detect the start of a group transaction */
180 if (!slot_cnt && !slots_per_op) {
181 slot_cnt = iter->slot_cnt;
182 slots_per_op = iter->slots_per_op;
183 if (slot_cnt <= slots_per_op) {
184 slot_cnt = 0;
185 slots_per_op = 0;
186 }
187 }
188
189 if (slot_cnt) {
190 pr_debug("\tgroup++\n");
191 if (!grp_start)
192 grp_start = iter;
193 slot_cnt -= slots_per_op;
194 }
195
196 /* all the members of a group are complete */
197 if (slots_per_op != 0 && slot_cnt == 0) {
198 struct iop_adma_desc_slot *grp_iter, *_grp_iter;
199 int end_of_chain = 0;
200 pr_debug("\tgroup end\n");
201
202 /* collect the total results */
203 if (grp_start->xor_check_result) {
204 u32 zero_sum_result = 0;
205 slot_cnt = grp_start->slot_cnt;
206 grp_iter = grp_start;
207
208 list_for_each_entry_from(grp_iter,
209 &iop_chan->chain, chain_node) {
210 zero_sum_result |=
211 iop_desc_get_zero_result(grp_iter);
212 pr_debug("\titer%d result: %d\n",
213 grp_iter->idx, zero_sum_result);
214 slot_cnt -= slots_per_op;
215 if (slot_cnt == 0)
216 break;
217 }
218 pr_debug("\tgrp_start->xor_check_result: %p\n",
219 grp_start->xor_check_result);
220 *grp_start->xor_check_result = zero_sum_result;
221 }
222
223 /* clean up the group */
224 slot_cnt = grp_start->slot_cnt;
225 grp_iter = grp_start;
226 list_for_each_entry_safe_from(grp_iter, _grp_iter,
227 &iop_chan->chain, chain_node) {
228 cookie = iop_adma_run_tx_complete_actions(
229 grp_iter, iop_chan, cookie);
230
231 slot_cnt -= slots_per_op;
232 end_of_chain = iop_adma_clean_slot(grp_iter,
233 iop_chan);
234
235 if (slot_cnt == 0 || end_of_chain)
236 break;
237 }
238
239 /* the group should be complete at this point */
240 BUG_ON(slot_cnt);
241
242 slots_per_op = 0;
243 grp_start = NULL;
244 if (end_of_chain)
245 break;
246 else
247 continue;
248 } else if (slots_per_op) /* wait for group completion */
249 continue;
250
251 /* write back zero sum results (single descriptor case) */
252 if (iter->xor_check_result && iter->async_tx.cookie)
253 *iter->xor_check_result =
254 iop_desc_get_zero_result(iter);
255
256 cookie = iop_adma_run_tx_complete_actions(
257 iter, iop_chan, cookie);
258
259 if (iop_adma_clean_slot(iter, iop_chan))
260 break;
261 }
262
263 BUG_ON(!seen_current);
264
265 if (cookie > 0) {
266 iop_chan->completed_cookie = cookie;
267 pr_debug("\tcompleted cookie %d\n", cookie);
268 }
269 }
270
271 static void
272 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
273 {
274 spin_lock_bh(&iop_chan->lock);
275 __iop_adma_slot_cleanup(iop_chan);
276 spin_unlock_bh(&iop_chan->lock);
277 }
278
279 static void iop_adma_tasklet(unsigned long data)
280 {
281 struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
282
283 spin_lock(&iop_chan->lock);
284 __iop_adma_slot_cleanup(iop_chan);
285 spin_unlock(&iop_chan->lock);
286 }
287
288 static struct iop_adma_desc_slot *
289 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
290 int slots_per_op)
291 {
292 struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
293 LIST_HEAD(chain);
294 int slots_found, retry = 0;
295
296 /* start search from the last allocated descrtiptor
297 * if a contiguous allocation can not be found start searching
298 * from the beginning of the list
299 */
300 retry:
301 slots_found = 0;
302 if (retry == 0)
303 iter = iop_chan->last_used;
304 else
305 iter = list_entry(&iop_chan->all_slots,
306 struct iop_adma_desc_slot,
307 slot_node);
308
309 list_for_each_entry_safe_continue(
310 iter, _iter, &iop_chan->all_slots, slot_node) {
311 prefetch(_iter);
312 prefetch(&_iter->async_tx);
313 if (iter->slots_per_op) {
314 /* give up after finding the first busy slot
315 * on the second pass through the list
316 */
317 if (retry)
318 break;
319
320 slots_found = 0;
321 continue;
322 }
323
324 /* start the allocation if the slot is correctly aligned */
325 if (!slots_found++) {
326 if (iop_desc_is_aligned(iter, slots_per_op))
327 alloc_start = iter;
328 else {
329 slots_found = 0;
330 continue;
331 }
332 }
333
334 if (slots_found == num_slots) {
335 struct iop_adma_desc_slot *alloc_tail = NULL;
336 struct iop_adma_desc_slot *last_used = NULL;
337 iter = alloc_start;
338 while (num_slots) {
339 int i;
340 dev_dbg(iop_chan->device->common.dev,
341 "allocated slot: %d "
342 "(desc %p phys: %#x) slots_per_op %d\n",
343 iter->idx, iter->hw_desc,
344 iter->async_tx.phys, slots_per_op);
345
346 /* pre-ack all but the last descriptor */
347 if (num_slots != slots_per_op)
348 async_tx_ack(&iter->async_tx);
349
350 list_add_tail(&iter->chain_node, &chain);
351 alloc_tail = iter;
352 iter->async_tx.cookie = 0;
353 iter->slot_cnt = num_slots;
354 iter->xor_check_result = NULL;
355 for (i = 0; i < slots_per_op; i++) {
356 iter->slots_per_op = slots_per_op - i;
357 last_used = iter;
358 iter = list_entry(iter->slot_node.next,
359 struct iop_adma_desc_slot,
360 slot_node);
361 }
362 num_slots -= slots_per_op;
363 }
364 alloc_tail->group_head = alloc_start;
365 alloc_tail->async_tx.cookie = -EBUSY;
366 list_splice(&chain, &alloc_tail->async_tx.tx_list);
367 iop_chan->last_used = last_used;
368 iop_desc_clear_next_desc(alloc_start);
369 iop_desc_clear_next_desc(alloc_tail);
370 return alloc_tail;
371 }
372 }
373 if (!retry++)
374 goto retry;
375
376 /* perform direct reclaim if the allocation fails */
377 __iop_adma_slot_cleanup(iop_chan);
378
379 return NULL;
380 }
381
382 static dma_cookie_t
383 iop_desc_assign_cookie(struct iop_adma_chan *iop_chan,
384 struct iop_adma_desc_slot *desc)
385 {
386 dma_cookie_t cookie = iop_chan->common.cookie;
387 cookie++;
388 if (cookie < 0)
389 cookie = 1;
390 iop_chan->common.cookie = desc->async_tx.cookie = cookie;
391 return cookie;
392 }
393
394 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
395 {
396 dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
397 iop_chan->pending);
398
399 if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
400 iop_chan->pending = 0;
401 iop_chan_append(iop_chan);
402 }
403 }
404
405 static dma_cookie_t
406 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
407 {
408 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
409 struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
410 struct iop_adma_desc_slot *grp_start, *old_chain_tail;
411 int slot_cnt;
412 int slots_per_op;
413 dma_cookie_t cookie;
414
415 grp_start = sw_desc->group_head;
416 slot_cnt = grp_start->slot_cnt;
417 slots_per_op = grp_start->slots_per_op;
418
419 spin_lock_bh(&iop_chan->lock);
420 cookie = iop_desc_assign_cookie(iop_chan, sw_desc);
421
422 old_chain_tail = list_entry(iop_chan->chain.prev,
423 struct iop_adma_desc_slot, chain_node);
424 list_splice_init(&sw_desc->async_tx.tx_list,
425 &old_chain_tail->chain_node);
426
427 /* fix up the hardware chain */
428 iop_desc_set_next_desc(old_chain_tail, grp_start->async_tx.phys);
429
430 /* 1/ don't add pre-chained descriptors
431 * 2/ dummy read to flush next_desc write
432 */
433 BUG_ON(iop_desc_get_next_desc(sw_desc));
434
435 /* increment the pending count by the number of slots
436 * memcpy operations have a 1:1 (slot:operation) relation
437 * other operations are heavier and will pop the threshold
438 * more often.
439 */
440 iop_chan->pending += slot_cnt;
441 iop_adma_check_threshold(iop_chan);
442 spin_unlock_bh(&iop_chan->lock);
443
444 dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
445 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
446
447 return cookie;
448 }
449
450 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
451 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
452
453 /**
454 * iop_adma_alloc_chan_resources - returns the number of allocated descriptors
455 * @chan - allocate descriptor resources for this channel
456 * @client - current client requesting the channel be ready for requests
457 *
458 * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To
459 * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
460 * greater than 2x the number slots needed to satisfy a device->max_xor
461 * request.
462 * */
463 static int iop_adma_alloc_chan_resources(struct dma_chan *chan,
464 struct dma_client *client)
465 {
466 char *hw_desc;
467 int idx;
468 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
469 struct iop_adma_desc_slot *slot = NULL;
470 int init = iop_chan->slots_allocated ? 0 : 1;
471 struct iop_adma_platform_data *plat_data =
472 iop_chan->device->pdev->dev.platform_data;
473 int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
474
475 /* Allocate descriptor slots */
476 do {
477 idx = iop_chan->slots_allocated;
478 if (idx == num_descs_in_pool)
479 break;
480
481 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
482 if (!slot) {
483 printk(KERN_INFO "IOP ADMA Channel only initialized"
484 " %d descriptor slots", idx);
485 break;
486 }
487 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
488 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
489
490 dma_async_tx_descriptor_init(&slot->async_tx, chan);
491 slot->async_tx.tx_submit = iop_adma_tx_submit;
492 INIT_LIST_HEAD(&slot->chain_node);
493 INIT_LIST_HEAD(&slot->slot_node);
494 INIT_LIST_HEAD(&slot->async_tx.tx_list);
495 hw_desc = (char *) iop_chan->device->dma_desc_pool;
496 slot->async_tx.phys =
497 (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
498 slot->idx = idx;
499
500 spin_lock_bh(&iop_chan->lock);
501 iop_chan->slots_allocated++;
502 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
503 spin_unlock_bh(&iop_chan->lock);
504 } while (iop_chan->slots_allocated < num_descs_in_pool);
505
506 if (idx && !iop_chan->last_used)
507 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
508 struct iop_adma_desc_slot,
509 slot_node);
510
511 dev_dbg(iop_chan->device->common.dev,
512 "allocated %d descriptor slots last_used: %p\n",
513 iop_chan->slots_allocated, iop_chan->last_used);
514
515 /* initialize the channel and the chain with a null operation */
516 if (init) {
517 if (dma_has_cap(DMA_MEMCPY,
518 iop_chan->device->common.cap_mask))
519 iop_chan_start_null_memcpy(iop_chan);
520 else if (dma_has_cap(DMA_XOR,
521 iop_chan->device->common.cap_mask))
522 iop_chan_start_null_xor(iop_chan);
523 else
524 BUG();
525 }
526
527 return (idx > 0) ? idx : -ENOMEM;
528 }
529
530 static struct dma_async_tx_descriptor *
531 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
532 {
533 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
534 struct iop_adma_desc_slot *sw_desc, *grp_start;
535 int slot_cnt, slots_per_op;
536
537 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
538
539 spin_lock_bh(&iop_chan->lock);
540 slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
541 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
542 if (sw_desc) {
543 grp_start = sw_desc->group_head;
544 iop_desc_init_interrupt(grp_start, iop_chan);
545 grp_start->unmap_len = 0;
546 sw_desc->async_tx.flags = flags;
547 }
548 spin_unlock_bh(&iop_chan->lock);
549
550 return sw_desc ? &sw_desc->async_tx : NULL;
551 }
552
553 static struct dma_async_tx_descriptor *
554 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
555 dma_addr_t dma_src, size_t len, unsigned long flags)
556 {
557 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
558 struct iop_adma_desc_slot *sw_desc, *grp_start;
559 int slot_cnt, slots_per_op;
560
561 if (unlikely(!len))
562 return NULL;
563 BUG_ON(unlikely(len > IOP_ADMA_MAX_BYTE_COUNT));
564
565 dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
566 __func__, len);
567
568 spin_lock_bh(&iop_chan->lock);
569 slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
570 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
571 if (sw_desc) {
572 grp_start = sw_desc->group_head;
573 iop_desc_init_memcpy(grp_start, flags);
574 iop_desc_set_byte_count(grp_start, iop_chan, len);
575 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
576 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
577 sw_desc->unmap_src_cnt = 1;
578 sw_desc->unmap_len = len;
579 sw_desc->async_tx.flags = flags;
580 }
581 spin_unlock_bh(&iop_chan->lock);
582
583 return sw_desc ? &sw_desc->async_tx : NULL;
584 }
585
586 static struct dma_async_tx_descriptor *
587 iop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest,
588 int value, size_t len, unsigned long flags)
589 {
590 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
591 struct iop_adma_desc_slot *sw_desc, *grp_start;
592 int slot_cnt, slots_per_op;
593
594 if (unlikely(!len))
595 return NULL;
596 BUG_ON(unlikely(len > IOP_ADMA_MAX_BYTE_COUNT));
597
598 dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
599 __func__, len);
600
601 spin_lock_bh(&iop_chan->lock);
602 slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op);
603 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
604 if (sw_desc) {
605 grp_start = sw_desc->group_head;
606 iop_desc_init_memset(grp_start, flags);
607 iop_desc_set_byte_count(grp_start, iop_chan, len);
608 iop_desc_set_block_fill_val(grp_start, value);
609 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
610 sw_desc->unmap_src_cnt = 1;
611 sw_desc->unmap_len = len;
612 sw_desc->async_tx.flags = flags;
613 }
614 spin_unlock_bh(&iop_chan->lock);
615
616 return sw_desc ? &sw_desc->async_tx : NULL;
617 }
618
619 static struct dma_async_tx_descriptor *
620 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
621 dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
622 unsigned long flags)
623 {
624 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
625 struct iop_adma_desc_slot *sw_desc, *grp_start;
626 int slot_cnt, slots_per_op;
627
628 if (unlikely(!len))
629 return NULL;
630 BUG_ON(unlikely(len > IOP_ADMA_XOR_MAX_BYTE_COUNT));
631
632 dev_dbg(iop_chan->device->common.dev,
633 "%s src_cnt: %d len: %u flags: %lx\n",
634 __func__, src_cnt, len, flags);
635
636 spin_lock_bh(&iop_chan->lock);
637 slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
638 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
639 if (sw_desc) {
640 grp_start = sw_desc->group_head;
641 iop_desc_init_xor(grp_start, src_cnt, flags);
642 iop_desc_set_byte_count(grp_start, iop_chan, len);
643 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
644 sw_desc->unmap_src_cnt = src_cnt;
645 sw_desc->unmap_len = len;
646 sw_desc->async_tx.flags = flags;
647 while (src_cnt--)
648 iop_desc_set_xor_src_addr(grp_start, src_cnt,
649 dma_src[src_cnt]);
650 }
651 spin_unlock_bh(&iop_chan->lock);
652
653 return sw_desc ? &sw_desc->async_tx : NULL;
654 }
655
656 static struct dma_async_tx_descriptor *
657 iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src,
658 unsigned int src_cnt, size_t len, u32 *result,
659 unsigned long flags)
660 {
661 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
662 struct iop_adma_desc_slot *sw_desc, *grp_start;
663 int slot_cnt, slots_per_op;
664
665 if (unlikely(!len))
666 return NULL;
667
668 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
669 __func__, src_cnt, len);
670
671 spin_lock_bh(&iop_chan->lock);
672 slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
673 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
674 if (sw_desc) {
675 grp_start = sw_desc->group_head;
676 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
677 iop_desc_set_zero_sum_byte_count(grp_start, len);
678 grp_start->xor_check_result = result;
679 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
680 __func__, grp_start->xor_check_result);
681 sw_desc->unmap_src_cnt = src_cnt;
682 sw_desc->unmap_len = len;
683 sw_desc->async_tx.flags = flags;
684 while (src_cnt--)
685 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
686 dma_src[src_cnt]);
687 }
688 spin_unlock_bh(&iop_chan->lock);
689
690 return sw_desc ? &sw_desc->async_tx : NULL;
691 }
692
693 static void iop_adma_free_chan_resources(struct dma_chan *chan)
694 {
695 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
696 struct iop_adma_desc_slot *iter, *_iter;
697 int in_use_descs = 0;
698
699 iop_adma_slot_cleanup(iop_chan);
700
701 spin_lock_bh(&iop_chan->lock);
702 list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
703 chain_node) {
704 in_use_descs++;
705 list_del(&iter->chain_node);
706 }
707 list_for_each_entry_safe_reverse(
708 iter, _iter, &iop_chan->all_slots, slot_node) {
709 list_del(&iter->slot_node);
710 kfree(iter);
711 iop_chan->slots_allocated--;
712 }
713 iop_chan->last_used = NULL;
714
715 dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
716 __func__, iop_chan->slots_allocated);
717 spin_unlock_bh(&iop_chan->lock);
718
719 /* one is ok since we left it on there on purpose */
720 if (in_use_descs > 1)
721 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
722 in_use_descs - 1);
723 }
724
725 /**
726 * iop_adma_is_complete - poll the status of an ADMA transaction
727 * @chan: ADMA channel handle
728 * @cookie: ADMA transaction identifier
729 */
730 static enum dma_status iop_adma_is_complete(struct dma_chan *chan,
731 dma_cookie_t cookie,
732 dma_cookie_t *done,
733 dma_cookie_t *used)
734 {
735 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
736 dma_cookie_t last_used;
737 dma_cookie_t last_complete;
738 enum dma_status ret;
739
740 last_used = chan->cookie;
741 last_complete = iop_chan->completed_cookie;
742
743 if (done)
744 *done = last_complete;
745 if (used)
746 *used = last_used;
747
748 ret = dma_async_is_complete(cookie, last_complete, last_used);
749 if (ret == DMA_SUCCESS)
750 return ret;
751
752 iop_adma_slot_cleanup(iop_chan);
753
754 last_used = chan->cookie;
755 last_complete = iop_chan->completed_cookie;
756
757 if (done)
758 *done = last_complete;
759 if (used)
760 *used = last_used;
761
762 return dma_async_is_complete(cookie, last_complete, last_used);
763 }
764
765 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
766 {
767 struct iop_adma_chan *chan = data;
768
769 dev_dbg(chan->device->common.dev, "%s\n", __func__);
770
771 tasklet_schedule(&chan->irq_tasklet);
772
773 iop_adma_device_clear_eot_status(chan);
774
775 return IRQ_HANDLED;
776 }
777
778 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
779 {
780 struct iop_adma_chan *chan = data;
781
782 dev_dbg(chan->device->common.dev, "%s\n", __func__);
783
784 tasklet_schedule(&chan->irq_tasklet);
785
786 iop_adma_device_clear_eoc_status(chan);
787
788 return IRQ_HANDLED;
789 }
790
791 static irqreturn_t iop_adma_err_handler(int irq, void *data)
792 {
793 struct iop_adma_chan *chan = data;
794 unsigned long status = iop_chan_get_status(chan);
795
796 dev_printk(KERN_ERR, chan->device->common.dev,
797 "error ( %s%s%s%s%s%s%s)\n",
798 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
799 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
800 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
801 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
802 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
803 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
804 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
805
806 iop_adma_device_clear_err_status(chan);
807
808 BUG();
809
810 return IRQ_HANDLED;
811 }
812
813 static void iop_adma_issue_pending(struct dma_chan *chan)
814 {
815 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
816
817 if (iop_chan->pending) {
818 iop_chan->pending = 0;
819 iop_chan_append(iop_chan);
820 }
821 }
822
823 /*
824 * Perform a transaction to verify the HW works.
825 */
826 #define IOP_ADMA_TEST_SIZE 2000
827
828 static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device)
829 {
830 int i;
831 void *src, *dest;
832 dma_addr_t src_dma, dest_dma;
833 struct dma_chan *dma_chan;
834 dma_cookie_t cookie;
835 struct dma_async_tx_descriptor *tx;
836 int err = 0;
837 struct iop_adma_chan *iop_chan;
838
839 dev_dbg(device->common.dev, "%s\n", __func__);
840
841 src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
842 if (!src)
843 return -ENOMEM;
844 dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
845 if (!dest) {
846 kfree(src);
847 return -ENOMEM;
848 }
849
850 /* Fill in src buffer */
851 for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
852 ((u8 *) src)[i] = (u8)i;
853
854 /* Start copy, using first DMA channel */
855 dma_chan = container_of(device->common.channels.next,
856 struct dma_chan,
857 device_node);
858 if (iop_adma_alloc_chan_resources(dma_chan, NULL) < 1) {
859 err = -ENODEV;
860 goto out;
861 }
862
863 dest_dma = dma_map_single(dma_chan->device->dev, dest,
864 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
865 src_dma = dma_map_single(dma_chan->device->dev, src,
866 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
867 tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
868 IOP_ADMA_TEST_SIZE,
869 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
870
871 cookie = iop_adma_tx_submit(tx);
872 iop_adma_issue_pending(dma_chan);
873 msleep(1);
874
875 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
876 DMA_SUCCESS) {
877 dev_printk(KERN_ERR, dma_chan->device->dev,
878 "Self-test copy timed out, disabling\n");
879 err = -ENODEV;
880 goto free_resources;
881 }
882
883 iop_chan = to_iop_adma_chan(dma_chan);
884 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
885 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
886 if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
887 dev_printk(KERN_ERR, dma_chan->device->dev,
888 "Self-test copy failed compare, disabling\n");
889 err = -ENODEV;
890 goto free_resources;
891 }
892
893 free_resources:
894 iop_adma_free_chan_resources(dma_chan);
895 out:
896 kfree(src);
897 kfree(dest);
898 return err;
899 }
900
901 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
902 static int __devinit
903 iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
904 {
905 int i, src_idx;
906 struct page *dest;
907 struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
908 struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
909 dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
910 dma_addr_t dma_addr, dest_dma;
911 struct dma_async_tx_descriptor *tx;
912 struct dma_chan *dma_chan;
913 dma_cookie_t cookie;
914 u8 cmp_byte = 0;
915 u32 cmp_word;
916 u32 zero_sum_result;
917 int err = 0;
918 struct iop_adma_chan *iop_chan;
919
920 dev_dbg(device->common.dev, "%s\n", __func__);
921
922 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
923 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
924 if (!xor_srcs[src_idx])
925 while (src_idx--) {
926 __free_page(xor_srcs[src_idx]);
927 return -ENOMEM;
928 }
929 }
930
931 dest = alloc_page(GFP_KERNEL);
932 if (!dest)
933 while (src_idx--) {
934 __free_page(xor_srcs[src_idx]);
935 return -ENOMEM;
936 }
937
938 /* Fill in src buffers */
939 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
940 u8 *ptr = page_address(xor_srcs[src_idx]);
941 for (i = 0; i < PAGE_SIZE; i++)
942 ptr[i] = (1 << src_idx);
943 }
944
945 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
946 cmp_byte ^= (u8) (1 << src_idx);
947
948 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
949 (cmp_byte << 8) | cmp_byte;
950
951 memset(page_address(dest), 0, PAGE_SIZE);
952
953 dma_chan = container_of(device->common.channels.next,
954 struct dma_chan,
955 device_node);
956 if (iop_adma_alloc_chan_resources(dma_chan, NULL) < 1) {
957 err = -ENODEV;
958 goto out;
959 }
960
961 /* test xor */
962 dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
963 PAGE_SIZE, DMA_FROM_DEVICE);
964 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
965 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
966 0, PAGE_SIZE, DMA_TO_DEVICE);
967 tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
968 IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
969 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
970
971 cookie = iop_adma_tx_submit(tx);
972 iop_adma_issue_pending(dma_chan);
973 msleep(8);
974
975 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
976 DMA_SUCCESS) {
977 dev_printk(KERN_ERR, dma_chan->device->dev,
978 "Self-test xor timed out, disabling\n");
979 err = -ENODEV;
980 goto free_resources;
981 }
982
983 iop_chan = to_iop_adma_chan(dma_chan);
984 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
985 PAGE_SIZE, DMA_FROM_DEVICE);
986 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
987 u32 *ptr = page_address(dest);
988 if (ptr[i] != cmp_word) {
989 dev_printk(KERN_ERR, dma_chan->device->dev,
990 "Self-test xor failed compare, disabling\n");
991 err = -ENODEV;
992 goto free_resources;
993 }
994 }
995 dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
996 PAGE_SIZE, DMA_TO_DEVICE);
997
998 /* skip zero sum if the capability is not present */
999 if (!dma_has_cap(DMA_ZERO_SUM, dma_chan->device->cap_mask))
1000 goto free_resources;
1001
1002 /* zero sum the sources with the destintation page */
1003 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1004 zero_sum_srcs[i] = xor_srcs[i];
1005 zero_sum_srcs[i] = dest;
1006
1007 zero_sum_result = 1;
1008
1009 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1010 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1011 zero_sum_srcs[i], 0, PAGE_SIZE,
1012 DMA_TO_DEVICE);
1013 tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
1014 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1015 &zero_sum_result,
1016 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1017
1018 cookie = iop_adma_tx_submit(tx);
1019 iop_adma_issue_pending(dma_chan);
1020 msleep(8);
1021
1022 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1023 dev_printk(KERN_ERR, dma_chan->device->dev,
1024 "Self-test zero sum timed out, disabling\n");
1025 err = -ENODEV;
1026 goto free_resources;
1027 }
1028
1029 if (zero_sum_result != 0) {
1030 dev_printk(KERN_ERR, dma_chan->device->dev,
1031 "Self-test zero sum failed compare, disabling\n");
1032 err = -ENODEV;
1033 goto free_resources;
1034 }
1035
1036 /* test memset */
1037 dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1038 PAGE_SIZE, DMA_FROM_DEVICE);
1039 tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1040 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1041
1042 cookie = iop_adma_tx_submit(tx);
1043 iop_adma_issue_pending(dma_chan);
1044 msleep(8);
1045
1046 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1047 dev_printk(KERN_ERR, dma_chan->device->dev,
1048 "Self-test memset timed out, disabling\n");
1049 err = -ENODEV;
1050 goto free_resources;
1051 }
1052
1053 for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1054 u32 *ptr = page_address(dest);
1055 if (ptr[i]) {
1056 dev_printk(KERN_ERR, dma_chan->device->dev,
1057 "Self-test memset failed compare, disabling\n");
1058 err = -ENODEV;
1059 goto free_resources;
1060 }
1061 }
1062
1063 /* test for non-zero parity sum */
1064 zero_sum_result = 0;
1065 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1066 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1067 zero_sum_srcs[i], 0, PAGE_SIZE,
1068 DMA_TO_DEVICE);
1069 tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
1070 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1071 &zero_sum_result,
1072 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1073
1074 cookie = iop_adma_tx_submit(tx);
1075 iop_adma_issue_pending(dma_chan);
1076 msleep(8);
1077
1078 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1079 dev_printk(KERN_ERR, dma_chan->device->dev,
1080 "Self-test non-zero sum timed out, disabling\n");
1081 err = -ENODEV;
1082 goto free_resources;
1083 }
1084
1085 if (zero_sum_result != 1) {
1086 dev_printk(KERN_ERR, dma_chan->device->dev,
1087 "Self-test non-zero sum failed compare, disabling\n");
1088 err = -ENODEV;
1089 goto free_resources;
1090 }
1091
1092 free_resources:
1093 iop_adma_free_chan_resources(dma_chan);
1094 out:
1095 src_idx = IOP_ADMA_NUM_SRC_TEST;
1096 while (src_idx--)
1097 __free_page(xor_srcs[src_idx]);
1098 __free_page(dest);
1099 return err;
1100 }
1101
1102 static int __devexit iop_adma_remove(struct platform_device *dev)
1103 {
1104 struct iop_adma_device *device = platform_get_drvdata(dev);
1105 struct dma_chan *chan, *_chan;
1106 struct iop_adma_chan *iop_chan;
1107 int i;
1108 struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1109
1110 dma_async_device_unregister(&device->common);
1111
1112 for (i = 0; i < 3; i++) {
1113 unsigned int irq;
1114 irq = platform_get_irq(dev, i);
1115 free_irq(irq, device);
1116 }
1117
1118 dma_free_coherent(&dev->dev, plat_data->pool_size,
1119 device->dma_desc_pool_virt, device->dma_desc_pool);
1120
1121 do {
1122 struct resource *res;
1123 res = platform_get_resource(dev, IORESOURCE_MEM, 0);
1124 release_mem_region(res->start, res->end - res->start);
1125 } while (0);
1126
1127 list_for_each_entry_safe(chan, _chan, &device->common.channels,
1128 device_node) {
1129 iop_chan = to_iop_adma_chan(chan);
1130 list_del(&chan->device_node);
1131 kfree(iop_chan);
1132 }
1133 kfree(device);
1134
1135 return 0;
1136 }
1137
1138 static int __devinit iop_adma_probe(struct platform_device *pdev)
1139 {
1140 struct resource *res;
1141 int ret = 0, i;
1142 struct iop_adma_device *adev;
1143 struct iop_adma_chan *iop_chan;
1144 struct dma_device *dma_dev;
1145 struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1146
1147 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1148 if (!res)
1149 return -ENODEV;
1150
1151 if (!devm_request_mem_region(&pdev->dev, res->start,
1152 res->end - res->start, pdev->name))
1153 return -EBUSY;
1154
1155 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1156 if (!adev)
1157 return -ENOMEM;
1158 dma_dev = &adev->common;
1159
1160 /* allocate coherent memory for hardware descriptors
1161 * note: writecombine gives slightly better performance, but
1162 * requires that we explicitly flush the writes
1163 */
1164 if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1165 plat_data->pool_size,
1166 &adev->dma_desc_pool,
1167 GFP_KERNEL)) == NULL) {
1168 ret = -ENOMEM;
1169 goto err_free_adev;
1170 }
1171
1172 dev_dbg(&pdev->dev, "%s: allocted descriptor pool virt %p phys %p\n",
1173 __func__, adev->dma_desc_pool_virt,
1174 (void *) adev->dma_desc_pool);
1175
1176 adev->id = plat_data->hw_id;
1177
1178 /* discover transaction capabilites from the platform data */
1179 dma_dev->cap_mask = plat_data->cap_mask;
1180
1181 adev->pdev = pdev;
1182 platform_set_drvdata(pdev, adev);
1183
1184 INIT_LIST_HEAD(&dma_dev->channels);
1185
1186 /* set base routines */
1187 dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1188 dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1189 dma_dev->device_is_tx_complete = iop_adma_is_complete;
1190 dma_dev->device_issue_pending = iop_adma_issue_pending;
1191 dma_dev->dev = &pdev->dev;
1192
1193 /* set prep routines based on capability */
1194 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1195 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1196 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1197 dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset;
1198 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1199 dma_dev->max_xor = iop_adma_get_max_xor();
1200 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1201 }
1202 if (dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask))
1203 dma_dev->device_prep_dma_zero_sum =
1204 iop_adma_prep_dma_zero_sum;
1205 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1206 dma_dev->device_prep_dma_interrupt =
1207 iop_adma_prep_dma_interrupt;
1208
1209 iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1210 if (!iop_chan) {
1211 ret = -ENOMEM;
1212 goto err_free_dma;
1213 }
1214 iop_chan->device = adev;
1215
1216 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1217 res->end - res->start);
1218 if (!iop_chan->mmr_base) {
1219 ret = -ENOMEM;
1220 goto err_free_iop_chan;
1221 }
1222 tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1223 iop_chan);
1224
1225 /* clear errors before enabling interrupts */
1226 iop_adma_device_clear_err_status(iop_chan);
1227
1228 for (i = 0; i < 3; i++) {
1229 irq_handler_t handler[] = { iop_adma_eot_handler,
1230 iop_adma_eoc_handler,
1231 iop_adma_err_handler };
1232 int irq = platform_get_irq(pdev, i);
1233 if (irq < 0) {
1234 ret = -ENXIO;
1235 goto err_free_iop_chan;
1236 } else {
1237 ret = devm_request_irq(&pdev->dev, irq,
1238 handler[i], 0, pdev->name, iop_chan);
1239 if (ret)
1240 goto err_free_iop_chan;
1241 }
1242 }
1243
1244 spin_lock_init(&iop_chan->lock);
1245 INIT_LIST_HEAD(&iop_chan->chain);
1246 INIT_LIST_HEAD(&iop_chan->all_slots);
1247 INIT_RCU_HEAD(&iop_chan->common.rcu);
1248 iop_chan->common.device = dma_dev;
1249 list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1250
1251 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1252 ret = iop_adma_memcpy_self_test(adev);
1253 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1254 if (ret)
1255 goto err_free_iop_chan;
1256 }
1257
1258 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1259 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1260 ret = iop_adma_xor_zero_sum_self_test(adev);
1261 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1262 if (ret)
1263 goto err_free_iop_chan;
1264 }
1265
1266 dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
1267 "( %s%s%s%s%s%s%s%s%s%s)\n",
1268 dma_has_cap(DMA_PQ_XOR, dma_dev->cap_mask) ? "pq_xor " : "",
1269 dma_has_cap(DMA_PQ_UPDATE, dma_dev->cap_mask) ? "pq_update " : "",
1270 dma_has_cap(DMA_PQ_ZERO_SUM, dma_dev->cap_mask) ? "pq_zero_sum " : "",
1271 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1272 dma_has_cap(DMA_DUAL_XOR, dma_dev->cap_mask) ? "dual_xor " : "",
1273 dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask) ? "xor_zero_sum " : "",
1274 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "",
1275 dma_has_cap(DMA_MEMCPY_CRC32C, dma_dev->cap_mask) ? "cpy+crc " : "",
1276 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1277 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1278
1279 dma_async_device_register(dma_dev);
1280 goto out;
1281
1282 err_free_iop_chan:
1283 kfree(iop_chan);
1284 err_free_dma:
1285 dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1286 adev->dma_desc_pool_virt, adev->dma_desc_pool);
1287 err_free_adev:
1288 kfree(adev);
1289 out:
1290 return ret;
1291 }
1292
1293 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1294 {
1295 struct iop_adma_desc_slot *sw_desc, *grp_start;
1296 dma_cookie_t cookie;
1297 int slot_cnt, slots_per_op;
1298
1299 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1300
1301 spin_lock_bh(&iop_chan->lock);
1302 slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1303 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1304 if (sw_desc) {
1305 grp_start = sw_desc->group_head;
1306
1307 list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
1308 async_tx_ack(&sw_desc->async_tx);
1309 iop_desc_init_memcpy(grp_start, 0);
1310 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1311 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1312 iop_desc_set_memcpy_src_addr(grp_start, 0);
1313
1314 cookie = iop_chan->common.cookie;
1315 cookie++;
1316 if (cookie <= 1)
1317 cookie = 2;
1318
1319 /* initialize the completed cookie to be less than
1320 * the most recently used cookie
1321 */
1322 iop_chan->completed_cookie = cookie - 1;
1323 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1324
1325 /* channel should not be busy */
1326 BUG_ON(iop_chan_is_busy(iop_chan));
1327
1328 /* clear any prior error-status bits */
1329 iop_adma_device_clear_err_status(iop_chan);
1330
1331 /* disable operation */
1332 iop_chan_disable(iop_chan);
1333
1334 /* set the descriptor address */
1335 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1336
1337 /* 1/ don't add pre-chained descriptors
1338 * 2/ dummy read to flush next_desc write
1339 */
1340 BUG_ON(iop_desc_get_next_desc(sw_desc));
1341
1342 /* run the descriptor */
1343 iop_chan_enable(iop_chan);
1344 } else
1345 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1346 "failed to allocate null descriptor\n");
1347 spin_unlock_bh(&iop_chan->lock);
1348 }
1349
1350 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1351 {
1352 struct iop_adma_desc_slot *sw_desc, *grp_start;
1353 dma_cookie_t cookie;
1354 int slot_cnt, slots_per_op;
1355
1356 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1357
1358 spin_lock_bh(&iop_chan->lock);
1359 slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1360 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1361 if (sw_desc) {
1362 grp_start = sw_desc->group_head;
1363 list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
1364 async_tx_ack(&sw_desc->async_tx);
1365 iop_desc_init_null_xor(grp_start, 2, 0);
1366 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1367 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1368 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1369 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1370
1371 cookie = iop_chan->common.cookie;
1372 cookie++;
1373 if (cookie <= 1)
1374 cookie = 2;
1375
1376 /* initialize the completed cookie to be less than
1377 * the most recently used cookie
1378 */
1379 iop_chan->completed_cookie = cookie - 1;
1380 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1381
1382 /* channel should not be busy */
1383 BUG_ON(iop_chan_is_busy(iop_chan));
1384
1385 /* clear any prior error-status bits */
1386 iop_adma_device_clear_err_status(iop_chan);
1387
1388 /* disable operation */
1389 iop_chan_disable(iop_chan);
1390
1391 /* set the descriptor address */
1392 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1393
1394 /* 1/ don't add pre-chained descriptors
1395 * 2/ dummy read to flush next_desc write
1396 */
1397 BUG_ON(iop_desc_get_next_desc(sw_desc));
1398
1399 /* run the descriptor */
1400 iop_chan_enable(iop_chan);
1401 } else
1402 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1403 "failed to allocate null descriptor\n");
1404 spin_unlock_bh(&iop_chan->lock);
1405 }
1406
1407 MODULE_ALIAS("platform:iop-adma");
1408
1409 static struct platform_driver iop_adma_driver = {
1410 .probe = iop_adma_probe,
1411 .remove = iop_adma_remove,
1412 .driver = {
1413 .owner = THIS_MODULE,
1414 .name = "iop-adma",
1415 },
1416 };
1417
1418 static int __init iop_adma_init (void)
1419 {
1420 return platform_driver_register(&iop_adma_driver);
1421 }
1422
1423 /* it's currently unsafe to unload this module */
1424 #if 0
1425 static void __exit iop_adma_exit (void)
1426 {
1427 platform_driver_unregister(&iop_adma_driver);
1428 return;
1429 }
1430 module_exit(iop_adma_exit);
1431 #endif
1432
1433 module_init(iop_adma_init);
1434
1435 MODULE_AUTHOR("Intel Corporation");
1436 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1437 MODULE_LICENSE("GPL");
linux-next