2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2015 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
18 #include "scif_main.h"
22 * struct scif_dma_comp_cb - SCIF DMA completion callback
24 * @dma_completion_func: DMA completion callback
25 * @cb_cookie: DMA completion callback cookie
26 * @temp_buf: Temporary buffer
27 * @temp_buf_to_free: Temporary buffer to be freed
28 * @is_cache: Is a kmem_cache allocated buffer
29 * @dst_offset: Destination registration offset
30 * @dst_window: Destination registration window
31 * @len: Length of the temp buffer
32 * @temp_phys: DMA address of the temp buffer
33 * @sdev: The SCIF device
34 * @header_padding: padding for cache line alignment
36 struct scif_dma_comp_cb
{
37 void (*dma_completion_func
)(void *cookie
);
43 struct scif_window
*dst_window
;
46 struct scif_dev
*sdev
;
51 * struct scif_copy_work - Work for DMA copy
53 * @src_offset: Starting source offset
54 * @dst_offset: Starting destination offset
55 * @src_window: Starting src registered window
56 * @dst_window: Starting dst registered window
57 * @loopback: true if this is a loopback DMA transfer
58 * @len: Length of the transfer
59 * @comp_cb: DMA copy completion callback
60 * @remote_dev: The remote SCIF peer device
61 * @fence_type: polling or interrupt based
62 * @ordered: is this a tail byte ordered DMA transfer
64 struct scif_copy_work
{
67 struct scif_window
*src_window
;
68 struct scif_window
*dst_window
;
71 struct scif_dma_comp_cb
*comp_cb
;
72 struct scif_dev
*remote_dev
;
77 #ifndef list_entry_next
78 #define list_entry_next(pos, member) \
79 list_entry(pos->member.next, typeof(*pos), member)
83 * scif_reserve_dma_chan:
84 * @ep: Endpoint Descriptor.
86 * This routine reserves a DMA channel for a particular
87 * endpoint. All DMA transfers for an endpoint are always
88 * programmed on the same DMA channel.
90 int scif_reserve_dma_chan(struct scif_endpt
*ep
)
93 struct scif_dev
*scifdev
;
94 struct scif_hw_dev
*sdev
;
95 struct dma_chan
*chan
;
97 /* Loopback DMAs are not supported on the management node */
98 if (!scif_info
.nodeid
&& scifdev_self(ep
->remote_dev
))
100 if (scif_info
.nodeid
)
101 scifdev
= &scif_dev
[0];
103 scifdev
= ep
->remote_dev
;
104 sdev
= scifdev
->sdev
;
105 if (!sdev
->num_dma_ch
)
107 chan
= sdev
->dma_ch
[scifdev
->dma_ch_idx
];
108 scifdev
->dma_ch_idx
= (scifdev
->dma_ch_idx
+ 1) % sdev
->num_dma_ch
;
109 mutex_lock(&ep
->rma_info
.rma_lock
);
110 ep
->rma_info
.dma_chan
= chan
;
111 mutex_unlock(&ep
->rma_info
.rma_lock
);
115 #ifdef CONFIG_MMU_NOTIFIER
117 * scif_rma_destroy_tcw:
119 * This routine destroys temporary cached windows
122 void __scif_rma_destroy_tcw(struct scif_mmu_notif
*mmn
,
123 struct scif_endpt
*ep
,
126 struct list_head
*item
, *tmp
;
127 struct scif_window
*window
;
128 u64 start_va
, end_va
;
129 u64 end
= start
+ len
;
134 list_for_each_safe(item
, tmp
, &mmn
->tc_reg_list
) {
135 window
= list_entry(item
, struct scif_window
, list
);
136 ep
= (struct scif_endpt
*)window
->ep
;
139 start_va
= window
->va_for_temp
;
140 end_va
= start_va
+ (window
->nr_pages
<< PAGE_SHIFT
);
141 if (start
< start_va
&& end
<= start_va
)
145 __scif_rma_destroy_tcw_helper(window
);
149 static void scif_rma_destroy_tcw(struct scif_mmu_notif
*mmn
, u64 start
, u64 len
)
151 struct scif_endpt
*ep
= mmn
->ep
;
153 spin_lock(&ep
->rma_info
.tc_lock
);
154 __scif_rma_destroy_tcw(mmn
, ep
, start
, len
);
155 spin_unlock(&ep
->rma_info
.tc_lock
);
158 static void scif_rma_destroy_tcw_ep(struct scif_endpt
*ep
)
160 struct list_head
*item
, *tmp
;
161 struct scif_mmu_notif
*mmn
;
163 list_for_each_safe(item
, tmp
, &ep
->rma_info
.mmn_list
) {
164 mmn
= list_entry(item
, struct scif_mmu_notif
, list
);
165 scif_rma_destroy_tcw(mmn
, 0, ULONG_MAX
);
169 static void __scif_rma_destroy_tcw_ep(struct scif_endpt
*ep
)
171 struct list_head
*item
, *tmp
;
172 struct scif_mmu_notif
*mmn
;
174 spin_lock(&ep
->rma_info
.tc_lock
);
175 list_for_each_safe(item
, tmp
, &ep
->rma_info
.mmn_list
) {
176 mmn
= list_entry(item
, struct scif_mmu_notif
, list
);
177 __scif_rma_destroy_tcw(mmn
, ep
, 0, ULONG_MAX
);
179 spin_unlock(&ep
->rma_info
.tc_lock
);
182 static bool scif_rma_tc_can_cache(struct scif_endpt
*ep
, size_t cur_bytes
)
184 if ((cur_bytes
>> PAGE_SHIFT
) > scif_info
.rma_tc_limit
)
186 if ((atomic_read(&ep
->rma_info
.tcw_total_pages
)
187 + (cur_bytes
>> PAGE_SHIFT
)) >
188 scif_info
.rma_tc_limit
) {
189 dev_info(scif_info
.mdev
.this_device
,
190 "%s %d total=%d, current=%zu reached max\n",
192 atomic_read(&ep
->rma_info
.tcw_total_pages
),
193 (1 + (cur_bytes
>> PAGE_SHIFT
)));
194 scif_rma_destroy_tcw_invalid();
195 __scif_rma_destroy_tcw_ep(ep
);
200 static void scif_mmu_notifier_release(struct mmu_notifier
*mn
,
201 struct mm_struct
*mm
)
203 struct scif_mmu_notif
*mmn
;
205 mmn
= container_of(mn
, struct scif_mmu_notif
, ep_mmu_notifier
);
206 scif_rma_destroy_tcw(mmn
, 0, ULONG_MAX
);
207 schedule_work(&scif_info
.misc_work
);
210 static void scif_mmu_notifier_invalidate_page(struct mmu_notifier
*mn
,
211 struct mm_struct
*mm
,
212 unsigned long address
)
214 struct scif_mmu_notif
*mmn
;
216 mmn
= container_of(mn
, struct scif_mmu_notif
, ep_mmu_notifier
);
217 scif_rma_destroy_tcw(mmn
, address
, PAGE_SIZE
);
220 static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier
*mn
,
221 struct mm_struct
*mm
,
225 struct scif_mmu_notif
*mmn
;
227 mmn
= container_of(mn
, struct scif_mmu_notif
, ep_mmu_notifier
);
228 scif_rma_destroy_tcw(mmn
, start
, end
- start
);
231 static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier
*mn
,
232 struct mm_struct
*mm
,
237 * Nothing to do here, everything needed was done in
238 * invalidate_range_start.
242 static const struct mmu_notifier_ops scif_mmu_notifier_ops
= {
243 .release
= scif_mmu_notifier_release
,
244 .clear_flush_young
= NULL
,
245 .invalidate_page
= scif_mmu_notifier_invalidate_page
,
246 .invalidate_range_start
= scif_mmu_notifier_invalidate_range_start
,
247 .invalidate_range_end
= scif_mmu_notifier_invalidate_range_end
};
249 static void scif_ep_unregister_mmu_notifier(struct scif_endpt
*ep
)
251 struct scif_endpt_rma_info
*rma
= &ep
->rma_info
;
252 struct scif_mmu_notif
*mmn
= NULL
;
253 struct list_head
*item
, *tmp
;
255 mutex_lock(&ep
->rma_info
.mmn_lock
);
256 list_for_each_safe(item
, tmp
, &rma
->mmn_list
) {
257 mmn
= list_entry(item
, struct scif_mmu_notif
, list
);
258 mmu_notifier_unregister(&mmn
->ep_mmu_notifier
, mmn
->mm
);
262 mutex_unlock(&ep
->rma_info
.mmn_lock
);
265 static void scif_init_mmu_notifier(struct scif_mmu_notif
*mmn
,
266 struct mm_struct
*mm
, struct scif_endpt
*ep
)
270 mmn
->ep_mmu_notifier
.ops
= &scif_mmu_notifier_ops
;
271 INIT_LIST_HEAD(&mmn
->list
);
272 INIT_LIST_HEAD(&mmn
->tc_reg_list
);
275 static struct scif_mmu_notif
*
276 scif_find_mmu_notifier(struct mm_struct
*mm
, struct scif_endpt_rma_info
*rma
)
278 struct scif_mmu_notif
*mmn
;
279 struct list_head
*item
;
281 list_for_each(item
, &rma
->mmn_list
) {
282 mmn
= list_entry(item
, struct scif_mmu_notif
, list
);
289 static struct scif_mmu_notif
*
290 scif_add_mmu_notifier(struct mm_struct
*mm
, struct scif_endpt
*ep
)
292 struct scif_mmu_notif
*mmn
293 = kzalloc(sizeof(*mmn
), GFP_KERNEL
);
296 return ERR_PTR(ENOMEM
);
298 scif_init_mmu_notifier(mmn
, current
->mm
, ep
);
299 if (mmu_notifier_register(&mmn
->ep_mmu_notifier
,
302 return ERR_PTR(EBUSY
);
304 list_add(&mmn
->list
, &ep
->rma_info
.mmn_list
);
309 * Called from the misc thread to destroy temporary cached windows and
310 * unregister the MMU notifier for the SCIF endpoint.
312 void scif_mmu_notif_handler(struct work_struct
*work
)
314 struct list_head
*pos
, *tmpq
;
315 struct scif_endpt
*ep
;
317 scif_rma_destroy_tcw_invalid();
318 spin_lock(&scif_info
.rmalock
);
319 list_for_each_safe(pos
, tmpq
, &scif_info
.mmu_notif_cleanup
) {
320 ep
= list_entry(pos
, struct scif_endpt
, mmu_list
);
321 list_del(&ep
->mmu_list
);
322 spin_unlock(&scif_info
.rmalock
);
323 scif_rma_destroy_tcw_ep(ep
);
324 scif_ep_unregister_mmu_notifier(ep
);
327 spin_unlock(&scif_info
.rmalock
);
330 static bool scif_is_set_reg_cache(int flags
)
332 return !!(flags
& SCIF_RMA_USECACHE
);
335 static struct scif_mmu_notif
*
336 scif_find_mmu_notifier(struct mm_struct
*mm
,
337 struct scif_endpt_rma_info
*rma
)
342 static struct scif_mmu_notif
*
343 scif_add_mmu_notifier(struct mm_struct
*mm
, struct scif_endpt
*ep
)
348 void scif_mmu_notif_handler(struct work_struct
*work
)
352 static bool scif_is_set_reg_cache(int flags
)
357 static bool scif_rma_tc_can_cache(struct scif_endpt
*ep
, size_t cur_bytes
)
364 * scif_register_temp:
365 * @epd: End Point Descriptor.
366 * @addr: virtual address to/from which to copy
367 * @len: length of range to copy
368 * @out_offset: computed offset returned by reference.
369 * @out_window: allocated registered window returned by reference.
371 * Create a temporary registered window. The peer will not know about this
372 * window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
375 scif_register_temp(scif_epd_t epd
, unsigned long addr
, size_t len
, int prot
,
376 off_t
*out_offset
, struct scif_window
**out_window
)
378 struct scif_endpt
*ep
= (struct scif_endpt
*)epd
;
380 scif_pinned_pages_t pinned_pages
;
383 aligned_len
= ALIGN(len
, PAGE_SIZE
);
385 err
= __scif_pin_pages((void *)(addr
& PAGE_MASK
),
386 aligned_len
, &prot
, 0, &pinned_pages
);
390 pinned_pages
->prot
= prot
;
392 /* Compute the offset for this registration */
393 err
= scif_get_window_offset(ep
, 0, 0,
394 aligned_len
>> PAGE_SHIFT
,
399 /* Allocate and prepare self registration window */
400 *out_window
= scif_create_window(ep
, aligned_len
>> PAGE_SHIFT
,
403 scif_free_window_offset(ep
, NULL
, *out_offset
);
408 (*out_window
)->pinned_pages
= pinned_pages
;
409 (*out_window
)->nr_pages
= pinned_pages
->nr_pages
;
410 (*out_window
)->prot
= pinned_pages
->prot
;
412 (*out_window
)->va_for_temp
= addr
& PAGE_MASK
;
413 err
= scif_map_window(ep
->remote_dev
, *out_window
);
415 /* Something went wrong! Rollback */
416 scif_destroy_window(ep
, *out_window
);
419 *out_offset
|= (addr
- (*out_window
)->va_for_temp
);
424 dev_err(&ep
->remote_dev
->sdev
->dev
,
425 "%s %d err %d\n", __func__
, __LINE__
, err
);
426 scif_unpin_pages(pinned_pages
);
430 #define SCIF_DMA_TO (3 * HZ)
433 * scif_sync_dma - Program a DMA without an interrupt descriptor
435 * @dev - The address of the pointer to the device instance used
436 * for DMA registration.
437 * @chan - DMA channel to be used.
438 * @sync_wait: Wait for DMA to complete?
440 * Return 0 on success and -errno on error.
442 static int scif_sync_dma(struct scif_hw_dev
*sdev
, struct dma_chan
*chan
,
446 struct dma_async_tx_descriptor
*tx
= NULL
;
447 enum dma_ctrl_flags flags
= DMA_PREP_FENCE
;
449 struct dma_device
*ddev
;
453 dev_err(&sdev
->dev
, "%s %d err %d\n",
454 __func__
, __LINE__
, err
);
459 tx
= ddev
->device_prep_dma_memcpy(chan
, 0, 0, 0, flags
);
462 dev_err(&sdev
->dev
, "%s %d err %d\n",
463 __func__
, __LINE__
, err
);
466 cookie
= tx
->tx_submit(tx
);
468 if (dma_submit_error(cookie
)) {
470 dev_err(&sdev
->dev
, "%s %d err %d\n",
471 __func__
, __LINE__
, err
);
475 dma_async_issue_pending(chan
);
477 if (dma_sync_wait(chan
, cookie
) == DMA_COMPLETE
) {
481 dev_err(&sdev
->dev
, "%s %d err %d\n",
482 __func__
, __LINE__
, err
);
489 static void scif_dma_callback(void *arg
)
491 struct completion
*done
= (struct completion
*)arg
;
496 #define SCIF_DMA_SYNC_WAIT true
497 #define SCIF_DMA_POLL BIT(0)
498 #define SCIF_DMA_INTR BIT(1)
501 * scif_async_dma - Program a DMA with an interrupt descriptor
503 * @dev - The address of the pointer to the device instance used
504 * for DMA registration.
505 * @chan - DMA channel to be used.
506 * Return 0 on success and -errno on error.
508 static int scif_async_dma(struct scif_hw_dev
*sdev
, struct dma_chan
*chan
)
511 struct dma_device
*ddev
;
512 struct dma_async_tx_descriptor
*tx
= NULL
;
513 enum dma_ctrl_flags flags
= DMA_PREP_INTERRUPT
| DMA_PREP_FENCE
;
514 DECLARE_COMPLETION_ONSTACK(done_wait
);
516 enum dma_status status
;
520 dev_err(&sdev
->dev
, "%s %d err %d\n",
521 __func__
, __LINE__
, err
);
526 tx
= ddev
->device_prep_dma_memcpy(chan
, 0, 0, 0, flags
);
529 dev_err(&sdev
->dev
, "%s %d err %d\n",
530 __func__
, __LINE__
, err
);
533 reinit_completion(&done_wait
);
534 tx
->callback
= scif_dma_callback
;
535 tx
->callback_param
= &done_wait
;
536 cookie
= tx
->tx_submit(tx
);
538 if (dma_submit_error(cookie
)) {
540 dev_err(&sdev
->dev
, "%s %d err %d\n",
541 __func__
, __LINE__
, err
);
544 dma_async_issue_pending(chan
);
546 err
= wait_for_completion_timeout(&done_wait
, SCIF_DMA_TO
);
549 dev_err(&sdev
->dev
, "%s %d err %d\n",
550 __func__
, __LINE__
, err
);
554 status
= dma_async_is_tx_complete(chan
, cookie
, NULL
, NULL
);
555 if (status
!= DMA_COMPLETE
) {
557 dev_err(&sdev
->dev
, "%s %d err %d\n",
558 __func__
, __LINE__
, err
);
566 * scif_drain_dma_poll - Drain all outstanding DMA operations for a particular
567 * DMA channel via polling.
569 * @sdev - The SCIF device
570 * @chan - DMA channel
571 * Return 0 on success and -errno on error.
573 static int scif_drain_dma_poll(struct scif_hw_dev
*sdev
, struct dma_chan
*chan
)
577 return scif_sync_dma(sdev
, chan
, SCIF_DMA_SYNC_WAIT
);
581 * scif_drain_dma_intr - Drain all outstanding DMA operations for a particular
582 * DMA channel via interrupt based blocking wait.
584 * @sdev - The SCIF device
585 * @chan - DMA channel
586 * Return 0 on success and -errno on error.
588 int scif_drain_dma_intr(struct scif_hw_dev
*sdev
, struct dma_chan
*chan
)
592 return scif_async_dma(sdev
, chan
);
596 * scif_rma_destroy_windows:
598 * This routine destroys all windows queued for cleanup
600 void scif_rma_destroy_windows(void)
602 struct list_head
*item
, *tmp
;
603 struct scif_window
*window
;
604 struct scif_endpt
*ep
;
605 struct dma_chan
*chan
;
609 spin_lock(&scif_info
.rmalock
);
610 list_for_each_safe(item
, tmp
, &scif_info
.rma
) {
611 window
= list_entry(item
, struct scif_window
,
613 ep
= (struct scif_endpt
*)window
->ep
;
614 chan
= ep
->rma_info
.dma_chan
;
616 list_del_init(&window
->list
);
617 spin_unlock(&scif_info
.rmalock
);
618 if (!chan
|| !scifdev_alive(ep
) ||
619 !scif_drain_dma_intr(ep
->remote_dev
->sdev
,
620 ep
->rma_info
.dma_chan
))
621 /* Remove window from global list */
622 window
->unreg_state
= OP_COMPLETED
;
624 dev_warn(&ep
->remote_dev
->sdev
->dev
,
625 "DMA engine hung?\n");
626 if (window
->unreg_state
== OP_COMPLETED
) {
627 if (window
->type
== SCIF_WINDOW_SELF
)
628 scif_destroy_window(ep
, window
);
630 scif_destroy_remote_window(window
);
631 atomic_dec(&ep
->rma_info
.tw_refcount
);
635 spin_unlock(&scif_info
.rmalock
);
639 * scif_rma_destroy_tcw:
641 * This routine destroys temporary cached registered windows
642 * which have been queued for cleanup.
644 void scif_rma_destroy_tcw_invalid(void)
646 struct list_head
*item
, *tmp
;
647 struct scif_window
*window
;
648 struct scif_endpt
*ep
;
649 struct dma_chan
*chan
;
653 spin_lock(&scif_info
.rmalock
);
654 list_for_each_safe(item
, tmp
, &scif_info
.rma_tc
) {
655 window
= list_entry(item
, struct scif_window
, list
);
656 ep
= (struct scif_endpt
*)window
->ep
;
657 chan
= ep
->rma_info
.dma_chan
;
658 list_del_init(&window
->list
);
659 spin_unlock(&scif_info
.rmalock
);
660 mutex_lock(&ep
->rma_info
.rma_lock
);
661 if (!chan
|| !scifdev_alive(ep
) ||
662 !scif_drain_dma_intr(ep
->remote_dev
->sdev
,
663 ep
->rma_info
.dma_chan
)) {
664 atomic_sub(window
->nr_pages
,
665 &ep
->rma_info
.tcw_total_pages
);
666 scif_destroy_window(ep
, window
);
667 atomic_dec(&ep
->rma_info
.tcw_refcount
);
669 dev_warn(&ep
->remote_dev
->sdev
->dev
,
670 "DMA engine hung?\n");
672 mutex_unlock(&ep
->rma_info
.rma_lock
);
675 spin_unlock(&scif_info
.rmalock
);
679 void *_get_local_va(off_t off
, struct scif_window
*window
, size_t len
)
681 int page_nr
= (off
- window
->offset
) >> PAGE_SHIFT
;
682 off_t page_off
= off
& ~PAGE_MASK
;
685 if (window
->type
== SCIF_WINDOW_SELF
) {
686 struct page
**pages
= window
->pinned_pages
->pages
;
688 va
= page_address(pages
[page_nr
]) + page_off
;
694 void *ioremap_remote(off_t off
, struct scif_window
*window
,
695 size_t len
, struct scif_dev
*dev
,
696 struct scif_window_iter
*iter
)
698 dma_addr_t phys
= scif_off_to_dma_addr(window
, off
, NULL
, iter
);
701 * If the DMA address is not card relative then we need the DMA
702 * addresses to be an offset into the bar. The aperture base was already
703 * added so subtract it here since scif_ioremap is going to add it again
705 if (!scifdev_self(dev
) && window
->type
== SCIF_WINDOW_PEER
&&
706 dev
->sdev
->aper
&& !dev
->sdev
->card_rel_da
)
707 phys
= phys
- dev
->sdev
->aper
->pa
;
708 return scif_ioremap(phys
, len
, dev
);
712 iounmap_remote(void *virt
, size_t size
, struct scif_copy_work
*work
)
714 scif_iounmap(virt
, size
, work
->remote_dev
);
718 * Takes care of ordering issue caused by
719 * 1. Hardware: Only in the case of cpu copy from mgmt node to card
720 * because of WC memory.
721 * 2. Software: If memcpy reorders copy instructions for optimization.
722 * This could happen at both mgmt node and card.
725 scif_ordered_memcpy_toio(char *dst
, const char *src
, size_t count
)
730 memcpy_toio((void __iomem __force
*)dst
, src
, --count
);
731 /* Order the last byte with the previous stores */
733 *(dst
+ count
) = *(src
+ count
);
736 static inline void scif_unaligned_cpy_toio(char *dst
, const char *src
,
737 size_t count
, bool ordered
)
740 scif_ordered_memcpy_toio(dst
, src
, count
);
742 memcpy_toio((void __iomem __force
*)dst
, src
, count
);
746 void scif_ordered_memcpy_fromio(char *dst
, const char *src
, size_t count
)
751 memcpy_fromio(dst
, (void __iomem __force
*)src
, --count
);
752 /* Order the last byte with the previous loads */
754 *(dst
+ count
) = *(src
+ count
);
757 static inline void scif_unaligned_cpy_fromio(char *dst
, const char *src
,
758 size_t count
, bool ordered
)
761 scif_ordered_memcpy_fromio(dst
, src
, count
);
763 memcpy_fromio(dst
, (void __iomem __force
*)src
, count
);
766 #define SCIF_RMA_ERROR_CODE (~(dma_addr_t)0x0)
769 * scif_off_to_dma_addr:
770 * Obtain the dma_addr given the window and the offset.
771 * @window: Registered window.
772 * @off: Window offset.
773 * @nr_bytes: Return the number of contiguous bytes till next DMA addr index.
774 * @index: Return the index of the dma_addr array found.
775 * @start_off: start offset of index of the dma addr array found.
776 * The nr_bytes provides the callee an estimate of the maximum possible
777 * DMA xfer possible while the index/start_off provide faster lookups
778 * for the next iteration.
780 dma_addr_t
scif_off_to_dma_addr(struct scif_window
*window
, s64 off
,
781 size_t *nr_bytes
, struct scif_window_iter
*iter
)
787 if (window
->nr_pages
== window
->nr_contig_chunks
) {
788 page_nr
= (off
- window
->offset
) >> PAGE_SHIFT
;
789 page_off
= off
& ~PAGE_MASK
;
792 *nr_bytes
= PAGE_SIZE
- page_off
;
793 return window
->dma_addr
[page_nr
] | page_off
;
797 start
= iter
->offset
;
800 start
= window
->offset
;
802 for (; i
< window
->nr_contig_chunks
; i
++) {
803 end
= start
+ (window
->num_pages
[i
] << PAGE_SHIFT
);
804 if (off
>= start
&& off
< end
) {
807 iter
->offset
= start
;
810 *nr_bytes
= end
- off
;
811 return (window
->dma_addr
[i
] + (off
- start
));
813 start
+= (window
->num_pages
[i
] << PAGE_SHIFT
);
815 dev_err(scif_info
.mdev
.this_device
,
816 "%s %d BUG. Addr not found? window %p off 0x%llx\n",
817 __func__
, __LINE__
, window
, off
);
818 return SCIF_RMA_ERROR_CODE
;
822 * Copy between rma window and temporary buffer
824 static void scif_rma_local_cpu_copy(s64 offset
, struct scif_window
*window
,
825 u8
*temp
, size_t rem_len
, bool to_temp
)
832 offset_in_page
= offset
& ~PAGE_MASK
;
833 loop_len
= PAGE_SIZE
- offset_in_page
;
835 if (rem_len
< loop_len
)
838 window_virt
= _get_local_va(offset
, window
, loop_len
);
842 memcpy(temp
, window_virt
, loop_len
);
844 memcpy(window_virt
, temp
, loop_len
);
850 end_offset
= window
->offset
+
851 (window
->nr_pages
<< PAGE_SHIFT
);
853 if (offset
== end_offset
) {
854 window
= list_entry_next(window
, list
);
855 end_offset
= window
->offset
+
856 (window
->nr_pages
<< PAGE_SHIFT
);
858 loop_len
= min(PAGE_SIZE
, rem_len
);
859 window_virt
= _get_local_va(offset
, window
, loop_len
);
863 memcpy(temp
, window_virt
, loop_len
);
865 memcpy(window_virt
, temp
, loop_len
);
873 * scif_rma_completion_cb:
876 * RMA interrupt completion callback.
878 static void scif_rma_completion_cb(void *data
)
880 struct scif_dma_comp_cb
*comp_cb
= data
;
882 /* Free DMA Completion CB. */
883 if (comp_cb
->dst_window
)
884 scif_rma_local_cpu_copy(comp_cb
->dst_offset
,
887 comp_cb
->header_padding
,
888 comp_cb
->len
, false);
889 scif_unmap_single(comp_cb
->temp_phys
, comp_cb
->sdev
,
890 SCIF_KMEM_UNALIGNED_BUF_SIZE
);
891 if (comp_cb
->is_cache
)
892 kmem_cache_free(unaligned_cache
,
893 comp_cb
->temp_buf_to_free
);
895 kfree(comp_cb
->temp_buf_to_free
);
898 /* Copies between temporary buffer and offsets provided in work */
900 scif_rma_list_dma_copy_unaligned(struct scif_copy_work
*work
,
901 u8
*temp
, struct dma_chan
*chan
,
904 struct scif_dma_comp_cb
*comp_cb
= work
->comp_cb
;
905 dma_addr_t window_dma_addr
, temp_dma_addr
;
906 dma_addr_t temp_phys
= comp_cb
->temp_phys
;
907 size_t loop_len
, nr_contig_bytes
= 0, remaining_len
= work
->len
;
908 int offset_in_ca
, ret
= 0;
909 s64 end_offset
, offset
;
910 struct scif_window
*window
;
911 void *window_virt_addr
;
913 struct dma_async_tx_descriptor
*tx
;
914 struct dma_device
*dev
= chan
->device
;
918 offset
= work
->dst_offset
;
919 window
= work
->dst_window
;
921 offset
= work
->src_offset
;
922 window
= work
->src_window
;
925 offset_in_ca
= offset
& (L1_CACHE_BYTES
- 1);
927 loop_len
= L1_CACHE_BYTES
- offset_in_ca
;
928 loop_len
= min(loop_len
, remaining_len
);
929 window_virt_addr
= ioremap_remote(offset
, window
,
933 if (!window_virt_addr
)
936 scif_unaligned_cpy_toio(window_virt_addr
, temp
,
939 !(remaining_len
- loop_len
));
941 scif_unaligned_cpy_fromio(temp
, window_virt_addr
,
942 loop_len
, work
->ordered
&&
943 !(remaining_len
- loop_len
));
944 iounmap_remote(window_virt_addr
, loop_len
, work
);
948 temp_phys
+= loop_len
;
949 remaining_len
-= loop_len
;
952 offset_in_ca
= offset
& ~PAGE_MASK
;
953 end_offset
= window
->offset
+
954 (window
->nr_pages
<< PAGE_SHIFT
);
956 tail_len
= remaining_len
& (L1_CACHE_BYTES
- 1);
957 remaining_len
-= tail_len
;
958 while (remaining_len
) {
959 if (offset
== end_offset
) {
960 window
= list_entry_next(window
, list
);
961 end_offset
= window
->offset
+
962 (window
->nr_pages
<< PAGE_SHIFT
);
964 if (scif_is_mgmt_node())
965 temp_dma_addr
= temp_phys
;
967 /* Fix if we ever enable IOMMU on the card */
968 temp_dma_addr
= (dma_addr_t
)virt_to_phys(temp
);
969 window_dma_addr
= scif_off_to_dma_addr(window
, offset
,
972 loop_len
= min(nr_contig_bytes
, remaining_len
);
974 if (work
->ordered
&& !tail_len
&&
975 !(remaining_len
- loop_len
) &&
976 loop_len
!= L1_CACHE_BYTES
) {
978 * Break up the last chunk of the transfer into
979 * two steps. if there is no tail to guarantee
980 * DMA ordering. SCIF_DMA_POLLING inserts
981 * a status update descriptor in step 1 which
982 * acts as a double sided synchronization fence
983 * for the DMA engine to ensure that the last
984 * cache line in step 2 is updated last.
986 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
988 dev
->device_prep_dma_memcpy(chan
,
998 cookie
= tx
->tx_submit(tx
);
999 if (dma_submit_error(cookie
)) {
1003 dma_async_issue_pending(chan
);
1004 offset
+= (loop_len
- L1_CACHE_BYTES
);
1005 temp_dma_addr
+= (loop_len
- L1_CACHE_BYTES
);
1006 window_dma_addr
+= (loop_len
- L1_CACHE_BYTES
);
1007 remaining_len
-= (loop_len
- L1_CACHE_BYTES
);
1008 loop_len
= remaining_len
;
1010 /* Step 2) DMA: L1_CACHE_BYTES */
1012 dev
->device_prep_dma_memcpy(chan
,
1020 cookie
= tx
->tx_submit(tx
);
1021 if (dma_submit_error(cookie
)) {
1025 dma_async_issue_pending(chan
);
1028 dev
->device_prep_dma_memcpy(chan
,
1036 cookie
= tx
->tx_submit(tx
);
1037 if (dma_submit_error(cookie
)) {
1041 dma_async_issue_pending(chan
);
1044 tx
= dev
->device_prep_dma_memcpy(chan
, temp_dma_addr
,
1045 window_dma_addr
, loop_len
, 0);
1050 cookie
= tx
->tx_submit(tx
);
1051 if (dma_submit_error(cookie
)) {
1055 dma_async_issue_pending(chan
);
1061 temp_phys
+= loop_len
;
1062 remaining_len
-= loop_len
;
1066 if (offset
== end_offset
) {
1067 window
= list_entry_next(window
, list
);
1068 end_offset
= window
->offset
+
1069 (window
->nr_pages
<< PAGE_SHIFT
);
1071 window_virt_addr
= ioremap_remote(offset
, window
, tail_len
,
1074 if (!window_virt_addr
)
1077 * The CPU copy for the tail bytes must be initiated only once
1078 * previous DMA transfers for this endpoint have completed
1079 * to guarantee ordering.
1081 if (work
->ordered
) {
1082 struct scif_dev
*rdev
= work
->remote_dev
;
1084 ret
= scif_drain_dma_intr(rdev
->sdev
, chan
);
1089 scif_unaligned_cpy_toio(window_virt_addr
, temp
,
1090 tail_len
, work
->ordered
);
1092 scif_unaligned_cpy_fromio(temp
, window_virt_addr
,
1093 tail_len
, work
->ordered
);
1094 iounmap_remote(window_virt_addr
, tail_len
, work
);
1096 tx
= dev
->device_prep_dma_memcpy(chan
, 0, 0, 0, DMA_PREP_INTERRUPT
);
1101 tx
->callback
= &scif_rma_completion_cb
;
1102 tx
->callback_param
= comp_cb
;
1103 cookie
= tx
->tx_submit(tx
);
1105 if (dma_submit_error(cookie
)) {
1109 dma_async_issue_pending(chan
);
1112 dev_err(scif_info
.mdev
.this_device
,
1113 "%s %d Desc Prog Failed ret %d\n",
1114 __func__
, __LINE__
, ret
);
1119 * _scif_rma_list_dma_copy_aligned:
1121 * Traverse all the windows and perform DMA copy.
1123 static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work
*work
,
1124 struct dma_chan
*chan
)
1126 dma_addr_t src_dma_addr
, dst_dma_addr
;
1127 size_t loop_len
, remaining_len
, src_contig_bytes
= 0;
1128 size_t dst_contig_bytes
= 0;
1129 struct scif_window_iter src_win_iter
;
1130 struct scif_window_iter dst_win_iter
;
1131 s64 end_src_offset
, end_dst_offset
;
1132 struct scif_window
*src_window
= work
->src_window
;
1133 struct scif_window
*dst_window
= work
->dst_window
;
1134 s64 src_offset
= work
->src_offset
, dst_offset
= work
->dst_offset
;
1136 struct dma_async_tx_descriptor
*tx
;
1137 struct dma_device
*dev
= chan
->device
;
1138 dma_cookie_t cookie
;
1140 remaining_len
= work
->len
;
1142 scif_init_window_iter(src_window
, &src_win_iter
);
1143 scif_init_window_iter(dst_window
, &dst_win_iter
);
1144 end_src_offset
= src_window
->offset
+
1145 (src_window
->nr_pages
<< PAGE_SHIFT
);
1146 end_dst_offset
= dst_window
->offset
+
1147 (dst_window
->nr_pages
<< PAGE_SHIFT
);
1148 while (remaining_len
) {
1149 if (src_offset
== end_src_offset
) {
1150 src_window
= list_entry_next(src_window
, list
);
1151 end_src_offset
= src_window
->offset
+
1152 (src_window
->nr_pages
<< PAGE_SHIFT
);
1153 scif_init_window_iter(src_window
, &src_win_iter
);
1155 if (dst_offset
== end_dst_offset
) {
1156 dst_window
= list_entry_next(dst_window
, list
);
1157 end_dst_offset
= dst_window
->offset
+
1158 (dst_window
->nr_pages
<< PAGE_SHIFT
);
1159 scif_init_window_iter(dst_window
, &dst_win_iter
);
1162 /* compute dma addresses for transfer */
1163 src_dma_addr
= scif_off_to_dma_addr(src_window
, src_offset
,
1166 dst_dma_addr
= scif_off_to_dma_addr(dst_window
, dst_offset
,
1169 loop_len
= min(src_contig_bytes
, dst_contig_bytes
);
1170 loop_len
= min(loop_len
, remaining_len
);
1171 if (work
->ordered
&& !(remaining_len
- loop_len
)) {
1173 * Break up the last chunk of the transfer into two
1174 * steps to ensure that the last byte in step 2 is
1177 /* Step 1) DMA: Body Length - 1 */
1178 tx
= dev
->device_prep_dma_memcpy(chan
, dst_dma_addr
,
1186 cookie
= tx
->tx_submit(tx
);
1187 if (dma_submit_error(cookie
)) {
1191 src_offset
+= (loop_len
- 1);
1192 dst_offset
+= (loop_len
- 1);
1193 src_dma_addr
+= (loop_len
- 1);
1194 dst_dma_addr
+= (loop_len
- 1);
1195 remaining_len
-= (loop_len
- 1);
1196 loop_len
= remaining_len
;
1198 /* Step 2) DMA: 1 BYTES */
1199 tx
= dev
->device_prep_dma_memcpy(chan
, dst_dma_addr
,
1200 src_dma_addr
, loop_len
, 0);
1205 cookie
= tx
->tx_submit(tx
);
1206 if (dma_submit_error(cookie
)) {
1210 dma_async_issue_pending(chan
);
1212 tx
= dev
->device_prep_dma_memcpy(chan
, dst_dma_addr
,
1213 src_dma_addr
, loop_len
, 0);
1218 cookie
= tx
->tx_submit(tx
);
1219 if (dma_submit_error(cookie
)) {
1224 src_offset
+= loop_len
;
1225 dst_offset
+= loop_len
;
1226 remaining_len
-= loop_len
;
1230 dev_err(scif_info
.mdev
.this_device
,
1231 "%s %d Desc Prog Failed ret %d\n",
1232 __func__
, __LINE__
, ret
);
1237 * scif_rma_list_dma_copy_aligned:
1239 * Traverse all the windows and perform DMA copy.
1241 static int scif_rma_list_dma_copy_aligned(struct scif_copy_work
*work
,
1242 struct dma_chan
*chan
)
1244 dma_addr_t src_dma_addr
, dst_dma_addr
;
1245 size_t loop_len
, remaining_len
, tail_len
, src_contig_bytes
= 0;
1246 size_t dst_contig_bytes
= 0;
1248 s64 end_src_offset
, end_dst_offset
;
1249 struct scif_window_iter src_win_iter
;
1250 struct scif_window_iter dst_win_iter
;
1251 void *src_virt
, *dst_virt
;
1252 struct scif_window
*src_window
= work
->src_window
;
1253 struct scif_window
*dst_window
= work
->dst_window
;
1254 s64 src_offset
= work
->src_offset
, dst_offset
= work
->dst_offset
;
1256 struct dma_async_tx_descriptor
*tx
;
1257 struct dma_device
*dev
= chan
->device
;
1258 dma_cookie_t cookie
;
1260 remaining_len
= work
->len
;
1261 scif_init_window_iter(src_window
, &src_win_iter
);
1262 scif_init_window_iter(dst_window
, &dst_win_iter
);
1264 src_cache_off
= src_offset
& (L1_CACHE_BYTES
- 1);
1265 if (src_cache_off
!= 0) {
1267 loop_len
= L1_CACHE_BYTES
- src_cache_off
;
1268 loop_len
= min(loop_len
, remaining_len
);
1269 src_dma_addr
= __scif_off_to_dma_addr(src_window
, src_offset
);
1270 dst_dma_addr
= __scif_off_to_dma_addr(dst_window
, dst_offset
);
1271 if (src_window
->type
== SCIF_WINDOW_SELF
)
1272 src_virt
= _get_local_va(src_offset
, src_window
,
1275 src_virt
= ioremap_remote(src_offset
, src_window
,
1277 work
->remote_dev
, NULL
);
1280 if (dst_window
->type
== SCIF_WINDOW_SELF
)
1281 dst_virt
= _get_local_va(dst_offset
, dst_window
,
1284 dst_virt
= ioremap_remote(dst_offset
, dst_window
,
1286 work
->remote_dev
, NULL
);
1288 if (src_window
->type
!= SCIF_WINDOW_SELF
)
1289 iounmap_remote(src_virt
, loop_len
, work
);
1292 if (src_window
->type
== SCIF_WINDOW_SELF
)
1293 scif_unaligned_cpy_toio(dst_virt
, src_virt
, loop_len
,
1294 remaining_len
== loop_len
?
1295 work
->ordered
: false);
1297 scif_unaligned_cpy_fromio(dst_virt
, src_virt
, loop_len
,
1298 remaining_len
== loop_len
?
1299 work
->ordered
: false);
1300 if (src_window
->type
!= SCIF_WINDOW_SELF
)
1301 iounmap_remote(src_virt
, loop_len
, work
);
1302 if (dst_window
->type
!= SCIF_WINDOW_SELF
)
1303 iounmap_remote(dst_virt
, loop_len
, work
);
1304 src_offset
+= loop_len
;
1305 dst_offset
+= loop_len
;
1306 remaining_len
-= loop_len
;
1309 end_src_offset
= src_window
->offset
+
1310 (src_window
->nr_pages
<< PAGE_SHIFT
);
1311 end_dst_offset
= dst_window
->offset
+
1312 (dst_window
->nr_pages
<< PAGE_SHIFT
);
1313 tail_len
= remaining_len
& (L1_CACHE_BYTES
- 1);
1314 remaining_len
-= tail_len
;
1315 while (remaining_len
) {
1316 if (src_offset
== end_src_offset
) {
1317 src_window
= list_entry_next(src_window
, list
);
1318 end_src_offset
= src_window
->offset
+
1319 (src_window
->nr_pages
<< PAGE_SHIFT
);
1320 scif_init_window_iter(src_window
, &src_win_iter
);
1322 if (dst_offset
== end_dst_offset
) {
1323 dst_window
= list_entry_next(dst_window
, list
);
1324 end_dst_offset
= dst_window
->offset
+
1325 (dst_window
->nr_pages
<< PAGE_SHIFT
);
1326 scif_init_window_iter(dst_window
, &dst_win_iter
);
1329 /* compute dma addresses for transfer */
1330 src_dma_addr
= scif_off_to_dma_addr(src_window
, src_offset
,
1333 dst_dma_addr
= scif_off_to_dma_addr(dst_window
, dst_offset
,
1336 loop_len
= min(src_contig_bytes
, dst_contig_bytes
);
1337 loop_len
= min(loop_len
, remaining_len
);
1338 if (work
->ordered
&& !tail_len
&&
1339 !(remaining_len
- loop_len
)) {
1341 * Break up the last chunk of the transfer into two
1342 * steps. if there is no tail to gurantee DMA ordering.
1343 * Passing SCIF_DMA_POLLING inserts a status update
1344 * descriptor in step 1 which acts as a double sided
1345 * synchronization fence for the DMA engine to ensure
1346 * that the last cache line in step 2 is updated last.
1348 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
1349 tx
= dev
->device_prep_dma_memcpy(chan
, dst_dma_addr
,
1358 cookie
= tx
->tx_submit(tx
);
1359 if (dma_submit_error(cookie
)) {
1363 dma_async_issue_pending(chan
);
1364 src_offset
+= (loop_len
- L1_CACHE_BYTES
);
1365 dst_offset
+= (loop_len
- L1_CACHE_BYTES
);
1366 src_dma_addr
+= (loop_len
- L1_CACHE_BYTES
);
1367 dst_dma_addr
+= (loop_len
- L1_CACHE_BYTES
);
1368 remaining_len
-= (loop_len
- L1_CACHE_BYTES
);
1369 loop_len
= remaining_len
;
1371 /* Step 2) DMA: L1_CACHE_BYTES */
1372 tx
= dev
->device_prep_dma_memcpy(chan
, dst_dma_addr
,
1379 cookie
= tx
->tx_submit(tx
);
1380 if (dma_submit_error(cookie
)) {
1384 dma_async_issue_pending(chan
);
1386 tx
= dev
->device_prep_dma_memcpy(chan
, dst_dma_addr
,
1393 cookie
= tx
->tx_submit(tx
);
1394 if (dma_submit_error(cookie
)) {
1398 dma_async_issue_pending(chan
);
1400 src_offset
+= loop_len
;
1401 dst_offset
+= loop_len
;
1402 remaining_len
-= loop_len
;
1404 remaining_len
= tail_len
;
1405 if (remaining_len
) {
1406 loop_len
= remaining_len
;
1407 if (src_offset
== end_src_offset
)
1408 src_window
= list_entry_next(src_window
, list
);
1409 if (dst_offset
== end_dst_offset
)
1410 dst_window
= list_entry_next(dst_window
, list
);
1412 src_dma_addr
= __scif_off_to_dma_addr(src_window
, src_offset
);
1413 dst_dma_addr
= __scif_off_to_dma_addr(dst_window
, dst_offset
);
1415 * The CPU copy for the tail bytes must be initiated only once
1416 * previous DMA transfers for this endpoint have completed to
1417 * guarantee ordering.
1419 if (work
->ordered
) {
1420 struct scif_dev
*rdev
= work
->remote_dev
;
1422 ret
= scif_drain_dma_poll(rdev
->sdev
, chan
);
1426 if (src_window
->type
== SCIF_WINDOW_SELF
)
1427 src_virt
= _get_local_va(src_offset
, src_window
,
1430 src_virt
= ioremap_remote(src_offset
, src_window
,
1432 work
->remote_dev
, NULL
);
1436 if (dst_window
->type
== SCIF_WINDOW_SELF
)
1437 dst_virt
= _get_local_va(dst_offset
, dst_window
,
1440 dst_virt
= ioremap_remote(dst_offset
, dst_window
,
1442 work
->remote_dev
, NULL
);
1444 if (src_window
->type
!= SCIF_WINDOW_SELF
)
1445 iounmap_remote(src_virt
, loop_len
, work
);
1449 if (src_window
->type
== SCIF_WINDOW_SELF
)
1450 scif_unaligned_cpy_toio(dst_virt
, src_virt
, loop_len
,
1453 scif_unaligned_cpy_fromio(dst_virt
, src_virt
,
1454 loop_len
, work
->ordered
);
1455 if (src_window
->type
!= SCIF_WINDOW_SELF
)
1456 iounmap_remote(src_virt
, loop_len
, work
);
1458 if (dst_window
->type
!= SCIF_WINDOW_SELF
)
1459 iounmap_remote(dst_virt
, loop_len
, work
);
1460 remaining_len
-= loop_len
;
1464 dev_err(scif_info
.mdev
.this_device
,
1465 "%s %d Desc Prog Failed ret %d\n",
1466 __func__
, __LINE__
, ret
);
1471 * scif_rma_list_cpu_copy:
1473 * Traverse all the windows and perform CPU copy.
1475 static int scif_rma_list_cpu_copy(struct scif_copy_work
*work
)
1477 void *src_virt
, *dst_virt
;
1478 size_t loop_len
, remaining_len
;
1479 int src_page_off
, dst_page_off
;
1480 s64 src_offset
= work
->src_offset
, dst_offset
= work
->dst_offset
;
1481 struct scif_window
*src_window
= work
->src_window
;
1482 struct scif_window
*dst_window
= work
->dst_window
;
1483 s64 end_src_offset
, end_dst_offset
;
1485 struct scif_window_iter src_win_iter
;
1486 struct scif_window_iter dst_win_iter
;
1488 remaining_len
= work
->len
;
1490 scif_init_window_iter(src_window
, &src_win_iter
);
1491 scif_init_window_iter(dst_window
, &dst_win_iter
);
1492 while (remaining_len
) {
1493 src_page_off
= src_offset
& ~PAGE_MASK
;
1494 dst_page_off
= dst_offset
& ~PAGE_MASK
;
1495 loop_len
= min(PAGE_SIZE
-
1496 max(src_page_off
, dst_page_off
),
1499 if (src_window
->type
== SCIF_WINDOW_SELF
)
1500 src_virt
= _get_local_va(src_offset
, src_window
,
1503 src_virt
= ioremap_remote(src_offset
, src_window
,
1512 if (dst_window
->type
== SCIF_WINDOW_SELF
)
1513 dst_virt
= _get_local_va(dst_offset
, dst_window
,
1516 dst_virt
= ioremap_remote(dst_offset
, dst_window
,
1521 if (src_window
->type
== SCIF_WINDOW_PEER
)
1522 iounmap_remote(src_virt
, loop_len
, work
);
1527 if (work
->loopback
) {
1528 memcpy(dst_virt
, src_virt
, loop_len
);
1530 if (src_window
->type
== SCIF_WINDOW_SELF
)
1531 memcpy_toio((void __iomem __force
*)dst_virt
,
1532 src_virt
, loop_len
);
1534 memcpy_fromio(dst_virt
,
1535 (void __iomem __force
*)src_virt
,
1538 if (src_window
->type
== SCIF_WINDOW_PEER
)
1539 iounmap_remote(src_virt
, loop_len
, work
);
1541 if (dst_window
->type
== SCIF_WINDOW_PEER
)
1542 iounmap_remote(dst_virt
, loop_len
, work
);
1544 src_offset
+= loop_len
;
1545 dst_offset
+= loop_len
;
1546 remaining_len
-= loop_len
;
1547 if (remaining_len
) {
1548 end_src_offset
= src_window
->offset
+
1549 (src_window
->nr_pages
<< PAGE_SHIFT
);
1550 end_dst_offset
= dst_window
->offset
+
1551 (dst_window
->nr_pages
<< PAGE_SHIFT
);
1552 if (src_offset
== end_src_offset
) {
1553 src_window
= list_entry_next(src_window
, list
);
1554 scif_init_window_iter(src_window
,
1557 if (dst_offset
== end_dst_offset
) {
1558 dst_window
= list_entry_next(dst_window
, list
);
1559 scif_init_window_iter(dst_window
,
1568 static int scif_rma_list_dma_copy_wrapper(struct scif_endpt
*epd
,
1569 struct scif_copy_work
*work
,
1570 struct dma_chan
*chan
, off_t loffset
)
1572 int src_cache_off
, dst_cache_off
;
1573 s64 src_offset
= work
->src_offset
, dst_offset
= work
->dst_offset
;
1575 bool src_local
= true, dst_local
= false;
1576 struct scif_dma_comp_cb
*comp_cb
;
1577 dma_addr_t src_dma_addr
, dst_dma_addr
;
1580 if (is_dma_copy_aligned(chan
->device
, 1, 1, 1))
1581 return _scif_rma_list_dma_copy_aligned(work
, chan
);
1583 src_cache_off
= src_offset
& (L1_CACHE_BYTES
- 1);
1584 dst_cache_off
= dst_offset
& (L1_CACHE_BYTES
- 1);
1586 if (dst_cache_off
== src_cache_off
)
1587 return scif_rma_list_dma_copy_aligned(work
, chan
);
1590 return scif_rma_list_cpu_copy(work
);
1591 src_dma_addr
= __scif_off_to_dma_addr(work
->src_window
, src_offset
);
1592 dst_dma_addr
= __scif_off_to_dma_addr(work
->dst_window
, dst_offset
);
1593 src_local
= work
->src_window
->type
== SCIF_WINDOW_SELF
;
1594 dst_local
= work
->dst_window
->type
== SCIF_WINDOW_SELF
;
1596 dst_local
= dst_local
;
1597 /* Allocate dma_completion cb */
1598 comp_cb
= kzalloc(sizeof(*comp_cb
), GFP_KERNEL
);
1602 work
->comp_cb
= comp_cb
;
1603 comp_cb
->cb_cookie
= comp_cb
;
1604 comp_cb
->dma_completion_func
= &scif_rma_completion_cb
;
1606 if (work
->len
+ (L1_CACHE_BYTES
<< 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE
) {
1607 comp_cb
->is_cache
= false;
1608 /* Allocate padding bytes to align to a cache line */
1609 temp
= kmalloc(work
->len
+ (L1_CACHE_BYTES
<< 1),
1613 comp_cb
->temp_buf_to_free
= temp
;
1614 /* kmalloc(..) does not guarantee cache line alignment */
1615 if (!IS_ALIGNED((u64
)temp
, L1_CACHE_BYTES
))
1616 temp
= PTR_ALIGN(temp
, L1_CACHE_BYTES
);
1618 comp_cb
->is_cache
= true;
1619 temp
= kmem_cache_alloc(unaligned_cache
, GFP_KERNEL
);
1622 comp_cb
->temp_buf_to_free
= temp
;
1626 temp
+= dst_cache_off
;
1627 scif_rma_local_cpu_copy(work
->src_offset
, work
->src_window
,
1628 temp
, work
->len
, true);
1630 comp_cb
->dst_window
= work
->dst_window
;
1631 comp_cb
->dst_offset
= work
->dst_offset
;
1632 work
->src_offset
= work
->src_offset
- src_cache_off
;
1633 comp_cb
->len
= work
->len
;
1634 work
->len
= ALIGN(work
->len
+ src_cache_off
, L1_CACHE_BYTES
);
1635 comp_cb
->header_padding
= src_cache_off
;
1637 comp_cb
->temp_buf
= temp
;
1639 err
= scif_map_single(&comp_cb
->temp_phys
, temp
,
1640 work
->remote_dev
, SCIF_KMEM_UNALIGNED_BUF_SIZE
);
1643 comp_cb
->sdev
= work
->remote_dev
;
1644 if (scif_rma_list_dma_copy_unaligned(work
, temp
, chan
, src_local
) < 0)
1647 work
->fence_type
= SCIF_DMA_INTR
;
1650 if (comp_cb
->is_cache
)
1651 kmem_cache_free(unaligned_cache
, comp_cb
->temp_buf_to_free
);
1653 kfree(comp_cb
->temp_buf_to_free
);
1662 * @epd: end point descriptor.
1663 * @loffset: offset in local registered address space to/from which to copy
1664 * @addr: user virtual address to/from which to copy
1665 * @len: length of range to copy
1666 * @roffset: offset in remote registered address space to/from which to copy
1668 * @dir: LOCAL->REMOTE or vice versa.
1669 * @last_chunk: true if this is the last chunk of a larger transfer
1671 * Validate parameters, check if src/dst registered ranges requested for copy
1672 * are valid and initiate either CPU or DMA copy.
1674 static int scif_rma_copy(scif_epd_t epd
, off_t loffset
, unsigned long addr
,
1675 size_t len
, off_t roffset
, int flags
,
1676 enum scif_rma_dir dir
, bool last_chunk
)
1678 struct scif_endpt
*ep
= (struct scif_endpt
*)epd
;
1679 struct scif_rma_req remote_req
;
1680 struct scif_rma_req req
;
1681 struct scif_window
*local_window
= NULL
;
1682 struct scif_window
*remote_window
= NULL
;
1683 struct scif_copy_work copy_work
;
1686 struct dma_chan
*chan
;
1687 struct scif_mmu_notif
*mmn
= NULL
;
1689 struct device
*spdev
;
1691 err
= scif_verify_epd(ep
);
1695 if (flags
&& !(flags
& (SCIF_RMA_USECPU
| SCIF_RMA_USECACHE
|
1696 SCIF_RMA_SYNC
| SCIF_RMA_ORDERED
)))
1699 loopback
= scifdev_self(ep
->remote_dev
) ? true : false;
1700 copy_work
.fence_type
= ((flags
& SCIF_RMA_SYNC
) && last_chunk
) ?
1702 copy_work
.ordered
= !!((flags
& SCIF_RMA_ORDERED
) && last_chunk
);
1704 /* Use CPU for Mgmt node <-> Mgmt node copies */
1705 if (loopback
&& scif_is_mgmt_node()) {
1706 flags
|= SCIF_RMA_USECPU
;
1707 copy_work
.fence_type
= 0x0;
1710 cache
= scif_is_set_reg_cache(flags
);
1712 remote_req
.out_window
= &remote_window
;
1713 remote_req
.offset
= roffset
;
1714 remote_req
.nr_bytes
= len
;
1716 * If transfer is from local to remote then the remote window
1717 * must be writeable and vice versa.
1719 remote_req
.prot
= dir
== SCIF_LOCAL_TO_REMOTE
? VM_WRITE
: VM_READ
;
1720 remote_req
.type
= SCIF_WINDOW_PARTIAL
;
1721 remote_req
.head
= &ep
->rma_info
.remote_reg_list
;
1723 spdev
= scif_get_peer_dev(ep
->remote_dev
);
1724 if (IS_ERR(spdev
)) {
1725 err
= PTR_ERR(spdev
);
1729 if (addr
&& cache
) {
1730 mutex_lock(&ep
->rma_info
.mmn_lock
);
1731 mmn
= scif_find_mmu_notifier(current
->mm
, &ep
->rma_info
);
1733 scif_add_mmu_notifier(current
->mm
, ep
);
1734 mutex_unlock(&ep
->rma_info
.mmn_lock
);
1736 scif_put_peer_dev(spdev
);
1737 return PTR_ERR(mmn
);
1739 cache
= cache
&& !scif_rma_tc_can_cache(ep
, len
);
1741 mutex_lock(&ep
->rma_info
.rma_lock
);
1743 req
.out_window
= &local_window
;
1744 req
.nr_bytes
= ALIGN(len
+ (addr
& ~PAGE_MASK
),
1746 req
.va_for_temp
= addr
& PAGE_MASK
;
1747 req
.prot
= (dir
== SCIF_LOCAL_TO_REMOTE
?
1748 VM_READ
: VM_WRITE
| VM_READ
);
1749 /* Does a valid local window exist? */
1751 spin_lock(&ep
->rma_info
.tc_lock
);
1752 req
.head
= &mmn
->tc_reg_list
;
1753 err
= scif_query_tcw(ep
, &req
);
1754 spin_unlock(&ep
->rma_info
.tc_lock
);
1757 err
= scif_register_temp(epd
, req
.va_for_temp
,
1758 req
.nr_bytes
, req
.prot
,
1759 &loffset
, &local_window
);
1761 mutex_unlock(&ep
->rma_info
.rma_lock
);
1766 atomic_inc(&ep
->rma_info
.tcw_refcount
);
1767 atomic_add_return(local_window
->nr_pages
,
1768 &ep
->rma_info
.tcw_total_pages
);
1770 spin_lock(&ep
->rma_info
.tc_lock
);
1771 scif_insert_tcw(local_window
,
1773 spin_unlock(&ep
->rma_info
.tc_lock
);
1777 loffset
= local_window
->offset
+
1778 (addr
- local_window
->va_for_temp
);
1780 req
.out_window
= &local_window
;
1781 req
.offset
= loffset
;
1783 * If transfer is from local to remote then the self window
1784 * must be readable and vice versa.
1786 req
.prot
= dir
== SCIF_LOCAL_TO_REMOTE
? VM_READ
: VM_WRITE
;
1788 req
.type
= SCIF_WINDOW_PARTIAL
;
1789 req
.head
= &ep
->rma_info
.reg_list
;
1790 /* Does a valid local window exist? */
1791 err
= scif_query_window(&req
);
1793 mutex_unlock(&ep
->rma_info
.rma_lock
);
1798 /* Does a valid remote window exist? */
1799 err
= scif_query_window(&remote_req
);
1801 mutex_unlock(&ep
->rma_info
.rma_lock
);
1806 * Prepare copy_work for submitting work to the DMA kernel thread
1807 * or CPU copy routine.
1809 copy_work
.len
= len
;
1810 copy_work
.loopback
= loopback
;
1811 copy_work
.remote_dev
= ep
->remote_dev
;
1812 if (dir
== SCIF_LOCAL_TO_REMOTE
) {
1813 copy_work
.src_offset
= loffset
;
1814 copy_work
.src_window
= local_window
;
1815 copy_work
.dst_offset
= roffset
;
1816 copy_work
.dst_window
= remote_window
;
1818 copy_work
.src_offset
= roffset
;
1819 copy_work
.src_window
= remote_window
;
1820 copy_work
.dst_offset
= loffset
;
1821 copy_work
.dst_window
= local_window
;
1824 if (flags
& SCIF_RMA_USECPU
) {
1825 scif_rma_list_cpu_copy(©_work
);
1827 chan
= ep
->rma_info
.dma_chan
;
1828 err
= scif_rma_list_dma_copy_wrapper(epd
, ©_work
,
1832 atomic_inc(&ep
->rma_info
.tw_refcount
);
1834 mutex_unlock(&ep
->rma_info
.rma_lock
);
1837 struct scif_dev
*rdev
= ep
->remote_dev
;
1839 if (copy_work
.fence_type
== SCIF_DMA_POLL
)
1840 err
= scif_drain_dma_poll(rdev
->sdev
,
1841 ep
->rma_info
.dma_chan
);
1842 else if (copy_work
.fence_type
== SCIF_DMA_INTR
)
1843 err
= scif_drain_dma_intr(rdev
->sdev
,
1844 ep
->rma_info
.dma_chan
);
1848 scif_queue_for_cleanup(local_window
, &scif_info
.rma
);
1849 scif_put_peer_dev(spdev
);
1853 if (addr
&& local_window
&& !cache
)
1854 scif_destroy_window(ep
, local_window
);
1855 dev_err(scif_info
.mdev
.this_device
,
1856 "%s %d err %d len 0x%lx\n",
1857 __func__
, __LINE__
, err
, len
);
1859 scif_put_peer_dev(spdev
);
1863 int scif_readfrom(scif_epd_t epd
, off_t loffset
, size_t len
,
1864 off_t roffset
, int flags
)
1868 dev_dbg(scif_info
.mdev
.this_device
,
1869 "SCIFAPI readfrom: ep %p loffset 0x%lx len 0x%lx offset 0x%lx flags 0x%x\n",
1870 epd
, loffset
, len
, roffset
, flags
);
1871 if (scif_unaligned(loffset
, roffset
)) {
1872 while (len
> SCIF_MAX_UNALIGNED_BUF_SIZE
) {
1873 err
= scif_rma_copy(epd
, loffset
, 0x0,
1874 SCIF_MAX_UNALIGNED_BUF_SIZE
,
1876 SCIF_REMOTE_TO_LOCAL
, false);
1879 loffset
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1880 roffset
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1881 len
-= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1884 err
= scif_rma_copy(epd
, loffset
, 0x0, len
,
1885 roffset
, flags
, SCIF_REMOTE_TO_LOCAL
, true);
1889 EXPORT_SYMBOL_GPL(scif_readfrom
);
1891 int scif_writeto(scif_epd_t epd
, off_t loffset
, size_t len
,
1892 off_t roffset
, int flags
)
1896 dev_dbg(scif_info
.mdev
.this_device
,
1897 "SCIFAPI writeto: ep %p loffset 0x%lx len 0x%lx roffset 0x%lx flags 0x%x\n",
1898 epd
, loffset
, len
, roffset
, flags
);
1899 if (scif_unaligned(loffset
, roffset
)) {
1900 while (len
> SCIF_MAX_UNALIGNED_BUF_SIZE
) {
1901 err
= scif_rma_copy(epd
, loffset
, 0x0,
1902 SCIF_MAX_UNALIGNED_BUF_SIZE
,
1904 SCIF_LOCAL_TO_REMOTE
, false);
1907 loffset
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1908 roffset
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1909 len
-= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1912 err
= scif_rma_copy(epd
, loffset
, 0x0, len
,
1913 roffset
, flags
, SCIF_LOCAL_TO_REMOTE
, true);
1917 EXPORT_SYMBOL_GPL(scif_writeto
);
1919 int scif_vreadfrom(scif_epd_t epd
, void *addr
, size_t len
,
1920 off_t roffset
, int flags
)
1924 dev_dbg(scif_info
.mdev
.this_device
,
1925 "SCIFAPI vreadfrom: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1926 epd
, addr
, len
, roffset
, flags
);
1927 if (scif_unaligned((off_t __force
)addr
, roffset
)) {
1928 if (len
> SCIF_MAX_UNALIGNED_BUF_SIZE
)
1929 flags
&= ~SCIF_RMA_USECACHE
;
1931 while (len
> SCIF_MAX_UNALIGNED_BUF_SIZE
) {
1932 err
= scif_rma_copy(epd
, 0, (u64
)addr
,
1933 SCIF_MAX_UNALIGNED_BUF_SIZE
,
1935 SCIF_REMOTE_TO_LOCAL
, false);
1938 addr
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1939 roffset
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1940 len
-= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1943 err
= scif_rma_copy(epd
, 0, (u64
)addr
, len
,
1944 roffset
, flags
, SCIF_REMOTE_TO_LOCAL
, true);
1948 EXPORT_SYMBOL_GPL(scif_vreadfrom
);
1950 int scif_vwriteto(scif_epd_t epd
, void *addr
, size_t len
,
1951 off_t roffset
, int flags
)
1955 dev_dbg(scif_info
.mdev
.this_device
,
1956 "SCIFAPI vwriteto: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1957 epd
, addr
, len
, roffset
, flags
);
1958 if (scif_unaligned((off_t __force
)addr
, roffset
)) {
1959 if (len
> SCIF_MAX_UNALIGNED_BUF_SIZE
)
1960 flags
&= ~SCIF_RMA_USECACHE
;
1962 while (len
> SCIF_MAX_UNALIGNED_BUF_SIZE
) {
1963 err
= scif_rma_copy(epd
, 0, (u64
)addr
,
1964 SCIF_MAX_UNALIGNED_BUF_SIZE
,
1966 SCIF_LOCAL_TO_REMOTE
, false);
1969 addr
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1970 roffset
+= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1971 len
-= SCIF_MAX_UNALIGNED_BUF_SIZE
;
1974 err
= scif_rma_copy(epd
, 0, (u64
)addr
, len
,
1975 roffset
, flags
, SCIF_LOCAL_TO_REMOTE
, true);
1979 EXPORT_SYMBOL_GPL(scif_vwriteto
);