drivers/misc/mic/scif/scif_rma.h

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * Copyright(c) 2015 Intel Corporation.
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of version 2 of the GNU General Public License as
  13  * published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful, but
  16  * WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * General Public License for more details.
  19  *
  20  * BSD LICENSE
  21  *
  22  * Copyright(c) 2015 Intel Corporation.
  23  *
  24  * Redistribution and use in source and binary forms, with or without
  25  * modification, are permitted provided that the following conditions
  26  * are met:
  27  *
  28  * * Redistributions of source code must retain the above copyright
  29  *   notice, this list of conditions and the following disclaimer.
  30  * * Redistributions in binary form must reproduce the above copyright
  31  *   notice, this list of conditions and the following disclaimer in
  32  *   the documentation and/or other materials provided with the
  33  *   distribution.
  34  * * Neither the name of Intel Corporation nor the names of its
  35  *   contributors may be used to endorse or promote products derived
  36  *   from this software without specific prior written permission.
  37  *
  38  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  39  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  40  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  41  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  42  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  44  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  45  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  46  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  47  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  48  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  49  *
  50  * Intel SCIF driver.
  51  *
  52  */
  53 #ifndef SCIF_RMA_H
  54 #define SCIF_RMA_H
  55
  56 #include <linux/intel-iommu.h>
  57 #include <linux/mmu_notifier.h>
  58
  59 #include "../bus/scif_bus.h"
  60
  61 /* If this bit is set then the mark is a remote fence mark */
  62 #define SCIF_REMOTE_FENCE_BIT          31
  63 /* Magic value used to indicate a remote fence request */
  64 #define SCIF_REMOTE_FENCE BIT_ULL(SCIF_REMOTE_FENCE_BIT)
  65
  66 #define SCIF_MAX_UNALIGNED_BUF_SIZE (1024 * 1024ULL)
  67 #define SCIF_KMEM_UNALIGNED_BUF_SIZE (SCIF_MAX_UNALIGNED_BUF_SIZE + \
  68                                       (L1_CACHE_BYTES << 1))
  69
  70 #define SCIF_IOVA_START_PFN             (1)
  71 #define SCIF_IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
  72 #define SCIF_DMA_64BIT_PFN SCIF_IOVA_PFN(DMA_BIT_MASK(64))
  73 #define SCIF_DMA_63BIT_PFN SCIF_IOVA_PFN(DMA_BIT_MASK(63))
  74
  75 /*
  76  * struct scif_endpt_rma_info - Per Endpoint Remote Memory Access Information
  77  *
  78  * @reg_list: List of registration windows for self
  79  * @remote_reg_list: List of registration windows for peer
  80  * @iovad: Offset generator
  81  * @rma_lock: Synchronizes access to self/remote list and also protects the
  82  *            window from being destroyed while RMAs are in progress.
  83  * @tc_lock: Synchronizes access to temporary cached windows list
  84  *           for SCIF Registration Caching.
  85  * @mmn_lock: Synchronizes access to the list of MMU notifiers registered
  86  * @tw_refcount: Keeps track of number of outstanding temporary registered
  87  *               windows created by scif_vreadfrom/scif_vwriteto which have
  88  *               not been destroyed.
  89  * @tcw_refcount: Same as tw_refcount but for temporary cached windows
  90  * @tcw_total_pages: Same as tcw_refcount but in terms of pages pinned
  91  * @mmn_list: MMU notifier so that we can destroy the windows when required
  92  * @fence_refcount: Keeps track of number of outstanding remote fence
  93  *                  requests which have been received by the peer.
  94  * @dma_chan: DMA channel used for all DMA transfers for this endpoint.
  95  * @async_list_del: Detect asynchronous list entry deletion
  96  * @vma_list: List of vmas with remote memory mappings
  97  * @markwq: Wait queue used for scif_fence_mark/scif_fence_wait
  98 */
  99 struct scif_endpt_rma_info {
 100         struct list_head reg_list;
 101         struct list_head remote_reg_list;
 102         struct iova_domain iovad;
 103         struct mutex rma_lock;
 104         spinlock_t tc_lock;
 105         struct mutex mmn_lock;
 106         atomic_t tw_refcount;
 107         atomic_t tcw_refcount;
 108         atomic_t tcw_total_pages;
 109         struct list_head mmn_list;
 110         atomic_t fence_refcount;
 111         struct dma_chan *dma_chan;
 112         int async_list_del;
 113         struct list_head vma_list;
 114         wait_queue_head_t markwq;
 115 };
 116
 117 /*
 118  * struct scif_fence_info - used for tracking fence requests
 119  *
 120  * @state: State of this transfer
 121  * @wq: Fences wait on this queue
 122  * @dma_mark: Used for storing the DMA mark
 123  */
 124 struct scif_fence_info {
 125         enum scif_msg_state state;
 126         struct completion comp;
 127         int dma_mark;
 128 };
 129
 130 /*
 131  * struct scif_remote_fence_info - used for tracking remote fence requests
 132  *
 133  * @msg: List of SCIF node QP fence messages
 134  * @list: Link to list of remote fence requests
 135  */
 136 struct scif_remote_fence_info {
 137         struct scifmsg msg;
 138         struct list_head list;
 139 };
 140
 141 /*
 142  * Specifies whether an RMA operation can span across partial windows, a single
 143  * window or multiple contiguous windows. Mmaps can span across partial windows.
 144  * Unregistration can span across complete windows. scif_get_pages() can span a
 145  * single window. A window can also be of type self or peer.
 146  */
 147 enum scif_window_type {
 148         SCIF_WINDOW_PARTIAL,
 149         SCIF_WINDOW_SINGLE,
 150         SCIF_WINDOW_FULL,
 151         SCIF_WINDOW_SELF,
 152         SCIF_WINDOW_PEER
 153 };
 154
 155 /* The number of physical addresses that can be stored in a PAGE. */
 156 #define SCIF_NR_ADDR_IN_PAGE   (0x1000 >> 3)
 157
 158 /*
 159  * struct scif_rma_lookup - RMA lookup data structure for page list transfers
 160  *
 161  * Store an array of lookup offsets. Each offset in this array maps
 162  * one 4K page containing 512 physical addresses i.e. 2MB. 512 such
 163  * offsets in a 4K page will correspond to 1GB of registered address space.
 164
 165  * @lookup: Array of offsets
 166  * @offset: DMA offset of lookup array
 167  */
 168 struct scif_rma_lookup {
 169         dma_addr_t *lookup;
 170         dma_addr_t offset;
 171 };
 172
 173 /*
 174  * struct scif_pinned_pages - A set of pinned pages obtained with
 175  * scif_pin_pages() which could be part of multiple registered
 176  * windows across different end points.
 177  *
 178  * @nr_pages: Number of pages which is defined as a s64 instead of an int
 179  * to avoid sign extension with buffers >= 2GB
 180  * @prot: read/write protections
 181  * @map_flags: Flags specified during the pin operation
 182  * @ref_count: Reference count bumped in terms of number of pages
 183  * @magic: A magic value
 184  * @pages: Array of pointers to struct pages populated with get_user_pages(..)
 185  */
 186 struct scif_pinned_pages {
 187         s64 nr_pages;
 188         int prot;
 189         int map_flags;
 190         atomic_t ref_count;
 191         u64 magic;
 192         struct page **pages;
 193 };
 194
 195 /*
 196  * struct scif_status - Stores DMA status update information
 197  *
 198  * @src_dma_addr: Source buffer DMA address
 199  * @val: src location for value to be written to the destination
 200  * @ep: SCIF endpoint
 201  */
 202 struct scif_status {
 203         dma_addr_t src_dma_addr;
 204         u64 val;
 205         struct scif_endpt *ep;
 206 };
 207
 208 /*
 209  * struct scif_cb_arg - Stores the argument of the callback func
 210  *
 211  * @src_dma_addr: Source buffer DMA address
 212  * @status: DMA status
 213  * @ep: SCIF endpoint
 214  */
 215 struct scif_cb_arg {
 216         dma_addr_t src_dma_addr;
 217         struct scif_status *status;
 218         struct scif_endpt *ep;
 219 };
 220
 221 /*
 222  * struct scif_window - Registration Window for Self and Remote
 223  *
 224  * @nr_pages: Number of pages which is defined as a s64 instead of an int
 225  * to avoid sign extension with buffers >= 2GB
 226  * @nr_contig_chunks: Number of contiguous physical chunks
 227  * @prot: read/write protections
 228  * @ref_count: reference count in terms of number of pages
 229  * @magic: Cookie to detect corruption
 230  * @offset: registered offset
 231  * @va_for_temp: va address that this window represents
 232  * @dma_mark: Used to determine if all DMAs against the window are done
 233  * @ep: Pointer to EP. Useful for passing EP around with messages to
 234         avoid expensive list traversals.
 235  * @list: link to list of windows for the endpoint
 236  * @type: self or peer window
 237  * @peer_window: Pointer to peer window. Useful for sending messages to peer
 238  *               without requiring an extra list traversal
 239  * @unreg_state: unregistration state
 240  * @offset_freed: True if the offset has been freed
 241  * @temp: True for temporary windows created via scif_vreadfrom/scif_vwriteto
 242  * @mm: memory descriptor for the task_struct which initiated the RMA
 243  * @st: scatter gather table for DMA mappings with IOMMU enabled
 244  * @pinned_pages: The set of pinned_pages backing this window
 245  * @alloc_handle: Handle for sending ALLOC_REQ
 246  * @regwq: Wait Queue for an registration (N)ACK
 247  * @reg_state: Registration state
 248  * @unregwq: Wait Queue for an unregistration (N)ACK
 249  * @dma_addr_lookup: Lookup for physical addresses used for DMA
 250  * @nr_lookup: Number of entries in lookup
 251  * @mapped_offset: Offset used to map the window by the peer
 252  * @dma_addr: Array of physical addresses used for Mgmt node & MIC initiated DMA
 253  * @num_pages: Array specifying number of pages for each physical address
 254  */
 255 struct scif_window {
 256         s64 nr_pages;
 257         int nr_contig_chunks;
 258         int prot;
 259         int ref_count;
 260         u64 magic;
 261         s64 offset;
 262         unsigned long va_for_temp;
 263         int dma_mark;
 264         u64 ep;
 265         struct list_head list;
 266         enum scif_window_type type;
 267         u64 peer_window;
 268         enum scif_msg_state unreg_state;
 269         bool offset_freed;
 270         bool temp;
 271         struct mm_struct *mm;
 272         struct sg_table *st;
 273         union {
 274                 struct {
 275                         struct scif_pinned_pages *pinned_pages;
 276                         struct scif_allocmsg alloc_handle;
 277                         wait_queue_head_t regwq;
 278                         enum scif_msg_state reg_state;
 279                         wait_queue_head_t unregwq;
 280                 };
 281                 struct {
 282                         struct scif_rma_lookup dma_addr_lookup;
 283                         struct scif_rma_lookup num_pages_lookup;
 284                         int nr_lookup;
 285                         dma_addr_t mapped_offset;
 286                 };
 287         };
 288         dma_addr_t *dma_addr;
 289         u64 *num_pages;
 290 } __packed;
 291
 292 /*
 293  * scif_mmu_notif - SCIF mmu notifier information
 294  *
 295  * @mmu_notifier ep_mmu_notifier: MMU notifier operations
 296  * @tc_reg_list: List of temp registration windows for self
 297  * @mm: memory descriptor for the task_struct which initiated the RMA
 298  * @ep: SCIF endpoint
 299  * @list: link to list of MMU notifier information
 300  */
 301 struct scif_mmu_notif {
 302 #ifdef CONFIG_MMU_NOTIFIER
 303         struct mmu_notifier ep_mmu_notifier;
 304 #endif
 305         struct list_head tc_reg_list;
 306         struct mm_struct *mm;
 307         struct scif_endpt *ep;
 308         struct list_head list;
 309 };
 310
 311 enum scif_rma_dir {
 312         SCIF_LOCAL_TO_REMOTE,
 313         SCIF_REMOTE_TO_LOCAL
 314 };
 315
 316 extern struct kmem_cache *unaligned_cache;
 317 /* Initialize RMA for this EP */
 318 void scif_rma_ep_init(struct scif_endpt *ep);
 319 /* Check if epd can be uninitialized */
 320 int scif_rma_ep_can_uninit(struct scif_endpt *ep);
 321 /* Obtain a new offset. Callee must grab RMA lock */
 322 int scif_get_window_offset(struct scif_endpt *ep, int flags,
 323                            s64 offset, int nr_pages, s64 *out_offset);
 324 /* Free offset. Callee must grab RMA lock */
 325 void scif_free_window_offset(struct scif_endpt *ep,
 326                              struct scif_window *window, s64 offset);
 327 /* Create self registration window */
 328 struct scif_window *scif_create_window(struct scif_endpt *ep, int nr_pages,
 329                                        s64 offset, bool temp);
 330 /* Destroy self registration window.*/
 331 int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window);
 332 void scif_unmap_window(struct scif_dev *remote_dev, struct scif_window *window);
 333 /* Map pages of self window to Aperture/PCI */
 334 int scif_map_window(struct scif_dev *remote_dev,
 335                     struct scif_window *window);
 336 /* Unregister a self window */
 337 int scif_unregister_window(struct scif_window *window);
 338 /* Destroy remote registration window */
 339 void
 340 scif_destroy_remote_window(struct scif_window *window);
 341 /* remove valid remote memory mappings from process address space */
 342 void scif_zap_mmaps(int node);
 343 /* Query if any applications have remote memory mappings */
 344 bool scif_rma_do_apps_have_mmaps(int node);
 345 /* Cleanup remote registration lists for zombie endpoints */
 346 void scif_cleanup_rma_for_zombies(int node);
 347 /* Reserve a DMA channel for a particular endpoint */
 348 int scif_reserve_dma_chan(struct scif_endpt *ep);
 349 /* Setup a DMA mark for an endpoint */
 350 int _scif_fence_mark(scif_epd_t epd, int *mark);
 351 int scif_prog_signal(scif_epd_t epd, off_t offset, u64 val,
 352                      enum scif_window_type type);
 353 void scif_alloc_req(struct scif_dev *scifdev, struct scifmsg *msg);
 354 void scif_alloc_gnt_rej(struct scif_dev *scifdev, struct scifmsg *msg);
 355 void scif_free_virt(struct scif_dev *scifdev, struct scifmsg *msg);
 356 void scif_recv_reg(struct scif_dev *scifdev, struct scifmsg *msg);
 357 void scif_recv_unreg(struct scif_dev *scifdev, struct scifmsg *msg);
 358 void scif_recv_reg_ack(struct scif_dev *scifdev, struct scifmsg *msg);
 359 void scif_recv_reg_nack(struct scif_dev *scifdev, struct scifmsg *msg);
 360 void scif_recv_unreg_ack(struct scif_dev *scifdev, struct scifmsg *msg);
 361 void scif_recv_unreg_nack(struct scif_dev *scifdev, struct scifmsg *msg);
 362 void scif_recv_munmap(struct scif_dev *scifdev, struct scifmsg *msg);
 363 void scif_recv_mark(struct scif_dev *scifdev, struct scifmsg *msg);
 364 void scif_recv_mark_resp(struct scif_dev *scifdev, struct scifmsg *msg);
 365 void scif_recv_wait(struct scif_dev *scifdev, struct scifmsg *msg);
 366 void scif_recv_wait_resp(struct scif_dev *scifdev, struct scifmsg *msg);
 367 void scif_recv_sig_local(struct scif_dev *scifdev, struct scifmsg *msg);
 368 void scif_recv_sig_remote(struct scif_dev *scifdev, struct scifmsg *msg);
 369 void scif_recv_sig_resp(struct scif_dev *scifdev, struct scifmsg *msg);
 370 void scif_mmu_notif_handler(struct work_struct *work);
 371 void scif_rma_handle_remote_fences(void);
 372 void scif_rma_destroy_windows(void);
 373 void scif_rma_destroy_tcw_invalid(void);
 374 int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan);
 375
 376 struct scif_window_iter {
 377         s64 offset;
 378         int index;
 379 };
 380
 381 static inline void
 382 scif_init_window_iter(struct scif_window *window, struct scif_window_iter *iter)
 383 {
 384         iter->offset = window->offset;
 385         iter->index = 0;
 386 }
 387
 388 dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
 389                                 size_t *nr_bytes,
 390                                 struct scif_window_iter *iter);
 391 static inline
 392 dma_addr_t __scif_off_to_dma_addr(struct scif_window *window, s64 off)
 393 {
 394         return scif_off_to_dma_addr(window, off, NULL, NULL);
 395 }
 396
 397 static inline bool scif_unaligned(off_t src_offset, off_t dst_offset)
 398 {
 399         src_offset = src_offset & (L1_CACHE_BYTES - 1);
 400         dst_offset = dst_offset & (L1_CACHE_BYTES - 1);
 401         return !(src_offset == dst_offset);
 402 }
 403
 404 /*
 405  * scif_zalloc:
 406  * @size: Size of the allocation request.
 407  *
 408  * Helper API which attempts to allocate zeroed pages via
 409  * __get_free_pages(..) first and then falls back on
 410  * vzalloc(..) if that fails.
 411  */
 412 static inline void *scif_zalloc(size_t size)
 413 {
 414         void *ret = NULL;
 415         size_t align = ALIGN(size, PAGE_SIZE);
 416
 417         if (align && get_order(align) < MAX_ORDER)
 418                 ret = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 419                                                get_order(align));
 420         return ret ? ret : vzalloc(align);
 421 }
 422
 423 /*
 424  * scif_free:
 425  * @addr: Address to be freed.
 426  * @size: Size of the allocation.
 427  * Helper API which frees memory allocated via scif_zalloc().
 428  */
 429 static inline void scif_free(void *addr, size_t size)
 430 {
 431         size_t align = ALIGN(size, PAGE_SIZE);
 432
 433         if (is_vmalloc_addr(addr))
 434                 vfree(addr);
 435         else
 436                 free_pages((unsigned long)addr, get_order(align));
 437 }
 438
 439 static inline void scif_get_window(struct scif_window *window, int nr_pages)
 440 {
 441         window->ref_count += nr_pages;
 442 }
 443
 444 static inline void scif_put_window(struct scif_window *window, int nr_pages)
 445 {
 446         window->ref_count -= nr_pages;
 447 }
 448
 449 static inline void scif_set_window_ref(struct scif_window *window, int nr_pages)
 450 {
 451         window->ref_count = nr_pages;
 452 }
 453
 454 static inline void
 455 scif_queue_for_cleanup(struct scif_window *window, struct list_head *list)
 456 {
 457         spin_lock(&scif_info.rmalock);
 458         list_add_tail(&window->list, list);
 459         spin_unlock(&scif_info.rmalock);
 460         schedule_work(&scif_info.misc_work);
 461 }
 462
 463 static inline void __scif_rma_destroy_tcw_helper(struct scif_window *window)
 464 {
 465         list_del_init(&window->list);
 466         scif_queue_for_cleanup(window, &scif_info.rma_tc);
 467 }
 468
 469 static inline bool scif_is_iommu_enabled(void)
 470 {
 471 #ifdef CONFIG_INTEL_IOMMU
 472         return intel_iommu_enabled;
 473 #else
 474         return false;
 475 #endif
 476 }
 477 #endif /* SCIF_RMA_H */