2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Intel MIC Host driver.
21 #include <linux/pci.h>
23 #include "../common/mic_dev.h"
24 #include "mic_device.h"
27 static inline u64
mic_system_page_mask(struct mic_device
*mdev
)
29 return (1ULL << mdev
->smpt
->info
.page_shift
) - 1ULL;
32 static inline u8
mic_sys_addr_to_smpt(struct mic_device
*mdev
, dma_addr_t pa
)
34 return (pa
- mdev
->smpt
->info
.base
) >> mdev
->smpt
->info
.page_shift
;
37 static inline u64
mic_smpt_to_pa(struct mic_device
*mdev
, u8 index
)
39 return mdev
->smpt
->info
.base
+ (index
* mdev
->smpt
->info
.page_size
);
42 static inline u64
mic_smpt_offset(struct mic_device
*mdev
, dma_addr_t pa
)
44 return pa
& mic_system_page_mask(mdev
);
47 static inline u64
mic_smpt_align_low(struct mic_device
*mdev
, dma_addr_t pa
)
49 return ALIGN(pa
- mic_system_page_mask(mdev
),
50 mdev
->smpt
->info
.page_size
);
53 static inline u64
mic_smpt_align_high(struct mic_device
*mdev
, dma_addr_t pa
)
55 return ALIGN(pa
, mdev
->smpt
->info
.page_size
);
58 /* Total Cumulative system memory accessible by MIC across all SMPT entries */
59 static inline u64
mic_max_system_memory(struct mic_device
*mdev
)
61 return mdev
->smpt
->info
.num_reg
* mdev
->smpt
->info
.page_size
;
64 /* Maximum system memory address accessible by MIC */
65 static inline u64
mic_max_system_addr(struct mic_device
*mdev
)
67 return mdev
->smpt
->info
.base
+ mic_max_system_memory(mdev
) - 1ULL;
70 /* Check if the DMA address is a MIC system memory address */
72 mic_is_system_addr(struct mic_device
*mdev
, dma_addr_t pa
)
74 return pa
>= mdev
->smpt
->info
.base
&& pa
<= mic_max_system_addr(mdev
);
77 /* Populate an SMPT entry and update the reference counts. */
78 static void mic_add_smpt_entry(int spt
, s64
*ref
, u64 addr
,
79 int entries
, struct mic_device
*mdev
)
81 struct mic_smpt_info
*smpt_info
= mdev
->smpt
;
84 for (i
= spt
; i
< spt
+ entries
; i
++,
85 addr
+= smpt_info
->info
.page_size
) {
86 if (!smpt_info
->entry
[i
].ref_count
&&
87 (smpt_info
->entry
[i
].dma_addr
!= addr
)) {
88 mdev
->smpt_ops
->set(mdev
, addr
, i
);
89 smpt_info
->entry
[i
].dma_addr
= addr
;
91 smpt_info
->entry
[i
].ref_count
+= ref
[i
- spt
];
96 * Find an available MIC address in MIC SMPT address space
97 * for a given DMA address and size.
99 static dma_addr_t
mic_smpt_op(struct mic_device
*mdev
, u64 dma_addr
,
100 int entries
, s64
*ref
, size_t size
)
106 dma_addr_t mic_addr
= 0;
107 dma_addr_t addr
= dma_addr
;
108 struct mic_smpt_info
*smpt_info
= mdev
->smpt
;
110 spin_lock_irqsave(&smpt_info
->smpt_lock
, flags
);
112 /* find existing entries */
113 for (i
= 0; i
< smpt_info
->info
.num_reg
; i
++) {
114 if (smpt_info
->entry
[i
].dma_addr
== addr
) {
116 addr
+= smpt_info
->info
.page_size
;
117 } else if (ae
) /* cannot find contiguous entries */
124 /* find free entry */
125 for (ae
= 0, i
= 0; i
< smpt_info
->info
.num_reg
; i
++) {
126 ae
= (smpt_info
->entry
[i
].ref_count
== 0) ? ae
+ 1 : 0;
132 spin_unlock_irqrestore(&smpt_info
->smpt_lock
, flags
);
136 spt
= i
- entries
+ 1;
137 mic_addr
= mic_smpt_to_pa(mdev
, spt
);
138 mic_add_smpt_entry(spt
, ref
, dma_addr
, entries
, mdev
);
139 smpt_info
->map_count
++;
140 smpt_info
->ref_count
+= (s64
)size
;
141 spin_unlock_irqrestore(&smpt_info
->smpt_lock
, flags
);
146 * Returns number of smpt entries needed for dma_addr to dma_addr + size
147 * also returns the reference count array for each of those entries
148 * and the starting smpt address
150 static int mic_get_smpt_ref_count(struct mic_device
*mdev
, dma_addr_t dma_addr
,
151 size_t size
, s64
*ref
, u64
*smpt_start
)
153 u64 start
= dma_addr
;
154 u64 end
= dma_addr
+ size
;
157 while (start
< end
) {
158 ref
[i
++] = min(mic_smpt_align_high(mdev
, start
+ 1),
160 start
= mic_smpt_align_high(mdev
, start
+ 1);
164 *smpt_start
= mic_smpt_align_low(mdev
, dma_addr
);
170 * mic_to_dma_addr - Converts a MIC address to a DMA address.
172 * @mdev: pointer to mic_device instance.
173 * @mic_addr: MIC address.
175 * returns a DMA address.
177 dma_addr_t
mic_to_dma_addr(struct mic_device
*mdev
, dma_addr_t mic_addr
)
179 struct mic_smpt_info
*smpt_info
= mdev
->smpt
;
183 if (!mic_is_system_addr(mdev
, mic_addr
)) {
184 dev_err(mdev
->sdev
->parent
,
185 "mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr
);
188 spt
= mic_sys_addr_to_smpt(mdev
, mic_addr
);
189 dma_addr
= smpt_info
->entry
[spt
].dma_addr
+
190 mic_smpt_offset(mdev
, mic_addr
);
195 * mic_map - Maps a DMA address to a MIC physical address.
197 * @mdev: pointer to mic_device instance.
198 * @dma_addr: DMA address.
199 * @size: Size of the region to be mapped.
201 * This API converts the DMA address provided to a DMA address understood
202 * by MIC. Caller should check for errors by calling mic_map_error(..).
204 * returns DMA address as required by MIC.
206 dma_addr_t
mic_map(struct mic_device
*mdev
, dma_addr_t dma_addr
, size_t size
)
208 dma_addr_t mic_addr
= 0;
213 if (!size
|| size
> mic_max_system_memory(mdev
))
216 ref
= kmalloc_array(mdev
->smpt
->info
.num_reg
, sizeof(s64
), GFP_ATOMIC
);
220 num_entries
= mic_get_smpt_ref_count(mdev
, dma_addr
, size
,
223 /* Set the smpt table appropriately and get 16G aligned mic address */
224 mic_addr
= mic_smpt_op(mdev
, smpt_start
, num_entries
, ref
, size
);
229 * If mic_addr is zero then its an error case
230 * since mic_addr can never be zero.
231 * else generate mic_addr by adding the 16G offset in dma_addr
233 if (!mic_addr
&& MIC_FAMILY_X100
== mdev
->family
) {
234 dev_err(mdev
->sdev
->parent
,
235 "mic_map failed dma_addr 0x%llx size 0x%lx\n",
239 return mic_addr
+ mic_smpt_offset(mdev
, dma_addr
);
244 * mic_unmap - Unmaps a MIC physical address.
246 * @mdev: pointer to mic_device instance.
247 * @mic_addr: MIC physical address.
248 * @size: Size of the region to be unmapped.
250 * This API unmaps the mappings created by mic_map(..).
254 void mic_unmap(struct mic_device
*mdev
, dma_addr_t mic_addr
, size_t size
)
256 struct mic_smpt_info
*smpt_info
= mdev
->smpt
;
266 if (!mic_is_system_addr(mdev
, mic_addr
)) {
267 dev_err(mdev
->sdev
->parent
,
268 "invalid address: 0x%llx\n", mic_addr
);
272 spt
= mic_sys_addr_to_smpt(mdev
, mic_addr
);
273 ref
= kmalloc_array(mdev
->smpt
->info
.num_reg
, sizeof(s64
), GFP_ATOMIC
);
277 /* Get number of smpt entries to be mapped, ref count array */
278 num_smpt
= mic_get_smpt_ref_count(mdev
, mic_addr
, size
, ref
, NULL
);
280 spin_lock_irqsave(&smpt_info
->smpt_lock
, flags
);
281 smpt_info
->unmap_count
++;
282 smpt_info
->ref_count
-= (s64
)size
;
284 for (i
= spt
; i
< spt
+ num_smpt
; i
++) {
285 smpt_info
->entry
[i
].ref_count
-= ref
[i
- spt
];
286 if (smpt_info
->entry
[i
].ref_count
< 0)
287 dev_warn(mdev
->sdev
->parent
,
288 "ref count for entry %d is negative\n", i
);
290 spin_unlock_irqrestore(&smpt_info
->smpt_lock
, flags
);
295 * mic_map_single - Maps a virtual address to a MIC physical address.
297 * @mdev: pointer to mic_device instance.
298 * @va: Kernel direct mapped virtual address.
299 * @size: Size of the region to be mapped.
301 * This API calls pci_map_single(..) for the direct mapped virtual address
302 * and then converts the DMA address provided to a DMA address understood
303 * by MIC. Caller should check for errors by calling mic_map_error(..).
305 * returns DMA address as required by MIC.
307 dma_addr_t
mic_map_single(struct mic_device
*mdev
, void *va
, size_t size
)
309 dma_addr_t mic_addr
= 0;
310 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
311 struct pci_dev
, dev
);
312 dma_addr_t dma_addr
=
313 pci_map_single(pdev
, va
, size
, PCI_DMA_BIDIRECTIONAL
);
315 if (!pci_dma_mapping_error(pdev
, dma_addr
)) {
316 mic_addr
= mic_map(mdev
, dma_addr
, size
);
318 dev_err(mdev
->sdev
->parent
,
319 "mic_map failed dma_addr 0x%llx size 0x%lx\n",
321 pci_unmap_single(pdev
, dma_addr
,
322 size
, PCI_DMA_BIDIRECTIONAL
);
329 * mic_unmap_single - Unmaps a MIC physical address.
331 * @mdev: pointer to mic_device instance.
332 * @mic_addr: MIC physical address.
333 * @size: Size of the region to be unmapped.
335 * This API unmaps the mappings created by mic_map_single(..).
340 mic_unmap_single(struct mic_device
*mdev
, dma_addr_t mic_addr
, size_t size
)
342 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
343 struct pci_dev
, dev
);
344 dma_addr_t dma_addr
= mic_to_dma_addr(mdev
, mic_addr
);
345 mic_unmap(mdev
, mic_addr
, size
);
346 pci_unmap_single(pdev
, dma_addr
, size
, PCI_DMA_BIDIRECTIONAL
);
350 * mic_smpt_init - Initialize MIC System Memory Page Tables.
352 * @mdev: pointer to mic_device instance.
354 * returns 0 for success and -errno for error.
356 int mic_smpt_init(struct mic_device
*mdev
)
360 struct mic_smpt_info
*smpt_info
;
362 mdev
->smpt
= kmalloc(sizeof(*mdev
->smpt
), GFP_KERNEL
);
366 smpt_info
= mdev
->smpt
;
367 mdev
->smpt_ops
->init(mdev
);
368 smpt_info
->entry
= kmalloc_array(smpt_info
->info
.num_reg
,
369 sizeof(*smpt_info
->entry
), GFP_KERNEL
);
370 if (!smpt_info
->entry
) {
374 spin_lock_init(&smpt_info
->smpt_lock
);
375 for (i
= 0; i
< smpt_info
->info
.num_reg
; i
++) {
376 dma_addr
= i
* smpt_info
->info
.page_size
;
377 smpt_info
->entry
[i
].dma_addr
= dma_addr
;
378 smpt_info
->entry
[i
].ref_count
= 0;
379 mdev
->smpt_ops
->set(mdev
, dma_addr
, i
);
381 smpt_info
->ref_count
= 0;
382 smpt_info
->map_count
= 0;
383 smpt_info
->unmap_count
= 0;
391 * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables.
393 * @mdev: pointer to mic_device instance.
397 void mic_smpt_uninit(struct mic_device
*mdev
)
399 struct mic_smpt_info
*smpt_info
= mdev
->smpt
;
402 dev_dbg(mdev
->sdev
->parent
,
403 "nodeid %d SMPT ref count %lld map %lld unmap %lld\n",
404 mdev
->id
, smpt_info
->ref_count
,
405 smpt_info
->map_count
, smpt_info
->unmap_count
);
407 for (i
= 0; i
< smpt_info
->info
.num_reg
; i
++) {
408 dev_dbg(mdev
->sdev
->parent
,
409 "SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n",
410 i
, smpt_info
->entry
[i
].dma_addr
,
411 smpt_info
->entry
[i
].ref_count
);
412 if (smpt_info
->entry
[i
].ref_count
)
413 dev_warn(mdev
->sdev
->parent
,
414 "ref count for entry %d is not zero\n", i
);
416 kfree(smpt_info
->entry
);
421 * mic_smpt_restore - Restore MIC System Memory Page Tables.
423 * @mdev: pointer to mic_device instance.
425 * Restore the SMPT registers to values previously stored in the
426 * SW data structures. Some MIC steppings lose register state
427 * across resets and this API should be called for performing
428 * a restore operation if required.
432 void mic_smpt_restore(struct mic_device
*mdev
)
437 for (i
= 0; i
< mdev
->smpt
->info
.num_reg
; i
++) {
438 dma_addr
= mdev
->smpt
->entry
[i
].dma_addr
;
439 mdev
->smpt_ops
->set(mdev
, dma_addr
, i
);