drivers/misc/mic/host/mic_smpt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Intel MIC Platform Software Stack (MPSS)
   4  *
   5  * Copyright(c) 2013 Intel Corporation.
   6  *
   7  * Intel MIC Host driver.
   8  */
   9 #include <linux/pci.h>
  10
  11 #include "../common/mic_dev.h"
  12 #include "mic_device.h"
  13 #include "mic_smpt.h"
  14
  15 static inline u64 mic_system_page_mask(struct mic_device *mdev)
  16 {
  17         return (1ULL << mdev->smpt->info.page_shift) - 1ULL;
  18 }
  19
  20 static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)
  21 {
  22         return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;
  23 }
  24
  25 static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)
  26 {
  27         return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);
  28 }
  29
  30 static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)
  31 {
  32         return pa & mic_system_page_mask(mdev);
  33 }
  34
  35 static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)
  36 {
  37         return ALIGN(pa - mic_system_page_mask(mdev),
  38                 mdev->smpt->info.page_size);
  39 }
  40
  41 static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)
  42 {
  43         return ALIGN(pa, mdev->smpt->info.page_size);
  44 }
  45
  46 /* Total Cumulative system memory accessible by MIC across all SMPT entries */
  47 static inline u64 mic_max_system_memory(struct mic_device *mdev)
  48 {
  49         return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;
  50 }
  51
  52 /* Maximum system memory address accessible by MIC */
  53 static inline u64 mic_max_system_addr(struct mic_device *mdev)
  54 {
  55         return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;
  56 }
  57
  58 /* Check if the DMA address is a MIC system memory address */
  59 static inline bool
  60 mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)
  61 {
  62         return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);
  63 }
  64
  65 /* Populate an SMPT entry and update the reference counts. */
  66 static void mic_add_smpt_entry(int spt, s64 *ref, u64 addr,
  67                                int entries, struct mic_device *mdev)
  68 {
  69         struct mic_smpt_info *smpt_info = mdev->smpt;
  70         int i;
  71
  72         for (i = spt; i < spt + entries; i++,
  73                 addr += smpt_info->info.page_size) {
  74                 if (!smpt_info->entry[i].ref_count &&
  75                     (smpt_info->entry[i].dma_addr != addr)) {
  76                         mdev->smpt_ops->set(mdev, addr, i);
  77                         smpt_info->entry[i].dma_addr = addr;
  78                 }
  79                 smpt_info->entry[i].ref_count += ref[i - spt];
  80         }
  81 }
  82
  83 /*
  84  * Find an available MIC address in MIC SMPT address space
  85  * for a given DMA address and size.
  86  */
  87 static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,
  88                               int entries, s64 *ref, size_t size)
  89 {
  90         int spt;
  91         int ae = 0;
  92         int i;
  93         unsigned long flags;
  94         dma_addr_t mic_addr = 0;
  95         dma_addr_t addr = dma_addr;
  96         struct mic_smpt_info *smpt_info = mdev->smpt;
  97
  98         spin_lock_irqsave(&smpt_info->smpt_lock, flags);
  99
 100         /* find existing entries */
 101         for (i = 0; i < smpt_info->info.num_reg; i++) {
 102                 if (smpt_info->entry[i].dma_addr == addr) {
 103                         ae++;
 104                         addr += smpt_info->info.page_size;
 105                 } else if (ae) /* cannot find contiguous entries */
 106                         goto not_found;
 107
 108                 if (ae == entries)
 109                         goto found;
 110         }
 111
 112         /* find free entry */
 113         for (ae = 0, i = 0; i < smpt_info->info.num_reg; i++) {
 114                 ae = (smpt_info->entry[i].ref_count == 0) ? ae + 1 : 0;
 115                 if (ae == entries)
 116                         goto found;
 117         }
 118
 119 not_found:
 120         spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
 121         return mic_addr;
 122
 123 found:
 124         spt = i - entries + 1;
 125         mic_addr = mic_smpt_to_pa(mdev, spt);
 126         mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);
 127         smpt_info->map_count++;
 128         smpt_info->ref_count += (s64)size;
 129         spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
 130         return mic_addr;
 131 }
 132
 133 /*
 134  * Returns number of smpt entries needed for dma_addr to dma_addr + size
 135  * also returns the reference count array for each of those entries
 136  * and the starting smpt address
 137  */
 138 static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,
 139                                   size_t size, s64 *ref,  u64 *smpt_start)
 140 {
 141         u64 start =  dma_addr;
 142         u64 end = dma_addr + size;
 143         int i = 0;
 144
 145         while (start < end) {
 146                 ref[i++] = min(mic_smpt_align_high(mdev, start + 1),
 147                         end) - start;
 148                 start = mic_smpt_align_high(mdev, start + 1);
 149         }
 150
 151         if (smpt_start)
 152                 *smpt_start = mic_smpt_align_low(mdev, dma_addr);
 153
 154         return i;
 155 }
 156
 157 /*
 158  * mic_to_dma_addr - Converts a MIC address to a DMA address.
 159  *
 160  * @mdev: pointer to mic_device instance.
 161  * @mic_addr: MIC address.
 162  *
 163  * returns a DMA address.
 164  */
 165 dma_addr_t mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)
 166 {
 167         struct mic_smpt_info *smpt_info = mdev->smpt;
 168         int spt;
 169         dma_addr_t dma_addr;
 170
 171         if (!mic_is_system_addr(mdev, mic_addr)) {
 172                 dev_err(&mdev->pdev->dev,
 173                         "mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr);
 174                 return -EINVAL;
 175         }
 176         spt = mic_sys_addr_to_smpt(mdev, mic_addr);
 177         dma_addr = smpt_info->entry[spt].dma_addr +
 178                 mic_smpt_offset(mdev, mic_addr);
 179         return dma_addr;
 180 }
 181
 182 /**
 183  * mic_map - Maps a DMA address to a MIC physical address.
 184  *
 185  * @mdev: pointer to mic_device instance.
 186  * @dma_addr: DMA address.
 187  * @size: Size of the region to be mapped.
 188  *
 189  * This API converts the DMA address provided to a DMA address understood
 190  * by MIC. Caller should check for errors by calling mic_map_error(..).
 191  *
 192  * returns DMA address as required by MIC.
 193  */
 194 dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)
 195 {
 196         dma_addr_t mic_addr = 0;
 197         int num_entries;
 198         s64 *ref;
 199         u64 smpt_start;
 200
 201         if (!size || size > mic_max_system_memory(mdev))
 202                 return mic_addr;
 203
 204         ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);
 205         if (!ref)
 206                 return mic_addr;
 207
 208         num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,
 209                                              ref, &smpt_start);
 210
 211         /* Set the smpt table appropriately and get 16G aligned mic address */
 212         mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);
 213
 214         kfree(ref);
 215
 216         /*
 217          * If mic_addr is zero then its an error case
 218          * since mic_addr can never be zero.
 219          * else generate mic_addr by adding the 16G offset in dma_addr
 220          */
 221         if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {
 222                 dev_err(&mdev->pdev->dev,
 223                         "mic_map failed dma_addr 0x%llx size 0x%lx\n",
 224                         dma_addr, size);
 225                 return mic_addr;
 226         } else {
 227                 return mic_addr + mic_smpt_offset(mdev, dma_addr);
 228         }
 229 }
 230
 231 /**
 232  * mic_unmap - Unmaps a MIC physical address.
 233  *
 234  * @mdev: pointer to mic_device instance.
 235  * @mic_addr: MIC physical address.
 236  * @size: Size of the region to be unmapped.
 237  *
 238  * This API unmaps the mappings created by mic_map(..).
 239  *
 240  * returns None.
 241  */
 242 void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
 243 {
 244         struct mic_smpt_info *smpt_info = mdev->smpt;
 245         s64 *ref;
 246         int num_smpt;
 247         int spt;
 248         int i;
 249         unsigned long flags;
 250
 251         if (!size)
 252                 return;
 253
 254         if (!mic_is_system_addr(mdev, mic_addr)) {
 255                 dev_err(&mdev->pdev->dev,
 256                         "invalid address: 0x%llx\n", mic_addr);
 257                 return;
 258         }
 259
 260         spt = mic_sys_addr_to_smpt(mdev, mic_addr);
 261         ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);
 262         if (!ref)
 263                 return;
 264
 265         /* Get number of smpt entries to be mapped, ref count array */
 266         num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);
 267
 268         spin_lock_irqsave(&smpt_info->smpt_lock, flags);
 269         smpt_info->unmap_count++;
 270         smpt_info->ref_count -= (s64)size;
 271
 272         for (i = spt; i < spt + num_smpt; i++) {
 273                 smpt_info->entry[i].ref_count -= ref[i - spt];
 274                 if (smpt_info->entry[i].ref_count < 0)
 275                         dev_warn(&mdev->pdev->dev,
 276                                  "ref count for entry %d is negative\n", i);
 277         }
 278         spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
 279         kfree(ref);
 280 }
 281
 282 /**
 283  * mic_map_single - Maps a virtual address to a MIC physical address.
 284  *
 285  * @mdev: pointer to mic_device instance.
 286  * @va: Kernel direct mapped virtual address.
 287  * @size: Size of the region to be mapped.
 288  *
 289  * This API calls pci_map_single(..) for the direct mapped virtual address
 290  * and then converts the DMA address provided to a DMA address understood
 291  * by MIC. Caller should check for errors by calling mic_map_error(..).
 292  *
 293  * returns DMA address as required by MIC.
 294  */
 295 dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)
 296 {
 297         dma_addr_t mic_addr = 0;
 298         struct pci_dev *pdev = mdev->pdev;
 299         dma_addr_t dma_addr =
 300                 pci_map_single(pdev, va, size, PCI_DMA_BIDIRECTIONAL);
 301
 302         if (!pci_dma_mapping_error(pdev, dma_addr)) {
 303                 mic_addr = mic_map(mdev, dma_addr, size);
 304                 if (!mic_addr) {
 305                         dev_err(&mdev->pdev->dev,
 306                                 "mic_map failed dma_addr 0x%llx size 0x%lx\n",
 307                                 dma_addr, size);
 308                         pci_unmap_single(pdev, dma_addr,
 309                                          size, PCI_DMA_BIDIRECTIONAL);
 310                 }
 311         }
 312         return mic_addr;
 313 }
 314
 315 /**
 316  * mic_unmap_single - Unmaps a MIC physical address.
 317  *
 318  * @mdev: pointer to mic_device instance.
 319  * @mic_addr: MIC physical address.
 320  * @size: Size of the region to be unmapped.
 321  *
 322  * This API unmaps the mappings created by mic_map_single(..).
 323  *
 324  * returns None.
 325  */
 326 void
 327 mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
 328 {
 329         struct pci_dev *pdev = mdev->pdev;
 330         dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);
 331         mic_unmap(mdev, mic_addr, size);
 332         pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
 333 }
 334
 335 /**
 336  * mic_smpt_init - Initialize MIC System Memory Page Tables.
 337  *
 338  * @mdev: pointer to mic_device instance.
 339  *
 340  * returns 0 for success and -errno for error.
 341  */
 342 int mic_smpt_init(struct mic_device *mdev)
 343 {
 344         int i, err = 0;
 345         dma_addr_t dma_addr;
 346         struct mic_smpt_info *smpt_info;
 347
 348         mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);
 349         if (!mdev->smpt)
 350                 return -ENOMEM;
 351
 352         smpt_info = mdev->smpt;
 353         mdev->smpt_ops->init(mdev);
 354         smpt_info->entry = kmalloc_array(smpt_info->info.num_reg,
 355                                          sizeof(*smpt_info->entry), GFP_KERNEL);
 356         if (!smpt_info->entry) {
 357                 err = -ENOMEM;
 358                 goto free_smpt;
 359         }
 360         spin_lock_init(&smpt_info->smpt_lock);
 361         for (i = 0; i < smpt_info->info.num_reg; i++) {
 362                 dma_addr = i * smpt_info->info.page_size;
 363                 smpt_info->entry[i].dma_addr = dma_addr;
 364                 smpt_info->entry[i].ref_count = 0;
 365                 mdev->smpt_ops->set(mdev, dma_addr, i);
 366         }
 367         smpt_info->ref_count = 0;
 368         smpt_info->map_count = 0;
 369         smpt_info->unmap_count = 0;
 370         return 0;
 371 free_smpt:
 372         kfree(smpt_info);
 373         return err;
 374 }
 375
 376 /**
 377  * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables.
 378  *
 379  * @mdev: pointer to mic_device instance.
 380  *
 381  * returns None.
 382  */
 383 void mic_smpt_uninit(struct mic_device *mdev)
 384 {
 385         struct mic_smpt_info *smpt_info = mdev->smpt;
 386         int i;
 387
 388         dev_dbg(&mdev->pdev->dev,
 389                 "nodeid %d SMPT ref count %lld map %lld unmap %lld\n",
 390                 mdev->id, smpt_info->ref_count,
 391                 smpt_info->map_count, smpt_info->unmap_count);
 392
 393         for (i = 0; i < smpt_info->info.num_reg; i++) {
 394                 dev_dbg(&mdev->pdev->dev,
 395                         "SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n",
 396                         i, smpt_info->entry[i].dma_addr,
 397                         smpt_info->entry[i].ref_count);
 398                 if (smpt_info->entry[i].ref_count)
 399                         dev_warn(&mdev->pdev->dev,
 400                                  "ref count for entry %d is not zero\n", i);
 401         }
 402         kfree(smpt_info->entry);
 403         kfree(smpt_info);
 404 }
 405
 406 /**
 407  * mic_smpt_restore - Restore MIC System Memory Page Tables.
 408  *
 409  * @mdev: pointer to mic_device instance.
 410  *
 411  * Restore the SMPT registers to values previously stored in the
 412  * SW data structures. Some MIC steppings lose register state
 413  * across resets and this API should be called for performing
 414  * a restore operation if required.
 415  *
 416  * returns None.
 417  */
 418 void mic_smpt_restore(struct mic_device *mdev)
 419 {
 420         int i;
 421         dma_addr_t dma_addr;
 422
 423         for (i = 0; i < mdev->smpt->info.num_reg; i++) {
 424                 dma_addr = mdev->smpt->entry[i].dma_addr;
 425                 mdev->smpt_ops->set(mdev, dma_addr, i);
 426         }
 427 }