drivers/firmware/qcom/qcom_tzmem.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Memory allocator for buffers shared with the TrustZone.
   4  *
   5  * Copyright (C) 2023-2024 Linaro Ltd.
   6  */
   7
   8 #include <linux/bug.h>
   9 #include <linux/cleanup.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/err.h>
  12 #include <linux/firmware/qcom/qcom_tzmem.h>
  13 #include <linux/genalloc.h>
  14 #include <linux/gfp.h>
  15 #include <linux/kernel.h>
  16 #include <linux/list.h>
  17 #include <linux/mm.h>
  18 #include <linux/radix-tree.h>
  19 #include <linux/slab.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/types.h>
  22
  23 #include "qcom_tzmem.h"
  24
  25 struct qcom_tzmem_area {
  26         struct list_head list;
  27         void *vaddr;
  28         dma_addr_t paddr;
  29         size_t size;
  30         void *priv;
  31 };
  32
  33 struct qcom_tzmem_pool {
  34         struct gen_pool *genpool;
  35         struct list_head areas;
  36         enum qcom_tzmem_policy policy;
  37         size_t increment;
  38         size_t max_size;
  39         spinlock_t lock;
  40 };
  41
  42 struct qcom_tzmem_chunk {
  43         size_t size;
  44         struct qcom_tzmem_pool *owner;
  45 };
  46
  47 static struct device *qcom_tzmem_dev;
  48 static RADIX_TREE(qcom_tzmem_chunks, GFP_ATOMIC);
  49 static DEFINE_SPINLOCK(qcom_tzmem_chunks_lock);
  50
  51 #if IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_GENERIC)
  52
  53 static int qcom_tzmem_init(void)
  54 {
  55         return 0;
  56 }
  57
  58 static int qcom_tzmem_init_area(struct qcom_tzmem_area *area)
  59 {
  60         return 0;
  61 }
  62
  63 static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)
  64 {
  65
  66 }
  67
  68 #elif IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE)
  69
  70 #include <linux/firmware/qcom/qcom_scm.h>
  71 #include <linux/of.h>
  72
  73 #define QCOM_SHM_BRIDGE_NUM_VM_SHIFT 9
  74
  75 static bool qcom_tzmem_using_shm_bridge;
  76
  77 /* List of machines that are known to not support SHM bridge correctly. */
  78 static const char *const qcom_tzmem_blacklist[] = {
  79         "qcom,sc8180x",
  80         "qcom,sdm670", /* failure in GPU firmware loading */
  81         "qcom,sdm845", /* reset in rmtfs memory assignment */
  82         "qcom,sm8150", /* reset in rmtfs memory assignment */
  83         NULL
  84 };
  85
  86 static int qcom_tzmem_init(void)
  87 {
  88         const char *const *platform;
  89         int ret;
  90
  91         for (platform = qcom_tzmem_blacklist; *platform; platform++) {
  92                 if (of_machine_is_compatible(*platform))
  93                         goto notsupp;
  94         }
  95
  96         ret = qcom_scm_shm_bridge_enable();
  97         if (ret == -EOPNOTSUPP)
  98                 goto notsupp;
  99
 100         if (!ret)
 101                 qcom_tzmem_using_shm_bridge = true;
 102
 103         return ret;
 104
 105 notsupp:
 106         dev_info(qcom_tzmem_dev, "SHM Bridge not supported\n");
 107         return 0;
 108 }
 109
 110 static int qcom_tzmem_init_area(struct qcom_tzmem_area *area)
 111 {
 112         u64 pfn_and_ns_perm, ipfn_and_s_perm, size_and_flags;
 113         int ret;
 114
 115         if (!qcom_tzmem_using_shm_bridge)
 116                 return 0;
 117
 118         pfn_and_ns_perm = (u64)area->paddr | QCOM_SCM_PERM_RW;
 119         ipfn_and_s_perm = (u64)area->paddr | QCOM_SCM_PERM_RW;
 120         size_and_flags = area->size | (1 << QCOM_SHM_BRIDGE_NUM_VM_SHIFT);
 121
 122         u64 *handle __free(kfree) = kzalloc(sizeof(*handle), GFP_KERNEL);
 123         if (!handle)
 124                 return -ENOMEM;
 125
 126         ret = qcom_scm_shm_bridge_create(qcom_tzmem_dev, pfn_and_ns_perm,
 127                                          ipfn_and_s_perm, size_and_flags,
 128                                          QCOM_SCM_VMID_HLOS, handle);
 129         if (ret)
 130                 return ret;
 131
 132         area->priv = no_free_ptr(handle);
 133
 134         return 0;
 135 }
 136
 137 static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)
 138 {
 139         u64 *handle = area->priv;
 140
 141         if (!qcom_tzmem_using_shm_bridge)
 142                 return;
 143
 144         qcom_scm_shm_bridge_delete(qcom_tzmem_dev, *handle);
 145         kfree(handle);
 146 }
 147
 148 #endif /* CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE */
 149
 150 static int qcom_tzmem_pool_add_memory(struct qcom_tzmem_pool *pool,
 151                                       size_t size, gfp_t gfp)
 152 {
 153         int ret;
 154
 155         struct qcom_tzmem_area *area __free(kfree) = kzalloc(sizeof(*area),
 156                                                              gfp);
 157         if (!area)
 158                 return -ENOMEM;
 159
 160         area->size = PAGE_ALIGN(size);
 161
 162         area->vaddr = dma_alloc_coherent(qcom_tzmem_dev, area->size,
 163                                          &area->paddr, gfp);
 164         if (!area->vaddr)
 165                 return -ENOMEM;
 166
 167         ret = qcom_tzmem_init_area(area);
 168         if (ret) {
 169                 dma_free_coherent(qcom_tzmem_dev, area->size,
 170                                   area->vaddr, area->paddr);
 171                 return ret;
 172         }
 173
 174         ret = gen_pool_add_virt(pool->genpool, (unsigned long)area->vaddr,
 175                                 (phys_addr_t)area->paddr, size, -1);
 176         if (ret) {
 177                 dma_free_coherent(qcom_tzmem_dev, area->size,
 178                                   area->vaddr, area->paddr);
 179                 return ret;
 180         }
 181
 182         scoped_guard(spinlock_irqsave, &pool->lock)
 183                 list_add_tail(&area->list, &pool->areas);
 184
 185         area = NULL;
 186         return 0;
 187 }
 188
 189 /**
 190  * qcom_tzmem_pool_new() - Create a new TZ memory pool.
 191  * @config: Pool configuration.
 192  *
 193  * Create a new pool of memory suitable for sharing with the TrustZone.
 194  *
 195  * Must not be used in atomic context.
 196  *
 197  * Return: New memory pool address or ERR_PTR() on error.
 198  */
 199 struct qcom_tzmem_pool *
 200 qcom_tzmem_pool_new(const struct qcom_tzmem_pool_config *config)
 201 {
 202         int ret = -ENOMEM;
 203
 204         might_sleep();
 205
 206         switch (config->policy) {
 207         case QCOM_TZMEM_POLICY_STATIC:
 208                 if (!config->initial_size)
 209                         return ERR_PTR(-EINVAL);
 210                 break;
 211         case QCOM_TZMEM_POLICY_MULTIPLIER:
 212                 if (!config->increment)
 213                         return ERR_PTR(-EINVAL);
 214                 break;
 215         case QCOM_TZMEM_POLICY_ON_DEMAND:
 216                 break;
 217         default:
 218                 return ERR_PTR(-EINVAL);
 219         }
 220
 221         struct qcom_tzmem_pool *pool __free(kfree) = kzalloc(sizeof(*pool),
 222                                                              GFP_KERNEL);
 223         if (!pool)
 224                 return ERR_PTR(-ENOMEM);
 225
 226         pool->genpool = gen_pool_create(PAGE_SHIFT, -1);
 227         if (!pool->genpool)
 228                 return ERR_PTR(-ENOMEM);
 229
 230         gen_pool_set_algo(pool->genpool, gen_pool_best_fit, NULL);
 231
 232         pool->policy = config->policy;
 233         pool->increment = config->increment;
 234         pool->max_size = config->max_size;
 235         INIT_LIST_HEAD(&pool->areas);
 236         spin_lock_init(&pool->lock);
 237
 238         if (config->initial_size) {
 239                 ret = qcom_tzmem_pool_add_memory(pool, config->initial_size,
 240                                                  GFP_KERNEL);
 241                 if (ret) {
 242                         gen_pool_destroy(pool->genpool);
 243                         return ERR_PTR(ret);
 244                 }
 245         }
 246
 247         return_ptr(pool);
 248 }
 249 EXPORT_SYMBOL_GPL(qcom_tzmem_pool_new);
 250
 251 /**
 252  * qcom_tzmem_pool_free() - Destroy a TZ memory pool and free all resources.
 253  * @pool: Memory pool to free.
 254  *
 255  * Must not be called if any of the allocated chunks has not been freed.
 256  * Must not be used in atomic context.
 257  */
 258 void qcom_tzmem_pool_free(struct qcom_tzmem_pool *pool)
 259 {
 260         struct qcom_tzmem_area *area, *next;
 261         struct qcom_tzmem_chunk *chunk;
 262         struct radix_tree_iter iter;
 263         bool non_empty = false;
 264         void __rcu **slot;
 265
 266         might_sleep();
 267
 268         if (!pool)
 269                 return;
 270
 271         scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock) {
 272                 radix_tree_for_each_slot(slot, &qcom_tzmem_chunks, &iter, 0) {
 273                         chunk = radix_tree_deref_slot_protected(slot,
 274                                                 &qcom_tzmem_chunks_lock);
 275
 276                         if (chunk->owner == pool)
 277                                 non_empty = true;
 278                 }
 279         }
 280
 281         WARN(non_empty, "Freeing TZ memory pool with memory still allocated");
 282
 283         list_for_each_entry_safe(area, next, &pool->areas, list) {
 284                 list_del(&area->list);
 285                 qcom_tzmem_cleanup_area(area);
 286                 dma_free_coherent(qcom_tzmem_dev, area->size,
 287                                   area->vaddr, area->paddr);
 288                 kfree(area);
 289         }
 290
 291         gen_pool_destroy(pool->genpool);
 292         kfree(pool);
 293 }
 294 EXPORT_SYMBOL_GPL(qcom_tzmem_pool_free);
 295
 296 static void devm_qcom_tzmem_pool_free(void *data)
 297 {
 298         struct qcom_tzmem_pool *pool = data;
 299
 300         qcom_tzmem_pool_free(pool);
 301 }
 302
 303 /**
 304  * devm_qcom_tzmem_pool_new() - Managed variant of qcom_tzmem_pool_new().
 305  * @dev: Device managing this resource.
 306  * @config: Pool configuration.
 307  *
 308  * Must not be used in atomic context.
 309  *
 310  * Return: Address of the managed pool or ERR_PTR() on failure.
 311  */
 312 struct qcom_tzmem_pool *
 313 devm_qcom_tzmem_pool_new(struct device *dev,
 314                          const struct qcom_tzmem_pool_config *config)
 315 {
 316         struct qcom_tzmem_pool *pool;
 317         int ret;
 318
 319         pool = qcom_tzmem_pool_new(config);
 320         if (IS_ERR(pool))
 321                 return pool;
 322
 323         ret = devm_add_action_or_reset(dev, devm_qcom_tzmem_pool_free, pool);
 324         if (ret)
 325                 return ERR_PTR(ret);
 326
 327         return pool;
 328 }
 329 EXPORT_SYMBOL_GPL(devm_qcom_tzmem_pool_new);
 330
 331 static bool qcom_tzmem_try_grow_pool(struct qcom_tzmem_pool *pool,
 332                                      size_t requested, gfp_t gfp)
 333 {
 334         size_t current_size = gen_pool_size(pool->genpool);
 335
 336         if (pool->max_size && (current_size + requested) > pool->max_size)
 337                 return false;
 338
 339         switch (pool->policy) {
 340         case QCOM_TZMEM_POLICY_STATIC:
 341                 return false;
 342         case QCOM_TZMEM_POLICY_MULTIPLIER:
 343                 requested = current_size * pool->increment;
 344                 break;
 345         case QCOM_TZMEM_POLICY_ON_DEMAND:
 346                 break;
 347         }
 348
 349         return !qcom_tzmem_pool_add_memory(pool, requested, gfp);
 350 }
 351
 352 /**
 353  * qcom_tzmem_alloc() - Allocate a memory chunk suitable for sharing with TZ.
 354  * @pool: TZ memory pool from which to allocate memory.
 355  * @size: Number of bytes to allocate.
 356  * @gfp: GFP flags.
 357  *
 358  * Can be used in any context.
 359  *
 360  * Return:
 361  * Address of the allocated buffer or NULL if no more memory can be allocated.
 362  * The buffer must be released using qcom_tzmem_free().
 363  */
 364 void *qcom_tzmem_alloc(struct qcom_tzmem_pool *pool, size_t size, gfp_t gfp)
 365 {
 366         unsigned long vaddr;
 367         int ret;
 368
 369         if (!size)
 370                 return NULL;
 371
 372         size = PAGE_ALIGN(size);
 373
 374         struct qcom_tzmem_chunk *chunk __free(kfree) = kzalloc(sizeof(*chunk),
 375                                                                gfp);
 376         if (!chunk)
 377                 return NULL;
 378
 379 again:
 380         vaddr = gen_pool_alloc(pool->genpool, size);
 381         if (!vaddr) {
 382                 if (qcom_tzmem_try_grow_pool(pool, size, gfp))
 383                         goto again;
 384
 385                 return NULL;
 386         }
 387
 388         chunk->size = size;
 389         chunk->owner = pool;
 390
 391         scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock) {
 392                 ret = radix_tree_insert(&qcom_tzmem_chunks, vaddr, chunk);
 393                 if (ret) {
 394                         gen_pool_free(pool->genpool, vaddr, size);
 395                         return NULL;
 396                 }
 397
 398                 chunk = NULL;
 399         }
 400
 401         return (void *)vaddr;
 402 }
 403 EXPORT_SYMBOL_GPL(qcom_tzmem_alloc);
 404
 405 /**
 406  * qcom_tzmem_free() - Release a buffer allocated from a TZ memory pool.
 407  * @vaddr: Virtual address of the buffer.
 408  *
 409  * Can be used in any context.
 410  */
 411 void qcom_tzmem_free(void *vaddr)
 412 {
 413         struct qcom_tzmem_chunk *chunk;
 414
 415         scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock)
 416                 chunk = radix_tree_delete_item(&qcom_tzmem_chunks,
 417                                                (unsigned long)vaddr, NULL);
 418
 419         if (!chunk) {
 420                 WARN(1, "Virtual address %p not owned by TZ memory allocator",
 421                      vaddr);
 422                 return;
 423         }
 424
 425         scoped_guard(spinlock_irqsave, &chunk->owner->lock)
 426                 gen_pool_free(chunk->owner->genpool, (unsigned long)vaddr,
 427                               chunk->size);
 428         kfree(chunk);
 429 }
 430 EXPORT_SYMBOL_GPL(qcom_tzmem_free);
 431
 432 /**
 433  * qcom_tzmem_to_phys() - Map the virtual address of TZ memory to physical.
 434  * @vaddr: Virtual address of memory allocated from a TZ memory pool.
 435  *
 436  * Can be used in any context. The address must point to memory allocated
 437  * using qcom_tzmem_alloc().
 438  *
 439  * Returns:
 440  * Physical address mapped from the virtual or 0 if the mapping failed.
 441  */
 442 phys_addr_t qcom_tzmem_to_phys(void *vaddr)
 443 {
 444         struct qcom_tzmem_chunk *chunk;
 445         struct radix_tree_iter iter;
 446         void __rcu **slot;
 447         phys_addr_t ret;
 448
 449         guard(spinlock_irqsave)(&qcom_tzmem_chunks_lock);
 450
 451         radix_tree_for_each_slot(slot, &qcom_tzmem_chunks, &iter, 0) {
 452                 chunk = radix_tree_deref_slot_protected(slot,
 453                                                 &qcom_tzmem_chunks_lock);
 454
 455                 ret = gen_pool_virt_to_phys(chunk->owner->genpool,
 456                                             (unsigned long)vaddr);
 457                 if (ret == -1)
 458                         continue;
 459
 460                 return ret;
 461         }
 462
 463         return 0;
 464 }
 465 EXPORT_SYMBOL_GPL(qcom_tzmem_to_phys);
 466
 467 int qcom_tzmem_enable(struct device *dev)
 468 {
 469         if (qcom_tzmem_dev)
 470                 return -EBUSY;
 471
 472         qcom_tzmem_dev = dev;
 473
 474         return qcom_tzmem_init();
 475 }
 476 EXPORT_SYMBOL_GPL(qcom_tzmem_enable);
 477
 478 MODULE_DESCRIPTION("TrustZone memory allocator for Qualcomm firmware drivers");
 479 MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@linaro.org>");
 480 MODULE_LICENSE("GPL");