drivers/gpu/host1x/job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Job
   4  *
   5  * Copyright (c) 2010-2015, NVIDIA Corporation.
   6  */
   7
   8 #include <linux/dma-mapping.h>
   9 #include <linux/err.h>
  10 #include <linux/host1x.h>
  11 #include <linux/kref.h>
  12 #include <linux/module.h>
  13 #include <linux/scatterlist.h>
  14 #include <linux/slab.h>
  15 #include <linux/vmalloc.h>
  16 #include <trace/events/host1x.h>
  17
  18 #include "channel.h"
  19 #include "dev.h"
  20 #include "job.h"
  21 #include "syncpt.h"
  22
  23 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8
  24
  25 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  26                                     u32 num_cmdbufs, u32 num_relocs)
  27 {
  28         struct host1x_job *job = NULL;
  29         unsigned int num_unpins = num_cmdbufs + num_relocs;
  30         u64 total;
  31         void *mem;
  32
  33         /* Check that we're not going to overflow */
  34         total = sizeof(struct host1x_job) +
  35                 (u64)num_relocs * sizeof(struct host1x_reloc) +
  36                 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
  37                 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
  38                 (u64)num_unpins * sizeof(dma_addr_t) +
  39                 (u64)num_unpins * sizeof(u32 *);
  40         if (total > ULONG_MAX)
  41                 return NULL;
  42
  43         mem = job = kzalloc(total, GFP_KERNEL);
  44         if (!job)
  45                 return NULL;
  46
  47         kref_init(&job->ref);
  48         job->channel = ch;
  49
  50         /* Redistribute memory to the structs  */
  51         mem += sizeof(struct host1x_job);
  52         job->relocs = num_relocs ? mem : NULL;
  53         mem += num_relocs * sizeof(struct host1x_reloc);
  54         job->unpins = num_unpins ? mem : NULL;
  55         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  56         job->gathers = num_cmdbufs ? mem : NULL;
  57         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  58         job->addr_phys = num_unpins ? mem : NULL;
  59
  60         job->reloc_addr_phys = job->addr_phys;
  61         job->gather_addr_phys = &job->addr_phys[num_relocs];
  62
  63         return job;
  64 }
  65 EXPORT_SYMBOL(host1x_job_alloc);
  66
  67 struct host1x_job *host1x_job_get(struct host1x_job *job)
  68 {
  69         kref_get(&job->ref);
  70         return job;
  71 }
  72 EXPORT_SYMBOL(host1x_job_get);
  73
  74 static void job_free(struct kref *ref)
  75 {
  76         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  77
  78         kfree(job);
  79 }
  80
  81 void host1x_job_put(struct host1x_job *job)
  82 {
  83         kref_put(&job->ref, job_free);
  84 }
  85 EXPORT_SYMBOL(host1x_job_put);
  86
  87 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
  88                            unsigned int words, unsigned int offset)
  89 {
  90         struct host1x_job_gather *gather = &job->gathers[job->num_gathers];
  91
  92         gather->words = words;
  93         gather->bo = bo;
  94         gather->offset = offset;
  95
  96         job->num_gathers++;
  97 }
  98 EXPORT_SYMBOL(host1x_job_add_gather);
  99
 100 static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 101 {
 102         unsigned int i;
 103         int err;
 104
 105         job->num_unpins = 0;
 106
 107         for (i = 0; i < job->num_relocs; i++) {
 108                 struct host1x_reloc *reloc = &job->relocs[i];
 109                 struct sg_table *sgt;
 110                 dma_addr_t phys_addr;
 111
 112                 reloc->target.bo = host1x_bo_get(reloc->target.bo);
 113                 if (!reloc->target.bo) {
 114                         err = -EINVAL;
 115                         goto unpin;
 116                 }
 117
 118                 phys_addr = host1x_bo_pin(reloc->target.bo, &sgt);
 119
 120                 job->addr_phys[job->num_unpins] = phys_addr;
 121                 job->unpins[job->num_unpins].bo = reloc->target.bo;
 122                 job->unpins[job->num_unpins].sgt = sgt;
 123                 job->num_unpins++;
 124         }
 125
 126         for (i = 0; i < job->num_gathers; i++) {
 127                 struct host1x_job_gather *g = &job->gathers[i];
 128                 size_t gather_size = 0;
 129                 struct scatterlist *sg;
 130                 struct sg_table *sgt;
 131                 dma_addr_t phys_addr;
 132                 unsigned long shift;
 133                 struct iova *alloc;
 134                 unsigned int j;
 135
 136                 g->bo = host1x_bo_get(g->bo);
 137                 if (!g->bo) {
 138                         err = -EINVAL;
 139                         goto unpin;
 140                 }
 141
 142                 phys_addr = host1x_bo_pin(g->bo, &sgt);
 143
 144                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) {
 145                         for_each_sg(sgt->sgl, sg, sgt->nents, j)
 146                                 gather_size += sg->length;
 147                         gather_size = iova_align(&host->iova, gather_size);
 148
 149                         shift = iova_shift(&host->iova);
 150                         alloc = alloc_iova(&host->iova, gather_size >> shift,
 151                                            host->iova_end >> shift, true);
 152                         if (!alloc) {
 153                                 err = -ENOMEM;
 154                                 goto unpin;
 155                         }
 156
 157                         err = iommu_map_sg(host->domain,
 158                                         iova_dma_addr(&host->iova, alloc),
 159                                         sgt->sgl, sgt->nents, IOMMU_READ);
 160                         if (err == 0) {
 161                                 __free_iova(&host->iova, alloc);
 162                                 err = -EINVAL;
 163                                 goto unpin;
 164                         }
 165
 166                         job->addr_phys[job->num_unpins] =
 167                                 iova_dma_addr(&host->iova, alloc);
 168                         job->unpins[job->num_unpins].size = gather_size;
 169                 } else {
 170                         job->addr_phys[job->num_unpins] = phys_addr;
 171                 }
 172
 173                 job->gather_addr_phys[i] = job->addr_phys[job->num_unpins];
 174
 175                 job->unpins[job->num_unpins].bo = g->bo;
 176                 job->unpins[job->num_unpins].sgt = sgt;
 177                 job->num_unpins++;
 178         }
 179
 180         return 0;
 181
 182 unpin:
 183         host1x_job_unpin(job);
 184         return err;
 185 }
 186
 187 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g)
 188 {
 189         u32 last_page = ~0;
 190         void *cmdbuf_page_addr = NULL;
 191         struct host1x_bo *cmdbuf = g->bo;
 192         unsigned int i;
 193
 194         /* pin & patch the relocs for one gather */
 195         for (i = 0; i < job->num_relocs; i++) {
 196                 struct host1x_reloc *reloc = &job->relocs[i];
 197                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 198                                   reloc->target.offset) >> reloc->shift;
 199                 u32 *target;
 200
 201                 /* skip all other gathers */
 202                 if (cmdbuf != reloc->cmdbuf.bo)
 203                         continue;
 204
 205                 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) {
 206                         target = (u32 *)job->gather_copy_mapped +
 207                                         reloc->cmdbuf.offset / sizeof(u32) +
 208                                                 g->offset / sizeof(u32);
 209                         goto patch_reloc;
 210                 }
 211
 212                 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) {
 213                         if (cmdbuf_page_addr)
 214                                 host1x_bo_kunmap(cmdbuf, last_page,
 215                                                  cmdbuf_page_addr);
 216
 217                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 218                                         reloc->cmdbuf.offset >> PAGE_SHIFT);
 219                         last_page = reloc->cmdbuf.offset >> PAGE_SHIFT;
 220
 221                         if (unlikely(!cmdbuf_page_addr)) {
 222                                 pr_err("Could not map cmdbuf for relocation\n");
 223                                 return -ENOMEM;
 224                         }
 225                 }
 226
 227                 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK);
 228 patch_reloc:
 229                 *target = reloc_addr;
 230         }
 231
 232         if (cmdbuf_page_addr)
 233                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 234
 235         return 0;
 236 }
 237
 238 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 239                         unsigned int offset)
 240 {
 241         offset *= sizeof(u32);
 242
 243         if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset)
 244                 return false;
 245
 246         /* relocation shift value validation isn't implemented yet */
 247         if (reloc->shift)
 248                 return false;
 249
 250         return true;
 251 }
 252
 253 struct host1x_firewall {
 254         struct host1x_job *job;
 255         struct device *dev;
 256
 257         unsigned int num_relocs;
 258         struct host1x_reloc *reloc;
 259
 260         struct host1x_bo *cmdbuf;
 261         unsigned int offset;
 262
 263         u32 words;
 264         u32 class;
 265         u32 reg;
 266         u32 mask;
 267         u32 count;
 268 };
 269
 270 static int check_register(struct host1x_firewall *fw, unsigned long offset)
 271 {
 272         if (!fw->job->is_addr_reg)
 273                 return 0;
 274
 275         if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
 276                 if (!fw->num_relocs)
 277                         return -EINVAL;
 278
 279                 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
 280                         return -EINVAL;
 281
 282                 fw->num_relocs--;
 283                 fw->reloc++;
 284         }
 285
 286         return 0;
 287 }
 288
 289 static int check_class(struct host1x_firewall *fw, u32 class)
 290 {
 291         if (!fw->job->is_valid_class) {
 292                 if (fw->class != class)
 293                         return -EINVAL;
 294         } else {
 295                 if (!fw->job->is_valid_class(fw->class))
 296                         return -EINVAL;
 297         }
 298
 299         return 0;
 300 }
 301
 302 static int check_mask(struct host1x_firewall *fw)
 303 {
 304         u32 mask = fw->mask;
 305         u32 reg = fw->reg;
 306         int ret;
 307
 308         while (mask) {
 309                 if (fw->words == 0)
 310                         return -EINVAL;
 311
 312                 if (mask & 1) {
 313                         ret = check_register(fw, reg);
 314                         if (ret < 0)
 315                                 return ret;
 316
 317                         fw->words--;
 318                         fw->offset++;
 319                 }
 320                 mask >>= 1;
 321                 reg++;
 322         }
 323
 324         return 0;
 325 }
 326
 327 static int check_incr(struct host1x_firewall *fw)
 328 {
 329         u32 count = fw->count;
 330         u32 reg = fw->reg;
 331         int ret;
 332
 333         while (count) {
 334                 if (fw->words == 0)
 335                         return -EINVAL;
 336
 337                 ret = check_register(fw, reg);
 338                 if (ret < 0)
 339                         return ret;
 340
 341                 reg++;
 342                 fw->words--;
 343                 fw->offset++;
 344                 count--;
 345         }
 346
 347         return 0;
 348 }
 349
 350 static int check_nonincr(struct host1x_firewall *fw)
 351 {
 352         u32 count = fw->count;
 353         int ret;
 354
 355         while (count) {
 356                 if (fw->words == 0)
 357                         return -EINVAL;
 358
 359                 ret = check_register(fw, fw->reg);
 360                 if (ret < 0)
 361                         return ret;
 362
 363                 fw->words--;
 364                 fw->offset++;
 365                 count--;
 366         }
 367
 368         return 0;
 369 }
 370
 371 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 372 {
 373         u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
 374                 (g->offset / sizeof(u32));
 375         u32 job_class = fw->class;
 376         int err = 0;
 377
 378         fw->words = g->words;
 379         fw->cmdbuf = g->bo;
 380         fw->offset = 0;
 381
 382         while (fw->words && !err) {
 383                 u32 word = cmdbuf_base[fw->offset];
 384                 u32 opcode = (word & 0xf0000000) >> 28;
 385
 386                 fw->mask = 0;
 387                 fw->reg = 0;
 388                 fw->count = 0;
 389                 fw->words--;
 390                 fw->offset++;
 391
 392                 switch (opcode) {
 393                 case 0:
 394                         fw->class = word >> 6 & 0x3ff;
 395                         fw->mask = word & 0x3f;
 396                         fw->reg = word >> 16 & 0xfff;
 397                         err = check_class(fw, job_class);
 398                         if (!err)
 399                                 err = check_mask(fw);
 400                         if (err)
 401                                 goto out;
 402                         break;
 403                 case 1:
 404                         fw->reg = word >> 16 & 0xfff;
 405                         fw->count = word & 0xffff;
 406                         err = check_incr(fw);
 407                         if (err)
 408                                 goto out;
 409                         break;
 410
 411                 case 2:
 412                         fw->reg = word >> 16 & 0xfff;
 413                         fw->count = word & 0xffff;
 414                         err = check_nonincr(fw);
 415                         if (err)
 416                                 goto out;
 417                         break;
 418
 419                 case 3:
 420                         fw->mask = word & 0xffff;
 421                         fw->reg = word >> 16 & 0xfff;
 422                         err = check_mask(fw);
 423                         if (err)
 424                                 goto out;
 425                         break;
 426                 case 4:
 427                 case 14:
 428                         break;
 429                 default:
 430                         err = -EINVAL;
 431                         break;
 432                 }
 433         }
 434
 435 out:
 436         return err;
 437 }
 438
 439 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 440 {
 441         struct host1x_firewall fw;
 442         size_t size = 0;
 443         size_t offset = 0;
 444         unsigned int i;
 445
 446         fw.job = job;
 447         fw.dev = dev;
 448         fw.reloc = job->relocs;
 449         fw.num_relocs = job->num_relocs;
 450         fw.class = job->class;
 451
 452         for (i = 0; i < job->num_gathers; i++) {
 453                 struct host1x_job_gather *g = &job->gathers[i];
 454
 455                 size += g->words * sizeof(u32);
 456         }
 457
 458         /*
 459          * Try a non-blocking allocation from a higher priority pools first,
 460          * as awaiting for the allocation here is a major performance hit.
 461          */
 462         job->gather_copy_mapped = dma_alloc_wc(dev, size, &job->gather_copy,
 463                                                GFP_NOWAIT);
 464
 465         /* the higher priority allocation failed, try the generic-blocking */
 466         if (!job->gather_copy_mapped)
 467                 job->gather_copy_mapped = dma_alloc_wc(dev, size,
 468                                                        &job->gather_copy,
 469                                                        GFP_KERNEL);
 470         if (!job->gather_copy_mapped)
 471                 return -ENOMEM;
 472
 473         job->gather_copy_size = size;
 474
 475         for (i = 0; i < job->num_gathers; i++) {
 476                 struct host1x_job_gather *g = &job->gathers[i];
 477                 void *gather;
 478
 479                 /* Copy the gather */
 480                 gather = host1x_bo_mmap(g->bo);
 481                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 482                        g->words * sizeof(u32));
 483                 host1x_bo_munmap(g->bo, gather);
 484
 485                 /* Store the location in the buffer */
 486                 g->base = job->gather_copy;
 487                 g->offset = offset;
 488
 489                 /* Validate the job */
 490                 if (validate(&fw, g))
 491                         return -EINVAL;
 492
 493                 offset += g->words * sizeof(u32);
 494         }
 495
 496         /* No relocs should remain at this point */
 497         if (fw.num_relocs)
 498                 return -EINVAL;
 499
 500         return 0;
 501 }
 502
 503 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 504 {
 505         int err;
 506         unsigned int i, j;
 507         struct host1x *host = dev_get_drvdata(dev->parent);
 508
 509         /* pin memory */
 510         err = pin_job(host, job);
 511         if (err)
 512                 goto out;
 513
 514         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) {
 515                 err = copy_gathers(job, dev);
 516                 if (err)
 517                         goto out;
 518         }
 519
 520         /* patch gathers */
 521         for (i = 0; i < job->num_gathers; i++) {
 522                 struct host1x_job_gather *g = &job->gathers[i];
 523
 524                 /* process each gather mem only once */
 525                 if (g->handled)
 526                         continue;
 527
 528                 /* copy_gathers() sets gathers base if firewall is enabled */
 529                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL))
 530                         g->base = job->gather_addr_phys[i];
 531
 532                 for (j = i + 1; j < job->num_gathers; j++) {
 533                         if (job->gathers[j].bo == g->bo) {
 534                                 job->gathers[j].handled = true;
 535                                 job->gathers[j].base = g->base;
 536                         }
 537                 }
 538
 539                 err = do_relocs(job, g);
 540                 if (err)
 541                         break;
 542         }
 543
 544 out:
 545         if (err)
 546                 host1x_job_unpin(job);
 547         wmb();
 548
 549         return err;
 550 }
 551 EXPORT_SYMBOL(host1x_job_pin);
 552
 553 void host1x_job_unpin(struct host1x_job *job)
 554 {
 555         struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
 556         unsigned int i;
 557
 558         for (i = 0; i < job->num_unpins; i++) {
 559                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 560
 561                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) &&
 562                     unpin->size && host->domain) {
 563                         iommu_unmap(host->domain, job->addr_phys[i],
 564                                     unpin->size);
 565                         free_iova(&host->iova,
 566                                 iova_pfn(&host->iova, job->addr_phys[i]));
 567                 }
 568
 569                 host1x_bo_unpin(unpin->bo, unpin->sgt);
 570                 host1x_bo_put(unpin->bo);
 571         }
 572
 573         job->num_unpins = 0;
 574
 575         if (job->gather_copy_size)
 576                 dma_free_wc(job->channel->dev, job->gather_copy_size,
 577                             job->gather_copy_mapped, job->gather_copy);
 578 }
 579 EXPORT_SYMBOL(host1x_job_unpin);
 580
 581 /*
 582  * Debug routine used to dump job entries
 583  */
 584 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 585 {
 586         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 587         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 588         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 589         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 590         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 591         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 592 }