drivers/gpu/drm/i915/gt/intel_gt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "debugfs_gt.h"
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_gt.h"
  10 #include "intel_gt_buffer_pool.h"
  11 #include "intel_gt_clock_utils.h"
  12 #include "intel_gt_pm.h"
  13 #include "intel_gt_requests.h"
  14 #include "intel_mocs.h"
  15 #include "intel_rc6.h"
  16 #include "intel_renderstate.h"
  17 #include "intel_rps.h"
  18 #include "intel_uncore.h"
  19 #include "intel_pm.h"
  20 #include "shmem_utils.h"
  21
  22 void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
  23 {
  24         gt->i915 = i915;
  25         gt->uncore = &i915->uncore;
  26
  27         spin_lock_init(&gt->irq_lock);
  28
  29         INIT_LIST_HEAD(&gt->closed_vma);
  30         spin_lock_init(&gt->closed_lock);
  31
  32         intel_gt_init_buffer_pool(gt);
  33         intel_gt_init_reset(gt);
  34         intel_gt_init_requests(gt);
  35         intel_gt_init_timelines(gt);
  36         intel_gt_pm_init_early(gt);
  37
  38         intel_rps_init_early(&gt->rps);
  39         intel_uc_init_early(&gt->uc);
  40 }
  41
  42 void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt)
  43 {
  44         gt->ggtt = ggtt;
  45 }
  46
  47 int intel_gt_init_mmio(struct intel_gt *gt)
  48 {
  49         intel_uc_init_mmio(&gt->uc);
  50         intel_sseu_info_init(gt);
  51
  52         return intel_engines_init_mmio(gt);
  53 }
  54
  55 static void init_unused_ring(struct intel_gt *gt, u32 base)
  56 {
  57         struct intel_uncore *uncore = gt->uncore;
  58
  59         intel_uncore_write(uncore, RING_CTL(base), 0);
  60         intel_uncore_write(uncore, RING_HEAD(base), 0);
  61         intel_uncore_write(uncore, RING_TAIL(base), 0);
  62         intel_uncore_write(uncore, RING_START(base), 0);
  63 }
  64
  65 static void init_unused_rings(struct intel_gt *gt)
  66 {
  67         struct drm_i915_private *i915 = gt->i915;
  68
  69         if (IS_I830(i915)) {
  70                 init_unused_ring(gt, PRB1_BASE);
  71                 init_unused_ring(gt, SRB0_BASE);
  72                 init_unused_ring(gt, SRB1_BASE);
  73                 init_unused_ring(gt, SRB2_BASE);
  74                 init_unused_ring(gt, SRB3_BASE);
  75         } else if (IS_GEN(i915, 2)) {
  76                 init_unused_ring(gt, SRB0_BASE);
  77                 init_unused_ring(gt, SRB1_BASE);
  78         } else if (IS_GEN(i915, 3)) {
  79                 init_unused_ring(gt, PRB1_BASE);
  80                 init_unused_ring(gt, PRB2_BASE);
  81         }
  82 }
  83
  84 int intel_gt_init_hw(struct intel_gt *gt)
  85 {
  86         struct drm_i915_private *i915 = gt->i915;
  87         struct intel_uncore *uncore = gt->uncore;
  88         int ret;
  89
  90         gt->last_init_time = ktime_get();
  91
  92         /* Double layer security blanket, see i915_gem_init() */
  93         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
  94
  95         if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9)
  96                 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
  97
  98         if (IS_HASWELL(i915))
  99                 intel_uncore_write(uncore,
 100                                    MI_PREDICATE_RESULT_2,
 101                                    IS_HSW_GT3(i915) ?
 102                                    LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 103
 104         /* Apply the GT workarounds... */
 105         intel_gt_apply_workarounds(gt);
 106         /* ...and determine whether they are sticking. */
 107         intel_gt_verify_workarounds(gt, "init");
 108
 109         intel_gt_init_swizzling(gt);
 110
 111         /*
 112          * At least 830 can leave some of the unused rings
 113          * "active" (ie. head != tail) after resume which
 114          * will prevent c3 entry. Makes sure all unused rings
 115          * are totally idle.
 116          */
 117         init_unused_rings(gt);
 118
 119         ret = i915_ppgtt_init_hw(gt);
 120         if (ret) {
 121                 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
 122                 goto out;
 123         }
 124
 125         /* We can't enable contexts until all firmware is loaded */
 126         ret = intel_uc_init_hw(&gt->uc);
 127         if (ret) {
 128                 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
 129                 goto out;
 130         }
 131
 132         intel_mocs_init(gt);
 133
 134 out:
 135         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 136         return ret;
 137 }
 138
 139 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
 140 {
 141         intel_uncore_rmw(uncore, reg, 0, set);
 142 }
 143
 144 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
 145 {
 146         intel_uncore_rmw(uncore, reg, clr, 0);
 147 }
 148
 149 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
 150 {
 151         intel_uncore_rmw(uncore, reg, 0, 0);
 152 }
 153
 154 static void gen8_clear_engine_error_register(struct intel_engine_cs *engine)
 155 {
 156         GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
 157         GEN6_RING_FAULT_REG_POSTING_READ(engine);
 158 }
 159
 160 void
 161 intel_gt_clear_error_registers(struct intel_gt *gt,
 162                                intel_engine_mask_t engine_mask)
 163 {
 164         struct drm_i915_private *i915 = gt->i915;
 165         struct intel_uncore *uncore = gt->uncore;
 166         u32 eir;
 167
 168         if (!IS_GEN(i915, 2))
 169                 clear_register(uncore, PGTBL_ER);
 170
 171         if (INTEL_GEN(i915) < 4)
 172                 clear_register(uncore, IPEIR(RENDER_RING_BASE));
 173         else
 174                 clear_register(uncore, IPEIR_I965);
 175
 176         clear_register(uncore, EIR);
 177         eir = intel_uncore_read(uncore, EIR);
 178         if (eir) {
 179                 /*
 180                  * some errors might have become stuck,
 181                  * mask them.
 182                  */
 183                 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
 184                 rmw_set(uncore, EMR, eir);
 185                 intel_uncore_write(uncore, GEN2_IIR,
 186                                    I915_MASTER_ERROR_INTERRUPT);
 187         }
 188
 189         if (INTEL_GEN(i915) >= 12) {
 190                 rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
 191                 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
 192         } else if (INTEL_GEN(i915) >= 8) {
 193                 rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
 194                 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
 195         } else if (INTEL_GEN(i915) >= 6) {
 196                 struct intel_engine_cs *engine;
 197                 enum intel_engine_id id;
 198
 199                 for_each_engine_masked(engine, gt, engine_mask, id)
 200                         gen8_clear_engine_error_register(engine);
 201         }
 202 }
 203
 204 static void gen6_check_faults(struct intel_gt *gt)
 205 {
 206         struct intel_engine_cs *engine;
 207         enum intel_engine_id id;
 208         u32 fault;
 209
 210         for_each_engine(engine, gt, id) {
 211                 fault = GEN6_RING_FAULT_REG_READ(engine);
 212                 if (fault & RING_FAULT_VALID) {
 213                         drm_dbg(&engine->i915->drm, "Unexpected fault\n"
 214                                 "\tAddr: 0x%08lx\n"
 215                                 "\tAddress space: %s\n"
 216                                 "\tSource ID: %d\n"
 217                                 "\tType: %d\n",
 218                                 fault & PAGE_MASK,
 219                                 fault & RING_FAULT_GTTSEL_MASK ?
 220                                 "GGTT" : "PPGTT",
 221                                 RING_FAULT_SRCID(fault),
 222                                 RING_FAULT_FAULT_TYPE(fault));
 223                 }
 224         }
 225 }
 226
 227 static void gen8_check_faults(struct intel_gt *gt)
 228 {
 229         struct intel_uncore *uncore = gt->uncore;
 230         i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
 231         u32 fault;
 232
 233         if (INTEL_GEN(gt->i915) >= 12) {
 234                 fault_reg = GEN12_RING_FAULT_REG;
 235                 fault_data0_reg = GEN12_FAULT_TLB_DATA0;
 236                 fault_data1_reg = GEN12_FAULT_TLB_DATA1;
 237         } else {
 238                 fault_reg = GEN8_RING_FAULT_REG;
 239                 fault_data0_reg = GEN8_FAULT_TLB_DATA0;
 240                 fault_data1_reg = GEN8_FAULT_TLB_DATA1;
 241         }
 242
 243         fault = intel_uncore_read(uncore, fault_reg);
 244         if (fault & RING_FAULT_VALID) {
 245                 u32 fault_data0, fault_data1;
 246                 u64 fault_addr;
 247
 248                 fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
 249                 fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
 250
 251                 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
 252                              ((u64)fault_data0 << 12);
 253
 254                 drm_dbg(&uncore->i915->drm, "Unexpected fault\n"
 255                         "\tAddr: 0x%08x_%08x\n"
 256                         "\tAddress space: %s\n"
 257                         "\tEngine ID: %d\n"
 258                         "\tSource ID: %d\n"
 259                         "\tType: %d\n",
 260                         upper_32_bits(fault_addr), lower_32_bits(fault_addr),
 261                         fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
 262                         GEN8_RING_FAULT_ENGINE_ID(fault),
 263                         RING_FAULT_SRCID(fault),
 264                         RING_FAULT_FAULT_TYPE(fault));
 265         }
 266 }
 267
 268 void intel_gt_check_and_clear_faults(struct intel_gt *gt)
 269 {
 270         struct drm_i915_private *i915 = gt->i915;
 271
 272         /* From GEN8 onwards we only have one 'All Engine Fault Register' */
 273         if (INTEL_GEN(i915) >= 8)
 274                 gen8_check_faults(gt);
 275         else if (INTEL_GEN(i915) >= 6)
 276                 gen6_check_faults(gt);
 277         else
 278                 return;
 279
 280         intel_gt_clear_error_registers(gt, ALL_ENGINES);
 281 }
 282
 283 void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
 284 {
 285         struct intel_uncore *uncore = gt->uncore;
 286         intel_wakeref_t wakeref;
 287
 288         /*
 289          * No actual flushing is required for the GTT write domain for reads
 290          * from the GTT domain. Writes to it "immediately" go to main memory
 291          * as far as we know, so there's no chipset flush. It also doesn't
 292          * land in the GPU render cache.
 293          *
 294          * However, we do have to enforce the order so that all writes through
 295          * the GTT land before any writes to the device, such as updates to
 296          * the GATT itself.
 297          *
 298          * We also have to wait a bit for the writes to land from the GTT.
 299          * An uncached read (i.e. mmio) seems to be ideal for the round-trip
 300          * timing. This issue has only been observed when switching quickly
 301          * between GTT writes and CPU reads from inside the kernel on recent hw,
 302          * and it appears to only affect discrete GTT blocks (i.e. on LLC
 303          * system agents we cannot reproduce this behaviour, until Cannonlake
 304          * that was!).
 305          */
 306
 307         wmb();
 308
 309         if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
 310                 return;
 311
 312         intel_gt_chipset_flush(gt);
 313
 314         with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
 315                 unsigned long flags;
 316
 317                 spin_lock_irqsave(&uncore->lock, flags);
 318                 intel_uncore_posting_read_fw(uncore,
 319                                              RING_HEAD(RENDER_RING_BASE));
 320                 spin_unlock_irqrestore(&uncore->lock, flags);
 321         }
 322 }
 323
 324 void intel_gt_chipset_flush(struct intel_gt *gt)
 325 {
 326         wmb();
 327         if (INTEL_GEN(gt->i915) < 6)
 328                 intel_gtt_chipset_flush();
 329 }
 330
 331 void intel_gt_driver_register(struct intel_gt *gt)
 332 {
 333         intel_rps_driver_register(&gt->rps);
 334
 335         debugfs_gt_register(gt);
 336 }
 337
 338 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
 339 {
 340         struct drm_i915_private *i915 = gt->i915;
 341         struct drm_i915_gem_object *obj;
 342         struct i915_vma *vma;
 343         int ret;
 344
 345         obj = i915_gem_object_create_stolen(i915, size);
 346         if (IS_ERR(obj))
 347                 obj = i915_gem_object_create_internal(i915, size);
 348         if (IS_ERR(obj)) {
 349                 DRM_ERROR("Failed to allocate scratch page\n");
 350                 return PTR_ERR(obj);
 351         }
 352
 353         vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
 354         if (IS_ERR(vma)) {
 355                 ret = PTR_ERR(vma);
 356                 goto err_unref;
 357         }
 358
 359         ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
 360         if (ret)
 361                 goto err_unref;
 362
 363         gt->scratch = i915_vma_make_unshrinkable(vma);
 364
 365         return 0;
 366
 367 err_unref:
 368         i915_gem_object_put(obj);
 369         return ret;
 370 }
 371
 372 static void intel_gt_fini_scratch(struct intel_gt *gt)
 373 {
 374         i915_vma_unpin_and_release(&gt->scratch, 0);
 375 }
 376
 377 static struct i915_address_space *kernel_vm(struct intel_gt *gt)
 378 {
 379         if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
 380                 return &i915_ppgtt_create(gt)->vm;
 381         else
 382                 return i915_vm_get(&gt->ggtt->vm);
 383 }
 384
 385 static int __engines_record_defaults(struct intel_gt *gt)
 386 {
 387         struct i915_request *requests[I915_NUM_ENGINES] = {};
 388         struct intel_engine_cs *engine;
 389         enum intel_engine_id id;
 390         int err = 0;
 391
 392         /*
 393          * As we reset the gpu during very early sanitisation, the current
 394          * register state on the GPU should reflect its defaults values.
 395          * We load a context onto the hw (with restore-inhibit), then switch
 396          * over to a second context to save that default register state. We
 397          * can then prime every new context with that state so they all start
 398          * from the same default HW values.
 399          */
 400
 401         for_each_engine(engine, gt, id) {
 402                 struct intel_renderstate so;
 403                 struct intel_context *ce;
 404                 struct i915_request *rq;
 405
 406                 /* We must be able to switch to something! */
 407                 GEM_BUG_ON(!engine->kernel_context);
 408
 409                 ce = intel_context_create(engine);
 410                 if (IS_ERR(ce)) {
 411                         err = PTR_ERR(ce);
 412                         goto out;
 413                 }
 414
 415                 err = intel_renderstate_init(&so, ce);
 416                 if (err)
 417                         goto err;
 418
 419                 rq = i915_request_create(ce);
 420                 if (IS_ERR(rq)) {
 421                         err = PTR_ERR(rq);
 422                         goto err_fini;
 423                 }
 424
 425                 err = intel_engine_emit_ctx_wa(rq);
 426                 if (err)
 427                         goto err_rq;
 428
 429                 err = intel_renderstate_emit(&so, rq);
 430                 if (err)
 431                         goto err_rq;
 432
 433 err_rq:
 434                 requests[id] = i915_request_get(rq);
 435                 i915_request_add(rq);
 436 err_fini:
 437                 intel_renderstate_fini(&so, ce);
 438 err:
 439                 if (err) {
 440                         intel_context_put(ce);
 441                         goto out;
 442                 }
 443         }
 444
 445         /* Flush the default context image to memory, and enable powersaving. */
 446         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
 447                 err = -EIO;
 448                 goto out;
 449         }
 450
 451         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 452                 struct i915_request *rq;
 453                 struct file *state;
 454
 455                 rq = requests[id];
 456                 if (!rq)
 457                         continue;
 458
 459                 if (rq->fence.error) {
 460                         err = -EIO;
 461                         goto out;
 462                 }
 463
 464                 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
 465                 if (!rq->context->state)
 466                         continue;
 467
 468                 /* Keep a copy of the state's backing pages; free the obj */
 469                 state = shmem_create_from_object(rq->context->state->obj);
 470                 if (IS_ERR(state)) {
 471                         err = PTR_ERR(state);
 472                         goto out;
 473                 }
 474                 rq->engine->default_state = state;
 475         }
 476
 477 out:
 478         /*
 479          * If we have to abandon now, we expect the engines to be idle
 480          * and ready to be torn-down. The quickest way we can accomplish
 481          * this is by declaring ourselves wedged.
 482          */
 483         if (err)
 484                 intel_gt_set_wedged(gt);
 485
 486         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 487                 struct intel_context *ce;
 488                 struct i915_request *rq;
 489
 490                 rq = requests[id];
 491                 if (!rq)
 492                         continue;
 493
 494                 ce = rq->context;
 495                 i915_request_put(rq);
 496                 intel_context_put(ce);
 497         }
 498         return err;
 499 }
 500
 501 static int __engines_verify_workarounds(struct intel_gt *gt)
 502 {
 503         struct intel_engine_cs *engine;
 504         enum intel_engine_id id;
 505         int err = 0;
 506
 507         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 508                 return 0;
 509
 510         for_each_engine(engine, gt, id) {
 511                 if (intel_engine_verify_workarounds(engine, "load"))
 512                         err = -EIO;
 513         }
 514
 515         /* Flush and restore the kernel context for safety */
 516         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
 517                 err = -EIO;
 518
 519         return err;
 520 }
 521
 522 static void __intel_gt_disable(struct intel_gt *gt)
 523 {
 524         intel_gt_set_wedged_on_fini(gt);
 525
 526         intel_gt_suspend_prepare(gt);
 527         intel_gt_suspend_late(gt);
 528
 529         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 530 }
 531
 532 int intel_gt_init(struct intel_gt *gt)
 533 {
 534         int err;
 535
 536         err = i915_inject_probe_error(gt->i915, -ENODEV);
 537         if (err)
 538                 return err;
 539
 540         /*
 541          * This is just a security blanket to placate dragons.
 542          * On some systems, we very sporadically observe that the first TLBs
 543          * used by the CS may be stale, despite us poking the TLB reset. If
 544          * we hold the forcewake during initialisation these problems
 545          * just magically go away.
 546          */
 547         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 548
 549         intel_gt_init_clock_frequency(gt);
 550
 551         err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K);
 552         if (err)
 553                 goto out_fw;
 554
 555         intel_gt_pm_init(gt);
 556
 557         gt->vm = kernel_vm(gt);
 558         if (!gt->vm) {
 559                 err = -ENOMEM;
 560                 goto err_pm;
 561         }
 562
 563         err = intel_engines_init(gt);
 564         if (err)
 565                 goto err_engines;
 566
 567         err = intel_uc_init(&gt->uc);
 568         if (err)
 569                 goto err_engines;
 570
 571         err = intel_gt_resume(gt);
 572         if (err)
 573                 goto err_uc_init;
 574
 575         err = __engines_record_defaults(gt);
 576         if (err)
 577                 goto err_gt;
 578
 579         err = __engines_verify_workarounds(gt);
 580         if (err)
 581                 goto err_gt;
 582
 583         err = i915_inject_probe_error(gt->i915, -EIO);
 584         if (err)
 585                 goto err_gt;
 586
 587         goto out_fw;
 588 err_gt:
 589         __intel_gt_disable(gt);
 590         intel_uc_fini_hw(&gt->uc);
 591 err_uc_init:
 592         intel_uc_fini(&gt->uc);
 593 err_engines:
 594         intel_engines_release(gt);
 595         i915_vm_put(fetch_and_zero(&gt->vm));
 596 err_pm:
 597         intel_gt_pm_fini(gt);
 598         intel_gt_fini_scratch(gt);
 599 out_fw:
 600         if (err)
 601                 intel_gt_set_wedged_on_init(gt);
 602         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 603         return err;
 604 }
 605
 606 void intel_gt_driver_remove(struct intel_gt *gt)
 607 {
 608         __intel_gt_disable(gt);
 609
 610         intel_uc_driver_remove(&gt->uc);
 611
 612         intel_engines_release(gt);
 613 }
 614
 615 void intel_gt_driver_unregister(struct intel_gt *gt)
 616 {
 617         intel_wakeref_t wakeref;
 618
 619         intel_rps_driver_unregister(&gt->rps);
 620
 621         /*
 622          * Upon unregistering the device to prevent any new users, cancel
 623          * all in-flight requests so that we can quickly unbind the active
 624          * resources.
 625          */
 626         intel_gt_set_wedged(gt);
 627
 628         /* Scrub all HW state upon release */
 629         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
 630                 __intel_gt_reset(gt, ALL_ENGINES);
 631 }
 632
 633 void intel_gt_driver_release(struct intel_gt *gt)
 634 {
 635         struct i915_address_space *vm;
 636
 637         vm = fetch_and_zero(&gt->vm);
 638         if (vm) /* FIXME being called twice on error paths :( */
 639                 i915_vm_put(vm);
 640
 641         intel_gt_pm_fini(gt);
 642         intel_gt_fini_scratch(gt);
 643         intel_gt_fini_buffer_pool(gt);
 644 }
 645
 646 void intel_gt_driver_late_release(struct intel_gt *gt)
 647 {
 648         /* We need to wait for inflight RCU frees to release their grip */
 649         rcu_barrier();
 650
 651         intel_uc_driver_late_release(&gt->uc);
 652         intel_gt_fini_requests(gt);
 653         intel_gt_fini_reset(gt);
 654         intel_gt_fini_timelines(gt);
 655         intel_engines_free(gt);
 656 }
 657
 658 void intel_gt_info_print(const struct intel_gt_info *info,
 659                          struct drm_printer *p)
 660 {
 661         drm_printf(p, "available engines: %x\n", info->engine_mask);
 662
 663         intel_sseu_dump(&info->sseu, p);
 664 }