drivers/gpu/drm/i915/vlv_suspend.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <linux/kernel.h>
   7
   8 #include <drm/drm_print.h>
   9
  10 #include "i915_drv.h"
  11 #include "i915_reg.h"
  12 #include "i915_trace.h"
  13 #include "i915_utils.h"
  14 #include "intel_pm.h"
  15 #include "vlv_suspend.h"
  16
  17 struct vlv_s0ix_state {
  18         /* GAM */
  19         u32 wr_watermark;
  20         u32 gfx_prio_ctrl;
  21         u32 arb_mode;
  22         u32 gfx_pend_tlb0;
  23         u32 gfx_pend_tlb1;
  24         u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
  25         u32 media_max_req_count;
  26         u32 gfx_max_req_count;
  27         u32 render_hwsp;
  28         u32 ecochk;
  29         u32 bsd_hwsp;
  30         u32 blt_hwsp;
  31         u32 tlb_rd_addr;
  32
  33         /* MBC */
  34         u32 g3dctl;
  35         u32 gsckgctl;
  36         u32 mbctl;
  37
  38         /* GCP */
  39         u32 ucgctl1;
  40         u32 ucgctl3;
  41         u32 rcgctl1;
  42         u32 rcgctl2;
  43         u32 rstctl;
  44         u32 misccpctl;
  45
  46         /* GPM */
  47         u32 gfxpause;
  48         u32 rpdeuhwtc;
  49         u32 rpdeuc;
  50         u32 ecobus;
  51         u32 pwrdwnupctl;
  52         u32 rp_down_timeout;
  53         u32 rp_deucsw;
  54         u32 rcubmabdtmr;
  55         u32 rcedata;
  56         u32 spare2gh;
  57
  58         /* Display 1 CZ domain */
  59         u32 gt_imr;
  60         u32 gt_ier;
  61         u32 pm_imr;
  62         u32 pm_ier;
  63         u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
  64
  65         /* GT SA CZ domain */
  66         u32 tilectl;
  67         u32 gt_fifoctl;
  68         u32 gtlc_wake_ctrl;
  69         u32 gtlc_survive;
  70         u32 pmwgicz;
  71
  72         /* Display 2 CZ domain */
  73         u32 gu_ctl0;
  74         u32 gu_ctl1;
  75         u32 pcbr;
  76         u32 clock_gate_dis2;
  77 };
  78
  79 /*
  80  * Save all Gunit registers that may be lost after a D3 and a subsequent
  81  * S0i[R123] transition. The list of registers needing a save/restore is
  82  * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
  83  * registers in the following way:
  84  * - Driver: saved/restored by the driver
  85  * - Punit : saved/restored by the Punit firmware
  86  * - No, w/o marking: no need to save/restore, since the register is R/O or
  87  *                    used internally by the HW in a way that doesn't depend
  88  *                    keeping the content across a suspend/resume.
  89  * - Debug : used for debugging
  90  *
  91  * We save/restore all registers marked with 'Driver', with the following
  92  * exceptions:
  93  * - Registers out of use, including also registers marked with 'Debug'.
  94  *   These have no effect on the driver's operation, so we don't save/restore
  95  *   them to reduce the overhead.
  96  * - Registers that are fully setup by an initialization function called from
  97  *   the resume path. For example many clock gating and RPS/RC6 registers.
  98  * - Registers that provide the right functionality with their reset defaults.
  99  *
 100  * TODO: Except for registers that based on the above 3 criteria can be safely
 101  * ignored, we save/restore all others, practically treating the HW context as
 102  * a black-box for the driver. Further investigation is needed to reduce the
 103  * saved/restored registers even further, by following the same 3 criteria.
 104  */
 105 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915)
 106 {
 107         struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
 108         struct intel_uncore *uncore = &i915->uncore;
 109         int i;
 110
 111         if (!s)
 112                 return;
 113
 114         /* GAM 0x4000-0x4770 */
 115         s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK);
 116         s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL);
 117         s->arb_mode = intel_uncore_read(uncore, ARB_MODE);
 118         s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0);
 119         s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1);
 120
 121         for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
 122                 s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i));
 123
 124         s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT);
 125         s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT);
 126
 127         s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7);
 128         s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
 129         s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7);
 130         s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7);
 131
 132         s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR);
 133
 134         /* MBC 0x9024-0x91D0, 0x8500 */
 135         s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL);
 136         s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL);
 137         s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL);
 138
 139         /* GCP 0x9400-0x9424, 0x8100-0x810C */
 140         s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1);
 141         s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3);
 142         s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1);
 143         s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2);
 144         s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL);
 145         s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL);
 146
 147         /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
 148         s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE);
 149         s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC);
 150         s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC);
 151         s->ecobus = intel_uncore_read(uncore, ECOBUS);
 152         s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL);
 153         s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT);
 154         s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW);
 155         s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR);
 156         s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA);
 157         s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H);
 158
 159         /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
 160         s->gt_imr = intel_uncore_read(uncore, GTIMR);
 161         s->gt_ier = intel_uncore_read(uncore, GTIER);
 162         s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
 163         s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
 164
 165         for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
 166                 s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i));
 167
 168         /* GT SA CZ domain, 0x100000-0x138124 */
 169         s->tilectl = intel_uncore_read(uncore, TILECTL);
 170         s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL);
 171         s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
 172         s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
 173         s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ);
 174
 175         /* Gunit-Display CZ domain, 0x182028-0x1821CF */
 176         s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0);
 177         s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1);
 178         s->pcbr = intel_uncore_read(uncore, VLV_PCBR);
 179         s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2);
 180
 181         /*
 182          * Not saving any of:
 183          * DFT,         0x9800-0x9EC0
 184          * SARB,        0xB000-0xB1FC
 185          * GAC,         0x5208-0x524C, 0x14000-0x14C000
 186          * PCI CFG
 187          */
 188 }
 189
 190 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915)
 191 {
 192         struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
 193         struct intel_uncore *uncore = &i915->uncore;
 194         u32 val;
 195         int i;
 196
 197         if (!s)
 198                 return;
 199
 200         /* GAM 0x4000-0x4770 */
 201         intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark);
 202         intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
 203         intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16));
 204         intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
 205         intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
 206
 207         for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
 208                 intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]);
 209
 210         intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
 211         intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
 212
 213         intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp);
 214         intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk);
 215         intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp);
 216         intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp);
 217
 218         intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
 219
 220         /* MBC 0x9024-0x91D0, 0x8500 */
 221         intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl);
 222         intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl);
 223         intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl);
 224
 225         /* GCP 0x9400-0x9424, 0x8100-0x810C */
 226         intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1);
 227         intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3);
 228         intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1);
 229         intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2);
 230         intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl);
 231         intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl);
 232
 233         /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
 234         intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause);
 235         intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc);
 236         intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc);
 237         intel_uncore_write(uncore, ECOBUS, s->ecobus);
 238         intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl);
 239         intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout);
 240         intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw);
 241         intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr);
 242         intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata);
 243         intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh);
 244
 245         /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
 246         intel_uncore_write(uncore, GTIMR, s->gt_imr);
 247         intel_uncore_write(uncore, GTIER, s->gt_ier);
 248         intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr);
 249         intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier);
 250
 251         for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
 252                 intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
 253
 254         /* GT SA CZ domain, 0x100000-0x138124 */
 255         intel_uncore_write(uncore, TILECTL, s->tilectl);
 256         intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl);
 257         /*
 258          * Preserve the GT allow wake and GFX force clock bit, they are not
 259          * be restored, as they are used to control the s0ix suspend/resume
 260          * sequence by the caller.
 261          */
 262         val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
 263         val &= VLV_GTLC_ALLOWWAKEREQ;
 264         val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
 265         intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val);
 266
 267         val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
 268         val &= VLV_GFX_CLK_FORCE_ON_BIT;
 269         val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
 270         intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val);
 271
 272         intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz);
 273
 274         /* Gunit-Display CZ domain, 0x182028-0x1821CF */
 275         intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0);
 276         intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1);
 277         intel_uncore_write(uncore, VLV_PCBR, s->pcbr);
 278         intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
 279 }
 280
 281 static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
 282                                   u32 mask, u32 val)
 283 {
 284         i915_reg_t reg = VLV_GTLC_PW_STATUS;
 285         u32 reg_value;
 286         int ret;
 287
 288         /* The HW does not like us polling for PW_STATUS frequently, so
 289          * use the sleeping loop rather than risk the busy spin within
 290          * intel_wait_for_register().
 291          *
 292          * Transitioning between RC6 states should be at most 2ms (see
 293          * valleyview_enable_rps) so use a 3ms timeout.
 294          */
 295         ret = wait_for(((reg_value =
 296                          intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
 297                        == val, 3);
 298
 299         /* just trace the final value */
 300         trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
 301
 302         return ret;
 303 }
 304
 305 static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on)
 306 {
 307         struct intel_uncore *uncore = &i915->uncore;
 308         u32 val;
 309         int err;
 310
 311         val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
 312         val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
 313         if (force_on)
 314                 val |= VLV_GFX_CLK_FORCE_ON_BIT;
 315         intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val);
 316
 317         if (!force_on)
 318                 return 0;
 319
 320         err = intel_wait_for_register(uncore,
 321                                       VLV_GTLC_SURVIVABILITY_REG,
 322                                       VLV_GFX_CLK_STATUS_BIT,
 323                                       VLV_GFX_CLK_STATUS_BIT,
 324                                       20);
 325         if (err)
 326                 drm_err(&i915->drm,
 327                         "timeout waiting for GFX clock force-on (%08x)\n",
 328                         intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG));
 329
 330         return err;
 331 }
 332
 333 static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow)
 334 {
 335         struct intel_uncore *uncore = &i915->uncore;
 336         u32 mask;
 337         u32 val;
 338         int err;
 339
 340         val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
 341         val &= ~VLV_GTLC_ALLOWWAKEREQ;
 342         if (allow)
 343                 val |= VLV_GTLC_ALLOWWAKEREQ;
 344         intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val);
 345         intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL);
 346
 347         mask = VLV_GTLC_ALLOWWAKEACK;
 348         val = allow ? mask : 0;
 349
 350         err = vlv_wait_for_pw_status(i915, mask, val);
 351         if (err)
 352                 drm_err(&i915->drm, "timeout disabling GT waking\n");
 353
 354         return err;
 355 }
 356
 357 static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
 358                                   bool wait_for_on)
 359 {
 360         u32 mask;
 361         u32 val;
 362
 363         mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
 364         val = wait_for_on ? mask : 0;
 365
 366         /*
 367          * RC6 transitioning can be delayed up to 2 msec (see
 368          * valleyview_enable_rps), use 3 msec for safety.
 369          *
 370          * This can fail to turn off the rc6 if the GPU is stuck after a failed
 371          * reset and we are trying to force the machine to sleep.
 372          */
 373         if (vlv_wait_for_pw_status(dev_priv, mask, val))
 374                 drm_dbg(&dev_priv->drm,
 375                         "timeout waiting for GT wells to go %s\n",
 376                         onoff(wait_for_on));
 377 }
 378
 379 static void vlv_check_no_gt_access(struct drm_i915_private *i915)
 380 {
 381         struct intel_uncore *uncore = &i915->uncore;
 382
 383         if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
 384                 return;
 385
 386         drm_dbg(&i915->drm, "GT register access while GT waking disabled\n");
 387         intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
 388 }
 389
 390 int vlv_suspend_complete(struct drm_i915_private *dev_priv)
 391 {
 392         u32 mask;
 393         int err;
 394
 395         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
 396                 return 0;
 397
 398         /*
 399          * Bspec defines the following GT well on flags as debug only, so
 400          * don't treat them as hard failures.
 401          */
 402         vlv_wait_for_gt_wells(dev_priv, false);
 403
 404         mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
 405         drm_WARN_ON(&dev_priv->drm,
 406                     (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask);
 407
 408         vlv_check_no_gt_access(dev_priv);
 409
 410         err = vlv_force_gfx_clock(dev_priv, true);
 411         if (err)
 412                 goto err1;
 413
 414         err = vlv_allow_gt_wake(dev_priv, false);
 415         if (err)
 416                 goto err2;
 417
 418         vlv_save_gunit_s0ix_state(dev_priv);
 419
 420         err = vlv_force_gfx_clock(dev_priv, false);
 421         if (err)
 422                 goto err2;
 423
 424         return 0;
 425
 426 err2:
 427         /* For safety always re-enable waking and disable gfx clock forcing */
 428         vlv_allow_gt_wake(dev_priv, true);
 429 err1:
 430         vlv_force_gfx_clock(dev_priv, false);
 431
 432         return err;
 433 }
 434
 435 int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
 436 {
 437         int err;
 438         int ret;
 439
 440         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
 441                 return 0;
 442
 443         /*
 444          * If any of the steps fail just try to continue, that's the best we
 445          * can do at this point. Return the first error code (which will also
 446          * leave RPM permanently disabled).
 447          */
 448         ret = vlv_force_gfx_clock(dev_priv, true);
 449
 450         vlv_restore_gunit_s0ix_state(dev_priv);
 451
 452         err = vlv_allow_gt_wake(dev_priv, true);
 453         if (!ret)
 454                 ret = err;
 455
 456         err = vlv_force_gfx_clock(dev_priv, false);
 457         if (!ret)
 458                 ret = err;
 459
 460         vlv_check_no_gt_access(dev_priv);
 461
 462         if (rpm_resume)
 463                 intel_init_clock_gating(dev_priv);
 464
 465         return ret;
 466 }
 467
 468 int vlv_suspend_init(struct drm_i915_private *i915)
 469 {
 470         if (!IS_VALLEYVIEW(i915))
 471                 return 0;
 472
 473         /* we write all the values in the struct, so no need to zero it out */
 474         i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
 475                                        GFP_KERNEL);
 476         if (!i915->vlv_s0ix_state)
 477                 return -ENOMEM;
 478
 479         return 0;
 480 }
 481
 482 void vlv_suspend_cleanup(struct drm_i915_private *i915)
 483 {
 484         if (!i915->vlv_s0ix_state)
 485                 return;
 486
 487         kfree(i915->vlv_s0ix_state);
 488         i915->vlv_s0ix_state = NULL;
 489 }